diff --git a/src/atlas/CMakeLists.txt b/src/atlas/CMakeLists.txt index bc2be2afe..1ea3b58dd 100644 --- a/src/atlas/CMakeLists.txt +++ b/src/atlas/CMakeLists.txt @@ -706,6 +706,8 @@ linalg/sparse.h linalg/sparse/Backend.h linalg/sparse/Backend.cc linalg/sparse/MakeEckitSparseMatrix.h +linalg/sparse/MakeSparseMatrixStorageEckit.h +linalg/sparse/MakeSparseMatrixStorageEigen.h linalg/sparse/SparseMatrixMultiply.h linalg/sparse/SparseMatrixMultiply.tcc linalg/sparse/SparseMatrixMultiply_EckitLinalg.h @@ -714,8 +716,7 @@ linalg/sparse/SparseMatrixMultiply_OpenMP.h linalg/sparse/SparseMatrixMultiply_OpenMP.cc linalg/sparse/SparseMatrixStorage.cc linalg/sparse/SparseMatrixStorage.h -linalg/sparse/SparseMatrixConvertor.h -linalg/sparse/SparseMatrixAdaptor.h +linalg/sparse/SparseMatrixView.h linalg/dense.h linalg/dense/Backend.h linalg/dense/Backend.cc diff --git a/src/atlas/interpolation/method/Method.cc b/src/atlas/interpolation/method/Method.cc index 189485528..a11fc1c65 100644 --- a/src/atlas/interpolation/method/Method.cc +++ b/src/atlas/interpolation/method/Method.cc @@ -270,7 +270,7 @@ void Method::setup(const FunctionSpace& source, const FunctionSpace& target) { if (not matrix_->empty()) { eckit::linalg::SparseMatrix matrix_copy = make_eckit_sparse_matrix(*matrix_); // Makes a copy! matrix_copy.transpose(); // transpose the copy in place - matrix_transpose_ = atlas::linalg::SparseMatrixStorage(std::move(matrix_copy)); // Move the copy into storage + matrix_transpose_ = linalg::make_sparse_matrix_storage(std::move(matrix_copy)); // Move the copy into storage } } } diff --git a/src/atlas/interpolation/method/Method.h b/src/atlas/interpolation/method/Method.h index 8b1fef295..994ad9f7c 100644 --- a/src/atlas/interpolation/method/Method.h +++ b/src/atlas/interpolation/method/Method.h @@ -21,6 +21,7 @@ #include "atlas/util/Object.h" #include "eckit/config/Configuration.h" #include "eckit/linalg/SparseMatrix.h" +#include "atlas/linalg/sparse/MakeSparseMatrixStorageEckit.h" namespace atlas { class Field; @@ -122,7 +123,7 @@ class Method : public util::Object { } void setMatrix(eckit::linalg::SparseMatrix&& m, const std::string& uid = "") { - setMatrix( Matrix(std::move(m)), uid ); + setMatrix( linalg::make_sparse_matrix_storage(std::move(m)), uid ); } void setMatrix(std::size_t rows, std::size_t cols, const Triplets& triplets, const std::string& uid = "") { diff --git a/src/atlas/linalg/sparse.h b/src/atlas/linalg/sparse.h index b2228335a..4c6ebc02e 100644 --- a/src/atlas/linalg/sparse.h +++ b/src/atlas/linalg/sparse.h @@ -11,10 +11,10 @@ #include "sparse/Backend.h" #include "sparse/MakeEckitSparseMatrix.h" -#include "sparse/SparseMatrixAdaptor.h" -#include "sparse/SparseMatrixConvertor.h" +#include "sparse/MakeSparseMatrixStorageEckit.h" #include "sparse/SparseMatrixMultiply.h" #include "sparse/SparseMatrixStorage.h" +#include "sparse/SparseMatrixView.h" namespace atlas { namespace linalg {} // namespace linalg diff --git a/src/atlas/linalg/sparse/MakeSparseMatrixStorageEckit.h b/src/atlas/linalg/sparse/MakeSparseMatrixStorageEckit.h new file mode 100644 index 000000000..fffbf9eeb --- /dev/null +++ b/src/atlas/linalg/sparse/MakeSparseMatrixStorageEckit.h @@ -0,0 +1,135 @@ +/* + * (C) Copyright 2024- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include +#include +#include +#include + +#include "eckit/linalg/SparseMatrix.h" + +#include "atlas/array.h" +#include "atlas/linalg/sparse/SparseMatrixStorage.h" + +namespace atlas { +namespace linalg { + +//---------------------------------------------------------------------------------------------------------------------- + +template +void host_copy_eckit (const InputT* input_data, array::Array& output) { + auto size = output.size(); + OutputT* output_data = output.host_data(); + std::copy( input_data, input_data + size, output_data ); +} + +inline SparseMatrixStorage make_sparse_matrix_storage(eckit::linalg::SparseMatrix&& m) { + using Index = eckit::linalg::Index; + using Value = eckit::linalg::Scalar; + std::cout << "make_sparse_matrix_storage move" << std::endl; + + std::size_t rows = m.rows(); + std::size_t cols = m.cols(); + std::size_t nnz = m.nonZeros(); + auto* outer = array::Array::wrap(const_cast(m.outer()), array::make_shape(rows+1)); + auto* inner = array::Array::wrap(const_cast(m.inner()), array::make_shape(nnz)); + auto* value = array::Array::wrap(const_cast(m.data()), array::make_shape(nnz)); + + // We now move the eckit::linalg::SparseMatrix into a generic storage so + // the wrapped array data does not go out of scope + auto storage = std::make_any(std::move(m)); + + return SparseMatrixStorage::make( rows, cols, nnz, + std::unique_ptr(value), + std::unique_ptr(inner), + std::unique_ptr(outer), + std::move(storage) + ); +} + +template< typename value_type, typename index_type = eckit::linalg::Index> +SparseMatrixStorage make_sparse_matrix_storage(eckit::linalg::SparseMatrix&& m) { + std::cout << "make_sparse_matrix_storage move or convert" << std::endl; + + if (std::is_same_v && std::is_same_v) { + return make_sparse_matrix_storage(std::move(m)); + } + std::size_t rows = m.rows(); + std::size_t cols = m.cols(); + std::size_t nnz = m.nonZeros(); + auto* outer = array::Array::create(rows+1); + auto* inner = array::Array::create(nnz); + auto* value = array::Array::create(nnz); + + host_copy_eckit(m.outer(), *outer); + host_copy_eckit(m.inner(), *inner); + host_copy_eckit(m.data(), *value); + + return SparseMatrixStorage::make( rows, cols, nnz, + std::unique_ptr(value), + std::unique_ptr(inner), + std::unique_ptr(outer), + std::any() + ); +} + + +inline SparseMatrixStorage make_sparse_matrix_storage(const eckit::linalg::SparseMatrix& m) { + using Index = eckit::linalg::Index; + using Value = eckit::linalg::Scalar; + + std::cout << "make_sparse_matrix_storage copy" << std::endl; + std::size_t rows = m.rows(); + std::size_t cols = m.cols(); + std::size_t nnz = m.nonZeros(); + auto* outer = array::Array::create(rows+1); + auto* inner = array::Array::create(nnz); + auto* value = array::Array::create(nnz); + + host_copy_eckit(m.outer(), *outer); + host_copy_eckit(m.inner(), *inner); + host_copy_eckit(m.data(), *value); + + return SparseMatrixStorage::make( rows, cols, nnz, + std::unique_ptr(value), + std::unique_ptr(inner), + std::unique_ptr(outer), + std::any() + ); +} + +template< typename value_type, typename index_type = eckit::linalg::Index> +SparseMatrixStorage make_sparse_matrix_storage(const eckit::linalg::SparseMatrix& m) { + std::cout << "make_sparse_matrix_storage copy and convert" << std::endl; + + std::size_t rows = m.rows(); + std::size_t cols = m.cols(); + std::size_t nnz = m.nonZeros(); + auto* outer = array::Array::create(rows+1); + auto* inner = array::Array::create(nnz); + auto* value = array::Array::create(nnz); + host_copy_eckit(m.outer(), *outer); + host_copy_eckit(m.inner(), *inner); + host_copy_eckit(m.data(), *value); + + return SparseMatrixStorage::make( rows, cols, nnz, + std::unique_ptr(value), + std::unique_ptr(inner), + std::unique_ptr(outer), + std::any() + ); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace linalg +} // namespace atlas \ No newline at end of file diff --git a/src/atlas/linalg/sparse/MakeSparseMatrixStorageEigen.h b/src/atlas/linalg/sparse/MakeSparseMatrixStorageEigen.h new file mode 100644 index 000000000..f306e85f0 --- /dev/null +++ b/src/atlas/linalg/sparse/MakeSparseMatrixStorageEigen.h @@ -0,0 +1,142 @@ +/* + * (C) Copyright 2024- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#pragma once + +#include +#include +#include +#include + +#include "atlas/library/defines.h" +#if ATLAS_HAVE_EIGEN +#include +#endif + +#include "atlas/array.h" +#include "atlas/linalg/sparse/SparseMatrixStorage.h" + +namespace atlas { +namespace linalg { + +//---------------------------------------------------------------------------------------------------------------------- + +#if ATLAS_HAVE_EIGEN + +template +void host_copy_eigen (const InputT* input_data, atlas::array::Array& output) { + auto size = output.size(); + OutputT* output_data = output.host_data(); + std::copy( input_data, input_data + size, output_data ); +} + +template +SparseMatrixStorage make_sparse_matrix_storage(Eigen::SparseMatrix&& m) { + std::cout << "make_sparse_matrix_storage move" << std::endl; + + std::size_t rows = m.rows(); + std::size_t cols = m.cols(); + std::size_t nnz = m.nonZeros(); + auto* outer = atlas::array::Array::wrap(const_cast(m.outerIndexPtr()), atlas::array::make_shape(rows+1)); + auto* inner = atlas::array::Array::wrap(const_cast(m.innerIndexPtr()), atlas::array::make_shape(nnz)); + auto* value = atlas::array::Array::wrap(const_cast(m.valuePtr()), atlas::array::make_shape(nnz)); + + using EigenMatrix = Eigen::SparseMatrix; + auto m_ptr = std::make_shared(); + m_ptr->swap(m); + auto storage = std::make_any>(std::move(m_ptr)); + + return SparseMatrixStorage::make( rows, cols, nnz, + std::unique_ptr(value), + std::unique_ptr(inner), + std::unique_ptr(outer), + std::move(storage) + ); +} + +template< typename value_type, typename index_type = eckit::linalg::Index, typename Value, typename Index, + typename = std::enable_if_t < !std::is_same_v>> +SparseMatrixStorage make_sparse_matrix_storage(Eigen::SparseMatrix&& m) { + std::cout << "make_sparse_matrix_storage move or convert" << std::endl; + + if (std::is_same_v && std::is_same_v) { + return make_sparse_matrix_storage(std::move(m)); + } + std::size_t rows = m.rows(); + std::size_t cols = m.cols(); + std::size_t nnz = m.nonZeros(); + auto* outer = atlas::array::Array::create(rows+1); + auto* inner = atlas::array::Array::create(nnz); + auto* value = atlas::array::Array::create(nnz); + + host_copy_eigen(m.outerIndexPtr(), *outer); + host_copy_eigen(m.innerIndexPtr(), *inner); + host_copy_eigen(m.valuePtr(), *value); + + return SparseMatrixStorage::make( rows, cols, nnz, + std::unique_ptr(value), + std::unique_ptr(inner), + std::unique_ptr(outer), + std::any() + ); +} + + +template +SparseMatrixStorage make_sparse_matrix_storage(const Eigen::SparseMatrix& m) { + std::cout << "make_sparse_matrix_storage copy" << std::endl; + std::size_t rows = m.rows(); + std::size_t cols = m.cols(); + std::size_t nnz = m.nonZeros(); + auto* outer = atlas::array::Array::create(rows+1); + auto* inner = atlas::array::Array::create(nnz); + auto* value = atlas::array::Array::create(nnz); + + host_copy_eigen(m.outerIndexPtr(), *outer); + host_copy_eigen(m.innerIndexPtr(), *inner); + host_copy_eigen(m.valuePtr(), *value); + + return SparseMatrixStorage::make( rows, cols, nnz, + std::unique_ptr(value), + std::unique_ptr(inner), + std::unique_ptr(outer), + std::any() + ); +} + +template< typename value_type, typename index_type = eckit::linalg::Index, typename Value, typename Index, + typename = std::enable_if_t < !std::is_same_v>> +SparseMatrixStorage make_sparse_matrix_storage(const Eigen::SparseMatrix& m) { + std::cout << "make_sparse_matrix_storage copy and convert" << std::endl; + + std::size_t rows = m.rows(); + std::size_t cols = m.cols(); + std::size_t nnz = m.nonZeros(); + auto* outer = atlas::array::Array::create(rows+1); + auto* inner = atlas::array::Array::create(nnz); + auto* value = atlas::array::Array::create(nnz); + host_copy_eigen(m.outerIndexPtr(), *outer); + host_copy_eigen(m.innerIndexPtr(), *inner); + host_copy_eigen(m.valuePtr(), *value); + + return SparseMatrixStorage::make( rows, cols, nnz, + std::unique_ptr(value), + std::unique_ptr(inner), + std::unique_ptr(outer), + std::any() + ); +} + +#endif + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace linalg +} // namespace atlas \ No newline at end of file diff --git a/src/atlas/linalg/sparse/SparseMatrixAdaptor.h b/src/atlas/linalg/sparse/SparseMatrixAdaptor.h deleted file mode 100644 index d4f1baa39..000000000 --- a/src/atlas/linalg/sparse/SparseMatrixAdaptor.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * (C) Copyright 2024- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - -#pragma once - -#include "atlas/library/defines.h" -#if ATLAS_HAVE_EIGEN -#include -#endif -#include "eckit/linalg/SparseMatrix.h" - -#include "atlas/array.h" - -namespace atlas { -namespace linalg { - -//---------------------------------------------------------------------------------------------------------------------- - -/// @brief Wrap existing sparse matrix formats into SparseMatrixStorage. -/// This means that the host_data is not owned, but device_data is owned -class SparseMatrixAdaptor : public SparseMatrixStorage { -public: - SparseMatrixAdaptor(const eckit::linalg::SparseMatrix& m) { - nnz_ = m.nonZeros(); - rows_ = m.rows(); - cols_ = m.cols(); - outer_.reset(atlas::array::Array::wrap(const_cast(m.outer()), atlas::array::make_shape(rows_+1))); - inner_.reset(atlas::array::Array::wrap(const_cast(m.inner()), atlas::array::make_shape(nnz_))); - value_.reset(atlas::array::Array::wrap(const_cast(m.data()), atlas::array::make_shape(nnz_))); - } - -#if ATLAS_HAVE_EIGEN - template - SparseMatrixAdaptor(const Eigen::SparseMatrix& m) { - nnz_ = m.nonZeros(); - rows_ = m.rows(); - cols_ = m.cols(); - outer_.reset(atlas::array::Array::wrap(const_cast*>(m.outerIndexPtr()), atlas::array::make_shape(rows_+1))); - inner_.reset(atlas::array::Array::wrap(const_cast*>(m.innerIndexPtr()), atlas::array::make_shape(nnz_))); - value_.reset(atlas::array::Array::wrap(const_cast*>(m.valuePtr()), atlas::array::make_shape(nnz_))); - } -#endif -}; - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace linalg -} // namespace atlas \ No newline at end of file diff --git a/src/atlas/linalg/sparse/SparseMatrixConvertor.h b/src/atlas/linalg/sparse/SparseMatrixConvertor.h deleted file mode 100644 index 05950e635..000000000 --- a/src/atlas/linalg/sparse/SparseMatrixConvertor.h +++ /dev/null @@ -1,168 +0,0 @@ -/* - * (C) Copyright 2024- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation - * nor does it submit to any jurisdiction. - */ - -#pragma once - -#include -#include - -#include "atlas/library/defines.h" -#if ATLAS_HAVE_EIGEN -#include -#endif -#include "eckit/linalg/SparseMatrix.h" - -#include "atlas/array.h" - -namespace atlas { -namespace linalg { - -//---------------------------------------------------------------------------------------------------------------------- - -template -struct SparseMatrixConvertor : public SparseMatrixStorage { - using value_type = Value; - using index_type = Index; - -private: - - template - void host_copy(const InputT* input_data, atlas::array::Array& output) { - auto size = output.size(); - OutputT* output_data = output.host_data(); - std::copy( input_data, input_data + size, output_data ); - } - - template - void host_copy(const atlas::array::Array& input, atlas::array::Array& output) { - host_copy( input.host_data(), output ); - } - -public: - virtual ~SparseMatrixConvertor() = default; - - SparseMatrixConvertor() = default; - - explicit SparseMatrixConvertor(eckit::linalg::SparseMatrix&& m) { - nnz_ = m.nonZeros(); - rows_ = m.rows(); - cols_ = m.cols(); - - if constexpr(std::is_same_v && std::is_same_v) { - outer_.reset(atlas::array::Array::wrap(const_cast(m.outer()), atlas::array::make_shape(rows_+1))); - inner_.reset(atlas::array::Array::wrap(const_cast(m.inner()), atlas::array::make_shape(nnz_))); - value_.reset(atlas::array::Array::wrap(const_cast(m.data()), atlas::array::make_shape(nnz_))); - - // We now move the eckit::linalg::SparseMatrix into a generic storage so - // the wrapped array data does not go out of scope - storage_ = std::make_any(std::move(m)); - } - else { - outer_.reset(atlas::array::Array::create(rows_+1)); - inner_.reset(atlas::array::Array::create(nnz_)); - value_.reset(atlas::array::Array::create(nnz_)); - host_copy(m.outer(), *outer_); - host_copy(m.inner(), *inner_); - host_copy(m.data(), *value_); - } - } - - - /// Create copy of eckit::linalg::SparseMatrix - explicit SparseMatrixConvertor(const eckit::linalg::SparseMatrix& other) { - nnz_ = other.nonZeros(); - rows_ = other.rows(); - cols_ = other.cols(); - outer_.reset(atlas::array::Array::create(rows_+1)); - inner_.reset(atlas::array::Array::create(nnz_)); - value_.reset(atlas::array::Array::create(nnz_)); - host_copy(other.outer(), *outer_); - host_copy(other.inner(), *inner_); - host_copy(other.data(), *value_); - } - - - template - SparseMatrixConvertor(const SparseMatrixConvertor& other) { - nnz_ = other.nnz(); - rows_ = other.rows(); - cols_ = other.cols(); - outer_.reset(atlas::array::Array::create(rows_+1)); - inner_.reset(atlas::array::Array::create(nnz_)); - value_.reset(atlas::array::Array::create(nnz_)); - host_copy(other.outer(), *outer_); - host_copy(other.inner(), *inner_); - host_copy(other.value(), *value_); - } - - SparseMatrixConvertor(SparseMatrixConvertor&& other) { - nnz_ = other.nnz_; - rows_ = other.rows_; - cols_ = other.cols_; - outer_ = std::move(other.outer_); - inner_ = std::move(other.inner_); - value_ = std::move(other.value_); - other.nnz_ = 0; - other.rows_ = 0; - other.cols_ = 0; - storage_ = std::move(other.storage_); - } - -#if ATLAS_HAVE_EIGEN - template - SparseMatrixConvertor(const Eigen::SparseMatrix& m) { - nnz_ = m.nonZeros(); - rows_ = m.rows(); - cols_ = m.cols(); - outer_.reset(atlas::array::Array::create(rows_+1)); - inner_.reset(atlas::array::Array::create(nnz_)); - value_.reset(atlas::array::Array::create(nnz_)); - host_copy(m.outerIndexPtr(), *outer_); - host_copy(m.innerIndexPtr(), *inner_); - host_copy(m.valuePtr(), *value_); - } - - template - SparseMatrixConvertor(Eigen::SparseMatrix&& m) { - nnz_ = m.nonZeros(); - rows_ = m.rows(); - cols_ = m.cols(); - - if constexpr(std::is_same_v, std::decay_t> && std::is_same_v, std::decay_t>) { - - outer_.reset(atlas::array::Array::wrap(const_cast(m.outerIndexPtr()), atlas::array::make_shape(rows_+1))); - inner_.reset(atlas::array::Array::wrap(const_cast(m.innerIndexPtr()), atlas::array::make_shape(nnz_))); - value_.reset(atlas::array::Array::wrap(const_cast(m.valuePtr()), atlas::array::make_shape(nnz_))); - - // We now move the eckit::linalg::SparseMatrix into a generic storage so - // the wrapped array data does not go out of scope - // Note: Eigen move constructor only available since 3.5; use swap instead. - using EigenMatrix = Eigen::SparseMatrix; - auto m_ptr = std::make_shared(); - m_ptr->swap(m); - storage_ = std::make_any>(std::move(m_ptr)); - } - else { - outer_.reset(atlas::array::Array::create(rows_+1)); - inner_.reset(atlas::array::Array::create(nnz_)); - value_.reset(atlas::array::Array::create(nnz_)); - host_copy(m.outerIndexPtr(), *outer_); - host_copy(m.innerIndexPtr(), *inner_); - host_copy(m.valuePtr(), *value_); - } - } -#endif - -}; - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace linalg -} // namespace atlas \ No newline at end of file diff --git a/src/atlas/linalg/sparse/SparseMatrixStorage.cc b/src/atlas/linalg/sparse/SparseMatrixStorage.cc index c3634570e..2e956558a 100644 --- a/src/atlas/linalg/sparse/SparseMatrixStorage.cc +++ b/src/atlas/linalg/sparse/SparseMatrixStorage.cc @@ -40,32 +40,6 @@ SparseMatrixStorage::SparseMatrixStorage(const SparseMatrixStorage& other) { value_->copy(*other.value_); } -SparseMatrixStorage::SparseMatrixStorage(eckit::linalg::SparseMatrix&& m) { - nnz_ = m.nonZeros(); - rows_ = m.rows(); - cols_ = m.cols(); - outer_.reset(atlas::array::Array::wrap(const_cast(m.outer()), atlas::array::make_shape(rows_+1))); - inner_.reset(atlas::array::Array::wrap(const_cast(m.inner()), atlas::array::make_shape(nnz_))); - value_.reset(atlas::array::Array::wrap(const_cast(m.data()), atlas::array::make_shape(nnz_))); - - // We now move the eckit::linalg::SparseMatrix into a generic storage so - // the wrapped array data does not go out of scope - storage_ = std::make_any(std::move(m)); -} - -/// Create copy of eckit::linalg::SparseMatrix -SparseMatrixStorage::SparseMatrixStorage(const eckit::linalg::SparseMatrix& other) { - nnz_ = other.nonZeros(); - rows_ = other.rows(); - cols_ = other.cols(); - outer_.reset(atlas::array::Array::create(rows_+1)); - inner_.reset(atlas::array::Array::create(nnz_)); - value_.reset(atlas::array::Array::create(nnz_)); - host_copy(other.outer(), *outer_); - host_copy(other.inner(), *inner_); - host_copy(other.data(), *value_); -} - SparseMatrixStorage& SparseMatrixStorage::operator=(SparseMatrixStorage&& other) { nnz_ = other.nnz_; rows_ = other.rows_; diff --git a/src/atlas/linalg/sparse/SparseMatrixStorage.h b/src/atlas/linalg/sparse/SparseMatrixStorage.h index 73157eaad..624c09664 100644 --- a/src/atlas/linalg/sparse/SparseMatrixStorage.h +++ b/src/atlas/linalg/sparse/SparseMatrixStorage.h @@ -13,11 +13,8 @@ #include #include #include +#include -#include "atlas/library/defines.h" -#if ATLAS_HAVE_EIGEN -#include -#endif #include "eckit/linalg/SparseMatrix.h" #include "atlas/array.h" @@ -39,31 +36,31 @@ namespace linalg { /// /// To construct it, taking ownership of a constructed Eigen::SparseMatrix /// -/// SparseMatrixStorage s {std::move(eigen_matrix)}; +/// SparseMatrixStorage s = make_sparse_matrix_storage(std::move(eigen_matrix)); /// /// To construct it, taking a copy of a constructed Eigen::SparseMatrix /// -/// SparseMatrixStorage s {eigen_matrix}; +/// SparseMatrixStorage s = make_sparse_matrix_storage(eigen_matrix); /// /// To construct it, taking ownership of a constructed eckit::linalg::SparseMatrix, avoiding copies if data types match /// -/// SparseMatrixStorage s {std::move(eckit_matrix)}; +/// SparseMatrixStorage s = make_sparse_matrix_storage(std::move(eckit_matrix)); /// /// /// To construct it, taking a copy of a constructed eckit::linalg::SparseMatrix (no std::move) /// -/// SparseMatrixStorage s {eckit_matrix}; +/// SparseMatrixStorage s = make_sparse_matrix_storage(eckit_matrix); /// /// /// It is also possible to initialise empty and move into it at later stage: /// /// SparseMatrixStorage s; -/// s = SparseMatrixStorage{eckit_matrix}; +/// s = make_sparse_matrix_storage(eckit_matrix); /// /// /// To construct it, taking a single precision copy of a constructed eckit::linalg::SparseMatrix /// -/// SparseMatrixStorage s {SparseMatrixConvertor{eckit_matrix}}; +/// s = make_sparse_matrix_storage(eckit_matrix); /// class SparseMatrixStorage { @@ -85,20 +82,6 @@ class SparseMatrixStorage { template SparseMatrixStorage(const SparseMatrixView& host_view); - /// Move constructor from eckit::linalg::SparseMatrix, takes ownership! - explicit SparseMatrixStorage(eckit::linalg::SparseMatrix&& eckit_matrix); - - /// Copy constructor from eckit::linalg::SparseMatrix, makes copy! - explicit SparseMatrixStorage(const eckit::linalg::SparseMatrix& eckit_matrix); - - /// Move constructor from Eigen::SparseMatrix, takes ownership! - template - explicit SparseMatrixStorage(Eigen::SparseMatrix&& eigen_matrix); - - /// Copy constructor from Eigen::SparseMatrix, makes copy! - template - explicit SparseMatrixStorage(const Eigen::SparseMatrix& eigen_matrix); - /// Move assign from other SparseMatrixStorage, takes ownership! SparseMatrixStorage& operator=(SparseMatrixStorage&& other); @@ -135,6 +118,47 @@ class SparseMatrixStorage { void allocateDevice() const; void deallocateDevice() const; + + static SparseMatrixStorage make( + std::size_t rows, + std::size_t cols, + std::size_t nnz, + std::unique_ptr&& value, + std::unique_ptr&& inner, + std::unique_ptr&& outer, + std::any&& storage) { + SparseMatrixStorage S; + S.rows_ = rows; + S.cols_ = cols; + S.nnz_ = nnz; + S.outer_ = std::move(outer); + S.inner_ = std::move(inner); + S.value_ = std::move(value); + S.storage_ = std::move(storage); + return S; + } + + void swap(SparseMatrixStorage& other) { + std::swap(other.rows_, rows_); + std::swap(other.cols_, cols_); + std::swap(other.nnz_, nnz_); + std::swap(other.value_, value_); + std::swap(other.inner_, inner_); + std::swap(other.outer_, outer_); + std::swap(other.storage_, storage_); + } + + bool contains(DataType value_type, DataType index_type) { + if (empty()) { + return false; + } + return value_->datatype() == value_type && outer_->datatype() == index_type; + } + + template + bool contains() { + return contains(make_datatype(), make_datatype()); + } protected: std::size_t nnz_{0}; @@ -175,40 +199,80 @@ SparseMatrixStorage::SparseMatrixStorage(const SparseMatrixView& ho //---------------------------------------------------------------------------------------------------------------------- -#if ATLAS_HAVE_EIGEN -template -SparseMatrixStorage::SparseMatrixStorage(Eigen::SparseMatrix&& m) { - nnz_ = m.nonZeros(); - rows_ = m.rows(); - cols_ = m.cols(); - - outer_.reset(atlas::array::Array::wrap(const_cast(m.outerIndexPtr()), atlas::array::make_shape(rows_+1))); - inner_.reset(atlas::array::Array::wrap(const_cast(m.innerIndexPtr()), atlas::array::make_shape(nnz_))); - value_.reset(atlas::array::Array::wrap(const_cast(m.valuePtr()), atlas::array::make_shape(nnz_))); - - // We now move the eckit::linalg::SparseMatrix into a generic storage so - // the wrapped array data does not go out of scope - // Note: Eigen move constructor only available since 3.5; use swap instead. - using EigenMatrix = Eigen::SparseMatrix; - auto m_ptr = std::make_shared(); - m_ptr->swap(m); - storage_ = std::make_any>(std::move(m_ptr)); +namespace { +template +void host_copy(const InputT* input_data, array::Array& output) { + auto size = output.size(); + OutputT* output_data = output.host_data(); + std::copy( input_data, input_data + size, output_data ); } -template -SparseMatrixStorage::SparseMatrixStorage(const Eigen::SparseMatrix& m) { - nnz_ = m.nonZeros(); - rows_ = m.rows(); - cols_ = m.cols(); - - outer_.reset(atlas::array::Array::create(rows_+1)); - inner_.reset(atlas::array::Array::create(nnz_)); - value_.reset(atlas::array::Array::create(nnz_)); - host_copy(m.outerIndexPtr(), *outer_); - host_copy(m.innerIndexPtr(), *inner_); - host_copy(m.valuePtr(), *value_); +template +void host_copy(const array::Array& input, array::Array& output) { + host_copy( input.host_data(), output ); +} + +template +void host_copy(const array::Array& input, array::Array& output) { + switch(input.datatype().kind()) { + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + default: ATLAS_NOTIMPLEMENTED; + } +} +void host_copy(const array::Array& input, array::Array& output) { + switch(output.datatype().kind()) { + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + case DataType::kind(): return host_copy( input, output ); + default: ATLAS_NOTIMPLEMENTED; + } +} + +} + +template +SparseMatrixStorage make_sparse_matrix_storage(const SparseMatrixStorage& other) { + auto rows = other.rows(); + auto cols = other.cols(); + auto nnz = other.nnz(); + std::unique_ptr value(array::Array::create(nnz)); + std::unique_ptr inner(array::Array::create(nnz)); + std::unique_ptr outer(array::Array::create(rows+1)); + host_copy(other.value(), *value); + host_copy(other.inner(), *inner); + host_copy(other.outer(), *outer); + return SparseMatrixStorage::make(rows,cols,nnz,std::move(value), std::move(inner), std::move(outer), std::any()); +} + +template +SparseMatrixStorage make_sparse_matrix_storage(SparseMatrixStorage&& other) { + SparseMatrixStorage S; + + if (other.contains()) { + S = std::move(other); + } + else { + auto rows = other.rows(); + auto cols = other.cols(); + auto nnz = other.nnz(); + std::unique_ptr value(array::Array::create(nnz)); + std::unique_ptr inner(array::Array::create(nnz)); + std::unique_ptr outer(array::Array::create(rows+1)); + host_copy(other.value(), *value); + host_copy(other.inner(), *inner); + host_copy(other.outer(), *outer); + S = SparseMatrixStorage::make(rows,cols,nnz,std::move(value), std::move(inner), std::move(outer), std::any()); + } + return S; } -#endif //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/tests/interpolation/test_interpolation_finite_element_cached.cc b/src/tests/interpolation/test_interpolation_finite_element_cached.cc index 992e263a3..5d75a8dfd 100644 --- a/src/tests/interpolation/test_interpolation_finite_element_cached.cc +++ b/src/tests/interpolation/test_interpolation_finite_element_cached.cc @@ -114,7 +114,7 @@ CASE("extract cache, copy it, and move it for use") { EXPECT(not eckit_matrix_view.empty()); EXPECT(not eckit_matrix_copy.empty()); - auto cache = interpolation::MatrixCache(atlas::linalg::SparseMatrixStorage(std::move(eckit_matrix_copy))); + auto cache = interpolation::MatrixCache(atlas::linalg::make_sparse_matrix_storage(std::move(eckit_matrix_copy))); EXPECT(not eckit_matrix_view.empty()); // We didn't touch the view EXPECT(eckit_matrix_copy.empty()); // This has been moved into the new 'cache' diff --git a/src/tests/linalg/test_linalg_sparse_matrix.cc b/src/tests/linalg/test_linalg_sparse_matrix.cc index 85be4e2b6..f6c43645b 100644 --- a/src/tests/linalg/test_linalg_sparse_matrix.cc +++ b/src/tests/linalg/test_linalg_sparse_matrix.cc @@ -13,12 +13,11 @@ #include "atlas/linalg/sparse/SparseMatrixStorage.h" #include "atlas/linalg/sparse/SparseMatrixView.h" -#include "atlas/linalg/sparse/SparseMatrixConvertor.h" -#include "atlas/linalg/sparse/SparseMatrixAdaptor.h" +#include "atlas/linalg/sparse/MakeSparseMatrixStorageEigen.h" +#include "atlas/linalg/sparse/MakeSparseMatrixStorageEckit.h" using atlas::linalg::SparseMatrixStorage; -using atlas::linalg::SparseMatrixAdaptor; -using atlas::linalg::SparseMatrixConvertor; +using atlas::linalg::make_sparse_matrix_storage; namespace atlas { namespace test { @@ -58,8 +57,8 @@ eckit::linalg::SparseMatrix create_eckit_sparsematrix() { } template -SparseMatrixConvertor create_sparsematrix_storage() { - return SparseMatrixConvertor(create_eckit_sparsematrix()); +SparseMatrixStorage create_sparsematrix_storage() { + return make_sparse_matrix_storage(create_eckit_sparsematrix()); } template @@ -134,33 +133,39 @@ void check_matrix(const SparseMatrixStorage& m) { } CASE("Create SparseMatrix moving eckit::linalg::SparseMatrix") { - SparseMatrixStorage S( create_eckit_sparsematrix() ); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eckit_sparsematrix()); + EXPECT_NO_THROW(check_matrix(S)); +} + +CASE("Create SparseMatrix moving eckit::linalg::SparseMatrix using std::move") { + auto A = create_eckit_sparsematrix(); + SparseMatrixStorage S = make_sparse_matrix_storage(std::move(A)); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create SparseMatrix copying eckit::linalg::SparseMatrix") { auto A = create_eckit_sparsematrix(); - SparseMatrixStorage S(A); + SparseMatrixStorage S = make_sparse_matrix_storage(A); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create single precision SparseMatrix copying from double precision SparseMatrix ") { auto Sdouble = create_sparsematrix_storage(); - SparseMatrixConvertor S(Sdouble); + SparseMatrixStorage S = make_sparse_matrix_storage(Sdouble); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create single precision SparseMatrix moving from double precision SparseMatrix which causes copy") { - SparseMatrixConvertor S(create_sparsematrix_storage()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_sparsematrix_storage()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create double precision SparseMatrix copying from single precision SparseMatrix ") { auto Sfloat = create_sparsematrix_storage(); - SparseMatrixConvertor S(Sfloat); + SparseMatrixStorage S = make_sparse_matrix_storage(Sfloat); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create double precision SparseMatrix moving from single precision SparseMatrix which causes copy") { - SparseMatrixConvertor S(create_sparsematrix_storage()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_sparsematrix_storage()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create base with copy constructor from double precision") { @@ -183,69 +188,57 @@ CASE("Create base with move constructor from single precision") { EXPECT_NO_THROW(check_matrix(S)); } -CASE("Create SparseMatrix wrapping eckit::linalg::SparseMatrix") { - auto A = create_eckit_sparsematrix(); - SparseMatrixStorage S = SparseMatrixAdaptor( A ); - EXPECT_NO_THROW(check_matrix(S)); -} - - #if ATLAS_HAVE_EIGEN CASE("Copy from Eigen double") { auto eigen_matrix = create_eigen_sparsematrix(); - SparseMatrixStorage S(eigen_matrix); + SparseMatrixStorage S = make_sparse_matrix_storage(eigen_matrix); EXPECT_NO_THROW(check_matrix(S)); } CASE("Move from Eigen double, avoiding copy") { - SparseMatrixStorage S(create_eigen_sparsematrix()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Copy and convert double from Eigen to single precision") { auto eigen_matrix = create_eigen_sparsematrix(); - SparseMatrixConvertor S(eigen_matrix); + SparseMatrixStorage S = make_sparse_matrix_storage(eigen_matrix); EXPECT_NO_THROW(check_matrix(S)); } CASE("Move and convert double from Eigen to single precision, should cause copy") { - SparseMatrixConvertor S(create_eigen_sparsematrix()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Copy from Eigen single") { auto eigen_matrix = create_eigen_sparsematrix(); - SparseMatrixStorage S(eigen_matrix); + SparseMatrixStorage S = make_sparse_matrix_storage(eigen_matrix); EXPECT_NO_THROW(check_matrix(S)); } CASE("Move from Eigen single, avoiding copy") { - SparseMatrixStorage S(create_eigen_sparsematrix()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Copy and convert single from Eigen to double precision") { auto eigen_matrix = create_eigen_sparsematrix(); - SparseMatrixConvertor S(eigen_matrix); + SparseMatrixStorage S = make_sparse_matrix_storage(eigen_matrix); EXPECT_NO_THROW(check_matrix(S)); } CASE("Move and convert single from Eigen to double precision, should cause copy") { - SparseMatrixConvertor S(create_eigen_sparsematrix()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create chain of moves from eigen double") { - SparseMatrixStorage S( SparseMatrixConvertor( create_eigen_sparsematrix() ) ); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create chain of moves from eigen single") { - SparseMatrixStorage S( SparseMatrixConvertor( create_eigen_sparsematrix() ) ); - EXPECT_NO_THROW(check_matrix(S)); -} -CASE("Create SparseMatrix wrapping Eigen matrix: no copy, no move") { - auto A = create_eigen_sparsematrix(); - SparseMatrixStorage S = SparseMatrixAdaptor( A ); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } #endif diff --git a/src/tests/linalg/test_linalg_sparse_matrix_gpu.cc b/src/tests/linalg/test_linalg_sparse_matrix_gpu.cc index 8b804d4ea..5bb2da54d 100644 --- a/src/tests/linalg/test_linalg_sparse_matrix_gpu.cc +++ b/src/tests/linalg/test_linalg_sparse_matrix_gpu.cc @@ -18,12 +18,11 @@ #include "atlas/linalg/sparse/SparseMatrixStorage.h" #include "atlas/linalg/sparse/SparseMatrixView.h" -#include "atlas/linalg/sparse/SparseMatrixConvertor.h" -#include "atlas/linalg/sparse/SparseMatrixAdaptor.h" +#include "atlas/linalg/sparse/MakeSparseMatrixStorageEigen.h" +#include "atlas/linalg/sparse/MakeSparseMatrixStorageEckit.h" using atlas::linalg::SparseMatrixStorage; -using atlas::linalg::SparseMatrixAdaptor; -using atlas::linalg::SparseMatrixConvertor; +using atlas::linalg::make_sparse_matrix_storage; using atlas::linalg::make_host_view; using atlas::linalg::make_device_view; @@ -149,8 +148,8 @@ eckit::linalg::SparseMatrix create_eckit_sparsematrix() { } template -SparseMatrixConvertor create_sparsematrix_storage() { - return SparseMatrixConvertor(create_eckit_sparsematrix()); +SparseMatrixStorage create_sparsematrix_storage() { + return make_sparse_matrix_storage(create_eckit_sparsematrix()); } template @@ -240,33 +239,39 @@ void check_matrix(const SparseMatrixStorage& m) { } CASE("Create SparseMatrix moving eckit::linalg::SparseMatrix") { - SparseMatrixStorage S( create_eckit_sparsematrix() ); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eckit_sparsematrix()); + EXPECT_NO_THROW(check_matrix(S)); +} + +CASE("Create SparseMatrix moving eckit::linalg::SparseMatrix using std::move") { + auto A = create_eckit_sparsematrix(); + SparseMatrixStorage S = make_sparse_matrix_storage(std::move(A)); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create SparseMatrix copying eckit::linalg::SparseMatrix") { auto A = create_eckit_sparsematrix(); - SparseMatrixStorage S(A); + SparseMatrixStorage S = make_sparse_matrix_storage(A); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create single precision SparseMatrix copying from double precision SparseMatrix ") { auto Sdouble = create_sparsematrix_storage(); - SparseMatrixConvertor S(Sdouble); + SparseMatrixStorage S = make_sparse_matrix_storage(Sdouble); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create single precision SparseMatrix moving from double precision SparseMatrix which causes copy") { - SparseMatrixConvertor S(create_sparsematrix_storage()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_sparsematrix_storage()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create double precision SparseMatrix copying from single precision SparseMatrix ") { auto Sfloat = create_sparsematrix_storage(); - SparseMatrixConvertor S(Sfloat); + SparseMatrixStorage S = make_sparse_matrix_storage(Sfloat); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create double precision SparseMatrix moving from single precision SparseMatrix which causes copy") { - SparseMatrixConvertor S(create_sparsematrix_storage()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_sparsematrix_storage()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create base with copy constructor from double precision") { @@ -289,71 +294,60 @@ CASE("Create base with move constructor from single precision") { EXPECT_NO_THROW(check_matrix(S)); } -CASE("Create SparseMatrix wrapping eckit::linalg::SparseMatrix, no move, no copy") { - auto A = create_eckit_sparsematrix(); - SparseMatrixStorage S = SparseMatrixAdaptor( A ); - EXPECT_NO_THROW(check_matrix(S)); -} - - #if ATLAS_HAVE_EIGEN CASE("Copy from Eigen double") { auto eigen_matrix = create_eigen_sparsematrix(); - SparseMatrixConvertor S(eigen_matrix); + SparseMatrixStorage S = make_sparse_matrix_storage(eigen_matrix); EXPECT_NO_THROW(check_matrix(S)); } CASE("Move from Eigen double, avoiding copy") { - SparseMatrixConvertor S(create_eigen_sparsematrix()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Copy and convert double from Eigen to single precision") { auto eigen_matrix = create_eigen_sparsematrix(); - SparseMatrixConvertor S(eigen_matrix); + SparseMatrixStorage S = make_sparse_matrix_storage(eigen_matrix); EXPECT_NO_THROW(check_matrix(S)); } CASE("Move and convert double from Eigen to single precision, should cause copy") { - SparseMatrixConvertor S(create_eigen_sparsematrix()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Copy from Eigen single") { auto eigen_matrix = create_eigen_sparsematrix(); - SparseMatrixConvertor S(eigen_matrix); + SparseMatrixStorage S = make_sparse_matrix_storage(eigen_matrix); EXPECT_NO_THROW(check_matrix(S)); } CASE("Move from Eigen single, avoiding copy") { - SparseMatrixConvertor S(create_eigen_sparsematrix()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Copy and convert single from Eigen to double precision") { auto eigen_matrix = create_eigen_sparsematrix(); - SparseMatrixConvertor S(eigen_matrix); + SparseMatrixStorage S = make_sparse_matrix_storage(eigen_matrix); EXPECT_NO_THROW(check_matrix(S)); } CASE("Move and convert single from Eigen to double precision, should cause copy") { - SparseMatrixConvertor S(create_eigen_sparsematrix()); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create chain of moves from eigen double") { - SparseMatrixStorage S( SparseMatrixConvertor( create_eigen_sparsematrix() ) ); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } CASE("Create chain of moves from eigen single") { - SparseMatrixStorage S( SparseMatrixConvertor( create_eigen_sparsematrix() ) ); - EXPECT_NO_THROW(check_matrix(S)); -} -CASE("Create SparseMatrix wrapping Eigen matrix") { - auto A = create_eigen_sparsematrix(); - SparseMatrixStorage S = SparseMatrixAdaptor( A ); + SparseMatrixStorage S = make_sparse_matrix_storage(create_eigen_sparsematrix()); EXPECT_NO_THROW(check_matrix(S)); } + #endif CASE("Test exceptions in make_host_view") {