diff --git a/.travis.yml b/.travis.yml index 7321634ac..89b3b0f1f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,10 @@ os: - linux env: - - VERSION=8 - - VERSION=7 + - VERSION=8 EXTERNAL_GINKGO=OFF + - VERSION=8 EXTERNAL_GINKGO=ON + - VERSION=7 EXTERNAL_GINKGO=OFF + - VERSION=7 EXTERNAL_GINKGO=ON cache: directories: @@ -33,12 +35,13 @@ before_install: install: - sudo apt-get -y install openfoam$VERSION - source /opt/openfoam$VERSION/etc/bashrc + - ./scripts/travis_install_external_ginkgo.sh before_script: - - cmake --version + - cmake --version - mkdir build - cd build - - cmake .. + - cmake -DOGL_USE_EXTERNAL_GINKGO=$EXTERNAL_GINKGO .. script: - make -j4 diff --git a/CMakeLists.txt b/CMakeLists.txt index 059916a1d..8babb3cc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,11 @@ add_compile_definitions( target_sources(OGL PRIVATE +common/common.C +IOHandler/IOPtr/IOPtr.C +IOHandler/IOSortingIdxHandler/IOSortingIdxHandler.C +IOHandler/IOExecutorHandler/IOExecutorHandler.C +IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C lduMatrix/GKOlduBase/GKOlduBase.C lduMatrix/GKOCG/GKOCG.C lduMatrix/GKOIR/GKOIR.C @@ -99,6 +104,11 @@ lduMatrix/GKOBiCGStab/GKOBiCGStab.C LduMatrix/GKOLduBase/GKOLduBase.C LduMatrix/GKOACG/GKOACG.C PUBLIC +common/common.H +IOHandler/IOPtr/IOPtr.H +IOHandler/IOSortingIdxHandler/IOSortingIdxHandler.H +IOHandler/IOExecutorHandler/IOExecutorHandler.H +IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H lduMatrix/GKOlduBase/GKOlduBase.H lduMatrix/GKOCG/GKOCG.H lduMatrix/GKOIR/GKOIR.H @@ -110,13 +120,17 @@ LduMatrix/GKOACG/GKOACG.H target_include_directories(OGL PUBLIC - $ENV{FOAM_SRC}/OpenFOAM/lnInclude - $ENV{FOAM_SRC}/meshTools/lnInclude $ENV{FOAM_SRC}/finiteVolume/lnInclude + $ENV{FOAM_SRC}/meshTools/lnInclude + $ENV{FOAM_SRC}/OpenFOAM/lnInclude $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude - common/ - lduMatrix/GKOlduBase - LduMatrix/GKOLduBase + common/ + IOHandler/IOPtr + IOHandler/IOExecutorHandler/ + IOHandler/IOSortingIdxHandler/ + IOHandler/IOGKOMatrixHandler/ + lduMatrix/GKOlduBase + LduMatrix/GKOLduBase ) diff --git a/IOHandler/IOExecutorHandler/IOExecutorHandler.C b/IOHandler/IOExecutorHandler/IOExecutorHandler.C new file mode 100644 index 000000000..3f0b225bc --- /dev/null +++ b/IOHandler/IOExecutorHandler/IOExecutorHandler.C @@ -0,0 +1,82 @@ + +#include +#include + +#include "IOExecutorHandler.H" + +namespace Foam { + +IOExecutorHandler::IOExecutorHandler(const objectRegistry &db, + const dictionary &controlDict) + : device_executor_name_( + controlDict.lookupOrDefault("executor", word("reference"))), + app_executor_name_( + controlDict.lookupOrDefault("app_executor", word("reference"))) +{ + // create executors + bool app_exec_stored = db.foundObject(app_executor_name_); + + if (app_exec_stored) { + ref_exec_ptr_ = + &db.lookupObjectRef(app_executor_name_); + if (device_executor_name_ == app_executor_name_) { + return; + } + } + + + bool device_exec_stored = + db.foundObject(device_executor_name_); + if (device_exec_stored) { + if (device_executor_name_ == "omp") { + omp_exec_ptr_ = + &db.lookupObjectRef(device_executor_name_); + return; + } + if (device_executor_name_ == "cuda") { + cuda_exec_ptr_ = + &db.lookupObjectRef(device_executor_name_); + return; + } + if (device_executor_name_ == "hip") { + hip_exec_ptr_ = + &db.lookupObjectRef(device_executor_name_); + return; + } + } + + const fileName app_exec_store = app_executor_name_; + ref_exec_ptr_ = + new GKOReferenceExecPtr(IOobject(app_exec_store, db), + gko::give(gko::ReferenceExecutor::create())); + + const fileName device_exec_store = device_executor_name_; + const fileName omp_exec_store = "omp"; + omp_exec_ptr_ = new GKOOmpExecPtr(IOobject(omp_exec_store, db), + gko::OmpExecutor::create()); + + if (device_executor_name_ == "cuda") { + cuda_exec_ptr_ = + new GKOCudaExecPtr(IOobject(device_exec_store, db), + gko::give(gko::CudaExecutor::create( + 0, omp_exec_ptr_->get_ptr(), true))); + } + if (device_executor_name_ == "hip") { + hip_exec_ptr_ = + new GKOHipExecPtr(IOobject(device_exec_store, db), + gko::give(gko::HipExecutor::create( + 0, omp_exec_ptr_->get_ptr(), true))); + } + if (device_executor_name_ == "omp") { + omp_exec_ptr_ = new GKOOmpExecPtr(IOobject(device_exec_store, db), + omp_exec_ptr_->get_ptr()); + } +} + +defineTemplateTypeNameWithName(GKOExecPtr, "ExecIOPtr"); +defineTemplateTypeNameWithName(GKOCudaExecPtr, "CudaExecIOPtr"); +defineTemplateTypeNameWithName(GKOOmpExecPtr, "OmpExecIOPtr"); +defineTemplateTypeNameWithName(GKOHipExecPtr, "HipExecIOPtr"); +defineTemplateTypeNameWithName(GKOReferenceExecPtr, "ReferenceExecIOPtr"); + +} // namespace Foam diff --git a/IOHandler/IOExecutorHandler/IOExecutorHandler.H b/IOHandler/IOExecutorHandler/IOExecutorHandler.H new file mode 100644 index 000000000..6a4b45a9d --- /dev/null +++ b/IOHandler/IOExecutorHandler/IOExecutorHandler.H @@ -0,0 +1,110 @@ +/*---------------------------------------------------------------------------*\ +License + This file is part of OGL. + + OGL is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OGL. If not, see . + +Class + Foam::IOExecutorHandler + +Author: Gregor Olenik + +SourceFiles + IOExecutorHandler.C + +\*---------------------------------------------------------------------------*/ + +#ifndef OGL_IOExecutorHandler_INCLUDED_H +#define OGL_IOExecutorHandler_INCLUDED_H + +#include "../IOPtr/IOPtr.H" +#include "fvCFD.H" +#include "regIOobject.H" + +#include + +namespace Foam { + +using vec = gko::matrix::Dense; +using mtx = gko::matrix::Coo; + + +class IOExecutorHandler { +private: + const word device_executor_name_; + + const word app_executor_name_; + + // executor where Ginkgo will perform the computation + GKOExecPtr *device_exec_ptr_; + + // executor of the application + GKOExecPtr *app_exec_ptr_; + + GKOReferenceExecPtr *ref_exec_ptr_; + + GKOCudaExecPtr *cuda_exec_ptr_; + + GKOOmpExecPtr *omp_exec_ptr_; + + GKOHipExecPtr *hip_exec_ptr_; + +public: + IOExecutorHandler(const objectRegistry &db, const dictionary &controlDict); + + word get_device_executor_name() const { return device_executor_name_; } + + std::shared_ptr get_device_executor() const + { + const word device_executor_name{get_device_executor_name()}; + if (device_executor_name == "omp") { + return omp_exec(); + } + if (device_executor_name == "cuda") { + return cuda_exec(); + } + if (device_executor_name == "hip") { + return hip_exec(); + } + return ref_exec(); + }; + + std::shared_ptr app_exec() const + { + return app_exec_ptr_->get_ptr(); + }; + + std::shared_ptr omp_exec() const + { + return omp_exec_ptr_->get_ptr(); + }; + + std::shared_ptr cuda_exec() const + { + return cuda_exec_ptr_->get_ptr(); + }; + + std::shared_ptr hip_exec() const + { + return hip_exec_ptr_->get_ptr(); + }; + + std::shared_ptr ref_exec() const + { + return ref_exec_ptr_->get_ptr(); + }; +}; + +} // namespace Foam +#endif diff --git a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C new file mode 100644 index 000000000..cd25a3800 --- /dev/null +++ b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C @@ -0,0 +1,97 @@ +/*---------------------------------------------------------------------------*\ +License + This file is part of OGL. + + OGL is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OGL. If not, see . + +Class + Foam::GKOCG + +Author: Gregor Olenik + +SourceFiles + GKOCG.C + +\*---------------------------------------------------------------------------*/ +#include +#include "IOGKOMatrixHandler.H" + +namespace Foam { + +void IOGKOMatrixHandler::init_device_matrix( + const objectRegistry &db, std::vector &values_host, + std::vector