diff --git a/Tutorial/gray-scott/CMakeLists.txt b/Tutorial/gray-scott/CMakeLists.txt index f44e4a1..5ddaf92 100644 --- a/Tutorial/gray-scott/CMakeLists.txt +++ b/Tutorial/gray-scott/CMakeLists.txt @@ -32,55 +32,82 @@ add_executable(pdf_calc analysis/pdf_calc.cpp) target_link_libraries(pdf_calc adios2::adios2 MPI::MPI_C) option(VTK "Build VTK apps") -if (VTK_ROOT) +if(VTK_ROOT) set(VTK ON) -endif(VTK_ROOT) +endif() -if (VTK) +if(VTK) message(STATUS "Configuring VTK apps") - find_package(VTK COMPONENTS - vtkFiltersCore - vtkIOImage - vtkIOXML - ) - + find_package(VTK COMPONENTS vtkFiltersCore vtkIOImage vtkIOXML) if(VTK_FOUND) + message(STATUS "Adding isosurface") add_executable(isosurface analysis/isosurface.cpp) - target_link_libraries(isosurface adios2::adios2 ${VTK_LIBRARIES} - MPI::MPI_C) - endif(VTK_FOUND) + target_include_directories(isosurface PRIVATE ${VTK_INCLUDE_DIRS}) + target_link_libraries(isosurface + adios2::adios2 MPI::MPI_C ${VTK_LIBRARIES} + ) + endif() find_package(VTK COMPONENTS - vtkFiltersCore - vtkFiltersGeometry + vtkCommonCore vtkCommonDataModel vtkFiltersCore vtkFiltersGeneral + vtkIOImage vtkIOXML vtkIOParallelXML vtkParallelMPI ) - + if(VTK_FOUND) + message(STATUS "Adding libisosurface") + add_library(libisosurface analysis/lib-isosurface.cpp) + target_include_directories(libisosurface + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/analysis + PRIVATE ${VTK_INCLUDE_DIRS} + ) + target_link_libraries(libisosurface + PUBLIC adios2::adios2 MPI::MPI_C + PRIVATE ${VTK_LIBRARIES} + ) + + message(STATUS "Adding block-isosurface") + add_executable(block-isosurface analysis/block-isosurface.cpp) + target_link_libraries(block-isosurface + adios2::adios2 MPI::MPI_C libisosurface + ) + + message(STATUS "Adding gray-scott-iso") + add_executable(gray-scott-iso + simulation/main.cpp + simulation/gray-scott.cpp + simulation/settings.cpp + simulation/writer.cpp + ) + target_link_libraries(gray-scott-iso + adios2::adios2 MPI::MPI_C libisosurface + ) + target_compile_definitions(gray-scott-iso PRIVATE USE_INLINE_ISOSURFACE=1) + endif() + + find_package(VTK COMPONENTS vtkFiltersCore vtkFiltersGeometry vtkIOXML) if(VTK_FOUND) add_executable(find_blobs analysis/find_blobs.cpp) - target_link_libraries(find_blobs adios2::adios2 ${VTK_LIBRARIES} - MPI::MPI_C) - endif(VTK_FOUND) - - find_package(VTK COMPONENTS - vtkFiltersGeneral - ) + target_include_directories(find_blobs PRIVATE ${VTK_INCLUDE_DIRS}) + target_link_libraries(find_blobs + adios2::adios2 MPI::MPI_C ${VTK_LIBRARIES} + ) + endif() + find_package(VTK COMPONENTS vtkFiltersGeneral) if(VTK_FOUND) add_executable(compute_curvature analysis/curvature.cpp) - target_link_libraries(compute_curvature adios2::adios2 ${VTK_LIBRARIES} - MPI::MPI_C) - endif(VTK_FOUND) - - - find_package(VTK COMPONENTS - vtkRenderingOpenGL2 - vtkViewsInfovis - ) + target_include_directories(compute_curvature PRIVATE ${VTK_INCLUDE_DIRS}) + target_link_libraries(compute_curvature + adios2::adios2 MPI::MPI_C ${VTK_LIBRARIES} + ) + endif() + find_package(VTK COMPONENTS vtkRenderingOpenGL2 vtkViewsInfovis) if(VTK_FOUND) add_executable(render_isosurface plot/render_isosurface.cpp) - target_link_libraries(render_isosurface adios2::adios2 ${VTK_LIBRARIES} - MPI::MPI_C) - endif(VTK_FOUND) -endif(VTK) + target_include_directories(render_isosurface PRIVATE ${VTK_INCLUDE_DIRS}) + target_link_libraries(render_isosurface + adios2::adios2 MPI::MPI_C ${VTK_LIBRARIES} + ) + endif() +endif() diff --git a/Tutorial/gray-scott/analysis/block-isosurface.cpp b/Tutorial/gray-scott/analysis/block-isosurface.cpp new file mode 100644 index 0000000..45c025a --- /dev/null +++ b/Tutorial/gray-scott/analysis/block-isosurface.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +#include +#include + +#ifdef ENABLE_TIMERS +#include "../common/timer.hpp" +#endif + +#include + +int main(int argc, char **argv) +{ + int mpiRank; + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + + if(argc < 4) + { + if(mpiRank == 0) + { + std::cerr << "Error: Too few arguments\n" + << "Usage: " << argv[0] << " input output isovalues...\n"; + } + MPI_Abort(MPI_COMM_WORLD, -1); + } + + const std::string inputFName(argv[1]); + const std::string outputFName(argv[2]); + + std::vector isoValues; + for (int i = 3; i < argc; i++) + { + isoValues.push_back(std::stod(argv[i])); + } + + try + { + adios2::ADIOS adios("adios2.xml", MPI_COMM_WORLD, adios2::DebugON); + adios2::IO io = adios.DeclareIO("SimulationOutput"); + + IsoSurface analysis(MPI_COMM_WORLD, io, inputFName, outputFName, + isoValues, false); + + while(analysis.Step()); + } + catch(const std::exception &ex) + { + std::cerr << "Error: " << ex.what() << "\n"; + MPI_Abort(MPI_COMM_WORLD, -2); + } + + MPI_Finalize(); + + return 0; +} diff --git a/Tutorial/gray-scott/analysis/lib-isosurface.cpp b/Tutorial/gray-scott/analysis/lib-isosurface.cpp new file mode 100644 index 0000000..d694bc2 --- /dev/null +++ b/Tutorial/gray-scott/analysis/lib-isosurface.cpp @@ -0,0 +1,286 @@ +#include "lib-isosurface.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Helper functions +namespace +{ + // Zero-copy inline interface + void Adios2Vtk( + const adios2::Variable &var, + adios2::Variable::Info &block, + vtkSmartPointer image) + { + image->SetWholeExtent( + 0, var.Count()[2] - 1, 0, var.Count()[1] - 1, 0, + var.Count()[0] - 1); + image->SetDataSpacing(1, 1, 1); + image->SetDataExtent( + block.Start[2], block.Start[2] + block.Count[2] - 1, + block.Start[1], block.Start[1] + block.Count[1] - 1, + block.Start[0], block.Start[0] + block.Count[0] - 1); + image->SetDataScalarTypeToDouble(); + image->SetNumberOfScalarComponents(1); + image->Modified(); + } + + // Copied image data + void Adios2Vtk( + const adios2::Variable &var, + adios2::Variable::Info &block, + vtkSmartPointer image) + { + image->SetOrigin(0, 0, 0); + image->SetSpacing(1, 1, 1); + image->SetExtent( + block.Start[2], block.Start[2] + block.Count[2] - 1, + block.Start[1], block.Start[1] + block.Count[1] - 1, + block.Start[0], block.Start[0] + block.Count[0] - 1); + image->AllocateScalars(VTK_DOUBLE, 1); + image->Modified(); + } +} + +struct IsoSurface::IsoSurfaceImpl +{ + adios2::IO &IO; + adios2::Engine Reader; + struct + { + vtkSmartPointer Comm; + vtkSmartPointer Controller; + std::vector> InputImageImports; + std::vector> InputImages; + vtkSmartPointer Group; + vtkSmartPointer Contour; + vtkSmartPointer Writer; + } Pipeline; + + size_t BlockIdMin, BlockIdMax; + int MpiRank, MpiSize; + std::string OutFName; + bool IsInline; + + IsoSurfaceImpl(MPI_Comm comm, adios2::IO &io, const std::string &in, + const std::string &out, const std::vector &isoValues, + bool isInline) + : IO(io), Reader(io.Open(in, adios2::Mode::Read)), OutFName(out), + IsInline(isInline) + { + MPI_Comm_rank(comm, &this->MpiRank); + MPI_Comm_size(comm, &this->MpiSize); + + // Setup MPI fpr VTK + // + vtkMPICommunicatorOpaqueComm opaqueComm(&comm); + this->Pipeline.Comm = vtkMPICommunicator::New(); + this->Pipeline.Comm->InitializeExternal(&opaqueComm); + + this->Pipeline.Controller = vtkMPIController::New(); + this->Pipeline.Controller->SetCommunicator(this->Pipeline.Comm.Get()); + vtkMultiProcessController::SetGlobalController( + this->Pipeline.Controller.Get()); + + // Setup the pipeline stages (except for input, that's later) + // + this->Pipeline.Group = vtkMultiBlockDataGroupFilter::New(); + + this->Pipeline.Contour = vtkMarchingCubes::New(); + this->Pipeline.Contour->AddInputConnection( + this->Pipeline.Group->GetOutputPort()); + this->Pipeline.Contour->SetNumberOfContours(isoValues.size()); + for(size_t i = 0; i < isoValues.size(); ++i) + { + this->Pipeline.Contour->SetValue(i, isoValues[i]); + } + + this->Pipeline.Writer = vtkXMLPMultiBlockDataWriter::New(); + this->Pipeline.Writer->AddInputConnection( + this->Pipeline.Contour->GetOutputPort()); + } + + ~IsoSurfaceImpl() = default; + + void UpdateBlockRange(size_t nBlocks) + { + size_t localBlockCount; + if(this->IsInline) + { + if(nBlocks != 1) + { + throw std::logic_error( + "Inline engine requires exactly 1 block per rank"); + } + localBlockCount = 1; + this->BlockIdMin = 0; + } + else + { + if(nBlocks < this->MpiSize) + { + throw std::logic_error( + "Number of blocks must be >= number of MPI ranks"); + } + localBlockCount = nBlocks / this->MpiSize; + size_t leftOver = nBlocks % this->MpiSize; + + if(this->MpiRank < leftOver) + { + localBlockCount++; + this->BlockIdMin = this->MpiRank * localBlockCount; + } + else + { + this->BlockIdMin = leftOver * (localBlockCount + 1) + + (this->MpiRank - leftOver) * localBlockCount; + } + } + this->BlockIdMax = this->BlockIdMin + localBlockCount - 1; + + // The inline engine uses the vtkImageImport filter for a zero-copy + // interface while all other engines read into a vtkImageData + if(this->IsInline) + { + // Skip pipeline input adjustment if it's already setup + if(localBlockCount == this->Pipeline.InputImageImports.size()) + { + return; + } + + // Setup the pipeline plumbing + this->Pipeline.Group->RemoveAllInputConnections(0); + this->Pipeline.InputImageImports.clear(); + this->Pipeline.InputImageImports.resize(localBlockCount); + for(auto &input : this->Pipeline.InputImageImports) + { + input.TakeReference(vtkImageImport::New()); + this->Pipeline.Group->AddInputConnection( + input->GetOutputPort()); + } + } + else + { + // Skip pipeline input adjustment if it's already setup + if(localBlockCount == this->Pipeline.InputImages.size()) + { + return; + } + + // Setup the pipeline plumbing + this->Pipeline.Group->RemoveAllInputConnections(0); + this->Pipeline.InputImages.clear(); + this->Pipeline.InputImages.resize(localBlockCount); + for(auto &input : this->Pipeline.InputImages) + { + input.TakeReference(vtkImageData::New()); + this->Pipeline.Group->AddInputData(input); + } + } + } +}; + +IsoSurface::IsoSurface(MPI_Comm comm, adios2::IO &io, const std::string &in, + const std::string &out, + const std::vector &isoValues, + bool isInline) + : Impl(new IsoSurfaceImpl(comm, io, in, out, isoValues, isInline)) +{ +} + +IsoSurface::~IsoSurface() = default; + +bool IsoSurface::Step() +{ + // 1. Start the step + adios2::StepStatus status = this->Impl->Reader.BeginStep(); + if(status != adios2::StepStatus::OK) + { + return false; + } + if(this->Impl->MpiRank == 0) + { + std::cout << "IsoContour step " << this->Impl->Reader.CurrentStep() + << std::endl; + } + + // 2. Locate the variable + adios2::Variable varU = this->Impl->IO.InquireVariable("U"); + if(!varU) // Ignore steps where the variable doesn't exist + { + return true; + } + + // 3. Get the breakdown of block information + auto blocksU = this->Impl->Reader.BlocksInfo(varU, + this->Impl->Reader.CurrentStep()); + if(blocksU.size() == 0) // Ignore steps where the variable is empty + { + return true; + } + + // 4. Determine which blocks need to be read by the current process + this->Impl->UpdateBlockRange(blocksU.size()); + + // 5. Loop through all available blocks and populate their block infos + for(auto &infoU : blocksU) + { + // Skip the blocks we're not working on + if(infoU.BlockID < this->Impl->BlockIdMin || + infoU.BlockID > this->Impl->BlockIdMax) + { + continue; + } + + varU.SetBlockSelection(infoU.BlockID); + + if(this->Impl->IsInline) + { + // 7. Establish the zero-copy interface into VTK + vtkSmartPointer image = + this->Impl->Pipeline.InputImageImports[ + infoU.BlockID-this->Impl->BlockIdMin]; + Adios2Vtk(varU, infoU, image); + this->Impl->Reader.Get(varU, infoU); + image->SetImportVoidPointer(const_cast(infoU.Data())); + } + else + { + // 7. Read into vtkImageData + vtkSmartPointer image = + this->Impl->Pipeline.InputImages[ + infoU.BlockID-this->Impl->BlockIdMin]; + Adios2Vtk(varU, infoU, image); + this->Impl->Reader.Get(varU, + reinterpret_cast(image->GetScalarPointer())); + } + } + this->Impl->Reader.PerformGets(); + + + // 8. Update the output filename + std::ostringstream ss; + ss << this->Impl->OutFName << "-" + << std::setfill('0') << std::setw(5) << this->Impl->Reader.CurrentStep() + << ".vtm"; + this->Impl->Pipeline.Writer->SetFileName(ss.str().c_str()); + + // 9. Trigger the writer which will in turn invoke the entire pipeline + this->Impl->Pipeline.Writer->Write(); + + // 10. Release the ADIOS step + this->Impl->Reader.EndStep(); + + return true; +} diff --git a/Tutorial/gray-scott/analysis/lib-isosurface.h b/Tutorial/gray-scott/analysis/lib-isosurface.h new file mode 100644 index 0000000..e0444ca --- /dev/null +++ b/Tutorial/gray-scott/analysis/lib-isosurface.h @@ -0,0 +1,28 @@ +#ifndef libIsoSurface_h_ +#define libIsoSurface_h_ + +#include +#include +#include + +#include + +#include + +class IsoSurface +{ +public: + IsoSurface(MPI_Comm comm, adios2::IO &io, const std::string &in, + const std::string &out, const std::vector &isoValues, + bool isInline); + + ~IsoSurface(); + + bool Step(); + +private: + struct IsoSurfaceImpl; + std::unique_ptr Impl; +}; + +#endif // libIsoSurface_h_ diff --git a/Tutorial/gray-scott/plot/render_isosurface.cpp b/Tutorial/gray-scott/plot/render_isosurface.cpp index f7cdb38..e6fc9a3 100644 --- a/Tutorial/gray-scott/plot/render_isosurface.cpp +++ b/Tutorial/gray-scott/plot/render_isosurface.cpp @@ -10,6 +10,8 @@ #include +#include + #include #include #include diff --git a/Tutorial/gray-scott/simulation/main.cpp b/Tutorial/gray-scott/simulation/main.cpp index 3ca6bc2..1d16bf9 100644 --- a/Tutorial/gray-scott/simulation/main.cpp +++ b/Tutorial/gray-scott/simulation/main.cpp @@ -10,6 +10,10 @@ #include "gray-scott.h" #include "writer.h" +#ifdef USE_INLINE_ISOSURFACE +#include +#endif + void print_io_settings(const adios2::IO &io) { std::cout << "Simulation writes data using engine type: " @@ -71,6 +75,11 @@ int main(int argc, char **argv) adios2::IO io_main = adios.DeclareIO("SimulationOutput"); adios2::IO io_ckpt = adios.DeclareIO("SimulationCheckpoint"); +#ifdef USE_INLINE_ISOSURFACE + io_main.SetEngine("Inline"); + io_main.SetParameters({{"verbose", "4"}, {"writerID", settings.output}}); +#endif + Writer writer_main(settings, sim, io_main); Writer writer_ckpt(settings, sim, io_ckpt); @@ -84,6 +93,11 @@ int main(int argc, char **argv) std::cout << "========================================" << std::endl; } +#ifdef USE_INLINE_ISOSURFACE + IsoSurface iso(comm, io_main, "inline_reader", "gs-iso", {.25, .5, .75}, + true); +#endif + #ifdef ENABLE_TIMERS Timer timer_total; Timer timer_compute; @@ -137,6 +151,10 @@ int main(int argc, char **argv) log << i << "\t" << time_step << "\t" << time_compute << "\t" << time_write << std::endl; #endif + +#ifdef USE_INLINE_ISOSURFACE + iso.Step(); +#endif } writer_main.close(); diff --git a/Tutorial/gray-scott/simulation/settings-files.json b/Tutorial/gray-scott/simulation/settings-files.json index b8c7e5a..5c38b70 100644 --- a/Tutorial/gray-scott/simulation/settings-files.json +++ b/Tutorial/gray-scott/simulation/settings-files.json @@ -14,6 +14,7 @@ "checkpoint_output": "gs_ckpt.bp", "adios_config": "adios2.xml", "adios_span": false, - "adios_memory_selection": false, + "adios_memory_selection": true, + "adios_write_ghost" : false, "mesh_type": "image" } diff --git a/Tutorial/gray-scott/simulation/settings-inline.json b/Tutorial/gray-scott/simulation/settings-inline.json new file mode 100644 index 0000000..87d7f4f --- /dev/null +++ b/Tutorial/gray-scott/simulation/settings-inline.json @@ -0,0 +1,20 @@ +{ + "L": 64, + "Du": 0.2, + "Dv": 0.1, + "F": 0.01, + "k": 0.05, + "dt": 2.0, + "plotgap": 10, + "steps": 1000, + "noise": 0.0000001, + "output": "gs.bp", + "checkpoint": false, + "checkpoint_freq": 10, + "checkpoint_output": "gs_ckpt.bp", + "adios_config": "adios2.xml", + "adios_span": false, + "adios_memory_selection": false, + "adios_write_ghost" : true, + "mesh_type": "image" +} diff --git a/Tutorial/gray-scott/simulation/settings-staging.json b/Tutorial/gray-scott/simulation/settings-staging.json index ba66ff7..6c2f9a8 100644 --- a/Tutorial/gray-scott/simulation/settings-staging.json +++ b/Tutorial/gray-scott/simulation/settings-staging.json @@ -15,5 +15,6 @@ "adios_config": "adios2.xml", "adios_span": false, "adios_memory_selection": true, + "adios_write_ghost": false, "mesh_type": "image" } diff --git a/Tutorial/gray-scott/simulation/settings.cpp b/Tutorial/gray-scott/simulation/settings.cpp index 7f87ce5..b70fd15 100644 --- a/Tutorial/gray-scott/simulation/settings.cpp +++ b/Tutorial/gray-scott/simulation/settings.cpp @@ -21,6 +21,7 @@ void to_json(nlohmann::json &j, const Settings &s) {"adios_config", s.adios_config}, {"adios_span", s.adios_span}, {"adios_memory_selection", s.adios_memory_selection}, + {"adios_memory_selection", s.adios_write_ghost}, {"mesh_type", s.mesh_type}}; } @@ -42,6 +43,7 @@ void from_json(const nlohmann::json &j, Settings &s) j.at("adios_config").get_to(s.adios_config); j.at("adios_span").get_to(s.adios_span); j.at("adios_memory_selection").get_to(s.adios_memory_selection); + j.at("adios_write_ghost").get_to(s.adios_write_ghost); j.at("mesh_type").get_to(s.mesh_type); } @@ -63,6 +65,7 @@ Settings::Settings() adios_config = "adios2.xml"; adios_span = false; adios_memory_selection = false; + adios_write_ghost = false; mesh_type = "image"; } diff --git a/Tutorial/gray-scott/simulation/settings.h b/Tutorial/gray-scott/simulation/settings.h index cd8b937..e306963 100644 --- a/Tutorial/gray-scott/simulation/settings.h +++ b/Tutorial/gray-scott/simulation/settings.h @@ -20,6 +20,7 @@ struct Settings { std::string adios_config; bool adios_span; bool adios_memory_selection; + bool adios_write_ghost; std::string mesh_type; Settings(); diff --git a/Tutorial/gray-scott/simulation/writer.cpp b/Tutorial/gray-scott/simulation/writer.cpp index 868680f..dc575a9 100644 --- a/Tutorial/gray-scott/simulation/writer.cpp +++ b/Tutorial/gray-scott/simulation/writer.cpp @@ -52,21 +52,35 @@ Writer::Writer(const Settings &settings, const GrayScott &sim, adios2::IO io) define_bpvtk_attribute(settings, io); } - var_u = - io.DefineVariable("U", {settings.L, settings.L, settings.L}, - {sim.offset_z, sim.offset_y, sim.offset_x}, - {sim.size_z, sim.size_y, sim.size_x}); - - var_v = - io.DefineVariable("V", {settings.L, settings.L, settings.L}, - {sim.offset_z, sim.offset_y, sim.offset_x}, - {sim.size_z, sim.size_y, sim.size_x}); - - if (settings.adios_memory_selection) { - var_u.SetMemorySelection( - {{1, 1, 1}, {sim.size_z + 2, sim.size_y + 2, sim.size_x + 2}}); - var_v.SetMemorySelection( - {{1, 1, 1}, {sim.size_z + 2, sim.size_y + 2, sim.size_x + 2}}); + if(settings.adios_write_ghost) { + var_u = + io.DefineVariable("U", + {settings.L + 2, settings.L + 2, settings.L + 2}, + {sim.offset_z, sim.offset_y, sim.offset_x}, + {sim.size_z + 2, sim.size_y + 2, sim.size_x + 2}); + var_v = + io.DefineVariable("V", + {settings.L + 2, settings.L + 2, settings.L + 2}, + {sim.offset_z, sim.offset_y, sim.offset_x}, + {sim.size_z + 2, sim.size_y + 2, sim.size_x + 2}); + } else { + var_u = + io.DefineVariable("U", + {settings.L, settings.L, settings.L}, + {sim.offset_z, sim.offset_y, sim.offset_x}, + {sim.size_z, sim.size_y, sim.size_x}); + var_v = + io.DefineVariable("V", + {settings.L, settings.L, settings.L}, + {sim.offset_z, sim.offset_y, sim.offset_x}, + {sim.size_z, sim.size_y, sim.size_x}); + + if (settings.adios_memory_selection) { + var_u.SetMemorySelection( + {{1, 1, 1}, {sim.size_z + 2, sim.size_y + 2, sim.size_x + 2}}); + var_v.SetMemorySelection( + {{1, 1, 1}, {sim.size_z + 2, sim.size_y + 2, sim.size_x + 2}}); + } } var_step = io.DefineVariable("step"); @@ -79,7 +93,7 @@ void Writer::open(const std::string &fname) void Writer::write(int step, const GrayScott &sim) { - if (settings.adios_memory_selection) { + if (settings.adios_memory_selection || settings.adios_write_ghost) { const std::vector &u = sim.u_ghost(); const std::vector &v = sim.v_ghost();