From b18c8c402920cf768533164aa1fb0c9457a871a8 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Mon, 18 Sep 2023 12:45:02 -0500 Subject: [PATCH 1/9] Adding bittree interface to improve regridding performance in octree mode Co-authored-by: Tom Klosterman --- GNUmakefile.in | 3 + Src/AmrCore/AMReX_AmrCore.cpp | 4 +- Src/AmrCore/AMReX_AmrMesh.H | 11 + Src/AmrCore/AMReX_AmrMesh.cpp | 161 +++++++- Src/AmrCore/Make.package | 3 +- Src/Extern/Bittree/AMReX_Bittree.H | 78 ++++ Src/Extern/Bittree/AMReX_Bittree.cpp | 377 ++++++++++++++++++ Src/Extern/Bittree/CMakeLists.txt | 11 + Src/Extern/Bittree/Make.package | 5 + Tests/Amr/Advection_AmrCore/Exec/GNUmakefile | 1 + .../Amr/Advection_AmrCore/Exec/inputs_bittree | 81 ++++ Tools/GNUMake/Make.defs | 6 + Tools/GNUMake/packages/Make.bittree | 16 + Tools/libamrex/configure.py | 5 + 14 files changed, 755 insertions(+), 7 deletions(-) create mode 100644 Src/Extern/Bittree/AMReX_Bittree.H create mode 100644 Src/Extern/Bittree/AMReX_Bittree.cpp create mode 100644 Src/Extern/Bittree/CMakeLists.txt create mode 100644 Src/Extern/Bittree/Make.package create mode 100644 Tests/Amr/Advection_AmrCore/Exec/inputs_bittree create mode 100644 Tools/GNUMake/packages/Make.bittree diff --git a/GNUmakefile.in b/GNUmakefile.in index 21185a705f1..b85c2e0c35e 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -45,6 +45,9 @@ endif ifeq ($(USE_SUNDIALS),TRUE) Pdirs += Extern/SUNDIALS endif +ifeq ($(USE_BITTREE),TRUE) + Pdirs += Extern/Bittree +endif Ppack := $(foreach dir, $(Pdirs), $(AMREX_HOME)/Src/$(dir)/Make.package) include $(Ppack) diff --git a/Src/AmrCore/AMReX_AmrCore.cpp b/Src/AmrCore/AMReX_AmrCore.cpp index 502b3f5cb23..ed0cd5d1020 100644 --- a/Src/AmrCore/AMReX_AmrCore.cpp +++ b/Src/AmrCore/AMReX_AmrCore.cpp @@ -104,7 +104,7 @@ AmrCore::regrid (int lbase, Real time, bool) DistributionMapping level_dmap = dmap[lev]; if (ba_changed) { level_grids = new_grids[lev]; - level_dmap = DistributionMapping(level_grids); + level_dmap = MakeDistributionMap(lev, level_grids); } const auto old_num_setdm = num_setdm; RemakeLevel(lev, time, level_grids, level_dmap); @@ -117,7 +117,7 @@ AmrCore::regrid (int lbase, Real time, bool) } else // a new level { - DistributionMapping new_dmap(new_grids[lev]); + DistributionMapping new_dmap = MakeDistributionMap(lev, new_grids[lev]); const auto old_num_setdm = num_setdm; MakeNewLevelFromCoarse(lev, time, new_grids[lev], new_dmap); SetBoxArray(lev, new_grids[lev]); diff --git a/Src/AmrCore/AMReX_AmrMesh.H b/Src/AmrCore/AMReX_AmrMesh.H index a3c6fbc8f62..5dcdd371451 100644 --- a/Src/AmrCore/AMReX_AmrMesh.H +++ b/Src/AmrCore/AMReX_AmrMesh.H @@ -11,6 +11,10 @@ #include #include +#ifdef AMREX_USE_BITTREE +#include +#endif + namespace amrex { struct AmrInfo { @@ -253,6 +257,8 @@ public: [[nodiscard]] long CountCells (int lev) noexcept; + [[nodiscard]] virtual DistributionMapping MakeDistributionMap (int lev, BoxArray const& ba); + protected: int finest_level; //!< Current finest level. @@ -260,6 +266,11 @@ protected: Vector dmap; Vector grids; +#ifdef AMREX_USE_BITTREE + bool use_bittree; + std::shared_ptr btmesh; +#endif + unsigned int num_setdm = 0; unsigned int num_setba = 0; diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index 70a8df1dabf..924f3e38652 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -5,6 +5,13 @@ #include #include #include +#include + +#ifdef AMREX_USE_BITTREE +#include +#endif + +#include namespace amrex { @@ -376,6 +383,10 @@ AmrMesh::InitAmrMesh (int max_level_in, const Vector& n_cell_in, finest_level = -1; +#ifdef AMREX_USE_BITTREE + pp.get("use_bittree",use_bittree); +#endif + if (check_input) { checkInput(); } } @@ -437,6 +448,41 @@ AmrMesh::LevelDefined (int lev) noexcept return lev <= max_level && !grids[lev].empty() && !dmap[lev].empty(); } +DistributionMapping +AmrMesh::MakeDistributionMap (int lev, BoxArray const& ba) +{ + + BL_PROFILE("AmrMesh::MakeDistributionMap()"); + + // useful print statement to notify that a new distribution map is being created at lev + amrex::Print() << "creating new distribution map on level: " << lev + 1 << "\n"; + + // initialize new distribution mapping + DistributionMapping dm; + +#ifdef AMREX_USE_BITTREE + if(!use_bittree) + { +#endif + // create distribution mapping using boxarray + dm.define(ba); + +#ifdef AMREX_USE_BITTREE + } +#endif + +#ifdef AMREX_USE_BITTREE + // Bittree version + if(use_bittree) + { + // create distribution mapping + dm.define(ba); + } +#endif + + return dm; +} + void AmrMesh::ChopGrids (int lev, BoxArray& ba, int target_size) const { @@ -514,6 +560,10 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& if (new_grids.size() < max_crse+2) { new_grids.resize(max_crse+2); } +#ifdef AMREX_USE_BITTREE + if(!use_bittree) { +#endif + // // Construct problem domain at each level. // @@ -774,6 +824,72 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& } } } + +#ifdef AMREX_USE_BITTREE + } +#endif + +#ifdef AMREX_USE_BITTREE + // Bittree version + if(use_bittree) { + // Initialize BT refinement + btmesh->refine_init(); + + // ------------------------------------------------------------------- + // Use tagging data to mark BT for refinement, then use the new bitmap + // to calculate the new grids. + auto tree0 = btmesh->getTree(); + + // [1] Error Estimation and tagging + // btTags is indexed by bitid, Bittree's internal indexing scheme. + // For any id, btTags = 1 if should be parent, -1 if should not be parent (or not exist). + std::vector btTags(tree0->id_upper_bound(),0); + + for (int lev=max_crse; lev>=lbase; --lev) { + + TagBoxArray tags(grids[lev],dmap[lev], n_error_buf[lev]); + ErrorEst(lev, tags, time, 0); + tags.buffer(n_error_buf[lev]); + + for (MFIter mfi(tags); mfi.isValid(); ++mfi) { + auto const& tagbox = tags.const_array(mfi); + bool has_set_tags = amrex::Reduce::AnyOf(mfi.validbox(), + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + return tagbox(i,j,k)!=TagBox::CLEAR; + }); + + // Set the values of btTags. + int bitid = btUnit::getBitid(btmesh.get(),false,lev,mfi.index()); + // TODO Check lev == tree0->block_level(bitid) + if(has_set_tags) { + btTags[bitid] = 1; + } + else { + btTags[bitid] = -1; + } + } + } + + // [2] btRefine - check for proper octree nesting and update bitmap + MPI_Comm comm = ParallelContext::CommunicatorSub(); + int changed = btUnit::btRefine(btmesh.get(), btTags, max_crse, lbase, grids, dmap, comm); + + // [3] btCalculateGrids - use new bitmap to generate new grids + if (changed>0) { + btUnit::btCalculateGrids(btmesh.get(),lbase,new_finest,new_grids,max_grid_size); + } else { + new_finest = finest_level; + for(int i=0; i<=finest_level; ++i) { + new_grids[i] = grids[i]; + } + } + + // Finalize BT refinement + btmesh->refine_apply(); + } +#endif + } void @@ -783,11 +899,48 @@ AmrMesh::MakeNewGrids (Real time) { finest_level = 0; - const BoxArray& ba = MakeBaseGrids(); - DistributionMapping dm(ba); + BoxArray ba; + DistributionMapping dm; const auto old_num_setdm = num_setdm; const auto old_num_setba = num_setba; +#ifdef AMREX_USE_BITTREE + if(!use_bittree) { +#endif + ba = MakeBaseGrids(); + dm = MakeDistributionMap(0, ba); + +#ifdef AMREX_USE_BITTREE + } + else { + //Initialize Bittree + + // top = number of grids on coarsest level in each direction + std::vector top(AMREX_SPACEDIM,0); + IntVect ncells = geom[0].Domain().length(); + for(int i=0; i includes(ngrids,1); + + btmesh = std::make_shared(top.data(),includes.data()); + + // Set BCs + for(int d=0; d +#include +#include +#include + +namespace amrex { + +/* +Include in Make.local: +BITTREE_PATH = /path/to/bittree/installation +INCLUDE_LOCATIONS += $(BITTREE_PATH)/include +LIBRARY_LOCATIONS += $(BITTREE_PATH)/lib +LIBRARIES += -lbittree + +Include in inputs: +amr.use_bittree = true +amr.bt_derefine = true +amr.infer_bt_grids = false + + */ + +class btUnit { + // Functions used in AmrMesh + public: + static int btRefine(bittree::BittreeAmr* const mesh, + std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, + MPI_Comm comm); + static void btCalculateGrids(bittree::BittreeAmr* const mesh, + int lbase, + int& new_finest, + Vector& new_grids, + Vector const& max_grid_size); + static void btCalculateLevel(bittree::BittreeAmr* const mesh, + int lev, + BoxArray& ba, + IntVect const& max_grid_size); + // Utils + public: + static int getBitid(bittree::BittreeAmr* const mesh, bool updated, + int lev, int idx_on_lev); + static int getIndex(bittree::BittreeAmr* const mesh, bool updated, + int lev, int bitid); + + // Functions to implement strict octree logic + private: + static void btCheckRefine(bittree::BittreeAmr* const mesh, + std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, + MPI_Comm comm); + + static void btCheckDerefine(bittree::BittreeAmr* const mesh, + std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, + MPI_Comm comm); + + // Utility Functions + static bool checkNeighborsRefine(bittree::BittreeAmr* const mesh, + bittree::MortonTree::Block b); + static std::vector neighIntCoords(bittree::BittreeAmr* const mesh, + unsigned lev, unsigned* lcoord, + int* gCell); + + public: + // Represents whether domain has periodic BC in each direction + // true = Periodic, false = Non-Periodic + static bool bcPeriodic[AMREX_SPACEDIM]; +}; + + +} +#endif diff --git a/Src/Extern/Bittree/AMReX_Bittree.cpp b/Src/Extern/Bittree/AMReX_Bittree.cpp new file mode 100644 index 00000000000..a47496da069 --- /dev/null +++ b/Src/Extern/Bittree/AMReX_Bittree.cpp @@ -0,0 +1,377 @@ +#include +#include +#include +#include + +using namespace bittree; + +namespace amrex { +static constexpr int K1D = unsigned(AMREX_SPACEDIM>=1); +static constexpr int K2D = unsigned(AMREX_SPACEDIM>=2); +static constexpr int K3D = unsigned(AMREX_SPACEDIM>=3); + + +bool btUnit::bcPeriodic[AMREX_SPACEDIM]; + +/* +NOTE: Bittree object is created in AmrMesh::MakeNewGrids (Real time) + with + `mesh = std::make_shared(top,includes);` + +The functions here are called in the BT version of MakeNewGrids which has three steps: + 1. Error Estimation and tagging - btTagging + 2. Bitree's actual bitmap generated/updated - btRefine + 3. AMReX updates grids based on bitree - btCalculateGrids +*/ + + +/** New Bittree mesh is generated. + * + * This makes use of BT library functions and as well as routines adapted + * from Flash-X that enforce Octree nesting. + */ +int btUnit::btRefine( BittreeAmr* const mesh, std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, MPI_Comm comm) { + + BL_PROFILE("Bittree-btRefine"); + + // Tree before refinement. With only one rank, lnblocks = nblocks. + auto tree0 = mesh->getTree(); + + // Mark leaves to be refined + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + if (btTags[id]==1) { + if(!tree0->block_is_parent(id)) { + mesh->refine_mark(id, true); + } + } + } + } + + mesh->refine_reduce(comm); + mesh->refine_update(); + + btCheckRefine(mesh, btTags, max_crse, lbase, grids, dmap, comm); + + // Mark derefinement (parents who will nodetype change to leaf) + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + if (btTags[id]==-1) { + if(tree0->block_is_parent(id)) { + mesh->refine_mark(id, true); + } + } + } + } + + mesh->refine_reduce(comm); + mesh->refine_update(); + + btCheckDerefine(mesh, btTags, max_crse, lbase, grids, dmap, comm); + + // return delta count + return static_cast( mesh->delta_count() ); +} + +/** Creates new box arrays to match the new Bittree mesh. + */ +void btUnit::btCalculateGrids(BittreeAmr* const mesh, int lbase, + int& new_finest, + Vector& new_grids, + Vector const& max_grid_size) { + BL_PROFILE("Bittree-btCalculateGrids"); + + auto tree1 = mesh->getTree(true); + int nlevs = tree1->levels(); + new_finest = nlevs - 1; + +//--Calculate the new grid layout and distribution map based on Bittree + for(int lev=lbase; lev<=new_finest; ++lev) { + btCalculateLevel(mesh, lev, new_grids[lev], + max_grid_size[lev]); + } +} + +/** Creates a box array based on Bittree. + */ +void btUnit::btCalculateLevel(BittreeAmr* const mesh, int lev, + BoxArray& ba, + IntVect const& max_grid_size) { + auto tree1 = mesh->getTree(true); + + //Bittree has its own indices for blocks which I call bitid; get + //the range of bitids for the level being made. Bitid range is + //contiguous for each level. + int id0 = tree1->level_id0(lev); + int id1 = tree1->level_id1(lev); + int nblocks = tree1->level_blocks(lev); + + BoxList bl; + + for(int i=id0; ilocate(i); + + if(b.level != lev) { + std::string msg = "Error identifying block in btCalculateGrids"; + //throw error? + } + + IntVect coordVec{AMREX_D_DECL(static_cast(b.coord[0]), + static_cast(b.coord[1]), + static_cast(b.coord[2]))}; + IntVect lo = max_grid_size*coordVec; + IntVect hi = max_grid_size*(coordVec+1) - 1; + bl.push_back( Box{lo,hi} ); + } + + ba = BoxArray(bl); +} + +int btUnit::getBitid(BittreeAmr* const mesh, bool updated, + int lev, int idx_on_lev) { + return idx_on_lev + mesh->getTree(updated)->level_id0(lev); +} + +int btUnit::getIndex(BittreeAmr* const mesh, bool updated, + int lev, int bitid) { + return bitid - mesh->getTree(updated)->level_id0(lev); +} + + + +//--------------------------------------------------------------------- +// Local Routines +//--------------------------------------------------------------------- + +/** Implements the logic which ensures the generated Bittree adheres + * to a strict octree structure with no more than one level difference + * between surrounding leaf blocks. + */ +void btUnit::btCheckRefine(BittreeAmr* const mesh, std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, MPI_Comm comm) { + + BL_PROFILE("Bittree-btCheckRefine"); + + // Tree before refinement. + auto tree0 = mesh->getTree(); + + // Ref test is marked 1 if block needs a tag (and doesn't have one). + std::vector ref_test(tree0->id_upper_bound()); + + // Repeat is made true if another round is needed + bool repeat = false; + + do { + // Clear out ref_test + std::fill(ref_test.begin(),ref_test.end(),0); + + // Check neighbors - if any adjacent child of a neighbor is either a parent + // or marked for refinement, this block needs to be refined. + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + auto b = tree0->locate(id); + if( !b.is_parent && btTags[id]!=1 ) { + bool needsTag = checkNeighborsRefine( mesh, b); + //amrex::Print() << "needsTag for " << id << " : " << needsTag <=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + if( ref_test[id]==1 && btTags[id]!=1 ) { + repeat = true; + btTags[id] = 1; + mesh->refine_mark(id,true); + } + } + } + + // If only processing local blocks, check all processors to see if + // a repeat is necessary, then reduce bittree to update on all ranks. + ParallelDescriptor::ReduceBoolOr(repeat); + + if(repeat) { + mesh->refine_reduce(comm); + mesh->refine_update(); + } + + } while(repeat); +} + + +/** Implements the logic which ensures the generated Bittree adheres + * to a strict octree structure with no more than one level difference + * between surrounding leaf blocks. + */ +void btUnit::btCheckDerefine(BittreeAmr* const mesh, std::vector& btTags, + int max_crse, int lbase, + Vector& grids, Vector& dmap, MPI_Comm comm) { + + BL_PROFILE("Bittree-btCheckDerefine"); + + // Tree before refinement. With only one rank, lnblocks = nblocks. + auto tree0 = mesh->getTree(); + + std::vector deref_test(tree0->id_upper_bound()); + + // Repeat is made true if another round is needed + bool repeat = false; + + // Repeat is left true if another round is needed + do { + // Turn deref_test to default 0 if block can't be derefined + deref_test = btTags; + + // Check neighbors - if any adjacent child of neighbor is either a parent + // or marked for refinement, do not derefine. + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + auto b = tree0->locate(id); + if( btTags[id]==-1 ) { + bool cantDeref = checkNeighborsRefine( mesh, b); + if(cantDeref) { + deref_test[id] = 0; + } + } + } + } + + // Unmark any blocks who cannot derefine (as per above check). + repeat = false; + for (int lev=max_crse; lev>=lbase; --lev) { + for (MFIter mfi(grids[lev], dmap[lev]); mfi.isValid(); ++mfi) { + int id = getBitid(mesh,false,lev,mfi.index()); + if( deref_test[id]==0 && btTags[id]==-1 ) { + repeat = true; + btTags[id] = 0; + + // Unmark for derefinement + mesh->refine_mark(id, false); + } + } + } + + // If only processing local blocks, check all processors to see if + // a repeat is necessary, then reduce bittree to update on all ranks. + ParallelDescriptor::ReduceBoolOr(repeat); + + if(repeat) { + mesh->refine_reduce_and(comm); + mesh->refine_update(); + } + + } while(repeat); +} + + +// Check all neighbors to see if their adjacent children are parents or marked for refinement. +bool btUnit::checkNeighborsRefine(BittreeAmr* const mesh, MortonTree::Block b) { + + BL_PROFILE("Bittree-checkNeighborsRefine"); + + auto tree0 = mesh->getTree(); + auto tree1 = mesh->getTree(true); + int nIdx[3], cIdx[3]; + unsigned childCoord_u[AMREX_SPACEDIM]; + + // Loop over neighbors + for(nIdx[2]= -1*K3D; nIdx[2]<= K3D; ++nIdx[2]) { + for(nIdx[1]= -1*K2D; nIdx[1]<= K2D; ++nIdx[1]) { + for(nIdx[0]= -1*K1D; nIdx[0]<= K1D; ++nIdx[0]) { + std::vector nCoord = neighIntCoords(mesh, b.level, b.coord, nIdx); + + // If neighbor is outside domain or otherwise invalid, continue. + if(AMREX_D_TERM(nCoord[0]<0, || nCoord[1]<0, || nCoord[2]<0 )) { + continue; + } + + // Identify neighbor from Bittree. + unsigned neighCoord_u[AMREX_SPACEDIM]; + for(unsigned d=0; d(nCoord[d]); + } + auto n = tree0->identify(b.level, neighCoord_u); + if(b.level==n.level && n.is_parent) { + // Loop over children of neighbor. + for(cIdx[2]= 0; cIdx[2]<= K3D; ++cIdx[2]) { + for(cIdx[1]= 0; cIdx[1]<= K2D; ++cIdx[1]) { + for(cIdx[0]= 0; cIdx[0]<= K1D; ++cIdx[0]) { + + // Only check adjacent children + if (( ((1-nIdx[0])/2)==cIdx[0] || nIdx[0] == 0 ) && + ( ((1-nIdx[1])/2)==cIdx[1] || nIdx[1] == 0 ) && + ( ((1-nIdx[2])/2)==cIdx[2] || nIdx[2] == 0 )) { + + // Identify child on updated tree + for(unsigned d=0; d(cIdx[d]); + } + auto c = tree1->identify(n.level+1, childCoord_u); + + // If child WILL be parent, return true + if( c.level==(b.level+1) && c.is_parent) { + return true; + } + } + }}} + } + }}} + + // Return false otherwise + return false; +} + +/** Calculate integer coordinates of neighbors, taking into acount BCs. + * Currently assuming Periodic in all directions. + */ +std::vector btUnit::neighIntCoords(BittreeAmr* const mesh, + unsigned lev, unsigned* lcoord, int* gCell) { + auto tree = mesh->getTree(); + + std::vector neighCoord(AMREX_SPACEDIM); + +//--Calculate integer coordinates of neighbor in direction + for(unsigned d=0;d(lcoord[d]) + gCell[d]; + +//--Make sure not out-of-bounds. If periodic BCs, apply modulo + std::vector maxcoord(AMREX_SPACEDIM); + for(unsigned d=0;d(tree->top_size(d)) << lev; + + for(unsigned d=0;d= maxcoord[d]) { + if ( bcPeriodic[d] == true ) + neighCoord[d] = neighCoord[d] - maxcoord[d]; + else + neighCoord[d] = -1; + } + + } + + return neighCoord; +} + +} diff --git a/Src/Extern/Bittree/CMakeLists.txt b/Src/Extern/Bittree/CMakeLists.txt new file mode 100644 index 00000000000..d51454b2172 --- /dev/null +++ b/Src/Extern/Bittree/CMakeLists.txt @@ -0,0 +1,11 @@ +target_include_directories( amrex + PUBLIC + $) + +add_amrex_define(AMREX_USE_BITTREE NO_LEGACY) + +target_sources( amrex + PRIVATE + AMReX_Bittree.H + AMReX_Bittree.cpp + ) diff --git a/Src/Extern/Bittree/Make.package b/Src/Extern/Bittree/Make.package new file mode 100644 index 00000000000..0aefb6f300a --- /dev/null +++ b/Src/Extern/Bittree/Make.package @@ -0,0 +1,5 @@ +CEXE_headers += AMReX_Bittree.H +CEXE_sources += AMReX_Bittree.cpp + +VPATH_LOCATIONS += $(AMREX_HOME)/Src/Extern/Bittree +INCLUDE_LOCATIONS += $(AMREX_HOME)/Src/Extern/Bittree diff --git a/Tests/Amr/Advection_AmrCore/Exec/GNUmakefile b/Tests/Amr/Advection_AmrCore/Exec/GNUmakefile index 284f9942c50..b7242d85957 100644 --- a/Tests/Amr/Advection_AmrCore/Exec/GNUmakefile +++ b/Tests/Amr/Advection_AmrCore/Exec/GNUmakefile @@ -14,6 +14,7 @@ USE_MPI = TRUE USE_OMP = FALSE USE_CUDA = FALSE USE_PARTICLES = TRUE +USE_BITTREE = FALSE Bpack := ./Make.package Blocs := . diff --git a/Tests/Amr/Advection_AmrCore/Exec/inputs_bittree b/Tests/Amr/Advection_AmrCore/Exec/inputs_bittree new file mode 100644 index 00000000000..b136cbdc951 --- /dev/null +++ b/Tests/Amr/Advection_AmrCore/Exec/inputs_bittree @@ -0,0 +1,81 @@ +# ***************************************************************** +# Run until nsteps == max_step or time == stop_time, +# whichever comes first +# ***************************************************************** +max_step = 1000 +stop_time = 2.0 + +# ***************************************************************** +# Are we restarting from an existing checkpoint file? +# ***************************************************************** +#amr.restart = chk00060 # restart from this checkpoint file + +# ***************************************************************** +# Problem size and geometry +# ***************************************************************** +geometry.prob_lo = 0.0 0.0 0.0 +geometry.prob_hi = 1.0 1.0 0.125 +geometry.is_periodic = 1 1 1 + +# ***************************************************************** +# VERBOSITY +# ***************************************************************** +amr.v = 1 # verbosity in Amr + +# ***************************************************************** +# Resolution and refinement +# ***************************************************************** +amr.n_cell = 64 64 8 +amr.max_level = 2 # maximum level number allowed -- + # number of levels = max_level + 1 + +amr.ref_ratio = 2 2 2 2 # refinement ratio between levels + +# ***************************************************************** +# Control of grid creation +# ***************************************************************** +# Blocking factor for grid creation in each dimension -- +# this ensures that every grid is coarsenable by a factor of 8 -- +# this is mostly relevant for multigrid performance +amr.blocking_factor_x = 8 +amr.blocking_factor_y = 8 +amr.blocking_factor_z = 8 + +amr.max_grid_size = 16 + +amr.regrid_int = 2 # how often to regrid + +amr.use_bittree = 1 + +# ***************************************************************** +# Time step control +# ***************************************************************** +adv.cfl = 0.7 # CFL constraint for explicit advection + +adv.do_subcycle = 1 # Do we subcycle in time? + +# ***************************************************************** +# Should we reflux at coarse-fine boundaries? +# ***************************************************************** +adv.do_reflux = 1 + +# ***************************************************************** +# Tagging - if phi > 1.01 at level 0, then refine +# if phi > 1.1 at level 1, then refine +# if phi > 1.5 at level 2, then refine +# ***************************************************************** +adv.phierr = 1.01 1.1 1.5 + +# ***************************************************************** +# Plotfile name and frequency +# ***************************************************************** +amr.plot_file = plt # root name of plot file +amr.plot_int = 10 # number of timesteps between plot files + # if negative then no plot files will be written + +# ***************************************************************** +# Checkpoint name and frequency +# ***************************************************************** +amr.chk_file = chk # root name of checkpoint file +amr.chk_int = -1 # number of timesteps between checkpoint files + # if negative then no checkpoint files will be written diff --git a/Tools/GNUMake/Make.defs b/Tools/GNUMake/Make.defs index c1b5eca6bff..b41048ef647 100644 --- a/Tools/GNUMake/Make.defs +++ b/Tools/GNUMake/Make.defs @@ -1067,6 +1067,12 @@ ifeq ($(USE_HDF5),TRUE) include $(AMREX_HOME)/Tools/GNUMake/packages/Make.hdf5 endif +ifeq ($(USE_BITTREE),TRUE) + $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.bittree...) + include $(AMREX_HOME)/Tools/GNUMake/packages/Make.bittree +endif + + ifneq ("$(wildcard $(AMREX_HOME)/Tools/GNUMake/Make.local)","") $(info Loading $(AMREX_HOME)/Tools/GNUMake/Make.local...) include $(AMREX_HOME)/Tools/GNUMake/Make.local diff --git a/Tools/GNUMake/packages/Make.bittree b/Tools/GNUMake/packages/Make.bittree new file mode 100644 index 00000000000..98758a915c2 --- /dev/null +++ b/Tools/GNUMake/packages/Make.bittree @@ -0,0 +1,16 @@ + +CPPFLAGS += -DAMREX_USE_BITTREE +include $(AMREX_HOME)/Src/Extern/Bittree/Make.package + +ifndef AMREX_BITTREE_HOME +ifdef BITTREE_$(DIM)D_HOME + AMREX_BITTREE_HOME = $(BITTREE_$(DIM)D_HOME) +endif +endif + +ifdef AMREX_BITTREE_HOME + BITTREE_ABSPATH = $(abspath $(AMREX_BITTREE_HOME)) + INCLUDE_LOCATIONS += $(BITTREE_ABSPATH)/include + LIBRARY_LOCATIONS += $(BITTREE_ABSPATH)/lib + LIBRARIES += -Wl,-rpath,$(BITTREE_ABSPATH)/lib -lbittree +endif diff --git a/Tools/libamrex/configure.py b/Tools/libamrex/configure.py index 42493e4647e..1545f86dfb2 100755 --- a/Tools/libamrex/configure.py +++ b/Tools/libamrex/configure.py @@ -116,6 +116,10 @@ def configure(argv): help="Only relevant to Amr/AmrLevel based codes that need to read probin file or call amrex_probinit", choices=["yes","no"], default="yes") + parser.add_argument("--enable-bittree", + help="Enable Bittree mode [default=no]", + choices=["yes","no"], + default="no") args = parser.parse_args() if args.with_fortran == "no": @@ -154,6 +158,7 @@ def configure(argv): f.write("USE_COMPILE_PIC = {}\n".format("FALSE" if args.enable_pic == "no" else "TRUE")) f.write("CUDA_ARCH = " + args.cuda_arch.strip() + "\n") f.write("AMREX_NO_PROBINIT = {}\n".format("TRUE" if args.enable_probinit == "no" else "FALSE")) + f.write("USE_BITTREE = {}\n".format("TRUE" if args.enable_bittree == "yes" else "FALSE")) f.write("\n") fin = open("GNUmakefile.in","r") From ab1bfb62d5e65942c83e44cd30a2652d402a2b1e Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 28 Sep 2023 17:03:35 -0500 Subject: [PATCH 2/9] Update Src/AmrCore/AMReX_AmrMesh.H Co-authored-by: Weiqun Zhang --- Src/AmrCore/AMReX_AmrMesh.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/AmrCore/AMReX_AmrMesh.H b/Src/AmrCore/AMReX_AmrMesh.H index 5dcdd371451..17782acd835 100644 --- a/Src/AmrCore/AMReX_AmrMesh.H +++ b/Src/AmrCore/AMReX_AmrMesh.H @@ -267,7 +267,7 @@ protected: Vector grids; #ifdef AMREX_USE_BITTREE - bool use_bittree; + bool use_bittree = false; std::shared_ptr btmesh; #endif From 0d0d8fbc901d066482d5521ea41a7703f531c83c Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 28 Sep 2023 17:03:48 -0500 Subject: [PATCH 3/9] Update Src/AmrCore/AMReX_AmrMesh.H Co-authored-by: Weiqun Zhang --- Src/AmrCore/AMReX_AmrMesh.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/AmrCore/AMReX_AmrMesh.H b/Src/AmrCore/AMReX_AmrMesh.H index 17782acd835..2cb17d720f3 100644 --- a/Src/AmrCore/AMReX_AmrMesh.H +++ b/Src/AmrCore/AMReX_AmrMesh.H @@ -268,7 +268,7 @@ protected: #ifdef AMREX_USE_BITTREE bool use_bittree = false; - std::shared_ptr btmesh; + std::unique_ptr btmesh; #endif unsigned int num_setdm = 0; From 8675c4c3d56804624d8b956d5a9fc2dbfea497b0 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 28 Sep 2023 17:03:58 -0500 Subject: [PATCH 4/9] Update Src/AmrCore/AMReX_AmrMesh.cpp Co-authored-by: Weiqun Zhang --- Src/AmrCore/AMReX_AmrMesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index 924f3e38652..413c50ab141 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -927,7 +927,7 @@ AmrMesh::MakeNewGrids (Real time) int ngrids = AMREX_D_TERM(top[0],*top[1],*top[2]); std::vector includes(ngrids,1); - btmesh = std::make_shared(top.data(),includes.data()); + btmesh = std::make_unique(top.data(),includes.data()); // Set BCs for(int d=0; d Date: Thu, 28 Sep 2023 17:04:08 -0500 Subject: [PATCH 5/9] Update Src/AmrCore/AMReX_AmrMesh.cpp Co-authored-by: Weiqun Zhang --- Src/AmrCore/AMReX_AmrMesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index 413c50ab141..72069a66124 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -384,7 +384,7 @@ AmrMesh::InitAmrMesh (int max_level_in, const Vector& n_cell_in, finest_level = -1; #ifdef AMREX_USE_BITTREE - pp.get("use_bittree",use_bittree); + pp.queryAdd("use_bittree",use_bittree); #endif if (check_input) { checkInput(); } From ef3b5249852569669394c7231f0b7249bc4cb263 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 28 Sep 2023 17:04:32 -0500 Subject: [PATCH 6/9] Update Src/AmrCore/AMReX_AmrMesh.cpp Co-authored-by: Weiqun Zhang --- Src/AmrCore/AMReX_AmrMesh.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index 72069a66124..85b1c9d6c54 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -454,8 +454,9 @@ AmrMesh::MakeDistributionMap (int lev, BoxArray const& ba) BL_PROFILE("AmrMesh::MakeDistributionMap()"); - // useful print statement to notify that a new distribution map is being created at lev - amrex::Print() << "creating new distribution map on level: " << lev + 1 << "\n"; + if (verbose) { + amrex::Print() << "Creating new distribution map on level: " << lev << "\n"; + } // initialize new distribution mapping DistributionMapping dm; From 76381984ec7eaa910c8f076c005a9917d0a91329 Mon Sep 17 00:00:00 2001 From: Akash Dhruv Date: Thu, 28 Sep 2023 17:04:50 -0500 Subject: [PATCH 7/9] Update Src/AmrCore/AMReX_AmrMesh.cpp Co-authored-by: Weiqun Zhang --- Src/AmrCore/AMReX_AmrMesh.cpp | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index 85b1c9d6c54..efccf318315 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -458,30 +458,14 @@ AmrMesh::MakeDistributionMap (int lev, BoxArray const& ba) amrex::Print() << "Creating new distribution map on level: " << lev << "\n"; } - // initialize new distribution mapping - DistributionMapping dm; - -#ifdef AMREX_USE_BITTREE - if(!use_bittree) - { -#endif - // create distribution mapping using boxarray - dm.define(ba); - #ifdef AMREX_USE_BITTREE - } + // if (use_bittree) { + // return DistributionMapping(ba); + // } else #endif - -#ifdef AMREX_USE_BITTREE - // Bittree version - if(use_bittree) { - // create distribution mapping - dm.define(ba); + return DistributionMapping(ba); } -#endif - - return dm; } void From af0b7cfaeda1c53ed0bc00b36803718cf67e285c Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Thu, 28 Sep 2023 15:16:05 -0700 Subject: [PATCH 8/9] Update Src/Extern/Bittree/AMReX_Bittree.cpp --- Src/Extern/Bittree/AMReX_Bittree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/Extern/Bittree/AMReX_Bittree.cpp b/Src/Extern/Bittree/AMReX_Bittree.cpp index a47496da069..543b5688208 100644 --- a/Src/Extern/Bittree/AMReX_Bittree.cpp +++ b/Src/Extern/Bittree/AMReX_Bittree.cpp @@ -336,7 +336,7 @@ bool btUnit::checkNeighborsRefine(BittreeAmr* const mesh, MortonTree::Block b) { return false; } -/** Calculate integer coordinates of neighbors, taking into acount BCs. +/** Calculate integer coordinates of neighbors, taking into account BCs. * Currently assuming Periodic in all directions. */ std::vector btUnit::neighIntCoords(BittreeAmr* const mesh, From b176ddfa01b55785e826fe95abd5c0a697a6a6ca Mon Sep 17 00:00:00 2001 From: Tom Klosterman Date: Thu, 28 Sep 2023 17:40:57 -0500 Subject: [PATCH 9/9] Src/Extern/Bittree --- Src/Extern/Bittree/AMReX_Bittree.H | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Src/Extern/Bittree/AMReX_Bittree.H b/Src/Extern/Bittree/AMReX_Bittree.H index b40580c2003..744cd3b0b58 100644 --- a/Src/Extern/Bittree/AMReX_Bittree.H +++ b/Src/Extern/Bittree/AMReX_Bittree.H @@ -17,10 +17,7 @@ LIBRARIES += -lbittree Include in inputs: amr.use_bittree = true -amr.bt_derefine = true -amr.infer_bt_grids = false - - */ +*/ class btUnit { // Functions used in AmrMesh