From e45be581810e1b243704393ae51580ef9c517134 Mon Sep 17 00:00:00 2001 From: Stephan Schulz Date: Wed, 28 Jul 2021 18:38:52 +0200 Subject: [PATCH 1/9] Add Cabana load balancing --- .github/workflows/CI.yml | 16 +- .gitignore | 1 + CMakeLists.txt | 12 +- input/in.lj | 3 + src/CMakeLists.txt | 3 + src/CabanaMD_config.hpp.cmakein | 1 + src/cabanamd.h | 8 + src/cabanamd_impl.h | 71 +++++++- src/comm_mpi_impl.h | 2 +- src/inputFile.h | 8 +- src/inputFile_impl.h | 140 ++++++++++++++-- src/load_balancer.h | 107 ++++++++++++ src/system.h | 106 ++++++++---- src/vtk_writer.h | 286 ++++++++++++++++++++++++++++++++ 14 files changed, 711 insertions(+), 53 deletions(-) create mode 100644 src/load_balancer.h create mode 100644 src/vtk_writer.h diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2fc1b12e..f13a3873 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -94,11 +94,20 @@ jobs: -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} cmake --build build --parallel 2 cmake --install build + - name: Checkout ALL + run: | + git clone --depth 1 --branch master https://gitlab.jsc.fz-juelich.de/SLMS/loadbalancing ALL + - name: Build ALL + working-directory: ALL + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$HOME/ALL + cmake --build build --parallel 2 + cmake --install build - name: Checkout Cabana uses: actions/checkout@v3 with: - repository: ECP-CoPA/Cabana - ref: 0.6.1 + repository: aetx/Cabana + ref: cajita_loadbalancer path: cabana - name: Build Cabana working-directory: cabana @@ -106,7 +115,7 @@ jobs: cmake -B build \ -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \ -DMPIEXEC_MAX_NUMPROCS=2 -DMPIEXEC_PREFLAGS="--oversubscribe" \ - -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/arborx" \ + -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/arborx;$HOME/ALL" \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ -DCabana_DISABLE_CAJITA_DEPRECATION_WARNINGS=ON \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} @@ -140,6 +149,7 @@ jobs: -DCabanaMD_LAYOUT=${{ matrix.layout }} \ -DCabanaMD_VECTORLENGTH=${{ matrix.vector }} \ -DCabanaMD_ENABLE_NNP=${{ matrix.nnp }} \ + -DCabanaMD_ENABLE_LB=ON \ -DN2P2_DIR=$HOME/n2p2 \ -DCabanaMD_LAYOUT_NNP=${{ matrix.layout_nnp }} \ -DCabanaMD_VECTORLENGTH_NNP=${{ matrix.vector_nnp }} diff --git a/.gitignore b/.gitignore index 40e1f1ae..c429bde9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ build .cproject .project +*.swp diff --git a/CMakeLists.txt b/CMakeLists.txt index d9f8ad2d..9c7c296c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.11) +cmake_minimum_required(VERSION 3.14) project(CabanaMD LANGUAGES CXX VERSION 0.1.0) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -10,6 +10,7 @@ endif(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CXX_FLAGS) option(CMAKE_VERBOSE_MAKEFILE "Generate verbose Makefiles" OFF) include(GNUInstallDirs) +include(CMakeDependentOption) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) @@ -18,6 +19,15 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) ##---------------------------------------------------------------------------## find_package(Cabana REQUIRED) +if( NOT Cabana_ENABLE_MPI ) + message( FATAL_ERROR "Cabana must be compiled with MPI" ) +endif() +if( NOT Cabana_ENABLE_CAJITA ) + message( FATAL_ERROR "Cabana must be compiled with Cajita" ) +endif() + +cmake_dependent_option(CabanaMD_ENABLE_LB "Utilize Cabana load balancer" ON Cabana_ENABLE_ALL OFF) + ##---------------------------------------------------------------------------## # Set up optional libraries ##---------------------------------------------------------------------------## diff --git a/input/in.lj b/input/in.lj index 9b5664ac..6d804d10 100644 --- a/input/in.lj +++ b/input/in.lj @@ -17,7 +17,10 @@ pair_coeff 1 1 1.0 1.0 2.5 neighbor 0.3 bin neigh_modify every 20 one 50 +comm_modify cutoff * 20 fix 1 all nve thermo 10 +dump dmpvtk all vtk 10 dump_*%.vtu + run 100 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 48050394..e2cdaf11 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,6 +89,9 @@ target_include_directories(CabanaMD PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/system_types $ $) +if(CabanaMD_ENBALE_ALL) + target_link_libraries(CabanaMD ALL::ALL) +endif() #------------------------------------------------------------ diff --git a/src/CabanaMD_config.hpp.cmakein b/src/CabanaMD_config.hpp.cmakein index f14e285e..a12cec77 100644 --- a/src/CabanaMD_config.hpp.cmakein +++ b/src/CabanaMD_config.hpp.cmakein @@ -5,6 +5,7 @@ #define CabanaMD_GIT_COMMIT_HASH "@CabanaMD_GIT_COMMIT_HASH@" #cmakedefine CabanaMD_ENABLE_NNP +#cmakedefine CabanaMD_ENABLE_LB #cmakedefine CabanaMD_LAYOUT @CabanaMD_LAYOUT@ #cmakedefine CabanaMD_VECTORLENGTH "@CabanaMD_VECTORLENGTH@" diff --git a/src/cabanamd.h b/src/cabanamd.h index bfb4582f..75a87b27 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -57,6 +57,11 @@ #include #include +#ifdef CabanaMD_ENABLE_LB +#include +#include +#endif + class CabanaMD { public: @@ -81,6 +86,9 @@ class CbnMD : public CabanaMD Comm *comm; Binning *binning; InputFile *input; +#ifdef CabanaMD_ENABLE_LB + Cajita::Experimental::LoadBalancer> *lb; +#endif void init( InputCL cl ) override; void run() override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index a2c743da..367e5a79 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -51,15 +51,19 @@ #include #include +#include + #include #include #include #include #include +#include #include #include #include +#include #define MAXPATHLEN 1024 @@ -224,6 +228,11 @@ void CbnMD::init( InputCL commandline ) comm->update_force(); } +#ifdef CabanaMD_ENABLE_LB + lb = new Cajita::Experimental::LoadBalancer>( + MPI_COMM_WORLD, system->global_grid ); +#endif + // Initial output int step = 0; if ( input->thermo_rate > 0 ) @@ -236,10 +245,19 @@ void CbnMD::init( InputCL commandline ) auto KE = kine.compute( system ) / system->N; if ( !_print_lammps ) { +#ifdef CabanaMD_ENABLE_LB + log( out, "\n", std::fixed, std::setprecision( 6 ), + "#Timestep Temperature PotE ETot Time Atomsteps/s " + "LBImbalance\n", + step, " ", T, " ", PE, " ", PE + KE, " ", + std::setprecision( 2 ), 0.0, " ", std::scientific, 0.0, " ", + std::setprecision( 2 ), 0.0 ); +#else log( out, "\n", std::fixed, std::setprecision( 6 ), "#Timestep Temperature PotE ETot Time Atomsteps/s\n", step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), 0.0, " ", std::scientific, 0.0 ); +#endif } else { @@ -264,6 +282,7 @@ template void CbnMD::run() { std::ofstream out( input->output_file, std::ofstream::app ); + std::ofstream err( input->error_file, std::ofstream::app ); auto neigh_cutoff = input->force_cutoff + input->neighbor_skin; bool half_neigh = input->force_iteration_type == FORCE_ITER_NEIGH_HALF; @@ -272,15 +291,19 @@ void CbnMD::run() PotE pote( comm ); KinE kine( comm ); + std::string vtk_actual_domain_basename( "domain_act" ); + std::string vtk_lb_domain_basename( "domain_lb" ); + double force_time = 0; double comm_time = 0; double neigh_time = 0; double integrate_time = 0; + double lb_time = 0; double other_time = 0; double last_time = 0; Kokkos::Timer timer, force_timer, comm_timer, neigh_timer, integrate_timer, - other_timer; + lb_timer, other_timer; // Main timestep loop for ( int step = 1; step <= nsteps; step++ ) @@ -292,6 +315,16 @@ void CbnMD::run() if ( step % input->comm_exchange_rate == 0 && step > 0 ) { + // Update domain decomposition + lb_timer.reset(); +#ifdef CabanaMD_ENABLE_LB + double work = system->N_local + system->N_ghost; + auto new_global_grid = lb->createBalancedGlobalGrid( + system->global_mesh, *system->partitioner, work ); + system->update_global_grid( new_global_grid ); +#endif + lb_time += lb_timer.seconds(); + // Exchange atoms across MPI ranks comm_timer.reset(); comm->exchange(); @@ -361,9 +394,16 @@ void CbnMD::run() double time = timer.seconds(); double rate = 1.0 * system->N * input->thermo_rate / ( time - last_time ); +#ifdef CabanaMD_ENABLE_LB + log( out, std::fixed, std::setprecision( 6 ), step, " ", T, " ", + PE, " ", PE + KE, " ", std::setprecision( 2 ), time, " ", + std::scientific, rate, " ", std::setprecision( 2 ), + lb->getImbalance() ); +#else log( out, std::fixed, std::setprecision( 6 ), step, " ", T, " ", PE, " ", PE + KE, " ", std::setprecision( 2 ), time, " ", std::scientific, rate ); +#endif last_time = time; } else @@ -373,8 +413,22 @@ void CbnMD::run() " ", T, " ", PE, " ", PE + KE, " ", time ); last_time = time; } +#ifdef CabanaMD_ENABLE_LB + double work = system->N_local + system->N_ghost; + std::array vertices; + vertices = lb->getVertices(); + VTKWriter::writeDomain( MPI_COMM_WORLD, step, vertices, work, + vtk_actual_domain_basename ); + vertices = lb->getInternalVertices(); + VTKWriter::writeDomain( MPI_COMM_WORLD, step, vertices, work, + vtk_lb_domain_basename ); +#endif } + if ( step % input->vtk_rate == 0 ) + VTKWriter::writeParticles( MPI_COMM_WORLD, step, system, + input->vtk_file, err ); + if ( input->dumpbinaryflag ) dump_binary( step ); @@ -391,16 +445,18 @@ void CbnMD::run() { double steps_per_sec = 1.0 * nsteps / time; double atom_steps_per_sec = system->N * steps_per_sec; + // todo(sschulz): Properly remove lb timing if not enabled. log( out, std::fixed, std::setprecision( 2 ), - "\n#Procs Atoms | Time T_Force T_Neigh T_Comm T_Int ", + "\n#Procs Atoms | Time T_Force T_Neigh T_Comm T_Int T_lb ", "T_Other |\n", comm->num_processes(), " ", system->N, " | ", time, " ", force_time, " ", neigh_time, " ", comm_time, " ", - integrate_time, " ", other_time, " | PERFORMANCE\n", std::fixed, - comm->num_processes(), " ", system->N, " | ", 1.0, " ", + integrate_time, " ", lb_time, " ", other_time, " | PERFORMANCE\n", + std::fixed, comm->num_processes(), " ", system->N, " | ", 1.0, " ", force_time / time, " ", neigh_time / time, " ", comm_time / time, - " ", integrate_time / time, " ", other_time / time, - " | FRACTION\n\n", "#Steps/s Atomsteps/s Atomsteps/(proc*s)\n", - std::scientific, steps_per_sec, " ", atom_steps_per_sec, " ", + " ", integrate_time / time, " ", lb_time / time, " ", + other_time / time, " | FRACTION\n\n", + "#Steps/s Atomsteps/s Atomsteps/(proc*s)\n", std::scientific, + steps_per_sec, " ", atom_steps_per_sec, " ", atom_steps_per_sec / comm->num_processes() ); } else @@ -409,6 +465,7 @@ void CbnMD::run() " procs for ", nsteps, " steps with ", system->N, " atoms" ); } out.close(); + err.close(); if ( input->write_data_flag ) write_data( system, input->output_data_file ); diff --git a/src/comm_mpi_impl.h b/src/comm_mpi_impl.h index bb7a9243..0a1983e3 100644 --- a/src/comm_mpi_impl.h +++ b/src/comm_mpi_impl.h @@ -230,7 +230,7 @@ void Comm::exchange() // resized as well if ( pack_ranks_migrate_all.extent( 0 ) < x.size() ) { - max_local *= 1.1; + max_local = x.size() * 1.1; Kokkos::realloc( pack_ranks_migrate_all, max_local ); } pack_ranks_migrate = diff --git a/src/inputFile.h b/src/inputFile.h index 10503eb9..25b5d793 100644 --- a/src/inputFile.h +++ b/src/inputFile.h @@ -179,8 +179,9 @@ class InputFile int binning_type = BINNING_LINKEDCELL; - int comm_type = COMM_MPI; - int comm_exchange_rate = 20; + int comm_type; + int comm_exchange_rate; + double comm_ghost_cutoff; int force_type = FORCE_LJ; int force_iteration_type; @@ -203,6 +204,9 @@ class InputFile std::string output_data_file; bool read_data_flag = false; bool write_data_flag = false; + bool write_vtk_flag = false; + int vtk_rate; + std::string vtk_file; InputFile( InputCL cl, t_System *s ); void read_file( const char *filename = NULL ); diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index 93fcf9b7..61266b8b 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -50,6 +50,7 @@ #include #include +#include #include std::vector split( const std::string &line ) @@ -77,6 +78,47 @@ InputFile::InputFile( InputCL commandline_, t_System *system_ ) output_file = commandline.output_file; error_file = commandline.error_file; + + // set defaults (matches ExaMiniMD LJ example) + + nsteps = 0; + + thermo_rate = 0; + dumpbinary_rate = 0; + correctness_rate = 0; + vtk_rate = 0; + dumpbinaryflag = false; + correctnessflag = false; + timestepflag = false; + + lattice_offset_x = 0.0; + lattice_offset_y = 0.0; + lattice_offset_z = 0.0; + box[0] = 0; + box[2] = 0; + box[4] = 0; + box[1] = 40; + box[3] = 40; + box[5] = 40; + + units_style = UNITS_LJ; + lattice_style = LATTICE_FCC; + lattice_constant = 0.8442; + + temperature_target = 1.4; + temperature_seed = 87287; + + nsteps = 100; + thermo_rate = 10; + + neighbor_skin = 0.3; + neighbor_skin = 0.0; // for metal and real units + max_neigh_guess = 50; + comm_exchange_rate = 20; + comm_ghost_cutoff = std::pow( ( 4.0 / lattice_constant ), ( 1.0 / 3.0 ) ) * + 20.0; // 20 lattice constants + + force_cutoff = 2.5; } template @@ -291,6 +333,38 @@ void InputFile::check_lammps_command( std::string line, write_data_flag = true; output_data_file = words.at( 1 ); } + if ( keyword.compare( "dump" ) == 0 ) + { + if ( words.at( 3 ).compare( "vtk" ) == 0 ) + { + known = true; + write_vtk_flag = true; + vtk_file = words.at( 5 ); + vtk_rate = std::stod( words.at( 4 ) ); + if ( words.at( 2 ).compare( "all" ) != 0 ) + { + log_err( err, "LAMMPS-Command: 'dump' command only supports " + "dumping 'all' types in CabanaMD" ); + } + size_t pos = 0; + pos = vtk_file.find( "*", pos ); + if ( std::string::npos == pos ) + log_err( err, + "LAMMPS-Command: 'dump' requires '*' in file name, so " + "it can be replaced by the time step in CabanaMD" ); + pos = 0; + pos = vtk_file.find( "%", pos ); + if ( std::string::npos == pos ) + log_err( err, + "LAMMPS-Command: 'dump' requires '%' in file name, so " + "it can be replaced by the rank in CabanaMD" ); + } + else + { + log_err( err, "LAMMPS-Command: 'dump' command only supports 'vtk' " + "in CabanaMD" ); + } + } if ( keyword.compare( "pair_style" ) == 0 ) { if ( words.at( 1 ).compare( "lj/cut" ) == 0 ) @@ -372,6 +446,27 @@ void InputFile::check_lammps_command( std::string line, } } } + if ( keyword.compare( "comm_modify" ) == 0 ) + { + if ( words.at( 1 ).compare( "cutoff" ) == 0 ) + { + if ( words.at( 2 ).compare( "*" ) == 0 ) + { + known = true; + comm_ghost_cutoff = std::stod( words.at( 3 ) ); + } + else + { + log_err( err, "LAMMPS-Command: 'comm_modify' command only " + "supported for all atom types '*' in CabanaMD" ); + } + } + else + { + log_err( err, "LAMMPS-Command: 'comm_modify' command only supports " + "single cutoff 'cutoff' in CabanaMD" ); + } + } if ( keyword.compare( "fix" ) == 0 ) { if ( words.at( 3 ).compare( "nve" ) == 0 ) @@ -455,7 +550,11 @@ void InputFile::create_lattice( Comm *comm ) T_X_FLOAT max_z = lattice_constant * lattice_nz; std::array global_low = { 0.0, 0.0, 0.0 }; std::array global_high = { max_x, max_y, max_z }; - system->create_domain( global_low, global_high ); + // Uncomment the following to create a vacuum for an unbalanced system. + // global_high[0] *= 2; + // global_high[1] *= 2; + // global_high[2] *= 2; + system->create_domain( global_low, global_high, comm_ghost_cutoff ); s = *system; auto local_mesh_lo_x = s.local_mesh_lo_x; @@ -465,12 +564,27 @@ void InputFile::create_lattice( Comm *comm ) auto local_mesh_hi_y = s.local_mesh_hi_y; auto local_mesh_hi_z = s.local_mesh_hi_z; - T_INT ix_start = local_mesh_lo_x / s.global_mesh_x * lattice_nx - 0.5; - T_INT iy_start = local_mesh_lo_y / s.global_mesh_y * lattice_ny - 0.5; - T_INT iz_start = local_mesh_lo_z / s.global_mesh_z * lattice_nz - 0.5; - T_INT ix_end = local_mesh_hi_x / s.global_mesh_x * lattice_nx + 0.5; - T_INT iy_end = local_mesh_hi_y / s.global_mesh_y * lattice_ny + 0.5; - T_INT iz_end = local_mesh_hi_z / s.global_mesh_z * lattice_nz + 0.5; + T_INT ix_start = local_mesh_lo_x / lattice_constant - 0.5; + T_INT iy_start = local_mesh_lo_y / lattice_constant - 0.5; + T_INT iz_start = local_mesh_lo_z / lattice_constant - 0.5; + T_INT ix_end = + std::max( std::min( static_cast( lattice_nx ), + local_mesh_hi_x / lattice_constant + 0.5 ), + static_cast( ix_start ) ); + T_INT iy_end = + std::max( std::min( static_cast( lattice_ny ), + local_mesh_hi_y / lattice_constant + 0.5 ), + static_cast( iy_start ) ); + T_INT iz_end = + std::max( std::min( static_cast( lattice_nz ), + local_mesh_hi_z / lattice_constant + 0.5 ), + static_cast( iz_start ) ); + if ( ix_start == ix_end ) + ix_end -= 1; + if ( iy_start == iy_end ) + iy_end -= 1; + if ( iz_start == iz_end ) + iz_end -= 1; // Create Simple Cubic Lattice if ( lattice_style == LATTICE_SC ) @@ -491,7 +605,8 @@ void InputFile::create_lattice( Comm *comm ) ( ztmp >= local_mesh_lo_z ) && ( xtmp < local_mesh_hi_x ) && ( ytmp < local_mesh_hi_y ) && - ( ztmp < local_mesh_hi_z ) ) + ( ztmp < local_mesh_hi_z ) && ( xtmp < max_x ) && + ( ytmp < max_y ) && ( ztmp < max_z ) ) { n++; } @@ -524,7 +639,8 @@ void InputFile::create_lattice( Comm *comm ) ( ztmp >= local_mesh_lo_z ) && ( xtmp < local_mesh_hi_x ) && ( ytmp < local_mesh_hi_y ) && - ( ztmp < local_mesh_hi_z ) ) + ( ztmp < local_mesh_hi_z ) && ( xtmp < max_x ) && + ( ytmp < max_y ) && ( ztmp < max_z ) ) { x( n, 0 ) = xtmp; x( n, 1 ) = ytmp; @@ -590,7 +706,8 @@ void InputFile::create_lattice( Comm *comm ) ( ztmp >= local_mesh_lo_z ) && ( xtmp < local_mesh_hi_x ) && ( ytmp < local_mesh_hi_y ) && - ( ztmp < local_mesh_hi_z ) ) + ( ztmp < local_mesh_hi_z ) && ( xtmp < max_x ) && + ( ytmp < max_y ) && ( ztmp < max_z ) ) { n++; } @@ -630,7 +747,8 @@ void InputFile::create_lattice( Comm *comm ) ( ztmp >= local_mesh_lo_z ) && ( xtmp < local_mesh_hi_x ) && ( ytmp < local_mesh_hi_y ) && - ( ztmp < local_mesh_hi_z ) ) + ( ztmp < local_mesh_hi_z ) && ( xtmp < max_x ) && + ( ytmp < max_y ) && ( ztmp < max_z ) ) { h_x( n, 0 ) = xtmp; h_x( n, 1 ) = ytmp; diff --git a/src/load_balancer.h b/src/load_balancer.h new file mode 100644 index 00000000..0596cfa5 --- /dev/null +++ b/src/load_balancer.h @@ -0,0 +1,107 @@ +/**************************************************************************** + * Copyright (c) 2018-2021 by the Cabana authors * + * All rights reserved. * + * * + * This file is part of the Cabana library. Cabana is distributed under a * + * BSD 3-clause license. For the licensing terms see the LICENSE file in * + * the top-level directory. * + * * + * SPDX-License-Identifier: BSD-3-Clause * + ****************************************************************************/ + +#ifndef LOADBALANCER_H +#define LOADBALANCER_H + +#include +#include + +#include + +#include + +#include + +template +class LoadBalancer +{ + public: + LoadBalancer( t_System *system ) + : _comm( MPI_COMM_WORLD ) + , _system( system ) + { + MPI_Comm_rank( _comm, &_rank ); + _liball = + std::make_shared>( ALL::TENSOR, 3, 0 ); + std::vector rank_dim_pos( _system->rank_dim_pos.begin(), + _system->rank_dim_pos.end() ); + std::vector ranks_per_dim( _system->ranks_per_dim.begin(), + _system->ranks_per_dim.end() ); + _liball->setProcGridParams( rank_dim_pos, ranks_per_dim ); + // todo(sschulz): Do we need a minimum domain size? + // std::vector min_size( 3, 15 ); + //_liball->setMinDomainSize( min_size ); + _liball->setCommunicator( _comm ); + _liball->setProcTag( _rank ); + _liball->setup(); + + std::vector> lb_vertices( 2, + ALL::Point( 3 ) ); + lb_vertices.at( 0 )[0] = _system->local_mesh_lo_x; + lb_vertices.at( 0 )[1] = _system->local_mesh_lo_y; + lb_vertices.at( 0 )[2] = _system->local_mesh_lo_z; + lb_vertices.at( 1 )[0] = _system->local_mesh_hi_x; + lb_vertices.at( 1 )[1] = _system->local_mesh_hi_y; + lb_vertices.at( 1 )[2] = _system->local_mesh_hi_z; + _liball->setVertices( lb_vertices ); + } + + void balance() + { + int rank; + MPI_Comm_rank( _comm, &rank ); + double work = ( _system->N_local + _system->N_ghost ); + _liball->setWork( work ); + _liball->balance(); + std::vector> updated_vertices = + _liball->getVertices(); + std::array low_corner = { updated_vertices.at( 0 )[0], + updated_vertices.at( 0 )[1], + updated_vertices.at( 0 )[2] }; + std::array high_corner = { updated_vertices.at( 1 )[0], + updated_vertices.at( 1 )[1], + updated_vertices.at( 1 )[2] }; + _system->update_domain( low_corner, high_corner ); + } + + void output( const int t ) const + { + std::string vtk_actual_domain_basename( "domain_act" ); + std::string vtk_lb_domain_basename( "domain_lb" ); + // _liball->printVTKoutlines( t ); + double work = ( _system->N_local + _system->N_ghost ); + + std::array vertices = { + _system->local_mesh_lo_x, _system->local_mesh_lo_y, + _system->local_mesh_lo_z, _system->local_mesh_hi_x, + _system->local_mesh_hi_y, _system->local_mesh_hi_z }; + VTKWriter::writeDomain( _comm, t, vertices, work, + vtk_actual_domain_basename ); + + std::vector> updated_vertices = + _liball->getVertices(); + for ( std::size_t d = 0; d < 3; ++d ) + vertices[d] = updated_vertices.at( 0 )[d]; + for ( std::size_t d = 3; d < 6; ++d ) + vertices[d] = updated_vertices.at( 1 )[d - 3]; + VTKWriter::writeDomain( _comm, t, vertices, work, + vtk_lb_domain_basename ); + } + + private: + MPI_Comm _comm; + t_System *_system; + std::shared_ptr> _liball; + int _rank; +}; + +#endif diff --git a/src/system.h b/src/system.h index 5185f1f7..f1a89b68 100644 --- a/src/system.h +++ b/src/system.h @@ -55,7 +55,11 @@ #include +#include +#include +#include #include +#include #include template @@ -90,9 +94,14 @@ class SystemCommon T_X_FLOAT local_mesh_hi_x, local_mesh_hi_y, local_mesh_hi_z; T_X_FLOAT ghost_mesh_lo_x, ghost_mesh_lo_y, ghost_mesh_lo_z; T_X_FLOAT ghost_mesh_hi_x, ghost_mesh_hi_y, ghost_mesh_hi_z; - std::shared_ptr< - Cabana::Grid::LocalGrid>> + T_X_FLOAT halo_width; + std::shared_ptr partitioner; + std::shared_ptr>> + global_mesh; + std::shared_ptr>> local_grid; + std::shared_ptr>> + global_grid; // Only needed for current comm std::array ranks_per_dim; @@ -129,13 +138,32 @@ class SystemCommon void create_domain( std::array low_corner, std::array high_corner ) { + double ghost_cutoff = + std::max( std::max( high_corner[0] - low_corner[0], + high_corner[2] - low_corner[1] ), + high_corner[2] - low_corner[2] ); + create_domain( low_corner, high_corner, ghost_cutoff ); + } + void create_domain( std::array low_corner, + std::array high_corner, double ghost_cutoff ) + { + halo_width = ghost_cutoff; // Create the MPI partitions. - Cabana::Grid::DimBlockPartitioner<3> partitioner; - ranks_per_dim = partitioner.ranksPerDimension( MPI_COMM_WORLD, {} ); + partitioner = std::make_shared(); + ranks_per_dim = partitioner->ranksPerDimension( MPI_COMM_WORLD, {} ); + int cells_per_dim_per_rank = 1; + + // The load balancing will be able to change the local domains with a + // resolution of 1/cells_per_dim_per_rank + cells_per_dim_per_rank = 100; + std::array cells_per_rank = { + cells_per_dim_per_rank * ranks_per_dim[0], + cells_per_dim_per_rank * ranks_per_dim[1], + cells_per_dim_per_rank * ranks_per_dim[2] }; // Create global mesh of MPI partitions. - auto global_mesh = Cabana::Grid::createUniformGlobalMesh( - low_corner, high_corner, ranks_per_dim ); + global_mesh = Cajita::createUniformGlobalMesh( low_corner, high_corner, + cells_per_rank ); global_mesh_x = global_mesh->extent( 0 ); global_mesh_y = global_mesh->extent( 1 ); @@ -143,8 +171,8 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; - auto global_grid = Cabana::Grid::createGlobalGrid( - MPI_COMM_WORLD, global_mesh, is_periodic, partitioner ); + global_grid = Cajita::createGlobalGrid( MPI_COMM_WORLD, global_mesh, + is_periodic, *partitioner ); for ( int d = 0; d < 3; d++ ) { @@ -152,26 +180,25 @@ class SystemCommon } // Create a local mesh - int halo_width = 1; - local_grid = Cabana::Grid::createLocalGrid( global_grid, halo_width ); - auto local_mesh = - Cabana::Grid::createLocalMesh( *local_grid ); - - local_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Own(), 0 ); - local_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Own(), 1 ); - local_mesh_lo_z = local_mesh.lowCorner( Cabana::Grid::Own(), 2 ); - local_mesh_hi_x = local_mesh.highCorner( Cabana::Grid::Own(), 0 ); - local_mesh_hi_y = local_mesh.highCorner( Cabana::Grid::Own(), 1 ); - local_mesh_hi_z = local_mesh.highCorner( Cabana::Grid::Own(), 2 ); - ghost_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Ghost(), 0 ); - ghost_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Ghost(), 1 ); - ghost_mesh_lo_z = local_mesh.lowCorner( Cabana::Grid::Ghost(), 2 ); - ghost_mesh_hi_x = local_mesh.highCorner( Cabana::Grid::Ghost(), 0 ); - ghost_mesh_hi_y = local_mesh.highCorner( Cabana::Grid::Ghost(), 1 ); - ghost_mesh_hi_z = local_mesh.highCorner( Cabana::Grid::Ghost(), 2 ); - local_mesh_x = local_mesh.extent( Cabana::Grid::Own(), 0 ); - local_mesh_y = local_mesh.extent( Cabana::Grid::Own(), 1 ); - local_mesh_z = local_mesh.extent( Cabana::Grid::Own(), 2 ); + double minimum_cell_size = std::min( + std::min( global_mesh->cellSize( 0 ), global_mesh->cellSize( 1 ) ), + global_mesh->cellSize( 2 ) ); + halo_width = std::ceil( ghost_cutoff / minimum_cell_size ); + // Update local_mesh_* and ghost_mesh_* info as well as create + // local_grid. + update_global_grid( global_grid ); + } + + // Update domain info according to new global grid. We assume that the + // number of ranks (per dim) does not change. We also assume that the + // position of this rank in the cartesian grid of ranks does not change. + void update_global_grid( const std::shared_ptr< + Cajita::GlobalGrid>> + &new_global_grid ) + { + global_grid = new_global_grid; + local_grid = Cajita::createLocalGrid( global_grid, halo_width ); + update_mesh_info(); } void slice_all() @@ -215,6 +242,29 @@ class SystemCommon migrate( std::shared_ptr> distributor ) = 0; virtual void gather( std::shared_ptr> halo ) = 0; virtual const char *name() { return "SystemNone"; } + + private: + // Update local_mesh_* and ghost_mesh* info from global grid + void update_mesh_info() + { + auto local_mesh = Cajita::createLocalMesh( *local_grid ); + + local_mesh_lo_x = local_mesh.lowCorner( Cajita::Own(), 0 ); + local_mesh_lo_y = local_mesh.lowCorner( Cajita::Own(), 1 ); + local_mesh_lo_z = local_mesh.lowCorner( Cajita::Own(), 2 ); + local_mesh_hi_x = local_mesh.highCorner( Cajita::Own(), 0 ); + local_mesh_hi_y = local_mesh.highCorner( Cajita::Own(), 1 ); + local_mesh_hi_z = local_mesh.highCorner( Cajita::Own(), 2 ); + ghost_mesh_lo_x = local_mesh.lowCorner( Cajita::Ghost(), 0 ); + ghost_mesh_lo_y = local_mesh.lowCorner( Cajita::Ghost(), 1 ); + ghost_mesh_lo_z = local_mesh.lowCorner( Cajita::Ghost(), 2 ); + ghost_mesh_hi_x = local_mesh.highCorner( Cajita::Ghost(), 0 ); + ghost_mesh_hi_y = local_mesh.highCorner( Cajita::Ghost(), 1 ); + ghost_mesh_hi_z = local_mesh.highCorner( Cajita::Ghost(), 2 ); + local_mesh_x = local_mesh.extent( Cajita::Own(), 0 ); + local_mesh_y = local_mesh.extent( Cajita::Own(), 1 ); + local_mesh_z = local_mesh.extent( Cajita::Own(), 2 ); + } }; template diff --git a/src/vtk_writer.h b/src/vtk_writer.h new file mode 100644 index 00000000..73359755 --- /dev/null +++ b/src/vtk_writer.h @@ -0,0 +1,286 @@ +/**************************************************************************** + * Copyright (c) 2018-2021 by the Cabana authors * + * All rights reserved. * + * * + * This file is part of the Cabana library. Cabana is distributed under a * + * BSD 3-clause license. For the licensing terms see the LICENSE file in * + * the top-level directory. * + * * + * SPDX-License-Identifier: BSD-3-Clause * + ****************************************************************************/ + +#ifndef VTK_DOMAIN_WRITER_H +#define VTK_DOMAIN_WRITER_H +#include + +#include +#include + +namespace VTKWriter +{ +// Write PVTU +void writeDomainParallelFile( MPI_Comm comm, int time_step, + std::string &basename ) +{ + // Should only be called from a single rank + int size; + MPI_Comm_size( comm, &size ); + std::stringstream filename; + filename << basename << "_" << time_step << ".pvtu"; + FILE *file = fopen( filename.str().c_str(), "w" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\n" ); + for ( int i = 0; i < size; ++i ) + fprintf( file, "\t\n", basename.c_str(), + time_step, i ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fclose( file ); +} + +// Write VTU for domain (low corner, high corner) +// basename will be appended with the corresponding time step, rank and +// extension +void writeDomain( MPI_Comm comm, int time_step, + std::array &domain_vertices, double work, + std::string &basename ) +{ + int rank; + MPI_Comm_rank( comm, &rank ); + if ( rank == 1 ) + writeDomainParallelFile( comm, time_step, basename ); + std::stringstream filename; + // todo(sschulz): properly format, according to max rank + filename << basename << "_" << time_step << "_" << rank << ".vtu"; + FILE *file = fopen( filename.str().c_str(), "w" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + std::array vertices; + vertices[0 * 3 + 0] = domain_vertices[0]; + vertices[2 * 3 + 0] = domain_vertices[0]; + vertices[4 * 3 + 0] = domain_vertices[0]; + vertices[6 * 3 + 0] = domain_vertices[0]; + vertices[0 * 3 + 1] = domain_vertices[1]; + vertices[1 * 3 + 1] = domain_vertices[1]; + vertices[4 * 3 + 1] = domain_vertices[1]; + vertices[5 * 3 + 1] = domain_vertices[1]; + vertices[0 * 3 + 2] = domain_vertices[2]; + vertices[1 * 3 + 2] = domain_vertices[2]; + vertices[2 * 3 + 2] = domain_vertices[2]; + vertices[3 * 3 + 2] = domain_vertices[2]; + vertices[1 * 3 + 0] = domain_vertices[3]; + vertices[3 * 3 + 0] = domain_vertices[3]; + vertices[5 * 3 + 0] = domain_vertices[3]; + vertices[7 * 3 + 0] = domain_vertices[3]; + vertices[2 * 3 + 1] = domain_vertices[4]; + vertices[3 * 3 + 1] = domain_vertices[4]; + vertices[6 * 3 + 1] = domain_vertices[4]; + vertices[7 * 3 + 1] = domain_vertices[4]; + vertices[4 * 3 + 2] = domain_vertices[5]; + vertices[5 * 3 + 2] = domain_vertices[5]; + vertices[6 * 3 + 2] = domain_vertices[5]; + vertices[7 * 3 + 2] = domain_vertices[5]; + std::array connectivity = { 0, 1, 2, 3, 4, 5, 6, 7 }; + fprintf( file, "\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "%d", rank ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "%g", work ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + for ( const double &vert : vertices ) + fprintf( file, "%g ", vert ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + for ( const int &conn : connectivity ) + fprintf( file, "%d ", conn ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "8\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "11\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fclose( file ); +} + +void writeParticlesParallelFile( MPI_Comm comm, const int step, + std::string filename ) +{ + // Should only be called from a single rank + int size; + MPI_Comm_size( comm, &size ); + // Prepare actual filename + // todo(sschulz): Also separate filename construction into function + size_t pos = 0; + pos = filename.find( "*", pos ); + std::stringstream time_string; + time_string << step; + filename.replace( pos, 1, time_string.str() ); + std::string parallel_filename( filename ); + pos = 0; + pos = parallel_filename.find( "%", pos ); + std::string empty_string( "" ); + parallel_filename.replace( pos, 1, empty_string ); + pos = 0; + pos = parallel_filename.find( ".vtu", pos ); + parallel_filename.replace( pos, 4, ".pvtu" ); + FILE *file = fopen( parallel_filename.c_str(), "w" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\t\n" ); + for ( int i = 0; i < size; ++i ) + { + std::string piece_filename( filename ); + pos = 0; + pos = piece_filename.find( "%", pos ); + std::stringstream rank_string; + rank_string << "_" << i; + piece_filename.replace( pos, 1, rank_string.str() ); + fprintf( file, "\t\n", piece_filename.c_str() ); + } + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fclose( file ); +} + +// Write particles to vtu file +// filename must contain * and % which will be replaced by time step and _rank. +// The filename dump_*%.vtu will be create the file dump_43_325.vtu in time +// step 43 on rank 325. +template +void writeParticles( MPI_Comm comm, const int step, t_System *system, + std::string filename, std::ofstream &err ) +{ + int rank; + MPI_Comm_rank( comm, &rank ); + // Write parallel file + if ( rank == 1 ) + writeParticlesParallelFile( comm, step, filename ); + // Prepare actual filename + // todo(sschulz): Separate filename construction into function + size_t pos = 0; + pos = filename.find( "*", pos ); + if ( std::string::npos == pos ) + log_err( err, "VTK output file does not contain required '*'" ); + std::stringstream time_string; + time_string << step; + filename.replace( pos, 1, time_string.str() ); + pos = 0; + pos = filename.find( "%", pos ); + if ( std::string::npos == pos ) + log_err( err, "VTK output file does not contain required '%'" ); + std::stringstream rank_string; + rank_string << "_" << rank; + filename.replace( pos, 1, rank_string.str() ); + // Prepare data + System, + CabanaMD_LAYOUT> + host_system; + system->slice_x(); + auto x = system->x; + host_system.resize( x.size() ); + host_system.slice_x(); + auto host_x = host_system.x; + host_system.deep_copy( *system ); + host_system.slice_all(); + host_x = host_system.x; + auto host_id = host_system.id; + auto host_type = host_system.type; + auto host_v = host_system.v; + FILE *file = fopen( filename.c_str(), "w" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n", + system->N_local ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + for ( int n = 0; n < system->N_local; ++n ) + fprintf( file, "%g %g %g ", host_v( n, 0 ), host_v( n, 1 ), + host_v( n, 2 ) ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + for ( int n = 0; n < system->N_local; ++n ) + fprintf( file, "%d ", host_id( n ) ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + for ( int n = 0; n < system->N_local; ++n ) + fprintf( file, "%d ", host_type( n ) ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + for ( int n = 0; n < system->N_local; ++n ) + fprintf( file, "%g %g %g ", host_x( n, 0 ), host_x( n, 1 ), + host_x( n, 2 ) ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\t\n" ); + fprintf( file, "\n\t\t\n" ); + fprintf( file, "\t\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fprintf( file, "\n" ); + fclose( file ); +} + +} // end namespace VTKWriter +#endif From 6c3444679f4cfc33fe53db7299738aea16a8e028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 19 Jun 2024 23:18:31 +0200 Subject: [PATCH 2/9] Fix deprecated code and remaining warnings --- CMakeLists.txt | 10 +-------- src/cabanamd.h | 6 +++--- src/cabanamd_impl.h | 2 +- src/system.h | 52 ++++++++++++++++++++++----------------------- 4 files changed, 31 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c7c296c..8ad0e9f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,6 @@ endif(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CXX_FLAGS) option(CMAKE_VERBOSE_MAKEFILE "Generate verbose Makefiles" OFF) include(GNUInstallDirs) -include(CMakeDependentOption) set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) @@ -19,14 +18,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) ##---------------------------------------------------------------------------## find_package(Cabana REQUIRED) -if( NOT Cabana_ENABLE_MPI ) - message( FATAL_ERROR "Cabana must be compiled with MPI" ) -endif() -if( NOT Cabana_ENABLE_CAJITA ) - message( FATAL_ERROR "Cabana must be compiled with Cajita" ) -endif() - -cmake_dependent_option(CabanaMD_ENABLE_LB "Utilize Cabana load balancer" ON Cabana_ENABLE_ALL OFF) +option(CabanaMD_ENABLE_LB "Utilize Cabana load balancer" OFF) ##---------------------------------------------------------------------------## # Set up optional libraries diff --git a/src/cabanamd.h b/src/cabanamd.h index 75a87b27..b69ae9a3 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -58,8 +58,8 @@ #include #ifdef CabanaMD_ENABLE_LB -#include -#include +#include +#include #endif class CabanaMD @@ -87,7 +87,7 @@ class CbnMD : public CabanaMD Binning *binning; InputFile *input; #ifdef CabanaMD_ENABLE_LB - Cajita::Experimental::LoadBalancer> *lb; + Cabana::Grid::Experimental::LoadBalancer> *lb; #endif void init( InputCL cl ) override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index 367e5a79..beba2d2c 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -229,7 +229,7 @@ void CbnMD::init( InputCL commandline ) } #ifdef CabanaMD_ENABLE_LB - lb = new Cajita::Experimental::LoadBalancer>( + lb = new Cabana::Grid::Experimental::LoadBalancer>( MPI_COMM_WORLD, system->global_grid ); #endif diff --git a/src/system.h b/src/system.h index f1a89b68..f5ab153c 100644 --- a/src/system.h +++ b/src/system.h @@ -95,12 +95,12 @@ class SystemCommon T_X_FLOAT ghost_mesh_lo_x, ghost_mesh_lo_y, ghost_mesh_lo_z; T_X_FLOAT ghost_mesh_hi_x, ghost_mesh_hi_y, ghost_mesh_hi_z; T_X_FLOAT halo_width; - std::shared_ptr partitioner; - std::shared_ptr>> + std::shared_ptr> partitioner; + std::shared_ptr>> global_mesh; - std::shared_ptr>> + std::shared_ptr>> local_grid; - std::shared_ptr>> + std::shared_ptr>> global_grid; // Only needed for current comm @@ -149,7 +149,7 @@ class SystemCommon { halo_width = ghost_cutoff; // Create the MPI partitions. - partitioner = std::make_shared(); + partitioner = std::make_shared>(); ranks_per_dim = partitioner->ranksPerDimension( MPI_COMM_WORLD, {} ); int cells_per_dim_per_rank = 1; @@ -162,7 +162,7 @@ class SystemCommon cells_per_dim_per_rank * ranks_per_dim[2] }; // Create global mesh of MPI partitions. - global_mesh = Cajita::createUniformGlobalMesh( low_corner, high_corner, + global_mesh = Cabana::Grid::createUniformGlobalMesh( low_corner, high_corner, cells_per_rank ); global_mesh_x = global_mesh->extent( 0 ); @@ -171,7 +171,7 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; - global_grid = Cajita::createGlobalGrid( MPI_COMM_WORLD, global_mesh, + global_grid = Cabana::Grid::createGlobalGrid( MPI_COMM_WORLD, global_mesh, is_periodic, *partitioner ); for ( int d = 0; d < 3; d++ ) @@ -193,11 +193,11 @@ class SystemCommon // number of ranks (per dim) does not change. We also assume that the // position of this rank in the cartesian grid of ranks does not change. void update_global_grid( const std::shared_ptr< - Cajita::GlobalGrid>> + Cabana::Grid::GlobalGrid>> &new_global_grid ) { global_grid = new_global_grid; - local_grid = Cajita::createLocalGrid( global_grid, halo_width ); + local_grid = Cabana::Grid::createLocalGrid( global_grid, halo_width ); update_mesh_info(); } @@ -247,23 +247,23 @@ class SystemCommon // Update local_mesh_* and ghost_mesh* info from global grid void update_mesh_info() { - auto local_mesh = Cajita::createLocalMesh( *local_grid ); - - local_mesh_lo_x = local_mesh.lowCorner( Cajita::Own(), 0 ); - local_mesh_lo_y = local_mesh.lowCorner( Cajita::Own(), 1 ); - local_mesh_lo_z = local_mesh.lowCorner( Cajita::Own(), 2 ); - local_mesh_hi_x = local_mesh.highCorner( Cajita::Own(), 0 ); - local_mesh_hi_y = local_mesh.highCorner( Cajita::Own(), 1 ); - local_mesh_hi_z = local_mesh.highCorner( Cajita::Own(), 2 ); - ghost_mesh_lo_x = local_mesh.lowCorner( Cajita::Ghost(), 0 ); - ghost_mesh_lo_y = local_mesh.lowCorner( Cajita::Ghost(), 1 ); - ghost_mesh_lo_z = local_mesh.lowCorner( Cajita::Ghost(), 2 ); - ghost_mesh_hi_x = local_mesh.highCorner( Cajita::Ghost(), 0 ); - ghost_mesh_hi_y = local_mesh.highCorner( Cajita::Ghost(), 1 ); - ghost_mesh_hi_z = local_mesh.highCorner( Cajita::Ghost(), 2 ); - local_mesh_x = local_mesh.extent( Cajita::Own(), 0 ); - local_mesh_y = local_mesh.extent( Cajita::Own(), 1 ); - local_mesh_z = local_mesh.extent( Cajita::Own(), 2 ); + auto local_mesh = Cabana::Grid::createLocalMesh( *local_grid ); + + local_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Own(), 0 ); + local_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Own(), 1 ); + local_mesh_lo_z = local_mesh.lowCorner( Cabana::Grid::Own(), 2 ); + local_mesh_hi_x = local_mesh.highCorner( Cabana::Grid::Own(), 0 ); + local_mesh_hi_y = local_mesh.highCorner( Cabana::Grid::Own(), 1 ); + local_mesh_hi_z = local_mesh.highCorner( Cabana::Grid::Own(), 2 ); + ghost_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Ghost(), 0 ); + ghost_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Ghost(), 1 ); + ghost_mesh_lo_z = local_mesh.lowCorner( Cabana::Grid::Ghost(), 2 ); + ghost_mesh_hi_x = local_mesh.highCorner( Cabana::Grid::Ghost(), 0 ); + ghost_mesh_hi_y = local_mesh.highCorner( Cabana::Grid::Ghost(), 1 ); + ghost_mesh_hi_z = local_mesh.highCorner( Cabana::Grid::Ghost(), 2 ); + local_mesh_x = local_mesh.extent( Cabana::Grid::Own(), 0 ); + local_mesh_y = local_mesh.extent( Cabana::Grid::Own(), 1 ); + local_mesh_z = local_mesh.extent( Cabana::Grid::Own(), 2 ); } }; From 3e97c049049aca401b6ee6369d5c52f792fa8202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 20 Jun 2024 19:58:52 +0200 Subject: [PATCH 3/9] Apply clang-format --- src/cabanamd.h | 3 ++- src/cabanamd_impl.h | 5 +++-- src/system.h | 27 ++++++++++++++++----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/cabanamd.h b/src/cabanamd.h index b69ae9a3..d5d62224 100644 --- a/src/cabanamd.h +++ b/src/cabanamd.h @@ -87,7 +87,8 @@ class CbnMD : public CabanaMD Binning *binning; InputFile *input; #ifdef CabanaMD_ENABLE_LB - Cabana::Grid::Experimental::LoadBalancer> *lb; + Cabana::Grid::Experimental::LoadBalancer> + *lb; #endif void init( InputCL cl ) override; diff --git a/src/cabanamd_impl.h b/src/cabanamd_impl.h index beba2d2c..8b047ab6 100644 --- a/src/cabanamd_impl.h +++ b/src/cabanamd_impl.h @@ -229,8 +229,9 @@ void CbnMD::init( InputCL commandline ) } #ifdef CabanaMD_ENABLE_LB - lb = new Cabana::Grid::Experimental::LoadBalancer>( - MPI_COMM_WORLD, system->global_grid ); + lb = new Cabana::Grid::Experimental::LoadBalancer< + Cabana::Grid::UniformMesh>( MPI_COMM_WORLD, + system->global_grid ); #endif // Initial output diff --git a/src/system.h b/src/system.h index f5ab153c..9a812856 100644 --- a/src/system.h +++ b/src/system.h @@ -96,11 +96,14 @@ class SystemCommon T_X_FLOAT ghost_mesh_hi_x, ghost_mesh_hi_y, ghost_mesh_hi_z; T_X_FLOAT halo_width; std::shared_ptr> partitioner; - std::shared_ptr>> + std::shared_ptr< + Cabana::Grid::GlobalMesh>> global_mesh; - std::shared_ptr>> + std::shared_ptr< + Cabana::Grid::LocalGrid>> local_grid; - std::shared_ptr>> + std::shared_ptr< + Cabana::Grid::GlobalGrid>> global_grid; // Only needed for current comm @@ -162,8 +165,8 @@ class SystemCommon cells_per_dim_per_rank * ranks_per_dim[2] }; // Create global mesh of MPI partitions. - global_mesh = Cabana::Grid::createUniformGlobalMesh( low_corner, high_corner, - cells_per_rank ); + global_mesh = Cabana::Grid::createUniformGlobalMesh( + low_corner, high_corner, cells_per_rank ); global_mesh_x = global_mesh->extent( 0 ); global_mesh_y = global_mesh->extent( 1 ); @@ -171,8 +174,8 @@ class SystemCommon // Create the global grid. std::array is_periodic = { true, true, true }; - global_grid = Cabana::Grid::createGlobalGrid( MPI_COMM_WORLD, global_mesh, - is_periodic, *partitioner ); + global_grid = Cabana::Grid::createGlobalGrid( + MPI_COMM_WORLD, global_mesh, is_periodic, *partitioner ); for ( int d = 0; d < 3; d++ ) { @@ -192,9 +195,10 @@ class SystemCommon // Update domain info according to new global grid. We assume that the // number of ranks (per dim) does not change. We also assume that the // position of this rank in the cartesian grid of ranks does not change. - void update_global_grid( const std::shared_ptr< - Cabana::Grid::GlobalGrid>> - &new_global_grid ) + void update_global_grid( + const std::shared_ptr< + Cabana::Grid::GlobalGrid>> + &new_global_grid ) { global_grid = new_global_grid; local_grid = Cabana::Grid::createLocalGrid( global_grid, halo_width ); @@ -247,7 +251,8 @@ class SystemCommon // Update local_mesh_* and ghost_mesh* info from global grid void update_mesh_info() { - auto local_mesh = Cabana::Grid::createLocalMesh( *local_grid ); + auto local_mesh = + Cabana::Grid::createLocalMesh( *local_grid ); local_mesh_lo_x = local_mesh.lowCorner( Cabana::Grid::Own(), 0 ); local_mesh_lo_y = local_mesh.lowCorner( Cabana::Grid::Own(), 1 ); From 91eb289f12d02ede399119d4b3ad9109afbf34b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Tue, 25 Jun 2024 14:15:51 +0200 Subject: [PATCH 4/9] Add a CLI flag for creating an unbalanced system --- src/inputCL.cpp | 7 +++++++ src/inputCL.h | 1 + src/inputFile_impl.h | 11 +++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/inputCL.cpp b/src/inputCL.cpp index b975b291..df96366f 100644 --- a/src/inputCL.cpp +++ b/src/inputCL.cpp @@ -116,6 +116,8 @@ void InputCL::read_args( int argc, char *argv[] ) " (N = positive integer)\n", " (PATH = location of ", "directory)\n" ); + log( std::cout, " --sparse : " + "create a vacuum for an unbalanced system\n" ); } // Read Lammps input deck @@ -233,6 +235,11 @@ void InputCL::read_args( int argc, char *argv[] ) i += 3; } + else if ( ( strcmp( argv[i], "--sparse" ) == 0 ) ) + { + sparse = true; + } + else if ( ( strstr( argv[i], "--kokkos-" ) == NULL ) ) { log_err( std::cout, "Unknown command line argument: ", argv[i] ); diff --git a/src/inputCL.h b/src/inputCL.h index c39d3c0b..15a2c64f 100644 --- a/src/inputCL.h +++ b/src/inputCL.h @@ -63,6 +63,7 @@ class InputCL int layout_type; int nnp_layout_type; int device_type; + bool sparse = false; int dumpbinary_rate, correctness_rate; bool dumpbinaryflag, correctnessflag; diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index 61266b8b..b4172607 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -550,10 +550,13 @@ void InputFile::create_lattice( Comm *comm ) T_X_FLOAT max_z = lattice_constant * lattice_nz; std::array global_low = { 0.0, 0.0, 0.0 }; std::array global_high = { max_x, max_y, max_z }; - // Uncomment the following to create a vacuum for an unbalanced system. - // global_high[0] *= 2; - // global_high[1] *= 2; - // global_high[2] *= 2; + if ( commandline.sparse ) + { + // Create a vacuum for an unbalanced system. + global_high[0] *= 2; + global_high[1] *= 2; + global_high[2] *= 2; + } system->create_domain( global_low, global_high, comm_ghost_cutoff ); s = *system; From 063fffe497095952e4c315bffee6da91da9d295d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Fri, 9 Aug 2024 22:34:41 +0200 Subject: [PATCH 5/9] Remove unused file --- src/load_balancer.h | 107 -------------------------------------------- 1 file changed, 107 deletions(-) delete mode 100644 src/load_balancer.h diff --git a/src/load_balancer.h b/src/load_balancer.h deleted file mode 100644 index 0596cfa5..00000000 --- a/src/load_balancer.h +++ /dev/null @@ -1,107 +0,0 @@ -/**************************************************************************** - * Copyright (c) 2018-2021 by the Cabana authors * - * All rights reserved. * - * * - * This file is part of the Cabana library. Cabana is distributed under a * - * BSD 3-clause license. For the licensing terms see the LICENSE file in * - * the top-level directory. * - * * - * SPDX-License-Identifier: BSD-3-Clause * - ****************************************************************************/ - -#ifndef LOADBALANCER_H -#define LOADBALANCER_H - -#include -#include - -#include - -#include - -#include - -template -class LoadBalancer -{ - public: - LoadBalancer( t_System *system ) - : _comm( MPI_COMM_WORLD ) - , _system( system ) - { - MPI_Comm_rank( _comm, &_rank ); - _liball = - std::make_shared>( ALL::TENSOR, 3, 0 ); - std::vector rank_dim_pos( _system->rank_dim_pos.begin(), - _system->rank_dim_pos.end() ); - std::vector ranks_per_dim( _system->ranks_per_dim.begin(), - _system->ranks_per_dim.end() ); - _liball->setProcGridParams( rank_dim_pos, ranks_per_dim ); - // todo(sschulz): Do we need a minimum domain size? - // std::vector min_size( 3, 15 ); - //_liball->setMinDomainSize( min_size ); - _liball->setCommunicator( _comm ); - _liball->setProcTag( _rank ); - _liball->setup(); - - std::vector> lb_vertices( 2, - ALL::Point( 3 ) ); - lb_vertices.at( 0 )[0] = _system->local_mesh_lo_x; - lb_vertices.at( 0 )[1] = _system->local_mesh_lo_y; - lb_vertices.at( 0 )[2] = _system->local_mesh_lo_z; - lb_vertices.at( 1 )[0] = _system->local_mesh_hi_x; - lb_vertices.at( 1 )[1] = _system->local_mesh_hi_y; - lb_vertices.at( 1 )[2] = _system->local_mesh_hi_z; - _liball->setVertices( lb_vertices ); - } - - void balance() - { - int rank; - MPI_Comm_rank( _comm, &rank ); - double work = ( _system->N_local + _system->N_ghost ); - _liball->setWork( work ); - _liball->balance(); - std::vector> updated_vertices = - _liball->getVertices(); - std::array low_corner = { updated_vertices.at( 0 )[0], - updated_vertices.at( 0 )[1], - updated_vertices.at( 0 )[2] }; - std::array high_corner = { updated_vertices.at( 1 )[0], - updated_vertices.at( 1 )[1], - updated_vertices.at( 1 )[2] }; - _system->update_domain( low_corner, high_corner ); - } - - void output( const int t ) const - { - std::string vtk_actual_domain_basename( "domain_act" ); - std::string vtk_lb_domain_basename( "domain_lb" ); - // _liball->printVTKoutlines( t ); - double work = ( _system->N_local + _system->N_ghost ); - - std::array vertices = { - _system->local_mesh_lo_x, _system->local_mesh_lo_y, - _system->local_mesh_lo_z, _system->local_mesh_hi_x, - _system->local_mesh_hi_y, _system->local_mesh_hi_z }; - VTKWriter::writeDomain( _comm, t, vertices, work, - vtk_actual_domain_basename ); - - std::vector> updated_vertices = - _liball->getVertices(); - for ( std::size_t d = 0; d < 3; ++d ) - vertices[d] = updated_vertices.at( 0 )[d]; - for ( std::size_t d = 3; d < 6; ++d ) - vertices[d] = updated_vertices.at( 1 )[d - 3]; - VTKWriter::writeDomain( _comm, t, vertices, work, - vtk_lb_domain_basename ); - } - - private: - MPI_Comm _comm; - t_System *_system; - std::shared_ptr> _liball; - int _rank; -}; - -#endif From 4a1cd7207d8f64818b223e1e741972132e2caaef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Tue, 10 Sep 2024 14:27:24 +0200 Subject: [PATCH 6/9] Use upstream Cabana repository --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f13a3873..ed6f5fe6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -106,8 +106,8 @@ jobs: - name: Checkout Cabana uses: actions/checkout@v3 with: - repository: aetx/Cabana - ref: cajita_loadbalancer + repository: ECP-CoPA/Cabana + ref: 0.6.1 path: cabana - name: Build Cabana working-directory: cabana From c9282ee34fdee76a75768c9abf084f1ef0633800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Tue, 10 Sep 2024 14:29:33 +0200 Subject: [PATCH 7/9] Revert unnecessary changes --- .github/workflows/CI.yml | 1 - .gitignore | 1 - src/CMakeLists.txt | 3 --- src/inputFile.h | 4 ++-- src/inputFile_impl.h | 38 -------------------------------------- 5 files changed, 2 insertions(+), 45 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ed6f5fe6..3d8c352a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -117,7 +117,6 @@ jobs: -DMPIEXEC_MAX_NUMPROCS=2 -DMPIEXEC_PREFLAGS="--oversubscribe" \ -DCMAKE_PREFIX_PATH="$HOME/kokkos;$HOME/arborx;$HOME/ALL" \ -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \ - -DCabana_DISABLE_CAJITA_DEPRECATION_WARNINGS=ON \ -DCMAKE_BUILD_TYPE=${{ matrix.cmake_build_type }} cmake --build build --parallel 2 cmake --install build diff --git a/.gitignore b/.gitignore index c429bde9..40e1f1ae 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ build .cproject .project -*.swp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e2cdaf11..48050394 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,9 +89,6 @@ target_include_directories(CabanaMD PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/system_types $ $) -if(CabanaMD_ENBALE_ALL) - target_link_libraries(CabanaMD ALL::ALL) -endif() #------------------------------------------------------------ diff --git a/src/inputFile.h b/src/inputFile.h index 25b5d793..d70cd546 100644 --- a/src/inputFile.h +++ b/src/inputFile.h @@ -179,8 +179,8 @@ class InputFile int binning_type = BINNING_LINKEDCELL; - int comm_type; - int comm_exchange_rate; + int comm_type = COMM_MPI; + int comm_exchange_rate = 20; double comm_ghost_cutoff; int force_type = FORCE_LJ; diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index b4172607..9213a716 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -79,46 +79,8 @@ InputFile::InputFile( InputCL commandline_, t_System *system_ ) output_file = commandline.output_file; error_file = commandline.error_file; - // set defaults (matches ExaMiniMD LJ example) - - nsteps = 0; - - thermo_rate = 0; - dumpbinary_rate = 0; - correctness_rate = 0; - vtk_rate = 0; - dumpbinaryflag = false; - correctnessflag = false; - timestepflag = false; - - lattice_offset_x = 0.0; - lattice_offset_y = 0.0; - lattice_offset_z = 0.0; - box[0] = 0; - box[2] = 0; - box[4] = 0; - box[1] = 40; - box[3] = 40; - box[5] = 40; - - units_style = UNITS_LJ; - lattice_style = LATTICE_FCC; - lattice_constant = 0.8442; - - temperature_target = 1.4; - temperature_seed = 87287; - - nsteps = 100; - thermo_rate = 10; - - neighbor_skin = 0.3; - neighbor_skin = 0.0; // for metal and real units - max_neigh_guess = 50; - comm_exchange_rate = 20; comm_ghost_cutoff = std::pow( ( 4.0 / lattice_constant ), ( 1.0 / 3.0 ) ) * 20.0; // 20 lattice constants - - force_cutoff = 2.5; } template From c779d483d980d9a91b8a14a49885d7ebe3648a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 26 Jun 2024 22:56:59 +0200 Subject: [PATCH 8/9] Change file naming scheme Place rank before timestep and add leading zeroes in the filenames. This helps ParaView deduce the order correctly. --- input/in.lj | 2 +- src/vtk_writer.h | 45 ++++++++++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/input/in.lj b/input/in.lj index 6d804d10..20198ebc 100644 --- a/input/in.lj +++ b/input/in.lj @@ -21,6 +21,6 @@ comm_modify cutoff * 20 fix 1 all nve thermo 10 -dump dmpvtk all vtk 10 dump_*%.vtu +dump dmpvtk all vtk 10 dump%_*.vtu run 100 diff --git a/src/vtk_writer.h b/src/vtk_writer.h index 73359755..88d54a6b 100644 --- a/src/vtk_writer.h +++ b/src/vtk_writer.h @@ -13,13 +13,26 @@ #define VTK_DOMAIN_WRITER_H #include +#include #include #include +namespace +{ + +std::string set_width( const int value, const unsigned width = 3 ) +{ + std::ostringstream oss; + oss << std::setw( width ) << std::setfill( '0' ) << value; + return oss.str(); +} + +} // namespace + namespace VTKWriter { // Write PVTU -void writeDomainParallelFile( MPI_Comm comm, int time_step, +void writeDomainParallelFile( MPI_Comm comm, const std::string &time_step, std::string &basename ) { // Should only be called from a single rank @@ -41,8 +54,8 @@ void writeDomainParallelFile( MPI_Comm comm, int time_step, "NumberOfComponents=\"3\"/>\n" ); fprintf( file, "\t\n" ); for ( int i = 0; i < size; ++i ) - fprintf( file, "\t\n", basename.c_str(), - time_step, i ); + fprintf( file, "\t\n", basename.c_str(), + i, time_step.c_str() ); fprintf( file, "\n" ); fprintf( file, "\n" ); fclose( file ); @@ -57,11 +70,13 @@ void writeDomain( MPI_Comm comm, int time_step, { int rank; MPI_Comm_rank( comm, &rank ); + + auto time_step_fixed = set_width( time_step ); if ( rank == 1 ) - writeDomainParallelFile( comm, time_step, basename ); + writeDomainParallelFile( comm, time_step_fixed, basename ); std::stringstream filename; // todo(sschulz): properly format, according to max rank - filename << basename << "_" << time_step << "_" << rank << ".vtu"; + filename << basename << "_" << rank << "_" << time_step_fixed << ".vtu"; FILE *file = fopen( filename.str().c_str(), "w" ); fprintf( file, "\n" ); fprintf( file, " -void writeParticles( MPI_Comm comm, const int step, t_System *system, +void writeParticles( MPI_Comm comm, const int time_step, t_System *system, std::string filename, std::ofstream &err ) { int rank; MPI_Comm_rank( comm, &rank ); + auto time_step_fixed = set_width( time_step ); // Write parallel file if ( rank == 1 ) - writeParticlesParallelFile( comm, step, filename ); + writeParticlesParallelFile( comm, time_step_fixed, filename ); // Prepare actual filename // todo(sschulz): Separate filename construction into function size_t pos = 0; pos = filename.find( "*", pos ); if ( std::string::npos == pos ) log_err( err, "VTK output file does not contain required '*'" ); - std::stringstream time_string; - time_string << step; - filename.replace( pos, 1, time_string.str() ); + filename.replace( pos, 1, time_step_fixed ); pos = 0; pos = filename.find( "%", pos ); if ( std::string::npos == pos ) From 066cb1f5f32a7fc88a89a07fb512e13c1d7ecf0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Mon, 16 Sep 2024 18:15:09 +0200 Subject: [PATCH 9/9] Rework `--sparse` option --- src/inputCL.cpp | 18 ++++++++++++++---- src/inputCL.h | 3 ++- src/inputFile_impl.h | 8 ++++---- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/inputCL.cpp b/src/inputCL.cpp index df96366f..11ba6e2c 100644 --- a/src/inputCL.cpp +++ b/src/inputCL.cpp @@ -48,6 +48,7 @@ #include +#include #include #include #include @@ -116,8 +117,12 @@ void InputCL::read_args( int argc, char *argv[] ) " (N = positive integer)\n", " (PATH = location of ", "directory)\n" ); - log( std::cout, " --sparse : " - "create a vacuum for an unbalanced system\n" ); + log( std::cout, + " --vacuum [N]: Create a vacuum for " + "an unbalanced system, enlarging the simulation " + "box N times\n" + " (N = floating-point " + "multiplier, must be bigger than 1.0)\n" ); } // Read Lammps input deck @@ -235,9 +240,14 @@ void InputCL::read_args( int argc, char *argv[] ) i += 3; } - else if ( ( strcmp( argv[i], "--sparse" ) == 0 ) ) + else if ( ( strcmp( argv[i], "--vacuum" ) == 0 ) ) { - sparse = true; + vacuum = true; + vacuum_rate = std::atof( argv[i + 1] ); + if ( vacuum_rate <= 1.0 ) + log_err( std::cout, + "Vacuum multiplier must be bigger than 1.0" ); + i += 2; } else if ( ( strstr( argv[i], "--kokkos-" ) == NULL ) ) diff --git a/src/inputCL.h b/src/inputCL.h index 15a2c64f..b161bf09 100644 --- a/src/inputCL.h +++ b/src/inputCL.h @@ -63,7 +63,8 @@ class InputCL int layout_type; int nnp_layout_type; int device_type; - bool sparse = false; + bool vacuum = false; + double vacuum_rate = 1.0; int dumpbinary_rate, correctness_rate; bool dumpbinaryflag, correctnessflag; diff --git a/src/inputFile_impl.h b/src/inputFile_impl.h index 9213a716..4023615d 100644 --- a/src/inputFile_impl.h +++ b/src/inputFile_impl.h @@ -512,12 +512,12 @@ void InputFile::create_lattice( Comm *comm ) T_X_FLOAT max_z = lattice_constant * lattice_nz; std::array global_low = { 0.0, 0.0, 0.0 }; std::array global_high = { max_x, max_y, max_z }; - if ( commandline.sparse ) + if ( commandline.vacuum ) { // Create a vacuum for an unbalanced system. - global_high[0] *= 2; - global_high[1] *= 2; - global_high[2] *= 2; + global_high[0] *= commandline.vacuum_rate; + global_high[1] *= commandline.vacuum_rate; + global_high[2] *= commandline.vacuum_rate; } system->create_domain( global_low, global_high, comm_ghost_cutoff ); s = *system;