Skip to content

Commit

Permalink
PartSysBase and SimpleSOL+particles tweaks (#247)
Browse files Browse the repository at this point in the history 
* Move particle output frequency handling into base class.

* Remove some 'm_' prefixes from SOLWithParticlesSystem member vars.

* Fix a comment.

* Separate PartSysBase definitions/declarations to make the interface clearer.

* Explain the unintuitive meaning of the 'num_particles_total' parameter in the SimpleSOL/SOLWithParticles system.

* Move PartSysBase implementation into a separate file.

* Use PartSysBase::init_output in SimpleSOL.

* For particle write, just check pointer is non-null rather than using an extra boolean.

* Doxygen tweak.
  • Loading branch information
@oparry-ukaea
oparry-ukaea authored Jun 25, 2024
1 parent 3848dbd commit 614d190
Show file tree
Hide file tree
Showing 8 changed files with 211 additions and 187 deletions.
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ set(LIB_SRC_FILES
${SRC_DIR}/nektar_interface/particle_cell_mapping/map_particles_common.cpp
${SRC_DIR}/nektar_interface/particle_cell_mapping/map_particles_host.cpp
${SRC_DIR}/nektar_interface/particle_cell_mapping/nektar_graph_local_mapper.cpp
${SRC_DIR}/nektar_interface/solver_base/partsys_base.cpp
${SRC_DIR}/plasma.cpp
${SRC_DIR}/run_info.cpp
${SRC_DIR}/simulation.cpp
Expand Down
7 changes: 6 additions & 1 deletion examples/SimpleSOL/2DWithParticles/2DWithParticles_config.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@
<P> rhoInf = 1.0 </P>
<P> uInf = 1.0 </P>
<P> num_particle_steps_per_fluid_step = 1 </P>
<!--
Note that, for the SOLWithParticles equation system, num_particles_total sets the
number of particles added to the simulation per timestep per MPI rank, rather than
the total number of particles that will be added.
-->
<P> num_particles_total = 100 </P>
<P> num_particles_per_cell = -1 </P>
<P> particle_num_write_particle_steps = 4 </P>
<P> particle_output_freq = 4 </P>
<P> particle_thermal_velocity = 1.0 </P>
<P> particle_number_density = 3e18 </P>
<P> particle_source_region_count = 2 </P>
Expand Down
151 changes: 25 additions & 126 deletions include/nektar_interface/solver_base/partsys_base.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,59 +32,34 @@ class PartSysBase {

/// NESO-Particles ParticleGroup
ParticleGroupSharedPtr particle_group;

/// Compute target
SYCLTargetSharedPtr sycl_target;

/**
* @brief Write particle parameter values to stdout for any parameter.
* @see also report_param()
*/
inline void add_params_report() {
std::cout << "Particle settings:" << std::endl;
for (auto const &[param_lbl, param_str_val] : this->param_vals_to_report) {
std::cout << " " << param_lbl << ": " << param_str_val << std::endl;
}
std::cout << "============================================================="
"=========="
<< std::endl
<< std::endl;
}
/// @brief Report particle parameter values (writes to stdout).
void add_params_report();

/// @brief Clear up memory related to the particle system
void free();

/**
* @brief Clear up memory related to the particle system
* @brief Check whether particle output is scheduled for \p step.
*
* @param step
* @returns true if \p step is a scheduled output step, according to the
* frequency read from the config file, false otherwise
*/
inline void free() {
if (this->h5part_exists) {
this->h5part->close();
}
this->particle_group->free();
this->sycl_target->free();
this->particle_mesh_interface->free();
};
bool is_output_step(int step);

/**
* @brief Write particle properties to an output file.
*
* @param step Time step number.
*/
inline void write(const int step) {
if (this->h5part_exists) {
if (this->sycl_target->comm_pair.rank_parent == 0) {
nprint("Writing particle properties at step", step);
}
this->h5part->write();
} else {
if (this->sycl_target->comm_pair.rank_parent == 0) {
nprint("Ignoring call to write particle data because an output file "
"wasn't set up. init_output() not called?");
}
}
}
void write(const int step);

protected:
/**
* Protected constructor to prohibit direct instantiation.
*
* @brief Protected constructor to prohibit direct instantiation.
* @param session Nektar++ session to use for parameters and simulation
* specification.
* @param graph Nektar++ MeshGraph on which particles exist.
Expand All @@ -94,36 +69,7 @@ class PartSysBase {
PartSysBase(const LU::SessionReaderSharedPtr session,
const SD::MeshGraphSharedPtr graph, ParticleSpec particle_spec,
MPI_Comm comm = MPI_COMM_WORLD,
PartSysOptions options = PartSysOptions())
: session(session), graph(graph), comm(comm), h5part_exists(false),
ndim(graph->GetSpaceDimension()) {

read_params();

// Store options
this->options = options;

// Create interface between particles and nektar++
this->particle_mesh_interface =
std::make_shared<ParticleMeshInterface>(graph, 0, this->comm);
extend_halos_fixed_offset(this->options.extend_halos_offset,
this->particle_mesh_interface);
this->sycl_target =
std::make_shared<SYCLTarget>(0, particle_mesh_interface->get_comm());
this->nektar_graph_local_mapper = std::make_shared<NektarGraphLocalMapper>(
this->sycl_target, this->particle_mesh_interface);
this->domain = std::make_shared<Domain>(this->particle_mesh_interface,
this->nektar_graph_local_mapper);

// Create ParticleGroup
this->particle_group =
std::make_shared<ParticleGroup>(domain, particle_spec, sycl_target);

// Set up map between cell indices
this->cell_id_translation = std::make_shared<CellIDTranslation>(
this->sycl_target, this->particle_group->cell_id_dat,
this->particle_mesh_interface);
}
PartSysOptions options = PartSysOptions());

/// Object used to map to/from nektar geometry ids to 0,N-1
std::shared_ptr<CellIDTranslation> cell_id_translation;
Expand All @@ -135,8 +81,6 @@ class PartSysBase {
SD::MeshGraphSharedPtr graph;
/// HDF5 output file
std::shared_ptr<H5Part> h5part;
/// HDF5 output file flag
bool h5part_exists;
/// Number of spatial dimensions being used
const int ndim;
/// Mapping instance to map particles into nektar++ elements.
Expand All @@ -150,29 +94,28 @@ class PartSysBase {

/**
* @brief Set up per-step particle output
*
* @param fname Output filename. Default is 'particle_trajectory.h5part'.
* @param args Remaining arguments (variable length) should be sym instances
* indicating which ParticleDats are to be written.
*/
template <typename... T>
inline void init_output(std::string fname, T... args) {
if (this->h5part_exists) {
template <typename... T> void init_output(std::string fname, T... args) {
if (this->h5part) {
if (this->sycl_target->comm_pair.rank_parent == 0) {
nprint("Ignoring (duplicate?) call to init_output().");
}
} else {
// Create H5Part instance
this->h5part =
std::make_shared<H5Part>(fname, this->particle_group, args...);
this->h5part_exists = true;
}
}

/// @brief Read some parameters associated with all particle systems.
void read_params();

/**
* @brief Store particle param values in a map.Values are reported later via
* add_params_report()
*
* @param label Label to attach to the parameter
* @param value Value of the parameter that was set
*/
Expand All @@ -183,57 +126,13 @@ class PartSysBase {
param_vals_to_report[label] = ss.str();
}

private:
/// Output frequency read from config file
int output_freq;
/**
* @brief Read some parameters associated with all particle systems.
* Map containing parameter name,value pairs to be written to stdout when
* the nektar equation system is initialised. Populated with report_param().
*/
inline void read_params() {

// Read total number of particles / number per cell from config
int num_parts_per_cell, num_parts_tot;
this->session->LoadParameter(NUM_PARTS_TOT_STR, num_parts_tot, -1);
this->session->LoadParameter(NUM_PARTS_PER_CELL_STR, num_parts_per_cell,
-1);

if (num_parts_tot > 0) {
this->num_parts_tot = num_parts_tot;
if (num_parts_per_cell > 0) {
nprint("Ignoring value of '" + NUM_PARTS_PER_CELL_STR +
"' because "
"'" +
NUM_PARTS_TOT_STR + "' was specified.");
}
} else {
if (num_parts_per_cell > 0) {
// Determine the global number of elements
const int num_elements_local = this->graph->GetNumElements();
int num_elements_global;
MPICHK(MPI_Allreduce(&num_elements_local, &num_elements_global, 1,
MPI_INT, MPI_SUM, this->comm));

// compute the global number of particles
this->num_parts_tot =
((int64_t)num_elements_global) * num_parts_per_cell;

report_param("Number of particles per cell/element",
num_parts_per_cell);
} else {
nprint("Particles disabled (Neither '" + NUM_PARTS_TOT_STR +
"' or "
"'" +
NUM_PARTS_PER_CELL_STR + "' are set)");
}
}
report_param("Total number of particles", this->num_parts_tot);

// Output frequency
int particle_output_freq;
this->session->LoadParameter(PART_OUTPUT_FREQ_STR, particle_output_freq, 0);
report_param("Output frequency (steps)", particle_output_freq);
}

private:
/// Map containing parameter name,value pairs to be written to stdout when the
/// nektar equation system is initialised. Populated with report_param().
std::map<std::string, std::string> param_vals_to_report;
};

Expand Down
38 changes: 17 additions & 21 deletions solvers/SimpleSOL/EquationSystems/SOLWithParticlesSystem.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,42 +46,38 @@ void SOLWithParticlesSystem::v_InitObject(bool DeclareField) {

// Set particle timestep from params
m_session->LoadParameter("num_particle_steps_per_fluid_step",
m_num_part_substeps, 1);
m_session->LoadParameter("particle_num_write_particle_steps",
m_num_write_particle_steps, 0);
m_part_timestep = m_timestep / m_num_part_substeps;
this->num_part_substeps, 1);
this->part_timestep = m_timestep / this->num_part_substeps;

// Store DisContFieldSharedPtr casts of fields in a map, indexed by name, for
// use in particle project,evaluate operations
int idx = 0;
for (auto &field_name : m_session->GetVariables()) {
m_discont_fields[field_name] =
this->discont_fields[field_name] =
std::dynamic_pointer_cast<MR::DisContField>(m_fields[idx]);
idx++;
}

particle_sys->setup_project(
m_discont_fields["rho_src"], m_discont_fields["rhou_src"],
m_discont_fields["rhov_src"], m_discont_fields["E_src"]);
this->particle_sys->setup_project(
this->discont_fields["rho_src"], this->discont_fields["rhou_src"],
this->discont_fields["rhov_src"], this->discont_fields["E_src"]);

particle_sys->setup_evaluate_n(m_discont_fields["rho"]);
particle_sys->setup_evaluate_T(m_discont_fields["T"]);
this->particle_sys->setup_evaluate_n(this->discont_fields["rho"]);
this->particle_sys->setup_evaluate_T(this->discont_fields["T"]);

m_diag_mass_recording = std::make_shared<MassRecording<MR::DisContField>>(
m_session, this->particle_sys, m_discont_fields["rho"]);
this->diag_mass_recording = std::make_shared<MassRecording<MR::DisContField>>(
m_session, this->particle_sys, this->discont_fields["rho"]);
}

bool SOLWithParticlesSystem::v_PostIntegrate(int step) {
// Writes a step of the particle trajectory.
if (m_num_write_particle_steps > 0) {
if ((step % m_num_write_particle_steps) == 0) {
particle_sys->write(step);
particle_sys->write_source_fields();
}
if (this->particle_sys->is_output_step(step)) {
this->particle_sys->write(step);
this->particle_sys->write_source_fields();
}

if (this->mass_recording_enabled) {
m_diag_mass_recording->compute(step);
this->diag_mass_recording->compute(step);
}

this->solver_callback_handler.call_post_integrate(this);
Expand All @@ -92,14 +88,14 @@ bool SOLWithParticlesSystem::v_PreIntegrate(int step) {
this->solver_callback_handler.call_pre_integrate(this);

if (this->mass_recording_enabled) {
m_diag_mass_recording->compute_initial_fluid_mass();
this->diag_mass_recording->compute_initial_fluid_mass();
}
// Update Temperature field
update_temperature();
// Integrate the particle system to the requested time.
particle_sys->integrate(m_time + m_timestep, m_part_timestep);
this->particle_sys->integrate(m_time + m_timestep, this->part_timestep);
// Project onto the source fields
particle_sys->project_source_terms();
this->particle_sys->project_source_terms();

return SOLSystem::v_PreIntegrate(step);
}
Expand Down
22 changes: 10 additions & 12 deletions solvers/SimpleSOL/EquationSystems/SOLWithParticlesSystem.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ class SOLWithParticlesSystem : public SOLSystem {
/// Callback handler to call user defined callbacks.
SolverCallbackHandler<SOLWithParticlesSystem> solver_callback_handler;

// Object that allows optional recording of stats related to mass conservation
std::shared_ptr<MassRecording<MR::DisContField>> m_diag_mass_recording;
/// Object that allows optional recording of stats related to mass
/// conservation
std::shared_ptr<MassRecording<MR::DisContField>> diag_mass_recording;

/// Creates an instance of this class.
static SU::EquationSystemSharedPtr
Expand All @@ -39,20 +40,17 @@ class SOLWithParticlesSystem : public SOLSystem {
const SD::MeshGraphSharedPtr &graph);

protected:
// Flag to toggle mass conservation checking
bool mass_recording_enabled;
// Number of particle timesteps per fluid timestep.
int m_num_part_substeps;
// Number of time steps between particle trajectory step writes.
int m_num_write_particle_steps;
// Particle timestep size.
double m_part_timestep;

/*
Source fields cast to DisContFieldSharedPtr, indexed by name, for use in
particle evaluation/projection methods
*/
std::map<std::string, MR::DisContFieldSharedPtr> m_discont_fields;
std::map<std::string, MR::DisContFieldSharedPtr> discont_fields;
// Flag to toggle mass conservation checking
bool mass_recording_enabled;
// Number of particle timesteps per fluid timestep.
int num_part_substeps;
// Particle timestep size.
double part_timestep;

void update_temperature();

Expand Down
Loading

0 comments on commit 614d190

Please sign in to comment.