Skip to content

Commit

Permalink
Merge pull request #209 from ammarhakim/revert-surf-expand
Browse files Browse the repository at this point in the history 
Change surface expansions back to volume evaluations, fix LBO conservation
  • Loading branch information
@ammarhakim
ammarhakim authored Sep 20, 2023
2 parents 77c3d52 + d48ad9f commit c65aace
Show file tree
Hide file tree
Showing 157 changed files with 9,115 additions and 12,369 deletions.
20 changes: 15 additions & 5 deletions apps/gkyl_vlasov_priv.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,11 @@ struct vm_species {
// Pointers to updaters that apply BC.
struct gkyl_bc_basic *bc_lo[3];
struct gkyl_bc_basic *bc_up[3];
// To simplify BC application, store local skin and ghost ranges
struct gkyl_range lower_skin[GKYL_MAX_DIM];
struct gkyl_range lower_ghost[GKYL_MAX_DIM];
struct gkyl_range upper_skin[GKYL_MAX_DIM];
struct gkyl_range upper_ghost[GKYL_MAX_DIM];
bool bc_is_absorb; // boolean for absorbing BCs since 1/rho is undefined in absorbing BCs
// If BCs are *not* absorbing, primitive variables can be calculated on *extended* range

Expand Down Expand Up @@ -302,15 +307,10 @@ struct vm_field {
struct gkyl_array *cell_avg_magB2; // Integer array for whether |B|^2 *only* uses cell averages for weak division
// Determined when constructing the matrix if |B|^2 < 0.0 at control points
struct gkyl_array *bvar; // magnetic field unit vector and tensor (diagnostic and for use in pkpm model)
struct gkyl_array *ExB; // E x B velocity = E x B/|B|^2 (diagnostic and for use in relativistic pkpm model)
struct gkyl_array *cell_avg_magB2_surf; // Integer array for whether |B|^2 *only* uses cell averages for weak division on a surface
// 2*cdim components, defined on each individual surface (xl, xr, yl, yr, zl, & zr)
// Determined when constructing the matrix if |B|^2 < 0.0 at control points
struct gkyl_array *bvar_surf; // Surface expansion magnetic field unit vector and tensor (for use in pkpm model)
struct gkyl_array *div_b; // Volume expansion of div(b) (for use in pkpm model)
struct gkyl_array *max_b; // max(|b_i|) penalization (for use in pkpm model)
struct gkyl_dg_calc_em_vars *calc_bvar; // Updater to compute magnetic field unit vector and tensor
struct gkyl_dg_calc_em_vars *calc_ExB; // Updater to compute ExB velocity

gkyl_hyper_dg *slvr; // Maxwell solver

Expand All @@ -325,6 +325,11 @@ struct vm_field {
// Pointers to updaters that apply BC.
struct gkyl_bc_basic *bc_lo[3];
struct gkyl_bc_basic *bc_up[3];
// To simplify BC application, store local skin and ghost ranges
struct gkyl_range lower_skin[GKYL_MAX_DIM];
struct gkyl_range lower_ghost[GKYL_MAX_DIM];
struct gkyl_range upper_skin[GKYL_MAX_DIM];
struct gkyl_range upper_ghost[GKYL_MAX_DIM];

double* omegaCfl_ptr;
};
Expand Down Expand Up @@ -376,6 +381,11 @@ struct vm_fluid_species {
// Pointers to updaters that apply BC.
struct gkyl_bc_basic *bc_lo[3];
struct gkyl_bc_basic *bc_up[3];
// To simplify BC application, store local skin and ghost ranges
struct gkyl_range lower_skin[GKYL_MAX_DIM];
struct gkyl_range lower_ghost[GKYL_MAX_DIM];
struct gkyl_range upper_skin[GKYL_MAX_DIM];
struct gkyl_range upper_ghost[GKYL_MAX_DIM];
bool bc_is_absorb; // boolean for absorbing BCs since 1/rho is undefined in absorbing BCs
// If BCs are *not* absorbing, primitive variables can be calculated on *extended* range

Expand Down
40 changes: 9 additions & 31 deletions apps/vm_field.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,24 +69,21 @@ vm_field_new(struct gkyl_vm *vm, struct gkyl_vlasov_app *app)
// ExB = E x B velocity, E x B/|B|^2
f->cell_avg_magB2 = mk_int_arr(app->use_gpu, 1, app->local_ext.volume);
f->bvar = mkarr(app->use_gpu, 9*app->confBasis.num_basis, app->local_ext.volume);
f->ExB = mkarr(app->use_gpu, 3*app->confBasis.num_basis, app->local_ext.volume);
// Surface magnetic field vector organized as:
// [bx_xl, bx_xr, bxbx_xl, bxbx_xr, bxby_xl, bxby_xr, bxbz_xl, bxbz_xr,
// by_yl, by_yr, bxby_yl, bxby_yr, byby_yl, byby_yr, bybz_yl, bybz_yr,
// bz_zl, bz_zr, bxbz_zl, bxbz_zr, bybz_zl, bybz_zr, bzbz_zl, bzbz_zr]
int cdim = app->cdim;
int Ncomp_surf = 2*cdim*4;
int Nbasis_surf = app->confBasis.num_basis/(app->confBasis.poly_order + 1); // *only valid for tensor bases for cdim > 1*
f->cell_avg_magB2_surf = mk_int_arr(app->use_gpu, 2*cdim, app->local_ext.volume);
f->bvar_surf = mkarr(app->use_gpu, Ncomp_surf*Nbasis_surf, app->local_ext.volume);
// Volume expansion of div(b)
f->div_b = mkarr(app->use_gpu, app->confBasis.num_basis, app->local_ext.volume);
// Surface expansion of max b penalization for streaming in PKPM system max(|b_i_l|, |b_i_r|)
f->max_b = mkarr(app->use_gpu, 2*cdim*Nbasis_surf, app->local_ext.volume);

// Create updaters for bvar and ExB
// Create updaters for bvar (needed by PKPM model)
f->calc_bvar = gkyl_dg_calc_em_vars_new(&app->grid, &app->confBasis, &app->local_ext, 0, app->use_gpu);
f->calc_ExB = gkyl_dg_calc_em_vars_new(&app->grid, &app->confBasis, &app->local_ext, 1, app->use_gpu);

f->has_ext_em = false;
f->ext_em_evolve = false;
Expand Down Expand Up @@ -136,7 +133,7 @@ vm_field_new(struct gkyl_vm *vm, struct gkyl_vlasov_app *app)
long buff_sz = 0;
// compute buffer size needed
for (int d=0; d<app->cdim; ++d) {
long vol = app->skin_ghost.lower_skin[d].volume;
long vol = GKYL_MAX(app->skin_ghost.lower_skin[d].volume, app->skin_ghost.upper_skin[d].volume);
buff_sz = buff_sz > vol ? buff_sz : vol;
}
f->bc_buffer = mkarr(app->use_gpu, 8*app->confBasis.num_basis, buff_sz);
Expand Down Expand Up @@ -198,8 +195,10 @@ vm_field_new(struct gkyl_vm *vm, struct gkyl_vlasov_app *app)
else if (f->lower_bc[d] == GKYL_FIELD_RESERVOIR)
bctype = GKYL_BC_MAXWELL_RESERVOIR;

// Create local lower skin and ghost ranges
gkyl_skin_ghost_ranges(&f->lower_skin[d], &f->lower_ghost[d], d, GKYL_LOWER_EDGE, &app->local_ext, ghost);
f->bc_lo[d] = gkyl_bc_basic_new(d, GKYL_LOWER_EDGE, bctype, app->basis_on_dev.confBasis,
&app->skin_ghost.lower_skin[d], &app->skin_ghost.lower_ghost[d], f->em->ncomp, app->cdim, app->use_gpu);
&f->lower_skin[d], &f->lower_ghost[d], f->em->ncomp, app->cdim, app->use_gpu);

// Upper BC updater. Copy BCs by default.
if (f->upper_bc[d] == GKYL_FIELD_COPY)
Expand All @@ -211,8 +210,10 @@ vm_field_new(struct gkyl_vm *vm, struct gkyl_vlasov_app *app)
else if (f->upper_bc[d] == GKYL_FIELD_RESERVOIR)
bctype = GKYL_BC_MAXWELL_RESERVOIR;

// Create local upper skin and ghost ranges
gkyl_skin_ghost_ranges(&f->upper_skin[d], &f->upper_ghost[d], d, GKYL_UPPER_EDGE, &app->local_ext, ghost);
f->bc_up[d] = gkyl_bc_basic_new(d, GKYL_UPPER_EDGE, bctype, app->basis_on_dev.confBasis,
&app->skin_ghost.upper_skin[d], &app->skin_ghost.upper_ghost[d], f->em->ncomp, app->cdim, app->use_gpu);
&f->upper_skin[d], &f->upper_ghost[d], f->em->ncomp, app->cdim, app->use_gpu);
}

gkyl_dg_eqn_release(eqn);
Expand Down Expand Up @@ -277,9 +278,7 @@ vm_field_calc_bvar(gkyl_vlasov_app *app, struct vm_field *field,
// Assumes magnetic field boundary conditions applied so magnetic field
// unit vector and unit tensor are defined everywhere in the domain
gkyl_dg_calc_em_vars_advance(field->calc_bvar, field->tot_em,
field->cell_avg_magB2, field->bvar);
gkyl_dg_calc_em_vars_surf_advance(field->calc_bvar, field->tot_em,
field->cell_avg_magB2_surf, field->bvar_surf);
field->cell_avg_magB2, field->bvar, field->bvar_surf);

// Compute div(b) and max_b = max(|b_i_l|, |b_i_r|)
gkyl_array_clear(field->div_b, 0.0); // Incremented in each dimension, so clear beforehand
Expand All @@ -290,24 +289,6 @@ vm_field_calc_bvar(gkyl_vlasov_app *app, struct vm_field *field,
app->stat.field_em_vars_tm += gkyl_time_diff_now_sec(tm);
}

void
vm_field_calc_ExB(gkyl_vlasov_app *app, struct vm_field *field,
const struct gkyl_array *em)
{
struct timespec tm = gkyl_wall_clock();

gkyl_array_clear(field->tot_em, 0.0);
gkyl_array_set(field->tot_em, 1.0, em);
if (field->has_ext_em)
gkyl_array_accumulate(field->tot_em, 1.0, field->ext_em);
// Assumes electric field and magnetic field boundary conditions applied
// so E x B velocity is defined everywhere in the domain
gkyl_dg_calc_em_vars_advance(field->calc_ExB, field->tot_em,
field->cell_avg_magB2, field->ExB);

app->stat.field_em_vars_tm += gkyl_time_diff_now_sec(tm);
}

void
vm_field_accumulate_current(gkyl_vlasov_app *app,
const struct gkyl_array *fin[], const struct gkyl_array *fluidin[],
Expand Down Expand Up @@ -470,13 +451,10 @@ vm_field_release(const gkyl_vlasov_app* app, struct vm_field *f)

gkyl_array_release(f->cell_avg_magB2);
gkyl_array_release(f->bvar);
gkyl_array_release(f->ExB);
gkyl_array_release(f->cell_avg_magB2_surf);
gkyl_array_release(f->bvar_surf);
gkyl_array_release(f->div_b);
gkyl_array_release(f->max_b);
gkyl_dg_calc_em_vars_release(f->calc_bvar);
gkyl_dg_calc_em_vars_release(f->calc_ExB);

if (f->has_ext_em) {
gkyl_array_release(f->ext_em);
Expand Down
10 changes: 7 additions & 3 deletions apps/vm_fluid_species.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ vm_fluid_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm
long buff_sz = 0;
// compute buffer size needed
for (int d=0; d<app->cdim; ++d) {
long vol = app->skin_ghost.lower_skin[d].volume;
long vol = GKYL_MAX(app->skin_ghost.lower_skin[d].volume, app->skin_ghost.upper_skin[d].volume);
buff_sz = buff_sz > vol ? buff_sz : vol;
}
f->bc_buffer = mkarr(app->use_gpu, num_eqn*app->confBasis.num_basis, buff_sz);
Expand Down Expand Up @@ -235,8 +235,10 @@ vm_fluid_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm
bctype = GKYL_BC_PKPM_MOM_NO_SLIP;
}

// Create local lower skin and ghost ranges
gkyl_skin_ghost_ranges(&f->lower_skin[d], &f->lower_ghost[d], d, GKYL_LOWER_EDGE, &app->local_ext, ghost);
f->bc_lo[d] = gkyl_bc_basic_new(d, GKYL_LOWER_EDGE, bctype, app->basis_on_dev.confBasis,
&app->skin_ghost.lower_skin[d], &app->skin_ghost.lower_ghost[d], f->fluid->ncomp, app->cdim, app->use_gpu);
&f->lower_skin[d], &f->lower_ghost[d], f->fluid->ncomp, app->cdim, app->use_gpu);

// Upper BC updater. Copy BCs by default.
if (f->upper_bc[d] == GKYL_SPECIES_COPY) {
Expand All @@ -253,8 +255,10 @@ vm_fluid_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm
bctype = GKYL_BC_PKPM_MOM_NO_SLIP;
}

// Create local upper skin and ghost ranges
gkyl_skin_ghost_ranges(&f->upper_skin[d], &f->upper_ghost[d], d, GKYL_UPPER_EDGE, &app->local_ext, ghost);
f->bc_up[d] = gkyl_bc_basic_new(d, GKYL_UPPER_EDGE, bctype, app->basis_on_dev.confBasis,
&app->skin_ghost.upper_skin[d], &app->skin_ghost.upper_ghost[d], f->fluid->ncomp, app->cdim, app->use_gpu);
&f->upper_skin[d], &f->upper_ghost[d], f->fluid->ncomp, app->cdim, app->use_gpu);
}
}

Expand Down
39 changes: 17 additions & 22 deletions apps/vm_species.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ vm_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm_speci
long buff_sz = 0;
// compute buffer size needed
for (int d=0; d<cdim; ++d) {
long vol = s->skin_ghost.lower_skin[d].volume;
long vol = GKYL_MAX(s->skin_ghost.lower_skin[d].volume, s->skin_ghost.upper_skin[d].volume);
buff_sz = buff_sz > vol ? buff_sz : vol;
}

Expand Down Expand Up @@ -206,9 +206,9 @@ vm_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm_speci
s->pkpm_fluid_species = vm_find_fluid_species(app, s->info.pkpm_fluid_species);
s->pkpm_fluid_index = vm_find_fluid_species_idx(app, s->info.pkpm_fluid_species);

// pkpm moments for update (rho, p_par, p_perp)
// pkpm moments for update (rho, p_par, p_perp, M1)
vm_species_moment_init(app, s, &s->pkpm_moms, "PKPM");
// pkpm moments for diagnostics (rho, p_par, p_perp, q_par, q_perp, r_parpar, r_parperp)
// pkpm moments for diagnostics (rho, M1, p_par, p_perp, q_par, q_perp, r_parpar, r_parperp)
// For simplicity, is_integrated flag also used by PKPM to turn on diagnostics
vm_species_moment_init(app, s, &s->pkpm_moms_diag, "Integrated");

Expand All @@ -218,7 +218,7 @@ vm_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm_speci
s->pkpm_div_ppar = mkarr(app->use_gpu, app->confBasis.num_basis, app->local_ext.volume);
// allocate array to store primitive moments : [ux, uy, uz, 1/rho*div(p_par b), T_perp/m, m/T_perp]
// pressure p_ij : (p_par - p_perp) b_i b_j + p_perp g_ij
s->pkpm_prim = mkarr(app->use_gpu, 6*app->confBasis.num_basis, app->local_ext.volume);
s->pkpm_prim = mkarr(app->use_gpu, 9*app->confBasis.num_basis, app->local_ext.volume);
s->pkpm_p_ij = mkarr(app->use_gpu, 6*app->confBasis.num_basis, app->local_ext.volume);
// boolean array for if we are only using the cell average for primitive variables
s->cell_avg_prim = mk_int_arr(app->use_gpu, 1, app->local_ext.volume);
Expand Down Expand Up @@ -366,6 +366,7 @@ vm_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm_speci
s->upper_bc[dir] = bc[1];
}
}

// Certain operations fail if absorbing BCs used because absorbing BCs
// means the mass density is 0 in the ghost cells (divide by zero)
s->bc_is_absorb = false;
Expand All @@ -389,8 +390,10 @@ vm_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm_speci
bctype = GKYL_BC_FIXED_FUNC;
}

// Create local lower skin and ghost ranges
gkyl_skin_ghost_ranges(&s->lower_skin[d], &s->lower_ghost[d], d, GKYL_LOWER_EDGE, &s->local_ext, ghost);
s->bc_lo[d] = gkyl_bc_basic_new(d, GKYL_LOWER_EDGE, bctype, app->basis_on_dev.basis,
&s->skin_ghost.lower_skin[d], &s->skin_ghost.lower_ghost[d], s->f->ncomp, app->cdim, app->use_gpu);
&s->lower_skin[d], &s->lower_ghost[d], s->f->ncomp, app->cdim, app->use_gpu);
// Upper BC updater. Copy BCs by default.
if (s->upper_bc[d] == GKYL_SPECIES_COPY) {
bctype = GKYL_BC_COPY;
Expand All @@ -409,8 +412,10 @@ vm_species_init(struct gkyl_vm *vm, struct gkyl_vlasov_app *app, struct vm_speci
bctype = GKYL_BC_FIXED_FUNC;
}

// Create local upper skin and ghost ranges
gkyl_skin_ghost_ranges(&s->upper_skin[d], &s->upper_ghost[d], d, GKYL_UPPER_EDGE, &s->local_ext, ghost);
s->bc_up[d] = gkyl_bc_basic_new(d, GKYL_UPPER_EDGE, bctype, app->basis_on_dev.basis,
&s->skin_ghost.upper_skin[d], &s->skin_ghost.upper_ghost[d], s->f->ncomp, app->cdim, app->use_gpu);
&s->upper_skin[d], &s->upper_ghost[d], s->f->ncomp, app->cdim, app->use_gpu);
}
}

Expand Down Expand Up @@ -484,26 +489,16 @@ vm_species_calc_pkpm_vars(gkyl_vlasov_app *app, struct vm_species *species,
app->field->bvar, app->field->bvar_surf, species->pkpm_moms.marr,
species->pkpm_p_ij, species->pkpm_p_ij_surf);
// Compute primitive variables in both the volume and on surfaces
if (species->bc_is_absorb) {
if (species->bc_is_absorb)
gkyl_dg_calc_pkpm_vars_advance(species->calc_pkpm_vars,
species->pkpm_moms.marr, fluidin[species->pkpm_fluid_index],
species->pkpm_div_ppar, species->cell_avg_prim,
species->pkpm_prim);
gkyl_dg_calc_pkpm_vars_surf_advance(species->calc_pkpm_vars,
species->pkpm_moms.marr, fluidin[species->pkpm_fluid_index],
species->pkpm_p_ij_surf, species->cell_avg_prim,
species->pkpm_prim_surf);
}
else {
species->pkpm_p_ij, species->pkpm_div_ppar,
species->cell_avg_prim, species->pkpm_prim, species->pkpm_prim_surf);
else
gkyl_dg_calc_pkpm_vars_advance(species->calc_pkpm_vars_ext,
species->pkpm_moms.marr, fluidin[species->pkpm_fluid_index],
species->pkpm_div_ppar, species->cell_avg_prim,
species->pkpm_prim);
gkyl_dg_calc_pkpm_vars_surf_advance(species->calc_pkpm_vars_ext,
species->pkpm_moms.marr, fluidin[species->pkpm_fluid_index],
species->pkpm_p_ij_surf, species->cell_avg_prim,
species->pkpm_prim_surf);
}
species->pkpm_p_ij, species->pkpm_div_ppar,
species->cell_avg_prim, species->pkpm_prim, species->pkpm_prim_surf);
}
app->stat.species_pkpm_vars_tm += gkyl_time_diff_now_sec(tm);
}
Expand Down
Loading

0 comments on commit c65aace

Please sign in to comment.