Merge branch 'main' into revert-surf-expand

apps/gkyl_app_priv.h

@@ -46,8 +46,8 @@ static inline struct gkyl_array*
 array_combine(struct gkyl_array *out, double c1, const struct gkyl_array *arr1,
   double c2, const struct gkyl_array *arr2, const struct gkyl_range rng)
 {
-  return gkyl_array_accumulate_range(gkyl_array_set_range(out, c1, arr1, rng),
-    c2, arr2, rng);
+  return gkyl_array_accumulate_range(gkyl_array_set_range(out, c1, arr1, &rng),
+    c2, arr2, &rng);
 }
 
 // Create ghost and skin sub-ranges given a parent range

apps/mom_coupling.c

@@ -43,20 +43,11 @@ moment_coupling_init(const struct gkyl_moment_app *app, struct moment_coupling *
     src->non_ideal_cflrate[n] = mkarr(false, 1, app->local_ext.volume); 
   }
 
-  // create grid and ranges for non-ideal variables (grid is in computational space)
-  // this grid is the grid of node values
-  int ghost[3] = { 2, 2, 2 };
-  double non_ideal_lower[3] = {0.0};
-  double non_ideal_upper[3] = {0.0};
-  int non_ideal_cells[3] = {0};
-  // non-ideal terms (e.g., heat flux tensor) grid has one "extra" cell and is half a grid cell larger past the lower and upper domain
-  for (int d=0; d<app->ndim; ++d) {
-    non_ideal_lower[d] = app->grid.lower[d] - (app->grid.upper[d]-app->grid.lower[d])/(2.0* (double) app->grid.cells[d]);
-    non_ideal_upper[d] = app->grid.upper[d] + (app->grid.upper[d]-app->grid.lower[d])/(2.0* (double) app->grid.cells[d]);
-    non_ideal_cells[d] = app->grid.cells[d] + 1;
-  }
-  gkyl_rect_grid_init(&src->non_ideal_grid, app->ndim, non_ideal_lower, non_ideal_upper, non_ideal_cells);
-  gkyl_create_grid_ranges(&app->grid, ghost, &src->non_ideal_local_ext, &src->non_ideal_local);
+  int ghost[3] = { 1, 1, 1 };
+  // create non-ideal local extended range from local range
+  // has one additional cell in each direction because non-ideal variables are stored at cell vertices
+  gkyl_create_ranges(&app->local, ghost, &src->non_ideal_local_ext, &src->non_ideal_local);
+
   // In Gradient-closure case, non-ideal variables are 10 heat flux tensor components
   for (int n=0;  n<app->num_species; ++n) 
     src->non_ideal_vars[n] = mkarr(false, 10, src->non_ideal_local_ext.volume);
@@ -93,8 +84,10 @@ moment_coupling_update(gkyl_moment_app *app, struct moment_coupling *src,
     app_accels[i] = app->species[i].app_accel;
 
     if (app->species[i].eqn_type == GKYL_EQN_TEN_MOMENT && app->species[i].has_grad_closure) {
+      // non-ideal variables defined on an extended range with one additional "cell" in each direction
+      // this additional cell accounts for the fact that non-ideal variables are stored at cell vertices
       gkyl_ten_moment_grad_closure_advance(src->grad_closure_slvr[i], 
-        &src->non_ideal_local, &app->local, 
+        &src->non_ideal_local_ext, &app->local, 
         app->species[i].f[sidx[nstrang]], app->field.f[sidx[nstrang]], 
         src->non_ideal_cflrate[i], src->non_ideal_vars[i], src->pr_rhs[i]);
     }

apps/mom_field.c

@@ -298,7 +298,7 @@ moment_field_rhs(gkyl_moment_app *app, struct moment_field *fld,
     fld->cflrate, rhs);
 
   double omegaCfl[1];
-  gkyl_array_reduce_range(omegaCfl, fld->cflrate, GKYL_MAX, app->local);
+  gkyl_array_reduce_range(omegaCfl, fld->cflrate, GKYL_MAX, &(app->local));
 
   app->stat.field_rhs_tm += gkyl_time_diff_now_sec(tm);
 

apps/mom_priv.c

@@ -41,11 +41,11 @@ void
 moment_apply_periodic_bc(const gkyl_moment_app *app, struct gkyl_array *bc_buffer,
   int dir, struct gkyl_array *f)
 {
-  gkyl_array_copy_to_buffer(bc_buffer->data, f, app->skin_ghost.lower_skin[dir]);
-  gkyl_array_copy_from_buffer(f, bc_buffer->data, app->skin_ghost.upper_ghost[dir]);
+  gkyl_array_copy_to_buffer(bc_buffer->data, f,   &(app->skin_ghost.lower_skin[dir]));
+  gkyl_array_copy_from_buffer(f, bc_buffer->data, &(app->skin_ghost.upper_ghost[dir]));
 
-  gkyl_array_copy_to_buffer(bc_buffer->data, f, app->skin_ghost.upper_skin[dir]);
-  gkyl_array_copy_from_buffer(f, bc_buffer->data, app->skin_ghost.lower_ghost[dir]);
+  gkyl_array_copy_to_buffer(bc_buffer->data, f,   &(app->skin_ghost.upper_skin[dir]));
+  gkyl_array_copy_from_buffer(f, bc_buffer->data, &(app->skin_ghost.lower_ghost[dir]));
 }
 
 void
@@ -97,9 +97,9 @@ moment_apply_wedge_bc(const gkyl_moment_app *app, double tcurr,
   struct gkyl_array *f)
 {
   gkyl_wv_apply_bc_to_buff(lo, tcurr, update_rng, f, bc_buffer->data);
-  gkyl_array_copy_from_buffer(f, bc_buffer->data, app->skin_ghost.upper_ghost[dir]);
+  gkyl_array_copy_from_buffer(f, bc_buffer->data, &(app->skin_ghost.upper_ghost[dir]));
 
   gkyl_wv_apply_bc_to_buff(up, tcurr, update_rng, f, bc_buffer->data);
-  gkyl_array_copy_from_buffer(f, bc_buffer->data, app->skin_ghost.lower_ghost[dir]);
+  gkyl_array_copy_from_buffer(f, bc_buffer->data, &(app->skin_ghost.lower_ghost[dir]));
 }
 

apps/mom_species.c

@@ -359,7 +359,7 @@ moment_species_rhs(gkyl_moment_app *app, struct moment_species *species,
       species->cflrate, rhs);
 
   double omegaCfl[1];
-  gkyl_array_reduce_range(omegaCfl, species->cflrate, GKYL_MAX, app->local);
+  gkyl_array_reduce_range(omegaCfl, species->cflrate, GKYL_MAX, &(app->local));
 
   app->stat.species_rhs_tm += gkyl_time_diff_now_sec(tm);
 

apps/mom_update_ssp_rk.c

@@ -38,15 +38,15 @@ forward_euler(gkyl_moment_app* app, double tcurr, double dt,
 
   // complete update of species
   for (int i=0; i<app->num_species; ++i) {
-    gkyl_array_accumulate_range(gkyl_array_scale_range(fout[i], dta, app->local),
-      1.0, fin[i], app->local);
+    gkyl_array_accumulate_range(gkyl_array_scale_range(fout[i], dta, &(app->local)),
+      1.0, fin[i], &(app->local));
     moment_species_apply_bc(app, tcurr, &app->species[i], fout[i]);
   }
   if (app->has_field) {
     // complete update of field (even when field is static, it is
     // safest to do this accumulate as it ensure emout = emin)
-    gkyl_array_accumulate_range(gkyl_array_scale_range(emout, dta, app->local),
-      1.0, emin, app->local);
+    gkyl_array_accumulate_range(gkyl_array_scale_range(emout, dta, &(app->local)),
+      1.0, emin, &(app->local));
 
     moment_field_apply_bc(app, tcurr, &app->field, emout);
   }
@@ -138,12 +138,12 @@ moment_update_ssp_rk3(gkyl_moment_app* app, double dt0)
           for (int i=0; i<app->num_species; ++i) {
             array_combine(app->species[i].f1,
               1.0/3.0, app->species[i].f0, 2.0/3.0, app->species[i].fnew, app->local_ext);
-            gkyl_array_copy_range(app->species[i].f0, app->species[i].f1, app->local_ext);
+            gkyl_array_copy_range(app->species[i].f0, app->species[i].f1, &(app->local_ext));
           }
           if (app->has_field) {
             array_combine(app->field.f1,
               1.0/3.0, app->field.f0, 2.0/3.0, app->field.fnew, app->local_ext);
-            gkyl_array_copy_range(app->field.f0, app->field.f1, app->local_ext);
+            gkyl_array_copy_range(app->field.f0, app->field.f1, &(app->local_ext));
           }
 
           state = RK_COMPLETE;

apps/vlasov.c

@@ -328,13 +328,13 @@ gkyl_vlasov_app_calc_integrated_mom(gkyl_vlasov_app* app, double tm)
       gkyl_dg_calc_pkpm_integrated_vars(s->calc_pkpm_vars, &app->local, 
         s->pkpm_moms.marr, fluidin[s->pkpm_fluid_index], 
         s->pkpm_prim, s->integ_pkpm_mom);
-      gkyl_array_scale_range(s->integ_pkpm_mom, app->grid.cellVolume, app->local);
+      gkyl_array_scale_range(s->integ_pkpm_mom, app->grid.cellVolume, &(app->local));
       if (app->use_gpu) {
-        gkyl_array_reduce_range(s->red_integ_diag, s->integ_pkpm_mom, GKYL_SUM, app->local);
+        gkyl_array_reduce_range(s->red_integ_diag, s->integ_pkpm_mom, GKYL_SUM, &(app->local));
         gkyl_cu_memcpy(avals, s->red_integ_diag, sizeof(double[2+GKYL_MAX_DIM]), GKYL_CU_MEMCPY_D2H);
       }
       else { 
-        gkyl_array_reduce_range(avals, s->integ_pkpm_mom, GKYL_SUM, app->local);
+        gkyl_array_reduce_range(avals, s->integ_pkpm_mom, GKYL_SUM, &(app->local));
       }
     }
     else {
@@ -349,11 +349,11 @@ gkyl_vlasov_app_calc_integrated_mom(gkyl_vlasov_app* app, double tm)
       vm_species_moment_calc(&s->integ_moms, s->local, app->local, s->f);
       // reduce to compute sum over whole domain, append to diagnostics
       if (app->use_gpu) {
-        gkyl_array_reduce_range(s->red_integ_diag, s->integ_moms.marr, GKYL_SUM, app->local);
+        gkyl_array_reduce_range(s->red_integ_diag, s->integ_moms.marr, GKYL_SUM, &(app->local));
         gkyl_cu_memcpy(avals, s->red_integ_diag, sizeof(double[2+GKYL_MAX_DIM]), GKYL_CU_MEMCPY_D2H);
       }
       else {
-        gkyl_array_reduce_range(avals, s->integ_moms.marr_host, GKYL_SUM, app->local);
+        gkyl_array_reduce_range(avals, s->integ_moms.marr_host, GKYL_SUM, &(app->local));
       }
     }
 
@@ -882,17 +882,17 @@ rk3(gkyl_vlasov_app* app, double dt0)
           for (int i=0; i<app->num_species; ++i) {
             array_combine(app->species[i].f1,
               1.0/3.0, app->species[i].f, 2.0/3.0, app->species[i].fnew, app->species[i].local_ext);
-            gkyl_array_copy_range(app->species[i].f, app->species[i].f1, app->species[i].local_ext);
+            gkyl_array_copy_range(app->species[i].f, app->species[i].f1, &(app->species[i].local_ext));
           }
           for (int i=0; i<app->num_fluid_species; ++i) {
             array_combine(app->fluid_species[i].fluid1,
               1.0/3.0, app->fluid_species[i].fluid, 2.0/3.0, app->fluid_species[i].fluidnew, app->local_ext);
-            gkyl_array_copy_range(app->fluid_species[i].fluid, app->fluid_species[i].fluid1, app->local_ext);
+            gkyl_array_copy_range(app->fluid_species[i].fluid, app->fluid_species[i].fluid1, &(app->local_ext));
           }
           if (app->has_field) {
             array_combine(app->field->em1,
               1.0/3.0, app->field->em, 2.0/3.0, app->field->emnew, app->local_ext);
-            gkyl_array_copy_range(app->field->em, app->field->em1, app->local_ext);
+            gkyl_array_copy_range(app->field->em, app->field->em1, &(app->local_ext));
           }
 
           state = RK_COMPLETE;
@@ -945,7 +945,7 @@ gkyl_vlasov_app_species_ktm_rhs(gkyl_vlasov_app* app, int update_vol_term)
     //   gkyl_hyper_dg_set_update_vol_cu(species->slvr, update_vol_term);
     // else
     //   gkyl_hyper_dg_set_update_vol(species->slvr, update_vol_term);
-    gkyl_array_clear_range(rhs, 0.0, species->local);
+    gkyl_array_clear_range(rhs, 0.0, &(species->local));
 
     gkyl_dg_updater_vlasov_advance(species->slvr, &species->local, 
       fin, species->cflrate, rhs); 

apps/vm_field.c

@@ -300,17 +300,17 @@ vm_field_accumulate_current(gkyl_vlasov_app *app,
 
     if (s->model_id == GKYL_MODEL_PKPM) {
       // Need to divide out the mass in pkpm model since we evolve momentum
-      gkyl_array_set_range(s->m1i_pkpm, 1.0/s->info.mass, fluidin[s->pkpm_fluid_index], app->local);
-      gkyl_array_accumulate_range(emout, -qbyeps, s->m1i_pkpm, app->local);   
+      gkyl_array_set_range(s->m1i_pkpm, 1.0/s->info.mass, fluidin[s->pkpm_fluid_index], &(app->local));
+      gkyl_array_accumulate_range(emout, -qbyeps, s->m1i_pkpm, &(app->local));   
     }
     else {
       vm_species_moment_calc(&s->m1i, s->local, app->local, fin[i]);
-      gkyl_array_accumulate_range(emout, -qbyeps, s->m1i.marr, app->local);
+      gkyl_array_accumulate_range(emout, -qbyeps, s->m1i.marr, &(app->local));
     }
   } 
   // Accumulate applied current to electric field terms
   if (app->field->has_app_current)
-    gkyl_array_accumulate_range(emout, -1.0/app->field->info.epsilon0, app->field->app_current, app->local);
+    gkyl_array_accumulate_range(emout, -1.0/app->field->info.epsilon0, app->field->app_current, &(app->local));
 }
 
 // Compute the RHS for field update, returning maximum stable
@@ -332,7 +332,7 @@ vm_field_rhs(gkyl_vlasov_app *app, struct vm_field *field,
     else
       gkyl_hyper_dg_advance(field->slvr, &app->local, em, field->cflrate, rhs);
 
-    gkyl_array_reduce_range(field->omegaCfl_ptr, field->cflrate, GKYL_MAX, app->local);
+    gkyl_array_reduce_range(field->omegaCfl_ptr, field->cflrate, GKYL_MAX, &(app->local));
 
     app->stat.nfield_omega_cfl += 1;
     struct timespec tm = gkyl_wall_clock();
@@ -357,11 +357,11 @@ void
 vm_field_apply_periodic_bc(gkyl_vlasov_app *app, const struct vm_field *field,
   int dir, struct gkyl_array *f)
 {
-  gkyl_array_copy_to_buffer(field->bc_buffer->data, f, app->skin_ghost.lower_skin[dir]);
-  gkyl_array_copy_from_buffer(f, field->bc_buffer->data, app->skin_ghost.upper_ghost[dir]);
+  gkyl_array_copy_to_buffer(field->bc_buffer->data, f, &(app->skin_ghost.lower_skin[dir]));
+  gkyl_array_copy_from_buffer(f, field->bc_buffer->data, &(app->skin_ghost.upper_ghost[dir]));
 
-  gkyl_array_copy_to_buffer(field->bc_buffer->data, f, app->skin_ghost.upper_skin[dir]);
-  gkyl_array_copy_from_buffer(f, field->bc_buffer->data, app->skin_ghost.lower_ghost[dir]);
+  gkyl_array_copy_to_buffer(field->bc_buffer->data, f, &(app->skin_ghost.upper_skin[dir]));
+  gkyl_array_copy_from_buffer(f, field->bc_buffer->data, &(app->skin_ghost.lower_ghost[dir]));
 }
 
 // Determine which directions are periodic and which directions are not periodic,
@@ -418,15 +418,15 @@ vm_field_calc_energy(gkyl_vlasov_app *app, double tm, const struct vm_field *fie
 {
   for (int i=0; i<6; ++i)
     gkyl_dg_calc_l2_range(app->confBasis, i, field->em_energy, i, field->em, app->local);
-  gkyl_array_scale_range(field->em_energy, app->grid.cellVolume, app->local);
+  gkyl_array_scale_range(field->em_energy, app->grid.cellVolume, &(app->local));
 
   double energy[6] = { 0.0 };
   if (app->use_gpu) {
-    gkyl_array_reduce_range(field->em_energy_red, field->em_energy, GKYL_SUM, app->local);
+    gkyl_array_reduce_range(field->em_energy_red, field->em_energy, GKYL_SUM, &(app->local));
     gkyl_cu_memcpy(energy, field->em_energy_red, sizeof(double[6]), GKYL_CU_MEMCPY_D2H);
   }
   else { 
-    gkyl_array_reduce_range(energy, field->em_energy, GKYL_SUM, app->local);
+    gkyl_array_reduce_range(energy, field->em_energy, GKYL_SUM, &(app->local));
   }
 
   double energy_global[6] = { 0.0 };

apps/vm_fluid_species.c

@@ -330,7 +330,7 @@ vm_fluid_species_rhs(gkyl_vlasov_app *app, struct vm_fluid_species *fluid_specie
       fluid_species->pkpm_species->qmem, fluid_species->pkpm_species->pkpm_moms.marr, fluid, rhs);
   }
 
-  gkyl_array_reduce_range(fluid_species->omegaCfl_ptr, fluid_species->cflrate, GKYL_MAX, app->local);
+  gkyl_array_reduce_range(fluid_species->omegaCfl_ptr, fluid_species->cflrate, GKYL_MAX, &(app->local));
 
   double omegaCfl_ho[1];
   if (app->use_gpu)
@@ -349,11 +349,11 @@ void
 vm_fluid_species_apply_periodic_bc(gkyl_vlasov_app *app, const struct vm_fluid_species *fluid_species,
   int dir, struct gkyl_array *f)
 {
-  gkyl_array_copy_to_buffer(fluid_species->bc_buffer->data, f, app->skin_ghost.lower_skin[dir]);
-  gkyl_array_copy_from_buffer(f, fluid_species->bc_buffer->data, app->skin_ghost.upper_ghost[dir]);
+  gkyl_array_copy_to_buffer(fluid_species->bc_buffer->data, f, &(app->skin_ghost.lower_skin[dir]));
+  gkyl_array_copy_from_buffer(f, fluid_species->bc_buffer->data, &(app->skin_ghost.upper_ghost[dir]));
 
-  gkyl_array_copy_to_buffer(fluid_species->bc_buffer->data, f, app->skin_ghost.upper_skin[dir]);
-  gkyl_array_copy_from_buffer(f, fluid_species->bc_buffer->data, app->skin_ghost.lower_ghost[dir]);
+  gkyl_array_copy_to_buffer(fluid_species->bc_buffer->data, f, &(app->skin_ghost.upper_skin[dir]));
+  gkyl_array_copy_from_buffer(f, fluid_species->bc_buffer->data, &(app->skin_ghost.lower_ghost[dir]));
 }
 
 // Determine which directions are periodic and which directions are not periodic,

apps/vm_species.c

@@ -555,7 +555,7 @@ vm_species_rhs(gkyl_vlasov_app *app, struct vm_species *species,
 
   app->stat.nspecies_omega_cfl +=1;
   struct timespec tm = gkyl_wall_clock();
-  gkyl_array_reduce_range(species->omegaCfl_ptr, species->cflrate, GKYL_MAX, species->local);
+  gkyl_array_reduce_range(species->omegaCfl_ptr, species->cflrate, GKYL_MAX, &(species->local));
 
   double omegaCfl_ho[1];
   if (app->use_gpu)
@@ -574,11 +574,11 @@ void
 vm_species_apply_periodic_bc(gkyl_vlasov_app *app, const struct vm_species *species,
   int dir, struct gkyl_array *f)
 {
-  gkyl_array_copy_to_buffer(species->bc_buffer->data, f, species->skin_ghost.lower_skin[dir]);
-  gkyl_array_copy_from_buffer(f, species->bc_buffer->data, species->skin_ghost.upper_ghost[dir]);
+  gkyl_array_copy_to_buffer(species->bc_buffer->data, f, &(species->skin_ghost.lower_skin[dir]));
+  gkyl_array_copy_from_buffer(f, species->bc_buffer->data, &(species->skin_ghost.upper_ghost[dir]));
 
-  gkyl_array_copy_to_buffer(species->bc_buffer->data, f, species->skin_ghost.upper_skin[dir]);
-  gkyl_array_copy_from_buffer(f, species->bc_buffer->data, species->skin_ghost.lower_ghost[dir]);
+  gkyl_array_copy_to_buffer(species->bc_buffer->data, f, &(species->skin_ghost.upper_skin[dir]));
+  gkyl_array_copy_from_buffer(f, species->bc_buffer->data, &(species->skin_ghost.lower_ghost[dir]));
 }
 
 // Determine which directions are periodic and which directions are not periodic,
@@ -644,15 +644,15 @@ void
 vm_species_calc_L2(gkyl_vlasov_app *app, double tm, const struct vm_species *species)
 {
   gkyl_dg_calc_l2_range(app->basis, 0, species->L2_f, 0, species->f, species->local);
-  gkyl_array_scale_range(species->L2_f, species->grid.cellVolume, species->local);
+  gkyl_array_scale_range(species->L2_f, species->grid.cellVolume, &(species->local));
 
   double L2[1] = { 0.0 };
   if (app->use_gpu) {
-    gkyl_array_reduce_range(species->red_L2_f, species->L2_f, GKYL_SUM, species->local);
+    gkyl_array_reduce_range(species->red_L2_f, species->L2_f, GKYL_SUM, &(species->local));
     gkyl_cu_memcpy(L2, species->red_L2_f, sizeof(double), GKYL_CU_MEMCPY_D2H);
   }
   else { 
-    gkyl_array_reduce_range(L2, species->L2_f, GKYL_SUM, species->local);
+    gkyl_array_reduce_range(L2, species->L2_f, GKYL_SUM, &(species->local));
   }
   double L2_global[1] = { 0.0 };
   gkyl_comm_all_reduce(app->comm, GKYL_DOUBLE, GKYL_SUM, 1, L2, L2_global);

apps/vm_species_bflux.c

@@ -21,8 +21,8 @@ vm_species_bflux_rhs(gkyl_vlasov_app *app, const struct vm_species *species,
 {
   // zero ghost cells before calculation to ensure there's no residual data
   for (int j=0; j<app->cdim; ++j) {
-    gkyl_array_clear_range(rhs, 0.0, species->skin_ghost.lower_ghost[j]);
-    gkyl_array_clear_range(rhs, 0.0, species->skin_ghost.upper_ghost[j]);
+    gkyl_array_clear_range(rhs, 0.0, &(species->skin_ghost.lower_ghost[j]));
+    gkyl_array_clear_range(rhs, 0.0, &(species->skin_ghost.upper_ghost[j]));
   }
   // ghost cells of the rhs array are filled with the bflux
   // This is overwritten by the boundary conditions and is not being stored,