Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Example leaks #1651

Merged
merged 9 commits into from
Aug 29, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ noether-cpu:
- export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip-int64 && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
- echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
- make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search="petsc fluids-navierstokes solids"
# -- MFEM v4.6
- cd .. && export MFEM_VERSION=mfem-4.6 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.6 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
# -- MFEM v4.7
- cd .. && export MFEM_VERSION=mfem-4.7 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.7 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
- echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
- make -k -j$((NPROC_CPU / NPROC_POOL)) BACKENDS="$BACKENDS_CPU" JUNIT_BATCH="cpu" junit search=mfem
# -- Nek5000 v19.0
Expand Down Expand Up @@ -265,8 +265,8 @@ noether-rocm:
- export PETSC_DIR=/projects/petsc PETSC_ARCH=mpich-hip && git -C $PETSC_DIR -c safe.directory=$PETSC_DIR describe
- echo "-------------- PETSc ---------------" && make -C $PETSC_DIR info
- make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search="petsc fluids solids"
# -- MFEM v4.6
- cd .. && export MFEM_VERSION=mfem-4.6 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.6 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
# -- MFEM v4.7
- cd .. && export MFEM_VERSION=mfem-4.7 && { [[ -d $MFEM_VERSION ]] || { git clone --depth 1 --branch v4.7 https://github.com/mfem/mfem.git $MFEM_VERSION && make -C $MFEM_VERSION -j$(nproc) serial CXXFLAGS="-O -std=c++11"; }; } && export MFEM_DIR=$PWD/$MFEM_VERSION && cd libCEED
- echo "-------------- MFEM ----------------" && make -C $MFEM_DIR info
- make -k -j$((NPROC_GPU / NPROC_POOL)) BACKENDS="$BACKENDS_GPU" JUNIT_BATCH="hip" junit search=mfem
# -- Nek5000 v19.0
Expand Down
2 changes: 1 addition & 1 deletion examples/ceed/ex2-surface.c
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ int main(int argc, const char *argv[]) {
CeedQFunction qf_apply;
if (gallery) {
// This creates the QFunction via the gallery.
char name[16] = "";
char name[25] = "";
snprintf(name, sizeof name, "Poisson%" CeedInt_FMT "DApply", dim);
CeedQFunctionCreateInteriorByName(ceed, name, &qf_apply);
} else {
Expand Down
2 changes: 1 addition & 1 deletion examples/deal.II/bps.cc
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ main(int argc, char *argv[])
#ifdef DEAL_II_WITH_P4EST
parallel::distributed::Triangulation<dim> tria(MPI_COMM_WORLD);
#else
parallel::shared::Triangulation<dim> tria(MPI_COMM_WORLD, ::Triangulation<dim>::none, true);
Triangulation<dim> tria;
#endif

GridGenerator::hyper_cube(tria);
Expand Down
16 changes: 8 additions & 8 deletions examples/deal.II/bps.h
Original file line number Diff line number Diff line change
Expand Up @@ -160,13 +160,13 @@ class OperatorCeed : public OperatorBase<Number>
*/
~OperatorCeed()
{
CeedOperatorDestroy(&op_apply);
CeedQFunctionDestroy(&qf_apply);
CeedQFunctionContextDestroy(&build_ctx);
CeedVectorDestroy(&q_data);
CeedElemRestrictionDestroy(&q_data_restriction);
CeedElemRestrictionDestroy(&sol_restriction);
CeedBasisDestroy(&sol_basis);
CeedQFunctionContextDestroy(&build_ctx);
CeedQFunctionDestroy(&qf_apply);
CeedOperatorDestroy(&op_apply);
CeedDestroy(&ceed);
}

Expand Down Expand Up @@ -641,15 +641,15 @@ class OperatorCeed : public OperatorBase<Number>

CeedOperatorApply(op_build, node_coords, q_data, CEED_REQUEST_IMMEDIATE);

CeedOperatorDestroy(&op_build);
CeedQFunctionDestroy(&qf_build);
CeedQFunctionContextDestroy(&build_ctx);
CeedElemRestrictionDestroy(&geo_restriction);
CeedVectorDestroy(&node_coords);
CeedElemRestrictionDestroy(&q_data_restriction);
CeedVectorSyncArray(q_data, CEED_MEM_HOST);
CeedVectorDestroy(&q_data);
CeedElemRestrictionDestroy(&geo_restriction);
CeedElemRestrictionDestroy(&q_data_restriction);
CeedBasisDestroy(&geo_basis);
CeedQFunctionContextDestroy(&build_ctx);
CeedQFunctionDestroy(&qf_build);
CeedOperatorDestroy(&op_build);

return weights;
}
Expand Down
18 changes: 2 additions & 16 deletions examples/fluids/navierstokes.c
Original file line number Diff line number Diff line change
Expand Up @@ -328,22 +328,8 @@ int main(int argc, char **argv) {

PetscCall(PetscFree(app_ctx->amat_type));
PetscCall(PetscFree(app_ctx->wall_forces.walls));
{
const char *filename = NULL;
PetscBool is_stdout = PETSC_FALSE;

if (app_ctx->wall_forces.viewer) {
PetscCall(PetscViewerFileGetName(app_ctx->wall_forces.viewer, &filename));
if (filename) PetscCall(PetscStrncmp(filename, "stdout", 7, &is_stdout));
if (!is_stdout) PetscCall(PetscViewerDestroy(&app_ctx->wall_forces.viewer));
}

if (app_ctx->turb_spanstats_viewer) {
PetscCall(PetscViewerFileGetName(app_ctx->turb_spanstats_viewer, &filename));
if (filename) PetscCall(PetscStrncmp(filename, "stdout", 7, &is_stdout));
if (!is_stdout) PetscCall(PetscViewerDestroy(&app_ctx->turb_spanstats_viewer));
}
}
PetscCall(PetscViewerDestroy(&app_ctx->wall_forces.viewer));
PetscCall(PetscViewerDestroy(&app_ctx->turb_spanstats_viewer));

// -- Structs
for (PetscInt i = 0; i < problem->num_bc_defs; i++) {
Expand Down
1 change: 1 addition & 0 deletions examples/fluids/problems/advection.c
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ PetscErrorCode CreateKSPMassOperator_AdvectionStabilized(User user, CeedOperator
PetscCallCeed(ceed, CeedOperatorSetField(*op_mass, "v", elem_restr_q, basis_q, CEED_VECTOR_ACTIVE));
PetscCallCeed(ceed, CeedOperatorSetField(*op_mass, "Grad_v", elem_restr_q, basis_q, CEED_VECTOR_ACTIVE));

PetscCallCeed(ceed, CeedQFunctionContextDestroy(&qf_ctx));
PetscCallCeed(ceed, CeedQFunctionDestroy(&qf_mass));
PetscFunctionReturn(PETSC_SUCCESS);
}
Expand Down
1 change: 1 addition & 0 deletions examples/mfem/bp3.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@ int main(int argc, char *argv[]) {
delete fespace;
delete fec;
delete mesh;
delete D;
CeedDestroy(&ceed);
return 0;
}
2 changes: 1 addition & 1 deletion examples/petsc/area.c
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ int main(int argc, char **argv) {
// Setup libCEED's objects and apply setup operator
PetscCall(PetscMalloc1(1, &ceed_data));
PetscCall(SetupLibceedByDegree(dm, ceed, degree, topo_dim, q_extra, num_comp_x, num_comp_u, g_size, xl_size, problem_options[problem_choice],
ceed_data, false, (CeedVector)NULL, (CeedVector *)NULL));
ceed_data, false, true, (CeedVector)NULL, (CeedVector *)NULL));

// Setup output vector
PetscCall(VecZeroEntries(V_loc));
Expand Down
2 changes: 1 addition & 1 deletion examples/petsc/bps.c
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ static PetscErrorCode RunWithDM(RunParams rp, DM dm, const char *ceed_resource)

PetscCall(PetscMalloc1(1, &ceed_data));
PetscCall(SetupLibceedByDegree(dm, ceed, rp->degree, rp->dim, rp->q_extra, rp->dim, rp->num_comp_u, g_size, xl_size, bp_options[rp->bp_choice],
ceed_data, true, rhs_ceed, &target));
ceed_data, true, true, rhs_ceed, &target));

// Gather RHS
PetscCall(VecC2P(rhs_ceed, mem_type, rhs_loc));
Expand Down
16 changes: 7 additions & 9 deletions examples/petsc/bpsraw.c
Original file line number Diff line number Diff line change
Expand Up @@ -438,9 +438,6 @@ int main(int argc, char **argv) {
PetscCall(VecSetFromOptions(X));
PetscCall(VecSetUp(X));

// Set up libCEED
CeedInit(ceed_resource, &ceed);

// Print summary
PetscInt gsize;

Expand Down Expand Up @@ -798,21 +795,22 @@ int main(int argc, char **argv) {
CeedVectorDestroy(&op_apply_ctx->y_ceed);
CeedVectorDestroy(&op_apply_ctx->q_data);
CeedVectorDestroy(&target);
CeedOperatorDestroy(&op_setup_geo);
CeedOperatorDestroy(&op_setup_rhs);
CeedOperatorDestroy(&op_apply);
CeedOperatorDestroy(&op_error);
CeedElemRestrictionDestroy(&elem_restr_u);
CeedElemRestrictionDestroy(&elem_restr_x);
CeedElemRestrictionDestroy(&elem_restr_u_i);
CeedElemRestrictionDestroy(&elem_restr_qd_i);
CeedBasisDestroy(&basis_u);
CeedBasisDestroy(&basis_x);
CeedQFunctionDestroy(&qf_setup_geo);
CeedQFunctionDestroy(&qf_setup_rhs);
CeedQFunctionDestroy(&qf_apply);
CeedQFunctionDestroy(&qf_error);
CeedBasisDestroy(&basis_u);
CeedBasisDestroy(&basis_x);
CeedOperatorDestroy(&op_setup_geo);
CeedOperatorDestroy(&op_setup_rhs);
CeedOperatorDestroy(&op_apply);
CeedOperatorDestroy(&op_error);
CeedDestroy(&ceed);

PetscCall(PetscFree(op_apply_ctx));
return PetscFinalize();
}
2 changes: 1 addition & 1 deletion examples/petsc/bpssphere.c
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ int main(int argc, char **argv) {
// Setup libCEED's objects
PetscCall(PetscMalloc1(1, &ceed_data));
PetscCall(SetupLibceedByDegree(dm, ceed, degree, topo_dim, q_extra, num_comp_x, num_comp_u, g_size, xl_size, bp_options[bp_choice], ceed_data, true,
rhs_ceed, &target));
true, rhs_ceed, &target));

// Gather RHS
PetscCall(VecC2P(rhs_ceed, mem_type, rhs_loc));
Expand Down
4 changes: 2 additions & 2 deletions examples/petsc/include/libceedsetup.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

PetscErrorCode CeedDataDestroy(CeedInt i, CeedData data);
PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree, CeedInt topo_dim, CeedInt q_extra, PetscInt num_comp_x, PetscInt num_comp_u,
PetscInt g_size, PetscInt xl_size, BPData bp_data, CeedData data, PetscBool setup_rhs, CeedVector rhs_ceed,
CeedVector *target);
PetscInt g_size, PetscInt xl_size, BPData bp_data, CeedData data, PetscBool setup_rhs, PetscBool is_fine_level,
CeedVector rhs_ceed, CeedVector *target);
PetscErrorCode CeedLevelTransferSetup(DM dm, Ceed ceed, CeedInt level, CeedInt num_comp_u, CeedData *data, BPData bp_data, Vec fine_mult);
PetscErrorCode SetupErrorOperator(DM dm, Ceed ceed, BPData bp_data, CeedInt topo_dim, PetscInt num_comp_x, PetscInt num_comp_u,
CeedOperator *op_error);
20 changes: 10 additions & 10 deletions examples/petsc/multigrid.c
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,10 @@ int main(int argc, char **argv) {

switch (coarsen) {
case COARSEN_UNIFORM:
for (int i = 0; i < num_levels; i++) level_degrees[i] = i + 1;
for (PetscInt i = 0; i < num_levels; i++) level_degrees[i] = i + 1;
break;
case COARSEN_LOGARITHMIC:
for (int i = 0; i < num_levels - 1; i++) level_degrees[i] = pow(2, i);
for (PetscInt i = 0; i < num_levels - 1; i++) level_degrees[i] = pow(2, i);
level_degrees[fine_level] = degree;
break;
}
Expand Down Expand Up @@ -199,7 +199,6 @@ int main(int argc, char **argv) {

// Operator
PetscCall(PetscMalloc1(1, &op_apply_ctx[i]));
PetscCall(PetscMalloc1(1, &op_error_ctx));
PetscCall(MatCreateShell(comm, l_size[i], l_size[i], g_size[i], g_size[i], op_apply_ctx[i], &mat_O[i]));
PetscCall(MatShellSetOperation(mat_O[i], MATOP_MULT, (void (*)(void))MatMult_Ceed));
PetscCall(MatShellSetOperation(mat_O[i], MATOP_GET_DIAGONAL, (void (*)(void))MatGetDiag));
Expand Down Expand Up @@ -267,7 +266,7 @@ int main(int argc, char **argv) {
}
PetscCall(PetscMalloc1(1, &ceed_data[i]));
PetscCall(SetupLibceedByDegree(dm[i], ceed, level_degrees[i], dim, q_extra, dim, num_comp_u, g_size[i], xl_size[i], bp_options[bp_choice],
ceed_data[i], i == (fine_level), rhs_ceed, &target));
ceed_data[i], i == fine_level, i == fine_level, rhs_ceed, &target));
}

// Gather RHS
Expand All @@ -291,7 +290,7 @@ int main(int argc, char **argv) {
CeedOperatorSetField(op_error, "error", ceed_data[fine_level]->elem_restr_u, ceed_data[fine_level]->basis_u, CEED_VECTOR_ACTIVE);

// Calculate multiplicity
for (int i = 0; i < num_levels; i++) {
for (PetscInt i = 0; i < num_levels; i++) {
PetscMemType mem_type;

// CEED vector
Expand Down Expand Up @@ -322,7 +321,7 @@ int main(int argc, char **argv) {
}

// Set up Mat
for (int i = 0; i < num_levels; i++) {
for (PetscInt i = fine_level; i >= 0; i--) {
// Set up apply operator context
PetscCall(SetupApplyOperatorCtx(comm, dm[i], ceed, ceed_data[i], X_loc[i], op_apply_ctx[i]));

Expand All @@ -335,8 +334,8 @@ int main(int argc, char **argv) {
pr_restr_ctx[i]->loc_vec_c = X_loc[i - 1];
pr_restr_ctx[i]->loc_vec_f = op_apply_ctx[i]->Y_loc;
pr_restr_ctx[i]->mult_vec = mult[i];
pr_restr_ctx[i]->ceed_vec_c = op_apply_ctx[i - 1]->x_ceed;
pr_restr_ctx[i]->ceed_vec_f = op_apply_ctx[i]->y_ceed;
pr_restr_ctx[i]->ceed_vec_c = ceed_data[i - 1]->x_ceed;
pr_restr_ctx[i]->ceed_vec_f = ceed_data[i]->y_ceed;
pr_restr_ctx[i]->op_prolong = ceed_data[i]->op_prolong;
pr_restr_ctx[i]->op_restrict = ceed_data[i]->op_restrict;
pr_restr_ctx[i]->ceed = ceed;
Expand Down Expand Up @@ -393,7 +392,7 @@ int main(int argc, char **argv) {

// PCMG levels
PetscCall(PCMGSetLevels(pc, num_levels, NULL));
for (int i = 0; i < num_levels; i++) {
for (PetscInt i = 0; i < num_levels; i++) {
// Smoother
KSP smoother;
PC smoother_pc;
Expand Down Expand Up @@ -502,6 +501,7 @@ int main(int argc, char **argv) {
}
{
// Set up error operator context
PetscCall(PetscMalloc1(1, &op_error_ctx));
PetscCall(SetupErrorOperatorCtx(comm, dm[fine_level], ceed, ceed_data[fine_level], X_loc[fine_level], op_error, op_error_ctx));
PetscScalar l2_error;
PetscCall(ComputeL2Error(X[fine_level], &l2_error, op_error_ctx));
Expand Down Expand Up @@ -532,7 +532,7 @@ int main(int argc, char **argv) {
}

// Cleanup
for (int i = 0; i < num_levels; i++) {
for (PetscInt i = 0; i < num_levels; i++) {
PetscCall(VecDestroy(&X[i]));
PetscCall(VecDestroy(&X_loc[i]));
PetscCall(VecDestroy(&mult[i]));
Expand Down
73 changes: 38 additions & 35 deletions examples/petsc/src/libceedsetup.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,14 @@ PetscErrorCode CeedDataDestroy(CeedInt i, CeedData data) {
// Set up libCEED for a given degree
// -----------------------------------------------------------------------------
PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree, CeedInt topo_dim, CeedInt q_extra, PetscInt num_comp_x, PetscInt num_comp_u,
PetscInt g_size, PetscInt xl_size, BPData bp_data, CeedData data, PetscBool setup_rhs, CeedVector rhs_ceed,
CeedVector *target) {
PetscInt g_size, PetscInt xl_size, BPData bp_data, CeedData data, PetscBool setup_rhs, PetscBool is_fine_level,
CeedVector rhs_ceed, CeedVector *target) {
DM dm_coord;
Vec coords;
const PetscScalar *coord_array;
CeedBasis basis_x, basis_u;
CeedElemRestriction elem_restr_x, elem_restr_u, elem_restr_u_i, elem_restr_qd_i;
CeedQFunction qf_setup_geo, qf_apply;
CeedQFunction qf_setup_geo = NULL, qf_apply = NULL;
CeedOperator op_setup_geo, op_apply;
CeedVector x_coord, q_data, x_ceed, y_ceed;
PetscInt c_start, c_end, num_elem;
Expand Down Expand Up @@ -86,36 +86,42 @@ PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree, CeedInt to
CeedVectorCreate(ceed, xl_size, &x_ceed);
CeedVectorCreate(ceed, xl_size, &y_ceed);

// Create the QFunction that builds the context data
CeedQFunctionCreateInterior(ceed, 1, bp_data.setup_geo, bp_data.setup_geo_loc, &qf_setup_geo);
CeedQFunctionAddInput(qf_setup_geo, "x", num_comp_x, CEED_EVAL_INTERP);
CeedQFunctionAddInput(qf_setup_geo, "dx", num_comp_x * topo_dim, CEED_EVAL_GRAD);
CeedQFunctionAddInput(qf_setup_geo, "weight", 1, CEED_EVAL_WEIGHT);
CeedQFunctionAddOutput(qf_setup_geo, "qdata", q_data_size, CEED_EVAL_NONE);

// Create the operator that builds the quadrature data
CeedOperatorCreate(ceed, qf_setup_geo, NULL, NULL, &op_setup_geo);
CeedOperatorSetField(op_setup_geo, "x", elem_restr_x, basis_x, CEED_VECTOR_ACTIVE);
CeedOperatorSetField(op_setup_geo, "dx", elem_restr_x, basis_x, CEED_VECTOR_ACTIVE);
CeedOperatorSetField(op_setup_geo, "weight", CEED_ELEMRESTRICTION_NONE, basis_x, CEED_VECTOR_NONE);
CeedOperatorSetField(op_setup_geo, "qdata", elem_restr_qd_i, CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);

// Setup q_data
CeedOperatorApply(op_setup_geo, x_coord, q_data, CEED_REQUEST_IMMEDIATE);

// Set up PDE operator
CeedInt in_scale = bp_data.in_mode == CEED_EVAL_GRAD ? topo_dim : 1;
CeedInt out_scale = bp_data.out_mode == CEED_EVAL_GRAD ? topo_dim : 1;
CeedQFunctionCreateInterior(ceed, 1, bp_data.apply, bp_data.apply_loc, &qf_apply);
CeedQFunctionAddInput(qf_apply, "u", num_comp_u * in_scale, bp_data.in_mode);
CeedQFunctionAddInput(qf_apply, "qdata", q_data_size, CEED_EVAL_NONE);
CeedQFunctionAddOutput(qf_apply, "v", num_comp_u * out_scale, bp_data.out_mode);
if (is_fine_level) {
// Create the QFunction that builds the context data
CeedQFunctionCreateInterior(ceed, 1, bp_data.setup_geo, bp_data.setup_geo_loc, &qf_setup_geo);
CeedQFunctionAddInput(qf_setup_geo, "x", num_comp_x, CEED_EVAL_INTERP);
CeedQFunctionAddInput(qf_setup_geo, "dx", num_comp_x * topo_dim, CEED_EVAL_GRAD);
CeedQFunctionAddInput(qf_setup_geo, "weight", 1, CEED_EVAL_WEIGHT);
CeedQFunctionAddOutput(qf_setup_geo, "qdata", q_data_size, CEED_EVAL_NONE);

// Create the operator that builds the quadrature data
CeedOperatorCreate(ceed, qf_setup_geo, NULL, NULL, &op_setup_geo);
CeedOperatorSetField(op_setup_geo, "x", elem_restr_x, basis_x, CEED_VECTOR_ACTIVE);
CeedOperatorSetField(op_setup_geo, "dx", elem_restr_x, basis_x, CEED_VECTOR_ACTIVE);
CeedOperatorSetField(op_setup_geo, "weight", CEED_ELEMRESTRICTION_NONE, basis_x, CEED_VECTOR_NONE);
CeedOperatorSetField(op_setup_geo, "qdata", elem_restr_qd_i, CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);

// Setup q_data
CeedOperatorApply(op_setup_geo, x_coord, q_data, CEED_REQUEST_IMMEDIATE);

// Set up PDE operator
CeedInt in_scale = bp_data.in_mode == CEED_EVAL_GRAD ? topo_dim : 1;
CeedInt out_scale = bp_data.out_mode == CEED_EVAL_GRAD ? topo_dim : 1;
CeedQFunctionCreateInterior(ceed, 1, bp_data.apply, bp_data.apply_loc, &qf_apply);
CeedQFunctionAddInput(qf_apply, "u", num_comp_u * in_scale, bp_data.in_mode);
CeedQFunctionAddInput(qf_apply, "qdata", q_data_size, CEED_EVAL_NONE);
CeedQFunctionAddOutput(qf_apply, "v", num_comp_u * out_scale, bp_data.out_mode);

// Create the mass or diff operator
CeedOperatorCreate(ceed, qf_apply, NULL, NULL, &op_apply);
CeedOperatorSetField(op_apply, "u", elem_restr_u, basis_u, CEED_VECTOR_ACTIVE);
CeedOperatorSetField(op_apply, "qdata", elem_restr_qd_i, CEED_BASIS_NONE, q_data);
CeedOperatorSetField(op_apply, "v", elem_restr_u, basis_u, CEED_VECTOR_ACTIVE);

// Create the mass or diff operator
CeedOperatorCreate(ceed, qf_apply, NULL, NULL, &op_apply);
CeedOperatorSetField(op_apply, "u", elem_restr_u, basis_u, CEED_VECTOR_ACTIVE);
CeedOperatorSetField(op_apply, "qdata", elem_restr_qd_i, CEED_BASIS_NONE, q_data);
CeedOperatorSetField(op_apply, "v", elem_restr_u, basis_u, CEED_VECTOR_ACTIVE);
// Cleanup
CeedQFunctionDestroy(&qf_setup_geo);
CeedOperatorDestroy(&op_setup_geo);
}

// Set up RHS if needed
if (setup_rhs) {
Expand Down Expand Up @@ -151,10 +157,7 @@ PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree, CeedInt to
CeedQFunctionDestroy(&qf_setup_rhs);
CeedOperatorDestroy(&op_setup_rhs);
}

// Cleanup
CeedQFunctionDestroy(&qf_setup_geo);
CeedOperatorDestroy(&op_setup_geo);
CeedVectorDestroy(&x_coord);

// Save libCEED data required for level
Expand Down