Skip to content

Commit

Permalink
Merge pull request #1275 from CEED/jrwrigh/fluidsCeedCall
Browse files Browse the repository at this point in the history 
fluids: Add PetscCallCeed to capture ceed errors with Petsc
  • Loading branch information
@jrwrigh
jrwrigh authored Aug 3, 2023
2 parents 3843268 + a424bcd commit 71bbf5a
Show file tree
Hide file tree
Showing 23 changed files with 858 additions and 775 deletions.
96 changes: 49 additions & 47 deletions examples/fluids/navierstokes.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,22 +104,24 @@ int main(int argc, char **argv) {
// ---------------------------------------------------------------------------
// -- Initialize backend
Ceed ceed;
CeedInit(app_ctx->ceed_resource, &ceed);
PetscCheck(CeedInit(app_ctx->ceed_resource, &ceed) == CEED_ERROR_SUCCESS, comm, PETSC_ERR_LIB, "Ceed initialization failed");
user->ceed = ceed;

PetscCheck(CeedSetErrorHandler(ceed, CeedErrorStore) == CEED_ERROR_SUCCESS, comm, PETSC_ERR_LIB, "Setting libCEED error handler failed");

// -- Check preferred MemType
CeedMemType mem_type_backend;
CeedGetPreferredMemType(ceed, &mem_type_backend);
PetscCallCeed(ceed, CeedGetPreferredMemType(ceed, &mem_type_backend));

{
const char *resource;
CeedGetResource(ceed, &resource);
PetscCallCeed(ceed, CeedGetResource(ceed, &resource));
if (strstr(resource, "/gpu/sycl")) {
PetscDeviceContext dctx;
PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
void *stream_handle;
PetscCall(PetscDeviceContextGetStreamHandle(dctx, &stream_handle));
CeedSetStream(ceed, stream_handle);
PetscCallCeed(ceed, CeedSetStream(ceed, stream_handle));
}
}

Expand All @@ -136,7 +138,7 @@ int main(int argc, char **argv) {
break;
case CEED_MEM_DEVICE: {
const char *resolved;
CeedGetResource(ceed, &resolved);
PetscCallCeed(ceed, CeedGetResource(ceed, &resolved));
if (strstr(resolved, "/gpu/cuda")) vec_type = VECCUDA;
else if (strstr(resolved, "/gpu/hip")) vec_type = VECKOKKOS;
else if (strstr(resolved, "/gpu/sycl")) vec_type = VECKOKKOS;
Expand Down Expand Up @@ -256,65 +258,65 @@ int main(int argc, char **argv) {
PetscCall(DifferentialFilterDataDestroy(user->diff_filter));

// -- Vectors
CeedVectorDestroy(&ceed_data->x_coord);
CeedVectorDestroy(&ceed_data->q_data);
CeedVectorDestroy(&user->q_ceed);
CeedVectorDestroy(&user->q_dot_ceed);
CeedVectorDestroy(&user->g_ceed);
CeedVectorDestroy(&user->coo_values_amat);
CeedVectorDestroy(&user->coo_values_pmat);
PetscCallCeed(ceed, CeedVectorDestroy(&ceed_data->x_coord));
PetscCallCeed(ceed, CeedVectorDestroy(&ceed_data->q_data));
PetscCallCeed(ceed, CeedVectorDestroy(&user->q_ceed));
PetscCallCeed(ceed, CeedVectorDestroy(&user->q_dot_ceed));
PetscCallCeed(ceed, CeedVectorDestroy(&user->g_ceed));
PetscCallCeed(ceed, CeedVectorDestroy(&user->coo_values_amat));
PetscCallCeed(ceed, CeedVectorDestroy(&user->coo_values_pmat));

// -- Bases
CeedBasisDestroy(&ceed_data->basis_q);
CeedBasisDestroy(&ceed_data->basis_x);
CeedBasisDestroy(&ceed_data->basis_xc);
CeedBasisDestroy(&ceed_data->basis_q_sur);
CeedBasisDestroy(&ceed_data->basis_x_sur);
PetscCallCeed(ceed, CeedBasisDestroy(&ceed_data->basis_q));
PetscCallCeed(ceed, CeedBasisDestroy(&ceed_data->basis_x));
PetscCallCeed(ceed, CeedBasisDestroy(&ceed_data->basis_xc));
PetscCallCeed(ceed, CeedBasisDestroy(&ceed_data->basis_q_sur));
PetscCallCeed(ceed, CeedBasisDestroy(&ceed_data->basis_x_sur));

// -- Restrictions
CeedElemRestrictionDestroy(&ceed_data->elem_restr_q);
CeedElemRestrictionDestroy(&ceed_data->elem_restr_x);
CeedElemRestrictionDestroy(&ceed_data->elem_restr_qd_i);
PetscCallCeed(ceed, CeedElemRestrictionDestroy(&ceed_data->elem_restr_q));
PetscCallCeed(ceed, CeedElemRestrictionDestroy(&ceed_data->elem_restr_x));
PetscCallCeed(ceed, CeedElemRestrictionDestroy(&ceed_data->elem_restr_qd_i));

// Destroy QFunction contexts after using
// ToDo: Simplify tracked libCEED objects, smaller struct
{
CeedQFunctionContextDestroy(&problem->apply_inflow_jacobian.qfunction_context);
CeedQFunctionContextDestroy(&problem->apply_inflow_jacobian.qfunction_context);
CeedQFunctionContextDestroy(&problem->apply_outflow_jacobian.qfunction_context);
CeedQFunctionContextDestroy(&problem->apply_outflow_jacobian.qfunction_context);
CeedQFunctionContextDestroy(&problem->apply_freestream_jacobian.qfunction_context);
CeedQFunctionContextDestroy(&problem->apply_freestream_jacobian.qfunction_context);
CeedQFunctionContextDestroy(&problem->setup_sur.qfunction_context);
CeedQFunctionContextDestroy(&problem->setup_vol.qfunction_context);
CeedQFunctionContextDestroy(&problem->ics.qfunction_context);
CeedQFunctionContextDestroy(&problem->apply_vol_rhs.qfunction_context);
CeedQFunctionContextDestroy(&problem->apply_vol_ifunction.qfunction_context);
CeedQFunctionContextDestroy(&problem->apply_vol_ijacobian.qfunction_context);
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_inflow_jacobian.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_inflow_jacobian.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_outflow_jacobian.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_outflow_jacobian.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_freestream_jacobian.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_freestream_jacobian.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->setup_sur.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->setup_vol.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->ics.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_vol_rhs.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_vol_ifunction.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextDestroy(&problem->apply_vol_ijacobian.qfunction_context));
}

// -- QFunctions
CeedQFunctionDestroy(&ceed_data->qf_setup_vol);
CeedQFunctionDestroy(&ceed_data->qf_ics);
CeedQFunctionDestroy(&ceed_data->qf_rhs_vol);
CeedQFunctionDestroy(&ceed_data->qf_ifunction_vol);
CeedQFunctionDestroy(&ceed_data->qf_setup_sur);
CeedQFunctionDestroy(&ceed_data->qf_apply_inflow);
CeedQFunctionDestroy(&ceed_data->qf_apply_inflow_jacobian);
CeedQFunctionDestroy(&ceed_data->qf_apply_freestream);
CeedQFunctionDestroy(&ceed_data->qf_apply_freestream_jacobian);
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_setup_vol));
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_ics));
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_rhs_vol));
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_ifunction_vol));
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_setup_sur));
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_apply_inflow));
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_apply_inflow_jacobian));
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_apply_freestream));
PetscCallCeed(ceed, CeedQFunctionDestroy(&ceed_data->qf_apply_freestream_jacobian));

// -- Operators
CeedOperatorDestroy(&ceed_data->op_setup_vol);
PetscCallCeed(ceed, CeedOperatorDestroy(&ceed_data->op_setup_vol));
PetscCall(OperatorApplyContextDestroy(ceed_data->op_ics_ctx));
CeedOperatorDestroy(&user->op_rhs_vol);
CeedOperatorDestroy(&user->op_ifunction_vol);
PetscCallCeed(ceed, CeedOperatorDestroy(&user->op_rhs_vol));
PetscCallCeed(ceed, CeedOperatorDestroy(&user->op_ifunction_vol));
PetscCall(OperatorApplyContextDestroy(user->op_rhs_ctx));
CeedOperatorDestroy(&user->op_ifunction);
CeedOperatorDestroy(&user->op_ijacobian);
PetscCallCeed(ceed, CeedOperatorDestroy(&user->op_ifunction));
PetscCallCeed(ceed, CeedOperatorDestroy(&user->op_ijacobian));

// -- Ceed
CeedDestroy(&ceed);
PetscCheck(CeedDestroy(&ceed) == CEED_ERROR_SUCCESS, comm, PETSC_ERR_LIB, "Destroying Ceed object failed");

if (app_ctx->test_type != TESTTYPE_NONE) {
PetscInt num_options_left = 0;
Expand Down
30 changes: 21 additions & 9 deletions examples/fluids/navierstokes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,25 @@
#include "qfunctions/newtonian_types.h"
#include "qfunctions/stabilization_types.h"

// -----------------------------------------------------------------------------
// PETSc Version
// -----------------------------------------------------------------------------
#if PETSC_VERSION_LT(3, 19, 0)
#error "PETSc v3.19 or later is required"
#endif

#define PetscCeedChk(ceed, ierr) \
do { \
if (ierr != CEED_ERROR_SUCCESS) { \
const char *error_message; \
CeedGetErrorMessage(ceed, &error_message); \
SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "%s", error_message); \
} \
} while (0)

#define PetscCallCeed(ceed, ...) \
do { \
int ierr_q_ = __VA_ARGS__; \
PetscCeedChk(ceed, ierr_q_); \
} while (0)

// -----------------------------------------------------------------------------
// Enums
// -----------------------------------------------------------------------------
Expand Down Expand Up @@ -279,7 +291,7 @@ struct ProblemData_private {
apply_freestream, apply_inflow_jacobian, apply_outflow_jacobian, apply_freestream_jacobian;
bool non_zero_time;
PetscBool bc_from_ics, use_strong_bc_ceed;
PetscErrorCode (*print_info)(ProblemData *, AppCtx);
PetscErrorCode (*print_info)(User, ProblemData *, AppCtx);
};

extern int FreeContextPetsc(void *);
Expand All @@ -299,15 +311,15 @@ extern PetscErrorCode NS_ADVECTION(ProblemData *problem, DM dm, void *ctx, Simpl
extern PetscErrorCode NS_ADVECTION2D(ProblemData *problem, DM dm, void *ctx, SimpleBC bc);

// Print function for each problem
extern PetscErrorCode PRINT_NEWTONIAN(ProblemData *problem, AppCtx app_ctx);
extern PetscErrorCode PRINT_NEWTONIAN(User user, ProblemData *problem, AppCtx app_ctx);

extern PetscErrorCode PRINT_EULER_VORTEX(ProblemData *problem, AppCtx app_ctx);
extern PetscErrorCode PRINT_EULER_VORTEX(User user, ProblemData *problem, AppCtx app_ctx);

extern PetscErrorCode PRINT_SHOCKTUBE(ProblemData *problem, AppCtx app_ctx);
extern PetscErrorCode PRINT_SHOCKTUBE(User user, ProblemData *problem, AppCtx app_ctx);

extern PetscErrorCode PRINT_ADVECTION(ProblemData *problem, AppCtx app_ctx);
extern PetscErrorCode PRINT_ADVECTION(User user, ProblemData *problem, AppCtx app_ctx);

extern PetscErrorCode PRINT_ADVECTION2D(ProblemData *problem, AppCtx app_ctx);
extern PetscErrorCode PRINT_ADVECTION2D(User user, ProblemData *problem, AppCtx app_ctx);

PetscErrorCode PrintRunInfo(User user, Physics phys_ctx, ProblemData *problem, MPI_Comm comm);

Expand Down
33 changes: 18 additions & 15 deletions examples/fluids/problems/advection.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ PetscErrorCode NS_ADVECTION(ProblemData *problem, DM dm, void *ctx, SimpleBC bc)
StabilizationType stab;
SetupContextAdv setup_context;
User user = *(User *)ctx;
MPI_Comm comm = PETSC_COMM_WORLD;
MPI_Comm comm = user->comm;
Ceed ceed = user->ceed;
PetscBool implicit;
PetscBool has_curr_time = PETSC_FALSE;
AdvectionContext advection_ctx;
Expand Down Expand Up @@ -173,27 +174,29 @@ PetscErrorCode NS_ADVECTION(ProblemData *problem, DM dm, void *ctx, SimpleBC bc)
advection_ctx->strong_form = strong_form;
advection_ctx->stabilization = stab;

CeedQFunctionContextCreate(user->ceed, &problem->ics.qfunction_context);
CeedQFunctionContextSetData(problem->ics.qfunction_context, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*setup_context), setup_context);
CeedQFunctionContextSetDataDestroy(problem->ics.qfunction_context, CEED_MEM_HOST, FreeContextPetsc);
PetscCallCeed(ceed, CeedQFunctionContextCreate(user->ceed, &problem->ics.qfunction_context));
PetscCallCeed(ceed,
CeedQFunctionContextSetData(problem->ics.qfunction_context, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*setup_context), setup_context));
PetscCallCeed(ceed, CeedQFunctionContextSetDataDestroy(problem->ics.qfunction_context, CEED_MEM_HOST, FreeContextPetsc));

CeedQFunctionContextCreate(user->ceed, &advection_context);
CeedQFunctionContextSetData(advection_context, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*advection_ctx), advection_ctx);
CeedQFunctionContextSetDataDestroy(advection_context, CEED_MEM_HOST, FreeContextPetsc);
PetscCallCeed(ceed, CeedQFunctionContextCreate(user->ceed, &advection_context));
PetscCallCeed(ceed, CeedQFunctionContextSetData(advection_context, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*advection_ctx), advection_ctx));
PetscCallCeed(ceed, CeedQFunctionContextSetDataDestroy(advection_context, CEED_MEM_HOST, FreeContextPetsc));
problem->apply_vol_rhs.qfunction_context = advection_context;
CeedQFunctionContextReferenceCopy(advection_context, &problem->apply_vol_ifunction.qfunction_context);
CeedQFunctionContextReferenceCopy(advection_context, &problem->apply_inflow.qfunction_context);
PetscCallCeed(ceed, CeedQFunctionContextReferenceCopy(advection_context, &problem->apply_vol_ifunction.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextReferenceCopy(advection_context, &problem->apply_inflow.qfunction_context));
PetscFunctionReturn(PETSC_SUCCESS);
}

PetscErrorCode PRINT_ADVECTION(ProblemData *problem, AppCtx app_ctx) {
MPI_Comm comm = PETSC_COMM_WORLD;
PetscErrorCode PRINT_ADVECTION(User user, ProblemData *problem, AppCtx app_ctx) {
MPI_Comm comm = user->comm;
Ceed ceed = user->ceed;
SetupContextAdv setup_ctx;
AdvectionContext advection_ctx;

PetscFunctionBeginUser;
CeedQFunctionContextGetData(problem->ics.qfunction_context, CEED_MEM_HOST, &setup_ctx);
CeedQFunctionContextGetData(problem->apply_vol_rhs.qfunction_context, CEED_MEM_HOST, &advection_ctx);
PetscCallCeed(ceed, CeedQFunctionContextGetData(problem->ics.qfunction_context, CEED_MEM_HOST, &setup_ctx));
PetscCallCeed(ceed, CeedQFunctionContextGetData(problem->apply_vol_rhs.qfunction_context, CEED_MEM_HOST, &advection_ctx));
PetscCall(PetscPrintf(comm,
" Problem:\n"
" Problem Name : %s\n"
Expand All @@ -208,7 +211,7 @@ PetscErrorCode PRINT_ADVECTION(ProblemData *problem, AppCtx app_ctx) {
if (setup_ctx->wind_type == WIND_TRANSLATION) {
PetscCall(PetscPrintf(comm, " Background Wind : %f,%f,%f\n", setup_ctx->wind[0], setup_ctx->wind[1], setup_ctx->wind[2]));
}
CeedQFunctionContextRestoreData(problem->ics.qfunction_context, &setup_ctx);
CeedQFunctionContextRestoreData(problem->apply_vol_rhs.qfunction_context, &advection_ctx);
PetscCallCeed(ceed, CeedQFunctionContextRestoreData(problem->ics.qfunction_context, &setup_ctx));
PetscCallCeed(ceed, CeedQFunctionContextRestoreData(problem->apply_vol_rhs.qfunction_context, &advection_ctx));
PetscFunctionReturn(PETSC_SUCCESS);
}
33 changes: 18 additions & 15 deletions examples/fluids/problems/advection2d.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ PetscErrorCode NS_ADVECTION2D(ProblemData *problem, DM dm, void *ctx, SimpleBC b
StabilizationType stab;
SetupContextAdv2D setup_context;
User user = *(User *)ctx;
MPI_Comm comm = PETSC_COMM_WORLD;
MPI_Comm comm = user->comm;
Ceed ceed = user->ceed;
PetscBool implicit;
PetscBool has_curr_time = PETSC_FALSE;
AdvectionContext advection_ctx;
Expand Down Expand Up @@ -156,27 +157,29 @@ PetscErrorCode NS_ADVECTION2D(ProblemData *problem, DM dm, void *ctx, SimpleBC b
advection_ctx->strong_form = strong_form;
advection_ctx->stabilization = stab;

CeedQFunctionContextCreate(user->ceed, &problem->ics.qfunction_context);
CeedQFunctionContextSetData(problem->ics.qfunction_context, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*setup_context), setup_context);
CeedQFunctionContextSetDataDestroy(problem->ics.qfunction_context, CEED_MEM_HOST, FreeContextPetsc);
PetscCallCeed(ceed, CeedQFunctionContextCreate(user->ceed, &problem->ics.qfunction_context));
PetscCallCeed(ceed,
CeedQFunctionContextSetData(problem->ics.qfunction_context, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*setup_context), setup_context));
PetscCallCeed(ceed, CeedQFunctionContextSetDataDestroy(problem->ics.qfunction_context, CEED_MEM_HOST, FreeContextPetsc));

CeedQFunctionContextCreate(user->ceed, &advection_context);
CeedQFunctionContextSetData(advection_context, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*advection_ctx), advection_ctx);
CeedQFunctionContextSetDataDestroy(advection_context, CEED_MEM_HOST, FreeContextPetsc);
PetscCallCeed(ceed, CeedQFunctionContextCreate(user->ceed, &advection_context));
PetscCallCeed(ceed, CeedQFunctionContextSetData(advection_context, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*advection_ctx), advection_ctx));
PetscCallCeed(ceed, CeedQFunctionContextSetDataDestroy(advection_context, CEED_MEM_HOST, FreeContextPetsc));
problem->apply_vol_rhs.qfunction_context = advection_context;
CeedQFunctionContextReferenceCopy(advection_context, &problem->apply_vol_ifunction.qfunction_context);
CeedQFunctionContextReferenceCopy(advection_context, &problem->apply_inflow.qfunction_context);
PetscCallCeed(ceed, CeedQFunctionContextReferenceCopy(advection_context, &problem->apply_vol_ifunction.qfunction_context));
PetscCallCeed(ceed, CeedQFunctionContextReferenceCopy(advection_context, &problem->apply_inflow.qfunction_context));
PetscFunctionReturn(PETSC_SUCCESS);
}

PetscErrorCode PRINT_ADVECTION2D(ProblemData *problem, AppCtx app_ctx) {
MPI_Comm comm = PETSC_COMM_WORLD;
PetscErrorCode PRINT_ADVECTION2D(User user, ProblemData *problem, AppCtx app_ctx) {
MPI_Comm comm = user->comm;
Ceed ceed = user->ceed;
SetupContextAdv2D setup_ctx;
AdvectionContext advection_ctx;

PetscFunctionBeginUser;
CeedQFunctionContextGetData(problem->ics.qfunction_context, CEED_MEM_HOST, &setup_ctx);
CeedQFunctionContextGetData(problem->apply_vol_rhs.qfunction_context, CEED_MEM_HOST, &advection_ctx);
PetscCallCeed(ceed, CeedQFunctionContextGetData(problem->ics.qfunction_context, CEED_MEM_HOST, &setup_ctx));
PetscCallCeed(ceed, CeedQFunctionContextGetData(problem->apply_vol_rhs.qfunction_context, CEED_MEM_HOST, &advection_ctx));
PetscCall(PetscPrintf(comm,
" Problem:\n"
" Problem Name : %s\n"
Expand All @@ -187,7 +190,7 @@ PetscErrorCode PRINT_ADVECTION2D(ProblemData *problem, AppCtx app_ctx) {
if (setup_ctx->wind_type == WIND_TRANSLATION) {
PetscCall(PetscPrintf(comm, " Background Wind : %f,%f\n", setup_ctx->wind[0], setup_ctx->wind[1]));
}
CeedQFunctionContextRestoreData(problem->ics.qfunction_context, &setup_ctx);
CeedQFunctionContextRestoreData(problem->apply_vol_rhs.qfunction_context, &advection_ctx);
PetscCallCeed(ceed, CeedQFunctionContextRestoreData(problem->ics.qfunction_context, &setup_ctx));
PetscCallCeed(ceed, CeedQFunctionContextRestoreData(problem->apply_vol_rhs.qfunction_context, &advection_ctx));
PetscFunctionReturn(PETSC_SUCCESS);
}
Loading

0 comments on commit 71bbf5a

Please sign in to comment.