wip - bddc

examples/petsc/Makefile

@@ -46,6 +46,11 @@ area.o = $(area.c:%.c=$(OBJDIR)/%.o)
 area: $(area.o) | $(PETSc.pc) $(ceed.pc)
 	$(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@
 
+bddc.c := bddc.c $(utils.c)
+bddc.o = $(bddc.c:%.c=$(OBJDIR)/%.o)
+bddc: $(bddc.o) | $(PETSc.pc) $(ceed.pc)
+	$(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@
+
 bps.c := bps.c $(utils.c)
 bps.o = $(bps.c:%.c=$(OBJDIR)/%.o)
 bps: $(bps.o) | $(PETSc.pc) $(ceed.pc)
@@ -89,7 +94,7 @@ print: $(PETSc.pc) $(ceed.pc)
 	@true
 
 clean:
-	$(RM) -r $(OBJDIR) *.vtu area bps bpsraw bpssphere multigrid
+	$(RM) -r $(OBJDIR) *.vtu area bddc bps bpsraw bpssphere multigrid
 
 $(PETSc.pc):
 	$(if $(wildcard $@),,$(error \

examples/petsc/bddc.h

@@ -0,0 +1,56 @@
+// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
+// reserved. See files LICENSE and NOTICE for details.
+//
+// This file is part of CEED, a collection of benchmarks, miniapps, software
+// libraries and APIs for efficient high-order finite element and spectral
+// element discretizations for exascale applications. For more information and
+// source code availability see http://github.com/ceed.
+//
+// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
+// a collaborative effort of two U.S. Department of Energy organizations (Office
+// of Science and the National Nuclear Security Administration) responsible for
+// the planning and preparation of a capable exascale ecosystem, including
+// software, applications, hardware, advanced system engineering and early
+// testbed platforms, in support of the nation's exascale computing imperative.
+
+#ifndef bddc_h
+#define bddc_h
+
+#include "include/bpsproblemdata.h"
+#include "include/petscmacros.h"
+#include "include/petscutils.h"
+#include "include/matops.h"
+#include "include/structs.h"
+#include "include/libceedsetup.h"
+#include "bps.h"
+
+#include <ceed.h>
+#include <petsc.h>
+#include <petscdmplex.h>
+#include <petscfe.h>
+#include <petscsys.h>
+#include <stdbool.h>
+#include <string.h>
+
+#if PETSC_VERSION_LT(3,12,0)
+#ifdef PETSC_HAVE_CUDA
+#include <petsccuda.h>
+// Note: With PETSc prior to version 3.12.0, providing the source path to
+//       include 'cublas_v2.h' will be needed to use 'petsccuda.h'.
+#endif
+#endif
+
+// -----------------------------------------------------------------------------
+// Command Line Options
+// -----------------------------------------------------------------------------
+
+// Coarsening options
+typedef enum {
+  INJECTION_SCALED = 0, INJECTION_HARMONIC = 1
+} InjectionType;
+static const char *const injection_types [] = {"scaled", "harmonic",
+                                               "InjectionType", "INJECTION", 0
+                                              };
+
+#endif // bddc_h

examples/petsc/include/libceedsetup.h

@@ -7,6 +7,7 @@
 #include "structs.h"
 
 PetscErrorCode CeedDataDestroy(CeedInt i, CeedData data);
+PetscErrorCode CeedDataBDDCDestroy(CeedDataBDDC data);
 PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree,
                                     CeedInt topo_dim, CeedInt q_extra,
                                     PetscInt num_comp_x, PetscInt num_comp_u,
@@ -17,5 +18,8 @@ PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree,
 PetscErrorCode CeedLevelTransferSetup(Ceed ceed, CeedInt num_levels,
                                       CeedInt num_comp_u, CeedData *data, CeedInt *leveldegrees,
                                       CeedQFunction qf_restrict, CeedQFunction qf_prolong);
+PetscErrorCode SetupLibceedBDDC(DM dm_vertex, CeedData data_fine,
+                                CeedDataBDDC data_bddc, PetscInt g_vertex_size,
+                                PetscInt xl_vertex_size, BPData bp_data);
 
 #endif // libceedsetup_h

examples/petsc/include/matops.h

@@ -13,6 +13,9 @@ PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y);
 PetscErrorCode FormResidual_Ceed(SNES snes, Vec X, Vec Y, void *ctx);
 PetscErrorCode MatMult_Prolong(Mat A, Vec X, Vec Y);
 PetscErrorCode MatMult_Restrict(Mat A, Vec X, Vec Y);
+PetscErrorCode PCShellApply_BDDC(Mat A, Vec X, Vec Y);
+PetscErrorCode PCShellSetup_BDDC(PC pc);
+PetscErrorCode MatMult_BDDCSchur(Mat S, Vec X, Vec Y);
 PetscErrorCode ComputeErrorMax(UserO user, CeedOperator op_error,
                                Vec X, CeedVector target, PetscReal *max_error);
 

examples/petsc/include/petscutils.h

@@ -18,6 +18,8 @@ typedef PetscErrorCode (*BCFunction)(PetscInt dim, PetscReal time,
 PetscErrorCode SetupDMByDegree(DM dm, PetscInt degree, PetscInt num_comp_u,
                                PetscInt topo_dim,
                                bool enforce_bc,  BCFunction bc_func);
+PetscErrorCode SetupVertexDMFromDM(DM dm, DM dm_vertex, PetscInt num_comp_u,
+                                   bool enforce_bc, BCFunction bc_func);
 PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt P,
     CeedInt topo_dim, CeedInt height, DMLabel domain_label, CeedInt value,
     CeedElemRestriction *elem_restr);

examples/petsc/include/structs.h

@@ -4,32 +4,6 @@
 #include <ceed.h>
 #include <petsc.h>
 
-// -----------------------------------------------------------------------------
-// PETSc Operator Structs
-// -----------------------------------------------------------------------------
-
-// Data for PETSc Matshell
-typedef struct UserO_ *UserO;
-struct UserO_ {
-  MPI_Comm comm;
-  DM dm;
-  Vec X_loc, Y_loc, diag;
-  CeedVector x_ceed, y_ceed;
-  CeedOperator op;
-  Ceed ceed;
-};
-
-// Data for PETSc Prolong/Restrict Matshells
-typedef struct UserProlongRestr_ *UserProlongRestr;
-struct UserProlongRestr_ {
-  MPI_Comm comm;
-  DM dmc, dmf;
-  Vec loc_vec_c, loc_vec_f, mult_vec;
-  CeedVector ceed_vec_c, ceed_vec_f;
-  CeedOperator op_prolong, op_restrict;
-  Ceed ceed;
-};
-
 // -----------------------------------------------------------------------------
 // libCEED Data Structs
 // -----------------------------------------------------------------------------
@@ -45,6 +19,18 @@ struct CeedData_ {
   CeedVector q_data, x_ceed, y_ceed;
 };
 
+// libCEED data struct for BDDC
+typedef struct CeedDataBDDC_ *CeedDataBDDC;
+struct CeedDataBDDC_ {
+  CeedBasis basis_Pi, basis_Pi_r;
+  CeedInt strides[3];
+  CeedElemRestriction elem_restr_Pi, elem_restr_r;
+  CeedOperator op_Pi_r, op_r_Pi, op_Pi_Pi, op_r_r, op_r_r_inv,
+               op_inject_Pi, op_inject_r, op_restrict_Pi, op_restrict_r;
+  CeedVector x_ceed, y_ceed, x_Pi_ceed, y_Pi_ceed, x_r_ceed, y_r_ceed, z_r_ceed,
+             mult_ceed, mask_r_ceed, mask_Gamma_ceed, mask_I_ceed;
+};
+
 // BP specific data
 typedef struct {
   CeedInt num_comp_x, num_comp_u, topo_dim, q_data_size, q_extra;
@@ -57,4 +43,41 @@ typedef struct {
                             PetscInt, PetscScalar *, void *);
 } BPData;
 
+// -----------------------------------------------------------------------------
+// PETSc Operator Structs
+// -----------------------------------------------------------------------------
+
+// Data for PETSc Matshell
+typedef struct UserO_ *UserO;
+struct UserO_ {
+  MPI_Comm comm;
+  DM dm;
+  Vec X_loc, Y_loc, diag;
+  CeedVector x_ceed, y_ceed;
+  CeedOperator op;
+  Ceed ceed;
+};
+
+// Data for PETSc Prolong/Restrict Matshells
+typedef struct UserProlongRestr_ *UserProlongRestr;
+struct UserProlongRestr_ {
+  MPI_Comm comm;
+  DM dmc, dmf;
+  Vec loc_vec_c, loc_vec_f, mult_vec;
+  CeedVector ceed_vec_c, ceed_vec_f;
+  CeedOperator op_prolong, op_restrict;
+  Ceed ceed;
+};
+// Data for PETSc PCshell
+typedef struct UserBDDC_ *UserBDDC;
+struct UserBDDC_ {
+  MPI_Comm comm;
+  DM dm, dm_Pi;
+  SNES snes_Pi;
+  KSP ksp_S_Pi;
+  Mat mat_S_Pi;
+  Vec X_loc, Y_loc, X_Pi, Y_Pi, X_Pi_loc, Y_Pi_loc, mult;
+  CeedDataBDDC ceed_data_bddc;
+};
+
 #endif // structs_h