From b560e7cf86dc3278bd94279d11ae4f4ab5ac20be Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Sun, 13 Aug 2023 14:56:35 -0700 Subject: [PATCH] prom example with parallel mfem. --- docker/librom/Dockerfile | 3 +- examples/prom/poisson_global_rom.py | 75 +++++++++-------------------- 2 files changed, 24 insertions(+), 54 deletions(-) diff --git a/docker/librom/Dockerfile b/docker/librom/Dockerfile index 421316b..1d81b4a 100644 --- a/docker/librom/Dockerfile +++ b/docker/librom/Dockerfile @@ -23,7 +23,8 @@ RUN sudo git pull # pylibROM is currently based on a specific commit of the libROM. RUN sudo git checkout 0edcb6fb268ae06bf835c3476665862040eb512d WORKDIR ./build -RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_MFEM=${USE_MFEM} -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB} +# RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_MFEM=${USE_MFEM} -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB} +RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_MFEM=OFF RUN sudo make -j 16 # ENV LIBROM_DIR=$LIB_DIR/libROM diff --git a/examples/prom/poisson_global_rom.py b/examples/prom/poisson_global_rom.py index b890593..8542491 100644 --- a/examples/prom/poisson_global_rom.py +++ b/examples/prom/poisson_global_rom.py @@ -7,14 +7,16 @@ import sys import time try: - import mfem.ser as mfem + import mfem.par as mfem except ModuleNotFoundError: msg = "PyMFEM is not installed yet. Install PyMFEM:\n" - msg += "\tpip install mfem" + msg += "\tgit clone https://github.com/mfem/PyMFEM.git\n" + msg += "\tcd PyMFEM\n" + msg += "\tpython3 setup.py install --with-parallel\n" raise ModuleNotFoundError(msg) from ctypes import c_double -from mfem.ser import intArray +from mfem.par import intArray from os.path import expanduser, join, dirname import numpy as np from numpy import sin, cos, exp, sqrt, pi, abs, array, floor, log, sum @@ -130,20 +132,6 @@ def run(): precision = 8 - # ref_levels = args.refine - # ode_solver_type = args.ode_solver - # t_final = args.t_final - # dt = args.time_step - # speed = args.speed - # dirichlet = (not args.neumann) - # visit = args.visit_datafiles - # visualization = args.visualization - # vis_steps = args.visualization_steps - # ref_dir = args.reference - # ef = args.energy_fraction - # rdim = args.rdim - # windowNumSamples = args.numwindowsamples - kappa = freq * np.pi if (fom): if (not (fom and (not offline) and (not online))): @@ -178,9 +166,7 @@ def run(): # 6. Define a parallel mesh by a partitioning of the serial mesh. Refine # this mesh further in parallel to increase the resolution. Once the # parallel mesh is defined, the serial mesh can be deleted. - # TODO(kevin): figure out mfem parallel version install - # pmesh = mfem.ParMesh(MPI_COMM_WORLD, mesh) - pmesh = mfem.Mesh(mesh) + pmesh = mfem.ParMesh(comm, mesh) mesh.Clear() par_ref_levels = 2 for l in range(par_ref_levels): @@ -201,11 +187,8 @@ def run(): fec = mfem.H1_FECollection(1, dim) delete_fec = True - # TODO(kevin): figure out mfem parallel version install - # fespace = mfem.ParFiniteElementSpace(pmesh, fec) - # size = fespace.GlobalTrueVSize() - fespace = mfem.FiniteElementSpace(pmesh, fec) - size = fespace.GetTrueVSize() + fespace = mfem.ParFiniteElementSpace(pmesh, fec) + size = fespace.GlobalTrueVSize() if (myid == 0): print("Number of finite element unknowns: %d" % size) @@ -225,10 +208,6 @@ def run(): isIncremental = False basisName = "basis" basisFileName = "%s%d" % (basisName, id) - # const CAROM::Matrix* spatialbasis - # CAROM::Options* options; - # CAROM::BasisGenerator *generator; - # int numRowRB, numColumnRB; solveTimer, assembleTimer, mergeTimer = StopWatch(), StopWatch(), StopWatch() # 10. Set BasisGenerator if offline @@ -262,15 +241,13 @@ def run(): # (f,phi_i) where f is given by the function f_exact and phi_i are the # basis functions in the finite element fespace. assembleTimer.Start() - # TODO(kevin): figure out mfem parallel install - # b = mfem.ParLinearForm(fespace) + b = mfem.ParLinearForm(fespace) class RightHandSide(mfem.PyCoefficient): def EvalValue(self, x): if (dim == 3): return sin(kappa * (x[0] + x[1] + x[2])) else: return sin(kappa * (x[0] + x[1])) - b = mfem.LinearForm(fespace) f = RightHandSide() b.AddDomainIntegrator(mfem.DomainLFIntegrator(f)) b.Assemble() @@ -278,20 +255,16 @@ def EvalValue(self, x): # 13. Define the solution vector x as a parallel finite element grid function # corresponding to fespace. Initialize x with initial guess of zero, # which satisfies the boundary conditions. - # TODO(kevin): figure out mfem parallel install - # x = mfem.ParGridFunction(fespace) - x = mfem.GridFunction(fespace) + x = mfem.ParGridFunction(fespace) x.Assign(0.0) # 14. Set up the parallel bilinear form a(.,.) on the finite element space # corresponding to the Laplacian operator -Delta, by adding the Diffusion # domain integrator. - # TODO(kevin): figure out mfem parallel install - # a = mfem.ParBilinearForm(fespace) - a = mfem.BilinearForm(fespace) + a = mfem.ParBilinearForm(fespace) one = mfem.ConstantCoefficient(coef) - # if (pa): - # a.SetAssemblyLevel(mfem.AssemblyLevel.PARTIAL) + if (pa): + a.SetAssemblyLevel(mfem.AssemblyLevel_PARTIAL) a.AddDomainIntegrator(mfem.DiffusionIntegrator(one)) # 15. Assemble the parallel bilinear form and the corresponding linear @@ -302,7 +275,8 @@ def EvalValue(self, x): a.EnableStaticCondensation() a.Assemble() - A = mfem.OperatorPtr() + # A = mfem.OperatorPtr() + A = mfem.HypreParMatrix() B = mfem.Vector() X = mfem.Vector() a.FormLinearSystem(ess_tdof_list, x, b, A, X, B) @@ -315,19 +289,17 @@ def EvalValue(self, x): if pa: if mfem.UsesTensorBasis(fespace): prec = mfem.OperatorJacobiSmoother(a, ess_tdof_list) - # TODO(kevin): figure out mfem parallel install - # else: - # prec = mfem.HypreBoomerAMG() + else: + prec = mfem.HypreBoomerAMG(A) - # TODO(kevin): figure out mfem parallel install - # cg = mfem.CGSolver(MPI.COMM_WORLD) - cg = mfem.CGSolver() + cg = mfem.CGSolver(comm) cg.SetRelTol(1e-12) cg.SetMaxIter(2000) cg.SetPrintLevel(1) if (prec is not None): cg.SetPreconditioner(prec) - cg.SetOperator(A.Ptr()) + # cg.SetOperator(A.Ptr()) + cg.SetOperator(A) solveTimer.Start() cg.Mult(B, X) solveTimer.Stop() @@ -405,11 +377,8 @@ def EvalValue(self, x): mfem.subtract_vector(x, x_fom, diff_x) # Get norms - # TODO(kevin): figure out mfem parallel install - # tot_diff_norm = np.sqrt(mfem.InnerProduct(MPI.COMM_WORLD, diff_x, diff_x)) - # tot_fom_norm = np.sqrt(mfem.InnerProduct(MPI.COMM_WORLD, x_fom, x_fom)) - tot_diff_norm = np.sqrt(mfem.InnerProduct(diff_x, diff_x)) - tot_fom_norm = np.sqrt(mfem.InnerProduct(x_fom, x_fom)) + tot_diff_norm = np.sqrt(mfem.InnerProduct(comm, diff_x, diff_x)) + tot_fom_norm = np.sqrt(mfem.InnerProduct(comm, x_fom, x_fom)) if (myid == 0): print("Relative error of ROM solution = %.5E" % (tot_diff_norm / tot_fom_norm))