From ba8d5bc80985628f7c6ac9f9c7530b20606dd15f Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Thu, 14 Sep 2023 10:38:50 -0700 Subject: [PATCH 1/9] example work in progress. --- .../prom/nonlinear_elasticity_global_rom.py | 1548 +++++++++++++++++ 1 file changed, 1548 insertions(+) create mode 100644 examples/prom/nonlinear_elasticity_global_rom.py diff --git a/examples/prom/nonlinear_elasticity_global_rom.py b/examples/prom/nonlinear_elasticity_global_rom.py new file mode 100644 index 0000000..60c6bf6 --- /dev/null +++ b/examples/prom/nonlinear_elasticity_global_rom.py @@ -0,0 +1,1548 @@ +# // libROM MFEM Example: parametric ROM for nonlinear elasticity problem (adapted from ex10p.cpp) +# // +# // Compile with: make nonlinear_elasticity_global_rom +# // +# // Description: This examples solves a time dependent nonlinear elasticity +# // problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a +# // hyperelastic model and S is a viscosity operator of Laplacian +# // type. The geometry of the domain is assumed to be as follows: +# // +# // +---------------------+ +# // boundary --->| | +# // attribute 1 | | +# // (fixed) +---------------------+ +# // +# // The example demonstrates the use of hyper reduction to solve a +# // nonlinear elasticity problem. Time integration is done with various +# // explicit time integrator solvers. The basis for the velocity field +# // is either constructed using a separate velocity basis or using the +# // displacement basis. It is possible to set the initial condition in +# // terms of either velocity or deformation. The velocity initial condition +# // works better when both velocity and displacement bases are used. The +# // deformation initial condition is better when only the displacement +# // basis is used. The input flag -sc controls the scaling of the initial +# // condition applied. This is what parameterizes the ROM. If the scaling +# // factor is within the range +-10%, the results are generally accurate. + +# // ================================================================================= +# // +# // Sample runs and results for parametric ROM using displacement basis, velocity basis +# // and nonlinear term basis, with velocity initial condition: +# // +# // Offline phase: +# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.90 -id 0 +# // +# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.10 -id 1 +# // +# // Merge phase: +# // ./nonlinear_elasticity_global_rom -merge -ns 2 -dt 0.01 -tf 5.0 +# // +# // Create FOM comparison data: +# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.00 -id 2 +# // +# // Online phase with full sampling: +# // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 1170 -sc 1.00 +# // Output message: +# // Elapsed time for time integration loop 1.80759 +# // Relative error of ROM position (x) at t_final: 5 is 0.000231698 +# // Relative error of ROM velocity (v) at t_final: 5 is 0.466941 +# // +# // Online phase with strong hyper-reduction: +# // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 100 -sc 1.00 +# // Output message: +# // Elapsed time for time integration loop 1.08048 +# // Relative error of ROM position (x) at t_final: 5 is 0.00209877 +# // Relative error of ROM velocity (v) at t_final: 5 is 1.39472 +# // +# // ================================================================================= +# // +# // Sample runs and results for parametric ROM using only displacement basis +# // and nonlinear term basis: +# // Offline phase: +# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.90 -xbo -def-ic -id 0 +# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.10 -xbo -def-ic -id 1 +# // +# // Merge phase: +# // ./nonlinear_elasticity_global_rom -merge -ns 2 -dt 0.01 -tf 5.0 -xbo +# // +# // Create FOM comparison data: +# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.00 -xbo -def-ic -id 2 +# // +# // Online phase with full sampling: +# // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rxdim 57 -hdim 183 -nsr 1170 -sc 1.00 -xbo -def-ic +# // Output message: +# // Elapsed time for time integration loop 18.9874 +# // Relative error of ROM position (x) at t_final: 5 is 7.08272e-05 +# // Relative error of ROM velocity (v) at t_final: 5 is 0.00387647 +# // +# // Online phase with strong hyper reduction: +# // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rxdim 2 -hdim 4 -nsr 10 -sc 1.00 -xbo -def-ic +# // Output message: +# // Elapsed time for time integration loop 1.01136 +# // Relative error of ROM position (x) at t_final: 5 is 0.0130818 +# // Relative error of ROM velocity (v) at t_final: 5 is 0.979978 +# // +# // This example runs in parallel with MPI, by using the same number of MPI ranks +# // in all phases (offline, merge, online). + +import os +import io +import sys +import time +try: + import mfem.par as mfem +except ModuleNotFoundError: + msg = "PyMFEM is not installed yet. Install PyMFEM:\n" + msg += "\tgit clone https://github.com/mfem/PyMFEM.git\n" + msg += "\tcd PyMFEM\n" + msg += "\tpython3 setup.py install --with-parallel\n" + raise ModuleNotFoundError(msg) + +from ctypes import c_double +from mfem.par import intArray, add_vector +from os.path import expanduser, join, dirname +import numpy as np +from numpy import sin, cos, exp, sqrt, pi, abs, array, floor, log, sum +from scipy.special import erfc + +sys.path.append("../../build") +import pylibROM.linalg as linalg +import pylibROM.hyperreduction as hyper +import pylibROM.mfem as mfem_support +from pylibROM.mfem import ComputeCtAB +from pylibROM.utils import StopWatch + +class ReducedSystemOperator(mfem.PyOperator): + def __init__(self, M, S, H, ess_tdof_list): + mfem.PyOperator.__init__(self, M.ParFESpace().TrueVSize()) + self.M = M + self.S = S + self.H = H + self.Jacobian = None + h = M.ParFESpace().TrueVSize() + self.w = mfem.Vector(h) + self.z = mfem.Vector(h) + self.dt = 0.0 + self.ess_tdof_list = ess_tdof_list + + def SetParameters(self, dt, v, x): + self.dt = dt + self.v = v + self.x = x + + def Mult(self, k, y): + add_vector(self.v, self.dt, k, self.w) + add_vector(self.x, self.dt, self.w, self.z) + self.H.Mult(self.z, y) + self.M.TrueAddMult(k, y) + self.S.TrueAddMult(self.w, y) + y.SetSubVector(self.ess_tdof_list, 0.0) + + def GetGradient(self, k): + localJ = mfem.Add(1.0, self.M.SpMat(), self.dt, self.S.SpMat()) + add_vector(self.v, self.dt, k, self.w) + add_vector(self.x, self.dt, self.w, self.z) + localJ.Add(self.dt * self.dt, self.H.GetLocalGradient(self.z)) + Jacobian = self.M.ParallelAssemble(localJ) + Jacobian.EliminateRowsCols(self.ess_tdof_list) + return Jacobian + +class HyperelasticOperator(mfem.PyTimeDependentOperator): + def __init__(self, fespace, ess_bdr, visc, mu, K): + mfem.PyTimeDependentOperator.__init__(self, 2*fespace.TrueVSize(), 0.0) + + rel_tol = 1e-8 + skip_zero_entries = 0 + ref_density = 1.0 + + self.ess_tdof_list = intArray() + self.z = mfem.Vector(self.Height()//2) + self.fespace = fespace + self.viscosity = visc + self.newton_solver = mfem.NewtonSolver(fespace.GetComm()) + + M = mfem.ParBilinearForm(fespace) + S = mfem.ParBilinearForm(fespace) + H = mfem.ParNonlinearForm(fespace) + self.M = M + self.H = H + self.S = S + + rho = mfem.ConstantCoefficient(ref_density) + M.AddDomainIntegrator(mfem.VectorMassIntegrator(rho)) + M.Assemble(skip_zero_entries) + M.Finalize(skip_zero_entries) + self.Mmat = M.ParallelAssemble() + + fespace.GetEssentialTrueDofs(ess_bdr, self.ess_tdof_list) + self.Mmat.EliminateRowsCols(self.ess_tdof_list) + + M_solver = mfem.CGSolver(fespace.GetComm()) + M_prec = mfem.HypreSmoother() + M_solver.iterative_mode = False + M_solver.SetRelTol(rel_tol) + M_solver.SetAbsTol(0.0) + M_solver.SetMaxIter(30) + M_solver.SetPrintLevel(0) + M_prec.SetType(mfem.HypreSmoother.Jacobi) + M_solver.SetPreconditioner(M_prec) + M_solver.SetOperator(self.Mmat) + + self.M_solver = M_solver + self.M_prec = M_prec + + model = mfem.NeoHookeanModel(mu, K) + H.AddDomainIntegrator(mfem.HyperelasticNLFIntegrator(model)) + H.SetEssentialTrueDofs(self.ess_tdof_list) + self.model = model + + visc_coeff = mfem.ConstantCoefficient(visc) + S.AddDomainIntegrator(mfem.VectorDiffusionIntegrator(visc_coeff)) + S.Assemble(skip_zero_entries) + S.Finalize(skip_zero_entries) + + self.reduced_oper = ReducedSystemOperator(M, S, H, self.ess_tdof_list) + + J_hypreSmoother = mfem.HypreSmoother() + J_hypreSmoother.SetType(mfem.HypreSmoother.l1Jacobi) + J_hypreSmoother.SetPositiveDiagonal(True) + J_prec = J_hypreSmoother + + J_minres = mfem.MINRESSolver(fespace.GetComm()) + J_minres.SetRelTol(rel_tol) + J_minres.SetAbsTol(0.0) + J_minres.SetMaxIter(300) + J_minres.SetPrintLevel(-1) + J_minres.SetPreconditioner(J_prec) + + self.J_solver = J_minres + self.J_prec = J_prec + + newton_solver = mfem.NewtonSolver(fespace.GetComm()) + newton_solver.iterative_mode = False + newton_solver.SetSolver(self.J_solver) + newton_solver.SetOperator(self.reduced_oper) + newton_solver.SetPrintLevel(1) # print Newton iterations + newton_solver.SetRelTol(rel_tol) + newton_solver.SetAbsTol(0.0) + newton_solver.SetAdaptiveLinRtol(2, 0.5, 0.9) + newton_solver.SetMaxIter(10) + self.newton_solver = newton_solver + + def Mult(self, vx, vx_dt): + sc = self.Height()//2 + v = mfem.Vector(vx, 0, sc) + x = mfem.Vector(vx, sc, sc) + dv_dt = mfem.Vector(dvx_dt, 0, sc) + dx_dt = mfem.Vector(dvx_dt, sc, sc) + + self.H.Mult(x, z) + if (self.viscosity != 0.0): + S.TrueAddMult(v, z) + z.SetSubVector(self.ess_tdof_list, 0.0) + z.Neg() + self.M_solver.Mult(z, dv_dt) + dx_dt = v + + def ImplicitSolve(self, dt, vx, dvx_dt): + sc = self.Height()//2 + v = mfem.Vector(vx, 0, sc) + x = mfem.Vector(vx, sc, sc) + dv_dt = mfem.Vector(dvx_dt, 0, sc) + dx_dt = mfem.Vector(dvx_dt, sc, sc) + + # By eliminating kx from the coupled system: + # kv = -M^{-1}*[H(x + dt*kx) + S*(v + dt*kv)] + # kx = v + dt*kv + # we reduce it to a nonlinear equation for kv, represented by the + # backward_euler_oper. This equation is solved with the newton_solver + # object (using J_solver and J_prec internally). + self.reduced_oper.SetParameters(dt, v, x) + zero = mfem.Vector() # empty vector is interpreted as + # zero r.h.s. by NewtonSolver + self.newton_solver.Mult(zero, dv_dt) + add_vector(v, dt, dv_dt, dx_dt) + + def ElasticEnergy(self, x): + return self.H.GetEnergy(x) + + def KineticEnergy(self, v): + local_energy = 0.5*self.M.InnerProduct(v, v) + energy = MPI.COMM_WORLD.allreduce(local_energy, op=MPI.SUM) + return energy + + def GetElasticEnergyDensity(self, x, w): + w_coeff = ElasticEnergyCoefficient(self.model, x) + w.ProjectCoefficient(w_coeff) + +class RomOperator(mfem.PyTimeDependentOperator): + # RomOperator::RomOperator(HyperelasticOperator* fom_, + # HyperelasticOperator* fomSp_, const int rvdim_, const int rxdim_, + # const int hdim_, CAROM::SampleMeshManager* smm_, const Vector* v0_, + # const Vector* x0_, const Vector v0_fom_, const CAROM::Matrix* V_v_, + # const CAROM::Matrix* V_x_, const CAROM::Matrix* U_H_, + # const CAROM::Matrix* Hsinv_, const int myid, const bool oversampling_, + # const bool hyperreduce_, const bool x_base_only_) + def __init__(self, fom_, fomSp_, rvdim_, rxdim_, hdim_, smm_, v0_, + x0_, v0_fom_, V_v_, V_x_, U_H_, Hsinv_, myid, oversampling_, + hyperreduce_, x_base_only_): + mfem.PyTimeDependentOperator.__init__(self, rxdim_ + rvdim_, 0.0) + self.fom = fom_ + self.fomSp = fomSp_ + self.rxdim, self.rvdim, self.hdim = rxdim_, rvdim_, hdim_ + self.x0, self.v0 = x0_, v0_ + self.v0_fom = v0_fom_ + self.smm = smm_ + self.nsamp_H = smm_.GetNumVarSamples("H") + self.V_x, self.V_v = V_x_, V_v_ + self.U_H, self.Hsinv = U_H_, Hsinv_ + self.zN = linalg.Vector(max(self.nsamp_H, 1), False) + self.zX = linalg.Vector(max(self.nsamp_H, 1), False) + self.oversampling = oversampling_ + self.M_hat_solver = mfem.CGSolver(fom_.fespace.GetComm()) + self.z = mfem.Vector(self.height // 2) + self.hyperreduce = hyperreduce_ + self.x_base_only_ = x_base_only_ + + if (myid == 0): + self.V_v_sp = linalg.Matrix(self.fomSp.Height() // 2, self.rvdim, False) + self.V_x_sp = linalg.Matrix(self.fomSp.Height() // 2, self.rxdim, False) + # TODO(kevin): we might need to initialize for non-root processes as well. + + # Gather distributed vectors + if (self.x_base_only): + self.smm.GatherDistributedMatrixRows("X", self.V_v, self.rvdim, self.V_v_sp) + else: + self.smm.GatherDistributedMatrixRows("V", self.V_v, self.rvdim, self.V_v_sp) + + self.smm.GatherDistributedMatrixRows("X", self.V_x, self.rxdim, self.V_x_sp) + + # Create V_vTU_H, for hyperreduction + self.V_v.transposeMult(self.U_H, self.V_vTU_H) + + self.S_hat = linalg.Matrix(self.rvdim, self.rvdim, False) + self.S_hat_v0 = linalg.Vector(self.rvdim, False) + self.S_hat_v0_temp = mfem.Vector(self.v0_fom.Size()) + self.S_hat_v0_temp_librom = linalg.Vector(self.S_hat_v0_temp.GetDataArray(), True, False) + self.M_hat = linalg.Matrix(self.rvdim, self.rvdim, False) + self.M_hat_inv = linalg.Matrix(self.rvdim, self.rvdim, False) + + # Create S_hat + ComputeCtAB(self.fom.Smat, self.V_v, self.V_v, self.S_hat) + + # Apply S_hat to the initial velocity and store + self.fom.Smat.Mult(self.v0_fom, self.S_hat_v0_temp) + self.V_v.transposeMult(self.S_hat_v0_temp_librom, self.S_hat_v0) + + # Create M_hat + ComputeCtAB(self.fom.Mmat, self.V_v, self.V_v, self.M_hat) + + # Invert M_hat and store + self.M_hat.inverse(self.M_hat_inv) + + if (myid == 0): + self.spdim = self.fomSp.Height() # Reduced height + + self.zH.SetSize(self.spdim // 2) # Samples of H + + # Allocate auxillary vectors + self.z.SetSize(self.spdim // 2) + self.z_v.SetSize(self.spdim // 2) + self.z_x.SetSize(self.spdim // 2) + self.z_librom = linalg.Vector(self.z.GetDataArray(), False, False) + self.z_v_librom = linalg.Vector(self.z_v.GetDataArray(), False, False) + self.z_x_librom = linalg.Vector(self.z_x.GetDataArray(), False, False) + + # This is for saving the recreated predictions + self.psp = mfem.Vector(self.spdim) + self.psp_librom = linalg.Vector(self.psp.GetDataArray(), False, False) + + # Define sub-vectors of psp. + self.psp_x = mfem.Vector(self.psp.GetData(), self.spdim // 2) + self.psp_v = mfem.Vector(self.psp, self.spdim // 2, self.spdim // 2) + + self.psp_v_librom = linalg.Vector(self.psp_v.GetDataArray(), False, False) + + if (not self.hyperreduce): + fdim = self.fom.Height() # Unreduced height + + self.z.SetSize(fdim // 2) + self.z_v.SetSize(fdim // 2) + self.z_x.SetSize(fdim // 2) + self.z_librom = linalg.Vector(self.z.GetDataArray(), False, False) + self.z_v_librom = linalg.Vector(self.z_v.GetDataArray(), True, False) + self.z_x_librom = linalg.Vector(self.z_x.GetDataArray(), True, False) + + # This is for saving the recreated predictions + self.pfom = mfem.Vector(fdim) + self.pfom_librom = linalg.Vector(self.pfom.GetDataArray(), False, False) + + # Define sub-vectors of pfom. + self.pfom_x = mfem.Vector(self.pfom.GetData(), fdim // 2) + self.pfom_v = mfem.Vector(self.pfom, fdim // 2, fdim // 2) + self.zfom_x = mfem.Vector(fdim / 2) + self.zfom_x_librom = linalg.Vector(self.zfom_x.GetDataArray(), True, False) + + self.pfom_v_librom = linalg.Vector(self.pfom_v.GetDataArray(), True, False) + + def Mult(self, vx, dvx_dt): + if (self.hyperreduce): + self.Mult_Hyperreduced(vx, dvx_dt) + else: + self.Mult_FullOrder(vx, dvx_dt) + + def Mult_Hyperreduced(self, vx, dvx_dt): + # Check that the sizes match + assert((vx.Size() == self.rvdim + self.rxdim) and (dvx_dt.Size() == self.rvdim + self.rxdim)) + + # Create views to the sub-vectors v, x of vx, and dv_dt, dx_dt of dvx_dt + v = mfem.Vector(vx, 0, self.rvdim) + v_librom = linalg.Vector(vx.GetDataArray()[:self.rvdim], False, False) + x_librom = linalg.Vector(vx.GetDataArray()[self.rvdim:], False, False) + dv_dt = mfem.Vector(dvx_dt, 0, self.rvdim) + dx_dt = mfem.Vector(dvx_dt, self.rvdim, self.rxdim) + dv_dt_librom = linalg.Vector(dv_dt.GetDataArray(), False, False) + dx_dt_librom = linalg.Vector(dx_dt.GetDataArray(), False, False) + + # Lift the x-, and v-vectors + # I.e. perform v = v0 + V_v v^, where v^ is the input + self.V_v_sp.mult(v_librom, self.z_v_librom) + self.V_x_sp.mult(x_librom, self.z_x_librom) + + add_vector(self.z_v, self.v0, self.psp_v) # Store liftings + add_vector(self.z_x, self.x0, self.psp_x) + + # Hyperreduce H + # Apply H to x to get zH + self.fomSp.H.Mult(self.psp_x, self.zH) + + # Sample the values from zH + self.smm.GetSampledValues("H", self.zH, self.zN) + + # Apply inverse H-basis + if (self.oversampling): + self.Hsinv.transposeMult(self.zN, self.zX) + else: + self.Hsinv.mult(self.zN, self.zX) + + # Multiply by V_v^T * U_H + self.V_vTU_H.mult(self.zX, self.z_librom) + + if (self.fomSp.viscosity != 0.0): + # Apply S^, the reduced S operator, to v + self.S_hat.multPlus(self.z_librom, v_librom, 1.0) + self.z_librom += self.S_hat_v0 + + self.z.Neg() # z = -z + self.M_hat_inv.mult(self.z_librom, dv_dt_librom) # to invert reduced mass matrix operator. + + self.V_x_sp.transposeMult(self.psp_v_librom, dx_dt_librom) + + def Mult_FullOrder(self, vx, dvx_dt): + # Check that the sizes match + assert((vx.Size() == self.rvdim + self.rxdim) and (dvx_dt.Size() == self.rvdim + self.rxdim)) + + # Create views to the sub-vectors v, x of vx, and dv_dt, dx_dt of dvx_dt + v = mfem.Vector(vx, 0, self.rvdim) + v_librom = linalg.Vector(vx.GetDataArray()[:self.rvdim], False, False) + x_librom = linalg.Vector(vx.GetDataArray()[self.rvdim:], False, False) + dv_dt = mfem.Vector(dvx_dt, 0, self.rvdim) + dx_dt = mfem.Vector(dvx_dt, self.rvdim, self.rxdim) + dv_dt_librom = linalg.Vector(dv_dt.GetDataArray(), False, False) + dx_dt_librom = linalg.Vector(dx_dt.GetDataArray(), False, False) + + # Lift the x-, and v-vectors + # I.e. perform v = v0 + V_v v^, where v^ is the input + self.V_x.mult(x_librom, self.z_x_librom) + self.V_v.mult(v_librom, self.z_v_librom) + + add_vector(self.z_x, self.x0, self.pfom_x) # Store liftings + add_vector(self.z_v, self.v0, self.pfom_v) + + # Apply H to x to get z + self.fom.H.Mult(self.pfom_x, self.zfom_x) + + self.V_v.transposeMult(self.zfom_x_librom, self.z_librom) + + if (self.fom.viscosity != 0.0): + # Apply S^, the reduced S operator, to v + self.S_hat.multPlus(self.z_librom, v_librom, 1.0) + self.z_librom += self.S_hat_v0 + + self.z.Neg() # z = -z + self.M_hat_inv.mult(self.z_librom, dv_dt_librom) # to invert reduced mass matrix operator. + + self.V_x.transposeMult(self.pfom_v_librom, dx_dt_librom) + +# /** Function representing the elastic energy density for the given hyperelastic +# model+deformation. Used in HyperelasticOperator::GetElasticEnergyDensity. */ +class ElasticEnergyCoefficient(mfem.PyCoefficient): + def __init__(self, model, x): + self.x = x + self.model = model + self.J = mfem.DenseMatrix() + mfem.PyCoefficient.__init__(self) + + def Eval(self, T, ip): + self.model.SetTransformation(T) + self.x.GetVectorGradient(T, self.J) + return self.model.EvalW(self.J)/(self.J.Det()) + +class InitialDeformationIC1(mfem.VectorPyCoefficient): + def EvalValue(self, x): + from copy import deepcopy + y = deepcopy(x) + return y + +class InitialVelocityIC1(mfem.VectorPyCoefficient): + def EvalValue(self, x): + global s + s_eff = s / 80.0 + + v = np.zeros(len(x)) + v[-1] = -s_eff * sin(s * x[0]) + return v + +class InitialDeformationIC2(mfem.VectorPyCoefficient): + def EvalValue(self, x): + global s + s_eff = s + + from copy import deepcopy + y = deepcopy(x) + y[-1] = x[-1] + 0.25 * x[0] * s_eff + return y + +class InitialVelocityIC2(mfem.VectorPyCoefficient): + def EvalValue(self, x): + v = np.zeros(len(x)) + return v + + +# def GetFirstColumns(N, A): +# S = linalg.Matrix(A.numRows(), min(N, A.numColumns()), A.distributed()) +# for (int i = 0; i < S->numRows(); ++i) +# { +# for (int j = 0; j < S->numColumns(); ++j) +# (*S)(i, j) = (*A)(i, j); +# } + +# # delete A; # TODO: find a good solution for this. +# return S; + +# TODO: move this to the library? +def BasisGeneratorFinalSummary(bg, energyFraction, cutoff, cutoffOutputPath): + rom_dim = bg.getSpatialBasis().numColumns() + sing_vals = bg.getSingularValues() + + assert(rom_dim <= sing_vals.dim()) + + sum = 0.0 + for sv in range(sing_vals.dim()): + sum += sing_vals[sv] + + energy_fractions = [ 0.9999999, 0.999999, 0.99999, 0.9999, 0.999, 0.99, 0.9 ] + reached_cutoff = False + + outfile = open(cutoffOutputPath, 'w') + + partialSum = 0.0 + for sv in range(sing_vals.dim()): + partialSum += sing_vals[sv] + for i in range(energy_fractions.size()-1, -1, -1): + if (partialSum / sum > energy_fractions[i]): + outfile.write("For energy fraction: %.5E, take first %d of %d basis vectors" % (energy_fractions[i], sv+1, sing_vals.dim())) + energy_fractions.pop(-1) + else: + break + + if ((not reached_cutoff) and (partialSum / sum > energyFraction)): + cutoff = sv + 1 + reached_cutoff = True + + if (not reached_cutoff): cutoff = sing_vals.dim() + outfile.write("Take first %d of %d basis vectors" % (cutoff, sing_vals.dim())) + outfile.close() + + return cutoff + +def MergeBasis(dimFOM, nparam, max_num_snapshots, name): + if (not (nparam > 0)): + raise RuntimeError("Must specify a positive number of parameter sets") + + update_right_SV = False + isIncremental = False + + options = linalg.Options(dimFOM, nparam * max_num_snapshots, 1, update_right_SV) + generator = linalg.BasisGenerator(options, isIncremental, "basis" + name) + + for paramID in range(nparam): + snapshot_filename = "basis%d_%s_snapshot" % (paramID, name) + generator.loadSamples(snapshot_filename, "snapshot") + + generator.endSamples() # save the merged basis file + + cutoff = 0 + cutoff = BasisGeneratorFinalSummary(generator, 0.9999999, cutoff, "mergedSV_" + name + ".txt") + +# TODO: remove this by making online computation serial? +def BroadcastUndistributedRomVector(v): + N = v.dim() + assert(N > 0) + + from copy import deepcopy + d = deepcopy(v.getData()) + + from mpi4py import MPI + MPI.COMM_WORLD.Bcast([d, MPI.DOUBLE], root=0) + + for i in range(N): + v[i] = d[i] + +def visualize(out, mesh, deformed_nodes, field, + field_name='', init_vis=False): + nodes = deformed_nodes + owns_nodes = 0 + + nodes, owns_nodes = mesh.SwapNodes(nodes, owns_nodes) + + out.send_text("parallel " + str(mesh.GetNRanks()) + " " + str(mesh.GetMyRank())) + out.send_solution(mesh, field) + + nodes, owns_nodes = mesh.SwapNodes(nodes, owns_nodes) + + if (init_vis): + out.send_text("window_size 800 800") + out.send_text("window_title '" + field_name) + if (mesh.SpaceDimension() == 2): + out.send_text("view 0 0") # view from top + out.send_text("keys jl") # turn off perspective and light + out.send_text("keys cm") # show colorbar and mesh + # update value-range; keep mesh-extents fixed + out.send_text("autoscale value") + out.send_text("pause") + out.flush() + +# Scaling factor for parameterization +s = 1.0 + +def run(): + # 1. Initialize MPI. + from mpi4py import MPI + comm = MPI.COMM_WORLD + myid = comm.Get_rank() + num_procs = comm.Get_size() + + # 2. Parse command-line options. + from mfem.common.arg_parser import ArgParser + parser = ArgParser(description="Projection ROM - MFEM nonlinear elasticity equation example.") + parser.add_argument("-m", "--mesh", + action='store', dest='mesh_file', default="../data/beam-quad.mesh", type=str, + help="Mesh file to use.") + parser.add_argument("-rs", "--refine-serial", + action='store', dest='ser_ref_levels', default=2, type=int, + help="Number of times to refine the mesh uniformly in serial.") + parser.add_argument("-rp", "--refine-parallel", + action='store', dest='par_ref_levels', default=0, type=int, + help="Number of times to refine the mesh uniformly in parallel.") + parser.add_argument("-o", "--order", + action='store', default=2, type=int, + help="Order (degree) of the finite elements.") + parser.add_argument("-s", "--ode-solver", + action='store', dest='ode_solver_type', default=14, type=int, + help="ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t" + " 11 - Forward Euler, 12 - RK2,\n\t" + " 13 - RK3 SSP, 14 - RK4." + " 22 - Implicit Midpoint Method,\n\t" + " 23 - SDIRK23 (A-stable), 24 - SDIRK34") + parser.add_argument("-tf", "--t-final", + action='store', default=15.0, type=float, + help="Final time; start time is 0.") + parser.add_argument("-dt", "--time-step", + action='store', dest='dt', default=0.03, type=float, + help="Time step.") + parser.add_argument("-v", "--viscosity", + action='store', dest='visc', default=1e-2, type=float, + help="Viscosity coefficient.") + parser.add_argument("-mu", "--shear-modulus", + action='store', dest='mu', default=0.25, type=float, + help="Shear modulus in the Neo-Hookean hyperelastic model.") + parser.add_argument("-K", "--bulk-modulus", + action='store', dest='K', default=5.0, type=float, + help="Bulk modulus in the Neo-Hookean hyperelastic model.") + parser.add_argument("-alrtol", "--adaptive-lin-rtol", + action='store_true', default=True, dest='adaptive_lin_rtol', + help="Enable adaptive linear solver rtol.") + parser.add_argument("-no-alrtol", "--no-adaptive-lin-rtol", + action='store_false', dest='adaptive_lin_rtol', + help="Disable adaptive linear solver rtol.") + parser.add_argument("-vis", "--visualization", + action='store_true', default=True, dest='visualization', + help="Enable GLVis visualization.") + parser.add_argument("-no-vis", "--no-visualization", + action='store_false', dest='visualization', + help="Disable GLVis visualization.") + parser.add_argument("-visit", "--visit-datafiles", + action='store_true', default=False, dest='visit', + help="Save data files for VisIt (visit.llnl.gov) visualization.") + parser.add_argument("-no-visit", "--no-visit-datafiles", + action='store_false', dest='visit', + help="Save data files for VisIt (visit.llnl.gov) visualization.") + parser.add_argument("-vs", "--visualization-steps", + action='store', dest='vis_steps', default=1, type=int, + help="Visualize every n-th timestep.") + parser.add_argument("-ns", "--nset", + action='store', dest='nsets', default=0, type=int, + help="Number of parametric snapshot sets") + parser.add_argument("-offline", "--offline", + action='store_true', dest='offline', default=False, + help="Enable the offline phase.") + parser.add_argument("-no-offline", "--no-offline", + action='store_false', dest='offline', + help="Disable the offline phase.") + parser.add_argument("-online", "--online", + action='store_true', dest='online', default=False, + help="Enable the online phase.") + parser.add_argument("-no-online", "--no-online", + action='store_false', dest='online', + help="Disable the online phase.") + parser.add_argument("-merge", "--merge", + action='store_true', dest='merge', default=False, + help="Enable the merge phase.") + parser.add_argument("-no-merge", "--no-merge", + action='store_false', dest='merge', + help="Disable the merge phase.") + parser.add_argument("-sopt", "--sopt", + action='store_true', dest='use_sopt', default=False, + help="Use S-OPT sampling instead of DEIM for the hyperreduction.") + parser.add_argument("-no-sopt", "--no-sopt", + action='store_false', dest='use_sopt', + help="(disable) Use S-OPT sampling instead of DEIM for the hyperreduction.") + parser.add_argument("-nsr", "--nsr", + action='store', dest='num_samples_req', default=-1, type=int, + help="number of samples we want to select for the sampling algorithm.") + parser.add_argument("-rxdim", "--rxdim", + action='store', default=-1, type=int, + help="Basis dimension for displacement solution space.") + parser.add_argument("-rvdim", "--rvdim", + action='store', default=-1, type=int, + help="Basis dimension for velocity solution space.") + parser.add_argument("-hdim", "--hdim", + action='store', default=-1, type=int, + help="Basis dimension for the nonlinear term.") + parser.add_argument("-id", "--id", + action='store', dest='id_param', default=0, type=int, + help="Parametric index") + parser.add_argument("-hyp", "--hyperreduce", + action='store_true', dest='hyperreduce', default=False, + help="Enable Hyper reduce nonlinear term") + parser.add_argument("-no-hyp", "--no-hyperreduce", + action='store_false', dest='hyperreduce', + help="Disable Hyper reduce nonlinear term") + parser.add_argument("-xbo", "--xbase-only", + action='store_true', dest='x_base_only', default=False, + help="Use the displacement (X) basis to approximate velocity.") + parser.add_argument("-no-xbo", "--not-xbase-only", + action='store_false', dest='x_base_only', + help="not use the displacement (X) basis to approximate velocity.") + parser.add_argument("-def-ic", "--deformation-ic", + action='store_true', dest='def_ic', default=False, + help="Use a deformation initial condition. Default is velocity IC.") + parser.add_argument("-vel-ic", "--velocity-ic", + action='store_false', dest='def_ic', + help="Use velocity initial condition. Default is velocity IC.") + parser.add_argument("-sc", "--scaling", + action='store', dest='s', default=1.0, type=float, + help="Scaling factor for initial condition.") + + args = parser.parse_args() + if (myid == 0): parser.print_options(args) + + mesh_file = args.mesh_file + ser_ref_levels = args.ser_ref_levels + par_ref_levels = args.par_ref_levels + order = args.order + ode_solver_type = args.ode_solver_type + vis_steps = args.vis_steps + t_final = args.t_final + dt = args.dt + visc = args.visc + mu = args.mu + K = args.K + adaptive_lin_rtol = args.adaptive_lin_rtol + visualization = args.visualization + visit = args.visit + def_ic = args.def_ic + global s + s = args.s + + # ROM parameters + offline = args.offline + merge = args.merge + online = args.online + use_sopt = args.use_sopt + hyperreduce = args.hyperreduce + x_base_only = args.x_base_only + num_samples_req = args.num_samples_req + + nsets = args.nsets + id_param = args.id_param + + # number of basis vectors to use + rxdim = args.rxdim + rvdim = args.rvdim + hdim = args.hdim + + check = (offline and (not merge) and (not online)) or ((not offline) and merge and (not online)) or ((not offline) and (not merge) and online) + if not check: + raise RuntimeError("only one of offline, merge, or online must be true!") + + solveTimer, totalTimer = StopWatch(), StopWatch() + totalTimer.Start() + + # // 3. Read the serial mesh from the given mesh file on all processors. We can + # // handle triangular, quadrilateral, tetrahedral and hexahedral meshes + # // with the same code. + mesh = mfem.Mesh(mesh_file, 1, 1) + dim = mesh.Dimension() + + # // 4. Define the ODE solver used for time integration. Several implicit + # // singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as + # // explicit Runge-Kutta methods are available. + # if ode_solver_type == 1: + # ode_solver = BackwardEulerSolver() + # elif ode_solver_type == 2: + # ode_solver = mfem.SDIRK23Solver(2) + # elif ode_solver_type == 3: + # ode_solver = mfem.SDIRK33Solver() + if ode_solver_type == 11: + ode_solver = ForwardEulerSolver() + elif ode_solver_type == 12: + ode_solver = mfem.RK2Solver(0.5) + elif ode_solver_type == 13: + ode_solver = mfem.RK3SSPSolver() + elif ode_solver_type == 14: + ode_solver = mfem.RK4Solver() + elif ode_solver_type == 15: + ode_solver = mfem.GeneralizedAlphaSolver(0.5) + # elif ode_solver_type == 22: + # ode_solver = mfem.ImplicitMidpointSolver() + # elif ode_solver_type == 23: + # ode_solver = mfem.SDIRK23Solver() + # elif ode_solver_type == 24: + # ode_solver = mfem.SDIRK34Solver() + else: + if myid == 0: + print("Unknown ODE solver type: " + str(ode_solver_type)) + sys.exit() + + # // 5. Refine the mesh in serial to increase the resolution. In this example + # // we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is + # // a command-line parameter. + for lev in range(ser_ref_levels): + mesh.UniformRefinement() + + # // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine + # // this mesh further in parallel to increase the resolution. Once the + # // parallel mesh is defined, the serial mesh can be deleted. + pmesh = mfem.ParMesh(MPI.COMM_WORLD, mesh) + del mesh + + for lev in range(par_ref_levels): + pmesh.UniformRefinement() + + # // 7. Define the parallel vector finite element spaces representing the mesh + # // deformation x_gf, the velocity v_gf, and the initial configuration, + # // x_ref. Define also the elastic energy density, w_gf, which is in a + # // discontinuous higher-order space. Since x and v are integrated in time + # // as a system, we group them together in block vector vx, on the unique + # // parallel degrees of freedom, with offsets given by array true_offset. + fe_coll = mfem.H1_FECollection(order, dim) + fespace = mfem.ParFiniteElementSpace(pmesh, fe_coll, dim) + glob_size = fespace.GlobalTrueVSize() + if (myid == 0): + print('Number of velocity/deformation unknowns: ' + str(glob_size)) + + true_size = fespace.TrueVSize() + true_offset = mfem.intArray(3) + true_offset[0] = 0 + true_offset[1] = true_size + true_offset[2] = 2*true_size + + vx = mfem.BlockVector(true_offset) + + v_gf = mfem.ParGridFunction(fespace) + x_gf = mfem.ParGridFunction(fespace) + # v_gf.MakeTRef(&fespace, vx, + # true_offset[0]); // Associate a new FiniteElementSpace and new true-dof data with the GridFunction. + # x_gf.MakeTRef(&fespace, vx, true_offset[1]); + + x_ref = mfem.ParGridFunction(fespace) + pmesh.GetNodes(x_ref) + + w_fec = mfem.L2_FECollection(order + 1, dim) + w_fespace = mfem.ParFiniteElementSpace(pmesh, w_fec) + w_gf = mfem.ParGridFunction(w_fespace) + + # Basis params + update_right_SV = False + isIncremental = False + basisFileName = "basis" + str(id_param) + max_num_snapshots = int(t_final / dt) + 2 + + # The merge phase + if (merge): + totalTimer.Reset() + totalTimer.Start() + + # Merge bases + if (not x_base_only): + MergeBasis(true_size, nsets, max_num_snapshots, "V") + + MergeBasis(true_size, nsets, max_num_snapshots, "X") + MergeBasis(true_size, nsets, max_num_snapshots, "H") + + totalTimer.Stop() + if (myid == 0): + print("Elapsed time for merging and building ROM basis: %e second\n" % totalTimer.duration) + + return + + # // 8. Set the initial conditions for v_gf, x_gf and vx, and define the + # // boundary conditions on a beam-like mesh (see description above). + if (def_ic): + velo = InitialVelocityIC2(dim) + else: + velo = InitialVelocityIC1(dim) + + v_gf.ProjectCoefficient(velo) + v_gf.SetTrueVector() + + if (def_ic): + deform = InitialDeformationIC2(dim) + else: + deform = InitialDeformationIC1(dim) + + x_gf.ProjectCoefficient(deform) + x_gf.SetTrueVector() + + v_gf.SetFromTrueVector() + x_gf.SetFromTrueVector() + + v_gf.GetTrueDofs(vx.GetBlock(0)) + x_gf.GetTrueDofs(vx.GetBlock(1)) + + ess_bdr = mfem.intArray(fespace.GetMesh().bdr_attributes.Max()) + ess_bdr.Assign(0) + ess_bdr[0] = 1 # boundary attribute 1 (index 0) is fixed + + ess_tdof_list = mfem.intArray() + fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list) + + # Store initial vx + vx0 = mfem.BlockVector(vx) + vx_diff = mfem.BlockVector(vx) + + # Reduced order solution + # Vector* wMFEM = 0; + # CAROM::Vector* w = 0; + # CAROM::Vector* w_v = 0; + # CAROM::Vector* w_x = 0; + + # Initialize reconstructed solution + v_rec = mfem.Vector(v_gf.GetTrueVector()) + x_rec = mfem.Vector(x_gf.GetTrueVector()) + + v_rec_librom = linalg.Vector(v_rec.GetDataArray(), True, False) + x_rec_librom = linalg.Vector(x_rec.GetDataArray(), True, False) + + # // 9. Initialize the hyperelastic operator, the GLVis visualization and print + # // the initial energies. + oper = HyperelasticOperator(fespace, ess_tdof_list, visc, mu, K) + soper = None + + # Fill dvdt and dxdt + dvxdt = mfem.Vector(true_size * 2) + dvdt = mfem.Vector(dvxdt, 0, true_size) + dxdt = mfem.Vector(dvxdt, true_size, true_size) + + if (visualization): + vishost = "localhost" + visport = 19916 + vis_v = mfem.socketstream(vishost, visport) + vis_v.precision(8) + visualize(vis_v, pmesh, x_gf, v_gf, "Velocity", True) + # // Make sure all ranks have sent their 'v' solution before initiating + # // another set of GLVis connections (one from each rank): + pmesh.GetComm().Barrier() + vis_w = mfem.socketstream(vishost, visport) + if (vis_w.good()): + oper.GetElasticEnergyDensity(x_gf, w_gf) + vis_w.precision(8) + visualize(vis_w, pmesh, x_gf, w_gf, "Elastic energy density", True) + + # // Create data collection for solution output: either VisItDataCollection for + # // ascii data files, or SidreDataCollection for binary data files. + dc = None + if (visit): + if (offline): + dc = mfem.VisItDataCollection("nlelast-fom", pmesh) + else: + dc = mfem.VisItDataCollection("nlelast-rom", pmesh) + + dc.SetPrecision(8) + # // To save the mesh using MFEM's parallel mesh format: + # // dc->SetFormat(DataCollection::PARALLEL_FORMAT); + dc.RegisterField("x", x_gf) + dc.RegisterField("v", v_gf) + dc.SetCycle(0) + dc.SetTime(0.0) + dc.Save() + + ee0 = oper.ElasticEnergy(x_gf) + ke0 = oper.KineticEnergy(v_gf) + + if (myid == 0): + print("initial elastic energy (EE) = %e" % ee0) + print("initial kinetic energy (KE) = %e" % ke0) + print("initial total energy (TE) = %e" % (ee0 + ke0)) + + # // 10. Create pROM object. + if (offline): + options = linalg.Options(fespace.GetTrueVSize(), max_num_snapshots, 1, update_right_SV) + + if (not x_base_only): + basis_generator_v = linalg.BasisGenerator(options, isIncremental, basisFileName + "_V") + + basis_generator_x = linalg.BasisGenerator(options, isIncremental, basisFileName + "_X") + + basis_generator_H = linalg.BasisGenerator(options, isIncremental, basisFileName + "_H") + + # RomOperator* romop = 0; + + # const CAROM::Matrix* BV_librom = 0; + # const CAROM::Matrix* BX_librom = 0; + # const CAROM::Matrix* H_librom = 0; + # const CAROM::Matrix* Hsinv = 0; + + nsamp_H = -1 + + # CAROM::SampleMeshManager* smm = nullptr; + + # // 11. Initialize ROM operator + # // I guess most of this should be done on id =0 + if (online): + # Read bases + if (x_base_only): + readerV = linalg.BasisReader("basisX") # The basis for v uses the x basis instead. + rvdim = rxdim + else: + readerV = linalg.BasisReader("basisV") + + BV_librom = readerV.getSpatialBasis(0.0) + + if (rvdim == -1): # Change rvdim + rvdim = BV_librom.numColumns() + else: + BV_librom = BV_librom.getFirstNColumns(rvdim) + + assert(BV_librom.numRows() == true_size) + + if (myid == 0): + print("reduced V dim = %d\n" % rvdim) + + readerX = linalg.BasisReader("basisX") + BX_librom = readerX.getSpatialBasis(0.0) + + if (rxdim == -1): # Change rxdim + rxdim = BX_librom.numColumns() + else: + BX_librom = BX_librom.getFirstNColumns(rxdim) + + assert(BX_librom.numRows() == true_size) + + if (myid == 0): + print("reduced X dim = %d\n" % rxdim) + + # Hyper reduce H + readerH = linalg.BasisReader("basisH") + H_librom = readerH.getSpatialBasis(0.0) + + # Compute sample points + if (hdim == -1): + hdim = H_librom.numColumns() + + assert(H_librom.numColumns() >= hdim) + + if (H_librom.numColumns() > hdim): + H_librom = H_librom.getFirstNColumns(hdim) + + if (myid == 0): + print("reduced H dim = %d\n" % hdim) + + # vector num_sample_dofs_per_proc(num_procs); + + if (num_samples_req != -1): + nsamp_H = num_samples_req + else: + nsamp_H = hdim + + Hsinv = linalg.Matrix(nsamp_H, hdim, False) + # vector sample_dofs(nsamp_H); + if (use_sopt): + if (myid == 0): + print("Using S_OPT sampling\n") + sample_dofs, num_sample_dofs_per_proc = hyper.S_OPT(H_librom, hdim, Hsinv, + myid, num_procs, nsamp_H) + elif (nsamp_H != hdim): + if (myid == 0): + print("Using GNAT sampling\n") + sample_dofs, num_sample_dofs_per_proc = hyper.GNAT(H_librom, hdim, Hsinv, + myid, num_procs, nsamp_H) + else: + if (myid == 0): + print("Using DEIM sampling\n") + sample_dofs, num_sample_dofs_per_proc = hyper.DEIM(H_librom, hdim, Hsinv, + myid, num_procs) + + # Construct sample mesh + nspaces = 1 + spfespace = [None] * nspaces + spfespace[0] = fespace + + # ParFiniteElementSpace* sp_XV_space; + + smm = mfem_support.SampleMeshManager(spfespace) + + # vector sample_dofs_empty; + num_sample_dofs_per_proc_empty = [0] * num_procs + + smm.RegisterSampledVariable("V", 0, sample_dofs, + num_sample_dofs_per_proc) + smm.RegisterSampledVariable("X", 0, sample_dofs, + num_sample_dofs_per_proc) + smm.RegisterSampledVariable("H", 0, sample_dofs, + num_sample_dofs_per_proc) + + smm.ConstructSampleMesh() + + w = linalg.Vector(rxdim + rvdim, False) + w_v = linalg.Vector(rvdim, False) + w_x = linalg.Vector(rxdim, False) + w.fill(0.0) + + # Note that some of this could be done only on the ROM solver process, but it is tricky, since RomOperator assembles Bsp in parallel. + wMFEM = mfem.Vector(w.getData(), rxdim + rvdim) + + # Initial conditions + # Vector* w_v0 = 0; + # Vector* w_x0 = 0; + + sp_size = 0 + + if (myid == 0): + # NOTE(kevin): SampleMeshManager::GetSampleFESpace returns a pointer to a ParFiniteElementSpace, + # which is binded via SWIG, not pybind11. + # We need a SWIG object 'shell' to wrap the c++ pointer we return from pybind11 side. + # The following instantiation is simply creating SWIG object, + # to which GetSampleFESpace will return the c++ pointer. + # Instantiation can be of any type, since we only need the 'shell'. + sp_XV_space = mfem.ParFiniteElementSpace(pmesh, fe_coll, dim) + # NOTE(kevin): Unlike c++ libROM, GetSampleFESpace returns a deep copy. + smm.GetSampleFESpace(0, sp_XV_space) + + sp_size = sp_XV_space.TrueVSize() + sp_offset = mfem.intArray(3) + sp_offset[0] = 0 + sp_offset[1] = sp_size + sp_offset[2] = 2*sp_size + + # Initialize sp_p with initial conditions. + sp_vx = mfem.BlockVector(sp_offset) + sp_v_gf, sp_x_gf = mfem.ParGridFunction(), mfem.ParGridFunction() + + # // 12. Set the initial conditions for v_gf, x_gf and vx, and define the + # // boundary conditions on a beam-like mesh (see description above). + + sp_v_gf.MakeTRef(sp_XV_space, sp_vx, + sp_offset[0]) # Associate a new FiniteElementSpace and new true-dof data with the GridFunction. + sp_x_gf.MakeTRef(sp_XV_space, sp_vx, sp_offset[1]) + + # VectorFunctionCoefficient* velo = 0; + # VectorFunctionCoefficient* deform = 0; + + if (def_ic): + velo = InitialVelocityIC2(dim) + else: + velo = InitialVelocityIC1(dim) + + sp_v_gf.ProjectCoefficient(velo) + sp_v_gf.SetTrueVector() + + if (def_ic): + deform = InitialDeformationIC2(dim) + else: + deform = InitialDeformationIC1(dim) + + sp_x_gf.ProjectCoefficient(deform) + sp_x_gf.SetTrueVector() + + sp_v_gf.SetFromTrueVector() + sp_x_gf.SetFromTrueVector() + + # Get initial conditions + w_v0 = mfem.Vector(sp_v_gf.GetTrueVector()) + w_x0 = mfem.Vector(sp_x_gf.GetTrueVector()) + + // Convert essential boundary list from FOM mesh to sample mesh + // Create binary list where 1 means essential boundary element, 0 means nonessential. + CAROM::Matrix Ess_mat(true_size, 1, true); + for (size_t i = 0; i < true_size; i++) + { + Ess_mat(i,0) = 0; + for (size_t j = 0; j < ess_tdof_list.Size(); j++) + { + if (ess_tdof_list[j] == i ) + { + Ess_mat(i,0) = 1; + } + } + } + + // Project binary FOM list onto sampling space + MPI_Bcast(&sp_size, 1, MPI_INT, 0, MPI_COMM_WORLD); + CAROM::Matrix Ess_mat_sp(sp_size, 1, false); + smm->GatherDistributedMatrixRows("X", Ess_mat, 1, Ess_mat_sp); + + // Count number of true elements in new matrix + int num_ess_sp = 0; + + for (size_t i = 0; i < sp_size; i++) + { + if (Ess_mat_sp(i,0) == 1) + { + num_ess_sp += 1; + } + } + + // Initialize essential dof list in sampling space + Array ess_tdof_list_sp(num_ess_sp); + + // Add indices to list + int ctr = 0; + for (size_t i = 0; i < sp_size; i++) + { + if (Ess_mat_sp(i,0) == 1) + { + ess_tdof_list_sp[ctr] = i; + ctr += 1; + } + } + + if (myid == 0) + { + // Define operator in sample space + soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); + } + + if (hyperreduce) + { romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, + vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, + num_samples_req != -1, hyperreduce, x_base_only); + } + else + { + romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, + &(vx0.GetBlock(0)), + &(vx0.GetBlock(1)), vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, + myid, + num_samples_req != -1, hyperreduce, x_base_only); + } + + // Print lifted initial energies + BroadcastUndistributedRomVector(w); + + for (int i=0; iV_v.mult(*w_v, *v_rec_librom); + romop->V_x.mult(*w_x, *x_rec_librom); + + *v_rec += vx0.GetBlock(0); + *x_rec += vx0.GetBlock(1); + + v_gf.SetFromTrueDofs(*v_rec); + x_gf.SetFromTrueDofs(*x_rec); + + double ee = oper.ElasticEnergy(x_gf); + double ke = oper.KineticEnergy(v_gf); + + if (myid == 0) + { + cout << "Lifted initial energies, EE = " << ee + << ", KE = " << ke << ", ΔTE = " << (ee + ke) - (ee0 + ke0) << endl; + + } + + ode_solver.Init(romop) + else: + # fom + ode_solver.Init(oper) + + // 13. Perform time-integration + // (looping over the time iterations, ti, with a time-step dt). + // (taking samples and storing it into the pROM object) + + double t = 0.0; + vector ts; + oper.SetTime(t); + + bool last_step = false; + for (int ti = 1; !last_step; ti++) + { + double dt_real = min(dt, t_final - t); + + if (online) + { + if (myid == 0) + { + solveTimer.Start(); + ode_solver->Step(*wMFEM, t, dt_real); + solveTimer.Stop(); + } + + MPI_Bcast(&t, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); + } + else + { + solveTimer.Start(); + ode_solver->Step(vx, t, dt_real); + solveTimer.Stop(); + } + + last_step = (t >= t_final - 1e-8 * dt); + + if (offline) + { + + if (basis_generator_x->isNextSample(t) || x_base_only == false + && basis_generator_v->isNextSample(t)) + { + dvxdt = oper.dvxdt_sp.GetData(); + vx_diff = BlockVector(vx); + vx_diff -= vx0; + } + + // Take samples + if (x_base_only == false && basis_generator_v->isNextSample(t)) + { + basis_generator_v->takeSample(vx_diff.GetBlock(0), t, dt); + basis_generator_v->computeNextSampleTime(vx_diff.GetBlock(0), dvdt.GetData(), + t); + basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); + } + + if (basis_generator_x->isNextSample(t)) + { + basis_generator_x->takeSample(vx_diff.GetBlock(1), t, dt); + basis_generator_x->computeNextSampleTime(vx_diff.GetBlock(1), dxdt.GetData(), + t); + + if (x_base_only == true) + { + basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); + } + } + } + + if (last_step || (ti % vis_steps) == 0) + { + if (online) + { + BroadcastUndistributedRomVector(w); + + for (int i=0; iV_v.mult(*w_v, *v_rec_librom); + romop->V_x.mult(*w_x, *x_rec_librom); + + *v_rec += vx0.GetBlock(0); + *x_rec += vx0.GetBlock(1); + + v_gf.SetFromTrueDofs(*v_rec); + x_gf.SetFromTrueDofs(*x_rec); + + } + else + { + v_gf.SetFromTrueVector(); + x_gf.SetFromTrueVector(); + + } + + double ee = oper.ElasticEnergy(x_gf); + double ke = oper.KineticEnergy(v_gf); + + if (myid == 0) + { + cout << "step " << ti << ", t = " << t << ", EE = " << ee + << ", KE = " << ke << ", ΔTE = " << (ee + ke) - (ee0 + ke0) << endl; + } + + if (visualization) + { + visualize(vis_v, pmesh, &x_gf, &v_gf); + if (vis_w) + { + oper.GetElasticEnergyDensity(x_gf, w_gf); + visualize(vis_w, pmesh, &x_gf, &w_gf); + } + } + + if (visit) + { + GridFunction* nodes = &x_gf; + int owns_nodes = 0; + pmesh->SwapNodes(nodes, owns_nodes); + + dc->SetCycle(ti); + dc->SetTime(t); + dc->Save(); + } + } + + } // timestep loop + + if (myid == 0) cout << "Elapsed time for time integration loop " << + solveTimer.RealTime() << endl; + + ostringstream velo_name, pos_name; + + velo_name << "velocity_s"<< s << "." << setfill('0') << setw(6) << myid; + pos_name << "position_s"<< s << "." << setfill('0') << setw(6) << myid; + + if (offline) + { + // Sample final solution, to prevent extrapolation in ROM between the last sample and the end of the simulation. + dvxdt = oper.dvxdt_sp.GetData(); + vx_diff = BlockVector(vx); + vx_diff -= vx0; + + // Take samples + if (x_base_only == false) + { + basis_generator_v->takeSample(vx_diff.GetBlock(0), t, dt); + basis_generator_v->writeSnapshot(); + delete basis_generator_v; + } + + basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); + basis_generator_H->writeSnapshot(); + delete basis_generator_H; + + basis_generator_x->takeSample(vx_diff.GetBlock(1), t, dt); + basis_generator_x->writeSnapshot(); + delete basis_generator_x; + + // 14. Save the displaced mesh, the velocity and elastic energy. + GridFunction* nodes = &x_gf; + int owns_nodes = 0; + pmesh->SwapNodes(nodes, owns_nodes); + + ostringstream mesh_name, ee_name; + mesh_name << "deformed." << setfill('0') << setw(6) << myid; + ee_name << "elastic_energy." << setfill('0') << setw(6) << myid; + + ofstream mesh_ofs(mesh_name.str().c_str()); + mesh_ofs.precision(8); + pmesh->Print(mesh_ofs); + pmesh->SwapNodes(nodes, owns_nodes); + ofstream velo_ofs(velo_name.str().c_str()); + velo_ofs.precision(16); + + Vector v_final(vx.GetBlock(0)); + for (int i = 0; i < v_final.Size(); ++i) + { + velo_ofs << v_final[i] << std::endl; + } + + ofstream pos_ofs(pos_name.str().c_str()); + pos_ofs.precision(16); + + Vector x_final(vx.GetBlock(1)); + for (int i = 0; i < x_final.Size(); ++i) + { + pos_ofs << x_final[i] << std::endl; + } + + ofstream ee_ofs(ee_name.str().c_str()); + ee_ofs.precision(8); + oper.GetElasticEnergyDensity(x_gf, w_gf); + w_gf.Save(ee_ofs); + + } + + // 15. Calculate the relative error between the ROM final solution and the true solution. + if (online) + { + // Initialize FOM solution + Vector v_fom(v_rec->Size()); + Vector x_fom(x_rec->Size()); + + ifstream fom_v_file, fom_x_file; + + // Open and load file + fom_v_file.open(velo_name.str().c_str()); + fom_x_file.open(pos_name.str().c_str()); + + v_fom.Load(fom_v_file, v_rec->Size()); + x_fom.Load(fom_x_file, x_rec->Size()); + + fom_v_file.close(); + fom_x_file.close(); + + // Get difference vector + Vector diff_v(v_rec->Size()); + Vector diff_x(x_rec->Size()); + + subtract(*v_rec, v_fom, diff_v); + subtract(*x_rec, x_fom, diff_x); + + // Get norms + double tot_diff_norm_v = sqrt(InnerProduct(MPI_COMM_WORLD, diff_v, diff_v)); + double tot_diff_norm_x = sqrt(InnerProduct(MPI_COMM_WORLD, diff_x, diff_x)); + + double tot_v_fom_norm = sqrt(InnerProduct(MPI_COMM_WORLD, + v_fom, v_fom)); + double tot_x_fom_norm = sqrt(InnerProduct(MPI_COMM_WORLD, + x_fom, x_fom)); + + if (myid == 0) + { + cout << "Relative error of ROM position (x) at t_final: " << t_final << + " is " << tot_diff_norm_x / tot_x_fom_norm << endl; + cout << "Relative error of ROM velocity (v) at t_final: " << t_final << + " is " << tot_diff_norm_v / tot_v_fom_norm << endl; + } + } + + // 16. Free the used memory. + delete ode_solver; + delete pmesh; + + totalTimer.Stop(); + if (myid == 0) cout << "Elapsed time for entire simulation " << + totalTimer.RealTime() << endl; + + MPI_Finalize(); + return 0; + +if __name__ == "__main__": + run() \ No newline at end of file From d4be45901262fa4f2e1f35696f689905da361527 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 18 Sep 2023 16:59:20 -0700 Subject: [PATCH 2/9] initial translation. not tested yet. --- .../prom/nonlinear_elasticity_global_rom.py | 608 ++++++++---------- 1 file changed, 263 insertions(+), 345 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.py b/examples/prom/nonlinear_elasticity_global_rom.py index 60c6bf6..be0eb1a 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.py +++ b/examples/prom/nonlinear_elasticity_global_rom.py @@ -99,7 +99,7 @@ raise ModuleNotFoundError(msg) from ctypes import c_double -from mfem.par import intArray, add_vector +from mfem.par import intArray, add_vector, subtract_vector from os.path import expanduser, join, dirname import numpy as np from numpy import sin, cos, exp, sqrt, pi, abs, array, floor, log, sum @@ -110,7 +110,7 @@ import pylibROM.hyperreduction as hyper import pylibROM.mfem as mfem_support from pylibROM.mfem import ComputeCtAB -from pylibROM.utils import StopWatch +from pylibROM.python_utils import StopWatch class ReducedSystemOperator(mfem.PyOperator): def __init__(self, M, S, H, ess_tdof_list): @@ -1194,355 +1194,273 @@ def run(): w_v0 = mfem.Vector(sp_v_gf.GetTrueVector()) w_x0 = mfem.Vector(sp_x_gf.GetTrueVector()) - // Convert essential boundary list from FOM mesh to sample mesh - // Create binary list where 1 means essential boundary element, 0 means nonessential. - CAROM::Matrix Ess_mat(true_size, 1, true); - for (size_t i = 0; i < true_size; i++) - { - Ess_mat(i,0) = 0; - for (size_t j = 0; j < ess_tdof_list.Size(); j++) - { - if (ess_tdof_list[j] == i ) - { - Ess_mat(i,0) = 1; - } - } - } - - // Project binary FOM list onto sampling space - MPI_Bcast(&sp_size, 1, MPI_INT, 0, MPI_COMM_WORLD); - CAROM::Matrix Ess_mat_sp(sp_size, 1, false); - smm->GatherDistributedMatrixRows("X", Ess_mat, 1, Ess_mat_sp); - - // Count number of true elements in new matrix - int num_ess_sp = 0; - - for (size_t i = 0; i < sp_size; i++) - { - if (Ess_mat_sp(i,0) == 1) - { - num_ess_sp += 1; - } - } - - // Initialize essential dof list in sampling space - Array ess_tdof_list_sp(num_ess_sp); - - // Add indices to list - int ctr = 0; - for (size_t i = 0; i < sp_size; i++) - { - if (Ess_mat_sp(i,0) == 1) - { - ess_tdof_list_sp[ctr] = i; - ctr += 1; - } - } - - if (myid == 0) - { - // Define operator in sample space - soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); - } - - if (hyperreduce) - { romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, - vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, - num_samples_req != -1, hyperreduce, x_base_only); - } - else - { - romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, - &(vx0.GetBlock(0)), - &(vx0.GetBlock(1)), vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, - myid, - num_samples_req != -1, hyperreduce, x_base_only); - } - - // Print lifted initial energies - BroadcastUndistributedRomVector(w); - - for (int i=0; iV_v.mult(*w_v, *v_rec_librom); - romop->V_x.mult(*w_x, *x_rec_librom); - - *v_rec += vx0.GetBlock(0); - *x_rec += vx0.GetBlock(1); - - v_gf.SetFromTrueDofs(*v_rec); - x_gf.SetFromTrueDofs(*x_rec); - - double ee = oper.ElasticEnergy(x_gf); - double ke = oper.KineticEnergy(v_gf); - - if (myid == 0) - { - cout << "Lifted initial energies, EE = " << ee - << ", KE = " << ke << ", ΔTE = " << (ee + ke) - (ee0 + ke0) << endl; - - } + # Convert essential boundary list from FOM mesh to sample mesh + # Create binary list where 1 means essential boundary element, 0 means nonessential. + Ess_mat = linalg.Matrix(true_size, 1, True) + for i in range(true_size): + Ess_mat[i,0] = 0. + for j in range(ess_tdof_list.Size()): + if (ess_tdof_list[j] == i ): + Ess_mat[i,0] = 1. + + # Project binary FOM list onto sampling space + MPI.COMM_WORLD.Bcast([sp_size, MPI.INT], root=0) + Ess_mat_sp = linalg.Matrix(sp_size, 1, False) + smm.GatherDistributedMatrixRows("X", Ess_mat, 1, Ess_mat_sp) + + # Count number of true elements in new matrix + num_ess_sp = 0 + + for i in range(sp_size): + if (Ess_mat_sp[i,0] == 1): + num_ess_sp += 1 + + # Initialize essential dof list in sampling space + ess_tdof_list_sp = mfem.intArray(num_ess_sp) + + # Add indices to list + ctr = 0 + for i in range(sp_size): + if (Ess_mat_sp[i,0] == 1): + ess_tdof_list_sp[ctr] = i + ctr += 1 + + if (myid == 0): + # Define operator in sample space + soper = HyperelasticOperator(sp_XV_space, ess_tdof_list_sp, visc, mu, K) + + if (hyperreduce): + romop = RomOperator(oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, + vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, + (num_samples_req != -1), hyperreduce, x_base_only) + else: + romop = RomOperator(oper, soper, rvdim, rxdim, hdim, smm, + vx0.GetBlock(0), vx0.GetBlock(1), vx0.GetBlock(0), + BV_librom, BX_librom, H_librom, Hsinv, myid, + (num_samples_req != -1), hyperreduce, x_base_only) + + # Print lifted initial energies + BroadcastUndistributedRomVector(w) + + for i in range(rvdim): + w_v[i] = w[i] + + for i in range(rxdim): + w_x[i] = w[rvdim + i] + + romop.V_v.mult(w_v, v_rec_librom) + romop.V_x.mult(w_x, x_rec_librom) + + v_rec += vx0.GetBlock(0) + x_rec += vx0.GetBlock(1) + + v_gf.SetFromTrueDofs(v_rec) + x_gf.SetFromTrueDofs(x_rec) + + ee = oper.ElasticEnergy(x_gf) + ke = oper.KineticEnergy(v_gf) + + if (myid == 0): + print("Lifted initial energies, EE = %.e, KE = %.e, ΔTE = %.e" % (ee, ke, (ee + ke) - (ee0 + ke0))) ode_solver.Init(romop) else: # fom ode_solver.Init(oper) - // 13. Perform time-integration - // (looping over the time iterations, ti, with a time-step dt). - // (taking samples and storing it into the pROM object) - - double t = 0.0; - vector ts; - oper.SetTime(t); - - bool last_step = false; - for (int ti = 1; !last_step; ti++) - { - double dt_real = min(dt, t_final - t); - - if (online) - { - if (myid == 0) - { - solveTimer.Start(); - ode_solver->Step(*wMFEM, t, dt_real); - solveTimer.Stop(); - } - - MPI_Bcast(&t, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); - } - else - { - solveTimer.Start(); - ode_solver->Step(vx, t, dt_real); - solveTimer.Stop(); - } - - last_step = (t >= t_final - 1e-8 * dt); - - if (offline) - { - - if (basis_generator_x->isNextSample(t) || x_base_only == false - && basis_generator_v->isNextSample(t)) - { - dvxdt = oper.dvxdt_sp.GetData(); - vx_diff = BlockVector(vx); - vx_diff -= vx0; - } - - // Take samples - if (x_base_only == false && basis_generator_v->isNextSample(t)) - { - basis_generator_v->takeSample(vx_diff.GetBlock(0), t, dt); - basis_generator_v->computeNextSampleTime(vx_diff.GetBlock(0), dvdt.GetData(), - t); - basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); - } - - if (basis_generator_x->isNextSample(t)) - { - basis_generator_x->takeSample(vx_diff.GetBlock(1), t, dt); - basis_generator_x->computeNextSampleTime(vx_diff.GetBlock(1), dxdt.GetData(), - t); - - if (x_base_only == true) - { - basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); - } - } - } - - if (last_step || (ti % vis_steps) == 0) - { - if (online) - { - BroadcastUndistributedRomVector(w); - - for (int i=0; iV_v.mult(*w_v, *v_rec_librom); - romop->V_x.mult(*w_x, *x_rec_librom); - - *v_rec += vx0.GetBlock(0); - *x_rec += vx0.GetBlock(1); - - v_gf.SetFromTrueDofs(*v_rec); - x_gf.SetFromTrueDofs(*x_rec); - - } - else - { - v_gf.SetFromTrueVector(); - x_gf.SetFromTrueVector(); - - } - - double ee = oper.ElasticEnergy(x_gf); - double ke = oper.KineticEnergy(v_gf); - - if (myid == 0) - { - cout << "step " << ti << ", t = " << t << ", EE = " << ee - << ", KE = " << ke << ", ΔTE = " << (ee + ke) - (ee0 + ke0) << endl; - } - - if (visualization) - { - visualize(vis_v, pmesh, &x_gf, &v_gf); - if (vis_w) - { - oper.GetElasticEnergyDensity(x_gf, w_gf); - visualize(vis_w, pmesh, &x_gf, &w_gf); - } - } - - if (visit) - { - GridFunction* nodes = &x_gf; - int owns_nodes = 0; - pmesh->SwapNodes(nodes, owns_nodes); - - dc->SetCycle(ti); - dc->SetTime(t); - dc->Save(); - } - } - - } // timestep loop - - if (myid == 0) cout << "Elapsed time for time integration loop " << - solveTimer.RealTime() << endl; - - ostringstream velo_name, pos_name; - - velo_name << "velocity_s"<< s << "." << setfill('0') << setw(6) << myid; - pos_name << "position_s"<< s << "." << setfill('0') << setw(6) << myid; - - if (offline) - { - // Sample final solution, to prevent extrapolation in ROM between the last sample and the end of the simulation. - dvxdt = oper.dvxdt_sp.GetData(); - vx_diff = BlockVector(vx); - vx_diff -= vx0; - - // Take samples - if (x_base_only == false) - { - basis_generator_v->takeSample(vx_diff.GetBlock(0), t, dt); - basis_generator_v->writeSnapshot(); - delete basis_generator_v; - } - - basis_generator_H->takeSample(oper.H_sp.GetData(), t, dt); - basis_generator_H->writeSnapshot(); - delete basis_generator_H; - - basis_generator_x->takeSample(vx_diff.GetBlock(1), t, dt); - basis_generator_x->writeSnapshot(); - delete basis_generator_x; - - // 14. Save the displaced mesh, the velocity and elastic energy. - GridFunction* nodes = &x_gf; - int owns_nodes = 0; - pmesh->SwapNodes(nodes, owns_nodes); - - ostringstream mesh_name, ee_name; - mesh_name << "deformed." << setfill('0') << setw(6) << myid; - ee_name << "elastic_energy." << setfill('0') << setw(6) << myid; - - ofstream mesh_ofs(mesh_name.str().c_str()); - mesh_ofs.precision(8); - pmesh->Print(mesh_ofs); - pmesh->SwapNodes(nodes, owns_nodes); - ofstream velo_ofs(velo_name.str().c_str()); - velo_ofs.precision(16); - - Vector v_final(vx.GetBlock(0)); - for (int i = 0; i < v_final.Size(); ++i) - { - velo_ofs << v_final[i] << std::endl; - } - - ofstream pos_ofs(pos_name.str().c_str()); - pos_ofs.precision(16); - - Vector x_final(vx.GetBlock(1)); - for (int i = 0; i < x_final.Size(); ++i) - { - pos_ofs << x_final[i] << std::endl; - } - - ofstream ee_ofs(ee_name.str().c_str()); - ee_ofs.precision(8); - oper.GetElasticEnergyDensity(x_gf, w_gf); - w_gf.Save(ee_ofs); - - } - - // 15. Calculate the relative error between the ROM final solution and the true solution. - if (online) - { - // Initialize FOM solution - Vector v_fom(v_rec->Size()); - Vector x_fom(x_rec->Size()); - - ifstream fom_v_file, fom_x_file; - - // Open and load file - fom_v_file.open(velo_name.str().c_str()); - fom_x_file.open(pos_name.str().c_str()); - - v_fom.Load(fom_v_file, v_rec->Size()); - x_fom.Load(fom_x_file, x_rec->Size()); - - fom_v_file.close(); - fom_x_file.close(); - - // Get difference vector - Vector diff_v(v_rec->Size()); - Vector diff_x(x_rec->Size()); - - subtract(*v_rec, v_fom, diff_v); - subtract(*x_rec, x_fom, diff_x); - - // Get norms - double tot_diff_norm_v = sqrt(InnerProduct(MPI_COMM_WORLD, diff_v, diff_v)); - double tot_diff_norm_x = sqrt(InnerProduct(MPI_COMM_WORLD, diff_x, diff_x)); - - double tot_v_fom_norm = sqrt(InnerProduct(MPI_COMM_WORLD, - v_fom, v_fom)); - double tot_x_fom_norm = sqrt(InnerProduct(MPI_COMM_WORLD, - x_fom, x_fom)); - - if (myid == 0) - { - cout << "Relative error of ROM position (x) at t_final: " << t_final << - " is " << tot_diff_norm_x / tot_x_fom_norm << endl; - cout << "Relative error of ROM velocity (v) at t_final: " << t_final << - " is " << tot_diff_norm_v / tot_v_fom_norm << endl; - } - } - - // 16. Free the used memory. - delete ode_solver; - delete pmesh; - - totalTimer.Stop(); - if (myid == 0) cout << "Elapsed time for entire simulation " << - totalTimer.RealTime() << endl; - - MPI_Finalize(); - return 0; + # 13. Perform time-integration + # (looping over the time iterations, ti, with a time-step dt). + # (taking samples and storing it into the pROM object) + + t = 0.0 + ts = [] + oper.SetTime(t) + + last_step = False + ti = 1 + while (not last_step): + dt_real = min(dt, t_final - t) + + if (online): + if (myid == 0): + solveTimer.Start() + ode_solver.Step(wMFEM, t, dt_real) + solveTimer.Stop() + + MPI.COMM_WORLD.Bcast([t, MPI.DOUBLE], root=0) + else: + solveTimer.Start() + ode_solver.Step(vx, t, dt_real) + solveTimer.Stop() + + last_step = (t >= t_final - 1e-8 * dt) + + if (offline): + + if (basis_generator_x.isNextSample(t) or (x_base_only == False) + and basis_generator_v.isNextSample(t)): + dvxdt = oper.dvxdt_sp.GetData() + vx_diff = mfem.BlockVector(vx) + vx_diff -= vx0 + + # Take samples + if ((x_base_only == False) and basis_generator_v.isNextSample(t)): + basis_generator_v.takeSample(vx_diff.GetBlock(0).GetDataArray(), t, dt) + basis_generator_v.computeNextSampleTime(vx_diff.GetBlock(0), dvdt.GetData(), t) + basis_generator_H.takeSample(oper.H_sp.GetDataArray(), t, dt) + + if (basis_generator_x.isNextSample(t)): + basis_generator_x.takeSample(vx_diff.GetBlock(1).GetDataArray(), t, dt) + basis_generator_x.computeNextSampleTime(vx_diff.GetBlock(1).GetDataArray(), dxdt.GetDataArray(), t) + + if (x_base_only): + basis_generator_H.takeSample(oper.H_sp.GetDataArray(), t, dt) + + if (last_step or ((ti % vis_steps) == 0)): + if (online): + BroadcastUndistributedRomVector(w) + + for i in range(rvdim): + w_v[i] = w[i] + + for i in range(rxdim): + w_x[i] = w[rvdim + i] + + romop.V_v.mult(w_v, v_rec_librom) + romop.V_x.mult(w_x, x_rec_librom) + + v_rec += vx0.GetBlock(0) + x_rec += vx0.GetBlock(1) + + v_gf.SetFromTrueDofs(v_rec) + x_gf.SetFromTrueDofs(x_rec) + + else: + v_gf.SetFromTrueVector() + x_gf.SetFromTrueVector() + + ee = oper.ElasticEnergy(x_gf) + ke = oper.KineticEnergy(v_gf) + + if (myid == 0): + print("step %d, t = %f, EE = %.e, KE = %.e, ΔTE = %.e" % (ti, t, ee, ke, (ee + ke) - (ee0 + ke0))) + + if (visualization): + visualize(vis_v, pmesh, x_gf, v_gf) + if (vis_w): + oper.GetElasticEnergyDensity(x_gf, w_gf) + visualize(vis_w, pmesh, x_gf, w_gf) + + if (visit): + nodes = x_gf + owns_nodes = 0 + pmesh.SwapNodes(nodes, owns_nodes) + + dc.SetCycle(ti) + dc.SetTime(t) + dc.Save() + + ti += 1 + # timestep loop + + if (myid == 0): + print("Elapsed time for time integration loop %.e" % solveTimer.duration) + + velo_name = "velocity_s%f.%06d" % (s, myid) + pos_name = "position_s%f.%06d" % (s, myid) + + if (offline): + # Sample final solution, to prevent extrapolation in ROM between the last sample and the end of the simulation. + dvxdt = oper.dvxdt_sp.GetData() + vx_diff = mfem.BlockVector(vx) + vx_diff -= vx0 + + # Take samples + if (not x_base_only): + basis_generator_v.takeSample(vx_diff.GetBlock(0).GetDataArray(), t, dt) + basis_generator_v.writeSnapshot() + del basis_generator_v + + basis_generator_H.takeSample(oper.H_sp.GetDataArray(), t, dt) + basis_generator_H.writeSnapshot() + del basis_generator_H + + basis_generator_x.takeSample(vx_diff.GetBlock(1).GetDataArray(), t, dt) + basis_generator_x.writeSnapshot() + del basis_generator_x + + # 14. Save the displaced mesh, the velocity and elastic energy. + nodes = x_gf + owns_nodes = 0 + pmesh.SwapNodes(nodes, owns_nodes) + + mesh_name = "deformed.%06d" % myid + ee_name = "elastic_energy.%06d" % myid + + pmesh.Print(mesh_name, 8) + pmesh.SwapNodes(nodes, owns_nodes) + + with open(velo_name, 'w') as fid: + velo_ofs = io.StringIO() + velo_ofs.precision = 16 + v_final = mfem.Vector(vx.GetBlock(0)) + v_final.Save(velo_ofs) + fid.write(velo_ofs.getvalue()) + + with open(pos_name, 'w') as fid: + pos_ofs = io.StringIO() + pos_ofs.precision = 16 + x_final = mfem.Vector(vx.GetBlock(1)) + x_final.Save(pos_ofs) + fid.write(pos_ofs.getvalue()) + + with open(ee_name, 'w') as fid: + ee_ofs = io.StringIO() + ee_ofs.precision = 8 + oper.GetElasticEnergyDensity(x_gf, w_gf) + w_gf.Save(ee_ofs) + + # 15. Calculate the relative error between the ROM final solution and the true solution. + if (online): + # Initialize FOM solution + v_fom = mfem.Vector(v_rec.Size()) + x_fom = mfem.Vector(x_rec.Size()) + + # Open and load file + with open(velo_name, 'r') as fom_v_file: + v_fom.Load(fom_v_file, v_rec.Size()) + + with open(pos_name, 'r') as fom_x_file: + x_fom.Load(fom_x_file, x_rec.Size()) + + # Get difference vector + diff_v = mfem.Vector(v_rec.Size()) + diff_x = mfem.Vector(x_rec.Size()) + + subtract_vector(v_rec, v_fom, diff_v) + subtract_vector(x_rec, x_fom, diff_x) + + # Get norms + tot_diff_norm_v = sqrt(mfem.InnerProduct(MPI.COMM_WORLD, diff_v, diff_v)) + tot_diff_norm_x = sqrt(mfem.InnerProduct(MPI.COMM_WORLD, diff_x, diff_x)) + + tot_v_fom_norm = sqrt(mfem.InnerProduct(MPI.COMM_WORLD, v_fom, v_fom)) + tot_x_fom_norm = sqrt(mfem.InnerProduct(MPI.COMM_WORLD, x_fom, x_fom)) + + if (myid == 0): + print("Relative error of ROM position (x) at t_final: %f is %.8e" % (t_final, tot_diff_norm_x / tot_x_fom_norm)) + print("Relative error of ROM velocity (v) at t_final: %f is %.8e" % (t_final, tot_diff_norm_v / tot_v_fom_norm)) + + # 16. Free the used memory. + del ode_solver + del pmesh + + totalTimer.Stop() + if (myid == 0): + print("Elapsed time for entire simulation %.e" % totalTimer.duration) + + MPI.Finalize() + return if __name__ == "__main__": run() \ No newline at end of file From c3c13846391881b83d7b02af958350980edc82bf Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 18 Sep 2023 17:14:53 -0700 Subject: [PATCH 3/9] some minor fix. still needs debugging. --- .../prom/nonlinear_elasticity_global_rom.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.py b/examples/prom/nonlinear_elasticity_global_rom.py index be0eb1a..74906f7 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.py +++ b/examples/prom/nonlinear_elasticity_global_rom.py @@ -156,7 +156,10 @@ def __init__(self, fespace, ess_bdr, visc, mu, K): ref_density = 1.0 self.ess_tdof_list = intArray() - self.z = mfem.Vector(self.Height()//2) + self.z = mfem.Vector(self.Height() // 2) + self.z2 = mfem.Vector(self.Height() // 2) + self.H_sp = mfem.Vector(self.Height() // 2) + self.dvxdt_sp = mfem.Vector(self.Height() // 2) self.fespace = fespace self.viscosity = visc self.newton_solver = mfem.NewtonSolver(fespace.GetComm()) @@ -229,19 +232,19 @@ def __init__(self, fespace, ess_bdr, visc, mu, K): newton_solver.SetMaxIter(10) self.newton_solver = newton_solver - def Mult(self, vx, vx_dt): - sc = self.Height()//2 + def Mult(self, vx, dvx_dt): + sc = self.Height() // 2 v = mfem.Vector(vx, 0, sc) x = mfem.Vector(vx, sc, sc) dv_dt = mfem.Vector(dvx_dt, 0, sc) dx_dt = mfem.Vector(dvx_dt, sc, sc) - self.H.Mult(x, z) + self.H.Mult(x, self.z) if (self.viscosity != 0.0): - S.TrueAddMult(v, z) - z.SetSubVector(self.ess_tdof_list, 0.0) - z.Neg() - self.M_solver.Mult(z, dv_dt) + self.S.TrueAddMult(v, self.z) + self.z.SetSubVector(self.ess_tdof_list, 0.0) + self.z.Neg() + self.M_solver.Mult(self.z, dv_dt) dx_dt = v def ImplicitSolve(self, dt, vx, dvx_dt): @@ -267,6 +270,7 @@ def ElasticEnergy(self, x): return self.H.GetEnergy(x) def KineticEnergy(self, v): + from mpi4py import MPI local_energy = 0.5*self.M.InnerProduct(v, v) energy = MPI.COMM_WORLD.allreduce(local_energy, op=MPI.SUM) return energy @@ -1284,13 +1288,13 @@ def run(): if (online): if (myid == 0): solveTimer.Start() - ode_solver.Step(wMFEM, t, dt_real) + t, dt = ode_solver.Step(wMFEM, t, dt_real) solveTimer.Stop() MPI.COMM_WORLD.Bcast([t, MPI.DOUBLE], root=0) else: solveTimer.Start() - ode_solver.Step(vx, t, dt_real) + t, dt = ode_solver.Step(vx, t, dt_real) solveTimer.Stop() last_step = (t >= t_final - 1e-8 * dt) @@ -1306,7 +1310,7 @@ def run(): # Take samples if ((x_base_only == False) and basis_generator_v.isNextSample(t)): basis_generator_v.takeSample(vx_diff.GetBlock(0).GetDataArray(), t, dt) - basis_generator_v.computeNextSampleTime(vx_diff.GetBlock(0), dvdt.GetData(), t) + basis_generator_v.computeNextSampleTime(vx_diff.GetBlock(0).GetDataArray(), dvdt.GetDataArray(), t) basis_generator_H.takeSample(oper.H_sp.GetDataArray(), t, dt) if (basis_generator_x.isNextSample(t)): From f4ff23b77a0a27670fc8f0fffa98847d649f833c Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 19 Sep 2023 17:13:47 -0700 Subject: [PATCH 4/9] mfem binding fixes with PyMFEM interface. --- CMakeLists.txt | 1 - bindings/pylibROM/mfem/pySampleMesh.cpp | 9 ++++++--- bindings/pylibROM/python_utils/cpp_utils.cpp | 11 ----------- bindings/pylibROM/python_utils/cpp_utils.hpp | 3 --- 4 files changed, 6 insertions(+), 18 deletions(-) delete mode 100644 bindings/pylibROM/python_utils/cpp_utils.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 670ec96..5653a51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,7 +163,6 @@ pybind11_add_module(_pylibROM bindings/pylibROM/mfem/pySampleMesh.cpp bindings/pylibROM/python_utils/cpp_utils.hpp - bindings/pylibROM/python_utils/cpp_utils.cpp ) message("building pylibROM...") diff --git a/bindings/pylibROM/mfem/pySampleMesh.cpp b/bindings/pylibROM/mfem/pySampleMesh.cpp index fa3644c..137340d 100644 --- a/bindings/pylibROM/mfem/pySampleMesh.cpp +++ b/bindings/pylibROM/mfem/pySampleMesh.cpp @@ -47,7 +47,7 @@ void init_mfem_SampleMesh(pybind11::module_ &m) { ParFiniteElementSpace *spfes_target = self.GetSampleFESpace(space); // deep copy of the spfes_target. - void *spfes_address = extractSwigPtrAddress(spfespace); + void *spfes_address = extractSwigPtr(spfespace); ParFiniteElementSpace *spfes = new (spfes_address) ParFiniteElementSpace(*spfes_target); }) @@ -55,13 +55,16 @@ void init_mfem_SampleMesh(pybind11::module_ &m) { ParMesh *sppmesh_target = self.GetSampleMesh(); // deep copy of the spfes_target. - void *sppmesh_address = extractSwigPtrAddress(sppmesh); + void *sppmesh_address = extractSwigPtr(sppmesh); ParMesh *sample_pmesh = new (sppmesh_address) ParMesh(*sppmesh_target); }) .def("GetNumVarSamples", &CAROM::SampleMeshManager::GetNumVarSamples) - .def("GetSampledValues", &CAROM::SampleMeshManager::GetSampledValues) + .def("GetSampledValues", [](CAROM::SampleMeshManager &self, const std::string variable, py::object &v, CAROM::Vector &s){ + Vector *v_ptr = extractSwigPtr(v); + self.GetSampledValues(variable, *v_ptr, s); + }) .def("GetSampleElements", &CAROM::SampleMeshManager::GetSampleElements) diff --git a/bindings/pylibROM/python_utils/cpp_utils.cpp b/bindings/pylibROM/python_utils/cpp_utils.cpp deleted file mode 100644 index c52f380..0000000 --- a/bindings/pylibROM/python_utils/cpp_utils.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "python_utils/cpp_utils.hpp" - -namespace py = pybind11; - -void * -extractSwigPtrAddress(const py::handle &swig_target) -{ - // On python, swig_target.this.__int__() returns the memory address of the wrapped c++ object pointer. - // We execute the same command here in the c++ side, where the memory address is given as py::object. - return swig_target.attr("this").attr("__int__")().cast(); -} diff --git a/bindings/pylibROM/python_utils/cpp_utils.hpp b/bindings/pylibROM/python_utils/cpp_utils.hpp index de876a0..8e9ae70 100644 --- a/bindings/pylibROM/python_utils/cpp_utils.hpp +++ b/bindings/pylibROM/python_utils/cpp_utils.hpp @@ -88,9 +88,6 @@ get1DArrayFromPtr(T *ptr, const int nelem, bool free_when_done=false) get1DArrayBufferHandle(ptr, free_when_done)); } -void * -extractSwigPtrAddress(const py::handle &swig_target); - template T* extractSwigPtr(const py::handle &swig_target) From 4ec303488b4ad27e8feff6efb2004a57c9bbe457 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 19 Sep 2023 17:41:13 -0700 Subject: [PATCH 5/9] GNAT example demonstrated. --- .../prom/nonlinear_elasticity_global_rom.py | 156 ++++-------------- 1 file changed, 33 insertions(+), 123 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.py b/examples/prom/nonlinear_elasticity_global_rom.py index 74906f7..ab9a746 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.py +++ b/examples/prom/nonlinear_elasticity_global_rom.py @@ -112,57 +112,21 @@ from pylibROM.mfem import ComputeCtAB from pylibROM.python_utils import StopWatch -class ReducedSystemOperator(mfem.PyOperator): - def __init__(self, M, S, H, ess_tdof_list): - mfem.PyOperator.__init__(self, M.ParFESpace().TrueVSize()) - self.M = M - self.S = S - self.H = H - self.Jacobian = None - h = M.ParFESpace().TrueVSize() - self.w = mfem.Vector(h) - self.z = mfem.Vector(h) - self.dt = 0.0 - self.ess_tdof_list = ess_tdof_list - - def SetParameters(self, dt, v, x): - self.dt = dt - self.v = v - self.x = x - - def Mult(self, k, y): - add_vector(self.v, self.dt, k, self.w) - add_vector(self.x, self.dt, self.w, self.z) - self.H.Mult(self.z, y) - self.M.TrueAddMult(k, y) - self.S.TrueAddMult(self.w, y) - y.SetSubVector(self.ess_tdof_list, 0.0) - - def GetGradient(self, k): - localJ = mfem.Add(1.0, self.M.SpMat(), self.dt, self.S.SpMat()) - add_vector(self.v, self.dt, k, self.w) - add_vector(self.x, self.dt, self.w, self.z) - localJ.Add(self.dt * self.dt, self.H.GetLocalGradient(self.z)) - Jacobian = self.M.ParallelAssemble(localJ) - Jacobian.EliminateRowsCols(self.ess_tdof_list) - return Jacobian - class HyperelasticOperator(mfem.PyTimeDependentOperator): - def __init__(self, fespace, ess_bdr, visc, mu, K): + def __init__(self, fespace, ess_tdof_list_, visc, mu, K): mfem.PyTimeDependentOperator.__init__(self, 2*fespace.TrueVSize(), 0.0) rel_tol = 1e-8 skip_zero_entries = 0 ref_density = 1.0 - self.ess_tdof_list = intArray() + self.ess_tdof_list = ess_tdof_list_ self.z = mfem.Vector(self.Height() // 2) self.z2 = mfem.Vector(self.Height() // 2) self.H_sp = mfem.Vector(self.Height() // 2) self.dvxdt_sp = mfem.Vector(self.Height() // 2) self.fespace = fespace self.viscosity = visc - self.newton_solver = mfem.NewtonSolver(fespace.GetComm()) M = mfem.ParBilinearForm(fespace) S = mfem.ParBilinearForm(fespace) @@ -176,8 +140,6 @@ def __init__(self, fespace, ess_bdr, visc, mu, K): M.Assemble(skip_zero_entries) M.Finalize(skip_zero_entries) self.Mmat = M.ParallelAssemble() - - fespace.GetEssentialTrueDofs(ess_bdr, self.ess_tdof_list) self.Mmat.EliminateRowsCols(self.ess_tdof_list) M_solver = mfem.CGSolver(fespace.GetComm()) @@ -203,34 +165,8 @@ def __init__(self, fespace, ess_bdr, visc, mu, K): S.AddDomainIntegrator(mfem.VectorDiffusionIntegrator(visc_coeff)) S.Assemble(skip_zero_entries) S.Finalize(skip_zero_entries) - - self.reduced_oper = ReducedSystemOperator(M, S, H, self.ess_tdof_list) - - J_hypreSmoother = mfem.HypreSmoother() - J_hypreSmoother.SetType(mfem.HypreSmoother.l1Jacobi) - J_hypreSmoother.SetPositiveDiagonal(True) - J_prec = J_hypreSmoother - - J_minres = mfem.MINRESSolver(fespace.GetComm()) - J_minres.SetRelTol(rel_tol) - J_minres.SetAbsTol(0.0) - J_minres.SetMaxIter(300) - J_minres.SetPrintLevel(-1) - J_minres.SetPreconditioner(J_prec) - - self.J_solver = J_minres - self.J_prec = J_prec - - newton_solver = mfem.NewtonSolver(fespace.GetComm()) - newton_solver.iterative_mode = False - newton_solver.SetSolver(self.J_solver) - newton_solver.SetOperator(self.reduced_oper) - newton_solver.SetPrintLevel(1) # print Newton iterations - newton_solver.SetRelTol(rel_tol) - newton_solver.SetAbsTol(0.0) - newton_solver.SetAdaptiveLinRtol(2, 0.5, 0.9) - newton_solver.SetMaxIter(10) - self.newton_solver = newton_solver + self.Smat = mfem.HypreParMatrix() + S.FormSystemMatrix(self.ess_tdof_list, self.Smat) def Mult(self, vx, dvx_dt): sc = self.Height() // 2 @@ -240,31 +176,17 @@ def Mult(self, vx, dvx_dt): dx_dt = mfem.Vector(dvx_dt, sc, sc) self.H.Mult(x, self.z) + self.H_sp.Assign(self.z) + if (self.viscosity != 0.0): - self.S.TrueAddMult(v, self.z) - self.z.SetSubVector(self.ess_tdof_list, 0.0) + self.Smat.Mult(v, self.z2) + self.z += self.z2 + self.z.Neg() self.M_solver.Mult(self.z, dv_dt) - dx_dt = v - def ImplicitSolve(self, dt, vx, dvx_dt): - sc = self.Height()//2 - v = mfem.Vector(vx, 0, sc) - x = mfem.Vector(vx, sc, sc) - dv_dt = mfem.Vector(dvx_dt, 0, sc) - dx_dt = mfem.Vector(dvx_dt, sc, sc) - - # By eliminating kx from the coupled system: - # kv = -M^{-1}*[H(x + dt*kx) + S*(v + dt*kv)] - # kx = v + dt*kv - # we reduce it to a nonlinear equation for kv, represented by the - # backward_euler_oper. This equation is solved with the newton_solver - # object (using J_solver and J_prec internally). - self.reduced_oper.SetParameters(dt, v, x) - zero = mfem.Vector() # empty vector is interpreted as - # zero r.h.s. by NewtonSolver - self.newton_solver.Mult(zero, dv_dt) - add_vector(v, dt, dv_dt, dx_dt) + dx_dt.Assign(v) # this changes dvx_dt + self.dvxdt_sp.Assign(dvx_dt) def ElasticEnergy(self, x): return self.H.GetEnergy(x) @@ -304,9 +226,9 @@ def __init__(self, fom_, fomSp_, rvdim_, rxdim_, hdim_, smm_, v0_, self.zX = linalg.Vector(max(self.nsamp_H, 1), False) self.oversampling = oversampling_ self.M_hat_solver = mfem.CGSolver(fom_.fespace.GetComm()) - self.z = mfem.Vector(self.height // 2) + self.z = mfem.Vector(self.Height() // 2) self.hyperreduce = hyperreduce_ - self.x_base_only_ = x_base_only_ + self.x_base_only = x_base_only_ if (myid == 0): self.V_v_sp = linalg.Matrix(self.fomSp.Height() // 2, self.rvdim, False) @@ -322,7 +244,7 @@ def __init__(self, fom_, fomSp_, rvdim_, rxdim_, hdim_, smm_, v0_, self.smm.GatherDistributedMatrixRows("X", self.V_x, self.rxdim, self.V_x_sp) # Create V_vTU_H, for hyperreduction - self.V_v.transposeMult(self.U_H, self.V_vTU_H) + self.V_vTU_H = self.V_v.transposeMult(self.U_H) self.S_hat = linalg.Matrix(self.rvdim, self.rvdim, False) self.S_hat_v0 = linalg.Vector(self.rvdim, False) @@ -347,12 +269,12 @@ def __init__(self, fom_, fomSp_, rvdim_, rxdim_, hdim_, smm_, v0_, if (myid == 0): self.spdim = self.fomSp.Height() # Reduced height - self.zH.SetSize(self.spdim // 2) # Samples of H + self.zH = mfem.Vector(self.spdim // 2) # Samples of H # Allocate auxillary vectors self.z.SetSize(self.spdim // 2) - self.z_v.SetSize(self.spdim // 2) - self.z_x.SetSize(self.spdim // 2) + self.z_v = mfem.Vector(self.spdim // 2) + self.z_x = mfem.Vector(self.spdim // 2) self.z_librom = linalg.Vector(self.z.GetDataArray(), False, False) self.z_v_librom = linalg.Vector(self.z_v.GetDataArray(), False, False) self.z_x_librom = linalg.Vector(self.z_x.GetDataArray(), False, False) @@ -553,7 +475,7 @@ def BasisGeneratorFinalSummary(bg, energyFraction, cutoff, cutoffOutputPath): partialSum = 0.0 for sv in range(sing_vals.dim()): partialSum += sing_vals[sv] - for i in range(energy_fractions.size()-1, -1, -1): + for i in range(len(energy_fractions)-1, -1, -1): if (partialSum / sum > energy_fractions[i]): outfile.write("For energy fraction: %.5E, take first %d of %d basis vectors" % (energy_fractions[i], sv+1, sing_vals.dim())) energy_fractions.pop(-1) @@ -877,6 +799,8 @@ def run(): v_gf = mfem.ParGridFunction(fespace) x_gf = mfem.ParGridFunction(fespace) + v_gf.MakeTRef(fespace, vx, true_offset[0]) + x_gf.MakeTRef(fespace, vx, true_offset[1]) # v_gf.MakeTRef(&fespace, vx, # true_offset[0]); // Associate a new FiniteElementSpace and new true-dof data with the GridFunction. # x_gf.MakeTRef(&fespace, vx, true_offset[1]); @@ -1208,7 +1132,7 @@ def run(): Ess_mat[i,0] = 1. # Project binary FOM list onto sampling space - MPI.COMM_WORLD.Bcast([sp_size, MPI.INT], root=0) + MPI.COMM_WORLD.bcast(sp_size, root=0) Ess_mat_sp = linalg.Matrix(sp_size, 1, False) smm.GatherDistributedMatrixRows("X", Ess_mat, 1, Ess_mat_sp) @@ -1265,7 +1189,7 @@ def run(): ke = oper.KineticEnergy(v_gf) if (myid == 0): - print("Lifted initial energies, EE = %.e, KE = %.e, ΔTE = %.e" % (ee, ke, (ee + ke) - (ee0 + ke0))) + print("Lifted initial energies, EE = %.5e, KE = %.5e, ΔTE = %.5e" % (ee, ke, (ee + ke) - (ee0 + ke0))) ode_solver.Init(romop) else: @@ -1291,7 +1215,7 @@ def run(): t, dt = ode_solver.Step(wMFEM, t, dt_real) solveTimer.Stop() - MPI.COMM_WORLD.Bcast([t, MPI.DOUBLE], root=0) + MPI.COMM_WORLD.bcast(t, root=0) else: solveTimer.Start() t, dt = ode_solver.Step(vx, t, dt_real) @@ -1300,15 +1224,13 @@ def run(): last_step = (t >= t_final - 1e-8 * dt) if (offline): - - if (basis_generator_x.isNextSample(t) or (x_base_only == False) - and basis_generator_v.isNextSample(t)): + if (basis_generator_x.isNextSample(t) or (not x_base_only) and basis_generator_v.isNextSample(t)): dvxdt = oper.dvxdt_sp.GetData() vx_diff = mfem.BlockVector(vx) vx_diff -= vx0 # Take samples - if ((x_base_only == False) and basis_generator_v.isNextSample(t)): + if ((not x_base_only) and basis_generator_v.isNextSample(t)): basis_generator_v.takeSample(vx_diff.GetBlock(0).GetDataArray(), t, dt) basis_generator_v.computeNextSampleTime(vx_diff.GetBlock(0).GetDataArray(), dvdt.GetDataArray(), t) basis_generator_H.takeSample(oper.H_sp.GetDataArray(), t, dt) @@ -1347,7 +1269,7 @@ def run(): ke = oper.KineticEnergy(v_gf) if (myid == 0): - print("step %d, t = %f, EE = %.e, KE = %.e, ΔTE = %.e" % (ti, t, ee, ke, (ee + ke) - (ee0 + ke0))) + print("step %d, t = %f, EE = %.5e, KE = %.5e, ΔTE = %.5e" % (ti, t, ee, ke, (ee + ke) - (ee0 + ke0))) if (visualization): visualize(vis_v, pmesh, x_gf, v_gf) @@ -1368,7 +1290,7 @@ def run(): # timestep loop if (myid == 0): - print("Elapsed time for time integration loop %.e" % solveTimer.duration) + print("Elapsed time for time integration loop %.5e" % solveTimer.duration) velo_name = "velocity_s%f.%06d" % (s, myid) pos_name = "position_s%f.%06d" % (s, myid) @@ -1404,25 +1326,15 @@ def run(): pmesh.Print(mesh_name, 8) pmesh.SwapNodes(nodes, owns_nodes) - with open(velo_name, 'w') as fid: - velo_ofs = io.StringIO() - velo_ofs.precision = 16 - v_final = mfem.Vector(vx.GetBlock(0)) - v_final.Save(velo_ofs) - fid.write(velo_ofs.getvalue()) - - with open(pos_name, 'w') as fid: - pos_ofs = io.StringIO() - pos_ofs.precision = 16 - x_final = mfem.Vector(vx.GetBlock(1)) - x_final.Save(pos_ofs) - fid.write(pos_ofs.getvalue()) + np.savetxt(velo_name, vx.GetBlock(0).GetDataArray(), fmt='%.16e') + np.savetxt(pos_name, vx.GetBlock(1).GetDataArray(), fmt='%.16e') with open(ee_name, 'w') as fid: ee_ofs = io.StringIO() ee_ofs.precision = 8 oper.GetElasticEnergyDensity(x_gf, w_gf) w_gf.Save(ee_ofs) + fid.write(ee_ofs.getvalue()) # 15. Calculate the relative error between the ROM final solution and the true solution. if (online): @@ -1431,11 +1343,9 @@ def run(): x_fom = mfem.Vector(x_rec.Size()) # Open and load file - with open(velo_name, 'r') as fom_v_file: - v_fom.Load(fom_v_file, v_rec.Size()) + v_fom.Load(velo_name, v_rec.Size()) - with open(pos_name, 'r') as fom_x_file: - x_fom.Load(fom_x_file, x_rec.Size()) + x_fom.Load(pos_name, x_rec.Size()) # Get difference vector diff_v = mfem.Vector(v_rec.Size()) @@ -1461,7 +1371,7 @@ def run(): totalTimer.Stop() if (myid == 0): - print("Elapsed time for entire simulation %.e" % totalTimer.duration) + print("Elapsed time for entire simulation %.5e" % totalTimer.duration) MPI.Finalize() return From 63d4a11f790047df026473715aa33c95427dca46 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Tue, 19 Sep 2023 17:48:48 -0700 Subject: [PATCH 6/9] libROM with hypre 2.28.0. --- extern/libROM | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/libROM b/extern/libROM index bf19fec..e6b798b 160000 --- a/extern/libROM +++ b/extern/libROM @@ -1 +1 @@ -Subproject commit bf19fec99ea2c8f32f0ccd57fd047033adac90c0 +Subproject commit e6b798b0d08f04106c2dfc382af4afbd327bd015 From d92abac611bc4fe6364860d86abe32a3992fcf70 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 20 Sep 2023 13:42:18 -0700 Subject: [PATCH 7/9] updated comments. --- examples/prom/nonlinear_elasticity_global_rom.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.py b/examples/prom/nonlinear_elasticity_global_rom.py index ab9a746..ceef24c 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.py +++ b/examples/prom/nonlinear_elasticity_global_rom.py @@ -30,25 +30,25 @@ # // and nonlinear term basis, with velocity initial condition: # // # // Offline phase: -# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.90 -id 0 +# // python3 nonlinear_elasticity_global_rom.py -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.90 -id 0 # // -# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.10 -id 1 +# // python3 nonlinear_elasticity_global_rom.py -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.10 -id 1 # // # // Merge phase: -# // ./nonlinear_elasticity_global_rom -merge -ns 2 -dt 0.01 -tf 5.0 +# // python3 nonlinear_elasticity_global_rom.py -merge -ns 2 -dt 0.01 -tf 5.0 # // # // Create FOM comparison data: -# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.00 -id 2 +# // python3 nonlinear_elasticity_global_rom.py -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.00 -id 2 # // # // Online phase with full sampling: -# // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 1170 -sc 1.00 +# // python3 nonlinear_elasticity_global_rom.py -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 1170 -sc 1.00 # // Output message: # // Elapsed time for time integration loop 1.80759 # // Relative error of ROM position (x) at t_final: 5 is 0.000231698 # // Relative error of ROM velocity (v) at t_final: 5 is 0.466941 # // # // Online phase with strong hyper-reduction: -# // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 100 -sc 1.00 +# // python3 nonlinear_elasticity_global_rom.py -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 100 -sc 1.00 # // Output message: # // Elapsed time for time integration loop 1.08048 # // Relative error of ROM position (x) at t_final: 5 is 0.00209877 From d2d8014ed1792ac1fac00fda48a5b4886e812ebe Mon Sep 17 00:00:00 2001 From: Seung Whan Chung Date: Thu, 21 Sep 2023 19:04:24 -0700 Subject: [PATCH 8/9] example command and results update. --- README.md | 4 +- examples/data/beam-hex-nurbs.mesh | 143 ++++++++++++++++++ .../prom/nonlinear_elasticity_global_rom.py | 47 +++--- extern/libROM | 2 +- 4 files changed, 165 insertions(+), 31 deletions(-) create mode 100644 examples/data/beam-hex-nurbs.mesh diff --git a/README.md b/README.md index b29a4e4..bd4c8e5 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,11 @@ For parallel version, a manual installation is required: ``` git clone https://github.com/mfem/PyMFEM.git cd PyMFEM -python3 setup.py install --with-parallel +python3 setup.py install --with-parallel --with-gslib ``` On LC quartz, use `--user` flag: ``` -python3 setup.py install --with-parallel --user +python3 setup.py install --with-parallel --with-gslib --user ``` Make sure [`swig`](https://pypi.org/project/swig) is installed first. Also, the binary file must be located in `PATH` environment variable. diff --git a/examples/data/beam-hex-nurbs.mesh b/examples/data/beam-hex-nurbs.mesh new file mode 100644 index 0000000..f64a6da --- /dev/null +++ b/examples/data/beam-hex-nurbs.mesh @@ -0,0 +1,143 @@ +MFEM NURBS mesh v1.0 + +# +# MFEM Geometry Types (see mesh/geom.hpp): +# +# SEGMENT = 1 +# SQUARE = 3 +# CUBE = 5 +# + +dimension +3 + +elements +2 +1 5 0 1 4 5 6 7 10 11 +2 5 1 2 3 4 7 8 9 10 + +boundary +10 +3 3 5 4 1 0 +3 3 0 1 7 6 +3 3 4 5 11 10 +1 3 5 0 6 11 +3 3 6 7 10 11 +3 3 4 3 2 1 +3 3 1 2 8 7 +2 3 2 3 9 8 +3 3 3 4 10 9 +3 3 7 8 9 10 + +edges +20 +0 0 1 +0 5 4 +0 6 7 +0 11 10 +1 1 2 +1 4 3 +1 7 8 +1 10 9 +2 0 5 +2 1 4 +2 2 3 +2 6 11 +2 7 10 +2 8 9 +3 0 6 +3 1 7 +3 2 8 +3 3 9 +3 4 10 +3 5 11 + +vertices +12 + +knotvectors +4 +1 5 0 0 1 2 3 4 4 +1 5 0 0 1 2 3 4 4 +1 2 0 0 1 1 +1 2 0 0 1 1 + +weights +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 + +FiniteElementSpace +FiniteElementCollection: NURBS1 +VDim: 3 +Ordering: 1 + +0 0 0 +4 0 0 +8 0 0 +8 1 0 +4 1 0 +0 1 0 +0 0 1 +4 0 1 +8 0 1 +8 1 1 +4 1 1 +0 1 1 +1 0 0 +2 0 0 +3 0 0 +3 1 0 +2 1 0 +1 1 0 +1 0 1 +2 0 1 +3 0 1 +3 1 1 +2 1 1 +1 1 1 +5 0 0 +6 0 0 +7 0 0 +7 1 0 +6 1 0 +5 1 0 +5 0 1 +6 0 1 +7 0 1 +7 1 1 +6 1 1 +5 1 1 diff --git a/examples/prom/nonlinear_elasticity_global_rom.py b/examples/prom/nonlinear_elasticity_global_rom.py index ceef24c..f12152c 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.py +++ b/examples/prom/nonlinear_elasticity_global_rom.py @@ -43,44 +43,36 @@ # // Online phase with full sampling: # // python3 nonlinear_elasticity_global_rom.py -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 1170 -sc 1.00 # // Output message: -# // Elapsed time for time integration loop 1.80759 -# // Relative error of ROM position (x) at t_final: 5 is 0.000231698 -# // Relative error of ROM velocity (v) at t_final: 5 is 0.466941 -# // -# // Online phase with strong hyper-reduction: -# // python3 nonlinear_elasticity_global_rom.py -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 100 -sc 1.00 -# // Output message: -# // Elapsed time for time integration loop 1.08048 -# // Relative error of ROM position (x) at t_final: 5 is 0.00209877 -# // Relative error of ROM velocity (v) at t_final: 5 is 1.39472 +# Elapsed time for time integration loop 5.87029e+00 +# Relative error of ROM position (x) at t_final: 5.000000 is 2.31698489e-04 +# Relative error of ROM velocity (v) at t_final: 5.000000 is 4.66941161e-01 +# Elapsed time for entire simulation 9.86927e+00 # // # // ================================================================================= # // -# // Sample runs and results for parametric ROM using only displacement basis -# // and nonlinear term basis: +# // Sample runs and results for parametric ROM using displacement basis, velocity basis +# // and nonlinear term basis, with velocity initial condition: (3d case in librom.net) +# // # // Offline phase: -# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.90 -xbo -def-ic -id 0 -# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.10 -xbo -def-ic -id 1 +# // python3 nonlinear_elasticity_global_rom.py --mesh ../data/beam-hex-nurbs.mesh -offline -dt 0.01 -tf 5.0 -s 14 -vs 10 -sc 3.90 -id 0 +# // +# // python3 nonlinear_elasticity_global_rom.py --mesh ../data/beam-hex-nurbs.mesh -offline -dt 0.01 -tf 5.0 -s 14 -vs 10 -sc 4.10 -id 1 # // # // Merge phase: -# // ./nonlinear_elasticity_global_rom -merge -ns 2 -dt 0.01 -tf 5.0 -xbo +# // python3 nonlinear_elasticity_global_rom.py --mesh ../data/beam-hex-nurbs.mesh -merge -ns 2 -dt 0.01 -tf 5.0 # // # // Create FOM comparison data: -# // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.00 -xbo -def-ic -id 2 +# // python3 nonlinear_elasticity_global_rom.py --mesh ../data/beam-hex-nurbs.mesh -offline -dt 0.01 -tf 5.0 -s 14 -vs 5 -sc 3.92 -id 2 # // # // Online phase with full sampling: -# // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rxdim 57 -hdim 183 -nsr 1170 -sc 1.00 -xbo -def-ic +# // python3 nonlinear_elasticity_global_rom.py --mesh ../data/beam-hex-nurbs.mesh -online -dt 0.01 -tf 5.0 -s 14 -vs 5 -hyp -rvdim 40 -rxdim 10 -hdim 71 -nsr 200 -sc 3.92 # // Output message: -# // Elapsed time for time integration loop 18.9874 -# // Relative error of ROM position (x) at t_final: 5 is 7.08272e-05 -# // Relative error of ROM velocity (v) at t_final: 5 is 0.00387647 +# Elapsed time for time integration loop 3.47888e+01 +# Relative error of ROM position (x) at t_final: 5.000000 is 5.08981700e-03 +# Relative error of ROM velocity (v) at t_final: 5.000000 is 9.11743695e+00 +# Elapsed time for entire simulation 6.63521e+01 # // -# // Online phase with strong hyper reduction: -# // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -rxdim 2 -hdim 4 -nsr 10 -sc 1.00 -xbo -def-ic -# // Output message: -# // Elapsed time for time integration loop 1.01136 -# // Relative error of ROM position (x) at t_final: 5 is 0.0130818 -# // Relative error of ROM velocity (v) at t_final: 5 is 0.979978 +# // ================================================================================= # // # // This example runs in parallel with MPI, by using the same number of MPI ranks # // in all phases (offline, merge, online). @@ -1039,7 +1031,6 @@ def run(): spfespace[0] = fespace # ParFiniteElementSpace* sp_XV_space; - smm = mfem_support.SampleMeshManager(spfespace) # vector sample_dofs_empty; @@ -1377,4 +1368,4 @@ def run(): return if __name__ == "__main__": - run() \ No newline at end of file + run() diff --git a/extern/libROM b/extern/libROM index e6b798b..4509741 160000 --- a/extern/libROM +++ b/extern/libROM @@ -1 +1 @@ -Subproject commit e6b798b0d08f04106c2dfc382af4afbd327bd015 +Subproject commit 4509741f3c30275d49c4f9431acc6fa7c007efa3 From 68d359fe36729e1f085c6170a40af36e7ddc1ffb Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Wed, 27 Sep 2023 19:32:35 -0700 Subject: [PATCH 9/9] updated libROM branch. --- .gitmodules | 3 +-- docker/librom/Dockerfile | 2 +- extern/libROM | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index e3c5069..8ffb741 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,5 +4,4 @@ branch = stable [submodule "extern/libROM"] path = extern/libROM - url = https://github.com/LLNL/libROM.git - branch = mfem_fix \ No newline at end of file + url = https://github.com/LLNL/libROM.git \ No newline at end of file diff --git a/docker/librom/Dockerfile b/docker/librom/Dockerfile index e505296..d3def6e 100644 --- a/docker/librom/Dockerfile +++ b/docker/librom/Dockerfile @@ -23,7 +23,7 @@ RUN sudo git clone https://github.com/LLNL/libROM.git WORKDIR ./libROM RUN sudo git pull # pylibROM is currently based on a specific commit of the libROM. -RUN sudo git checkout mfem_fix +RUN sudo git checkout 0809d7d09dc24f0963c38fc8c0a2649948142ba0 WORKDIR ./build RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_MFEM=${USE_MFEM} -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB} # RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DUSE_MFEM=OFF diff --git a/extern/libROM b/extern/libROM index 4509741..0809d7d 160000 --- a/extern/libROM +++ b/extern/libROM @@ -1 +1 @@ -Subproject commit 4509741f3c30275d49c4f9431acc6fa7c007efa3 +Subproject commit 0809d7d09dc24f0963c38fc8c0a2649948142ba0