diff --git a/examples/flame1d.py b/examples/flame1d.py new file mode 100644 index 000000000..c3557dfe9 --- /dev/null +++ b/examples/flame1d.py @@ -0,0 +1,876 @@ +"""mirgecom driver for the 1D flame demonstration.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import logging +import sys +import gc +from warnings import warn +from functools import partial +import numpy as np +import cantera +from pytools.obj_array import make_obj_array + +from grudge.dof_desc import BoundaryDomainTag +from grudge.shortcuts import compiled_lsrk45_step +from mirgecom.integrators import rk4_step, euler_step +from grudge.shortcuts import make_visualizer +from grudge import op + +from logpyle import IntervalTimer, set_dt +from mirgecom.navierstokes import ns_operator, grad_cv_operator, grad_t_operator +from mirgecom.simutil import ( + check_step, check_naninf_local, check_range_local, + get_sim_timestep, + distribute_mesh, + write_visfile, +) +from mirgecom.utils import force_evaluation +from mirgecom.restart import write_restart_file, read_restart_data +from mirgecom.io import make_init_message +from mirgecom.mpi import mpi_entry_point +from mirgecom.steppers import advance_state +from mirgecom.fluid import make_conserved +from mirgecom.eos import PyrometheusMixture +from mirgecom.gas_model import ( + GasModel, make_fluid_state, make_operator_fluid_states) +from mirgecom.logging_quantities import ( + initialize_logmgr, logmgr_add_cl_device_info, logmgr_set_time, + logmgr_add_device_memory_usage) + + +class SingleLevelFilter(logging.Filter): + def __init__(self, passlevel, reject): + self.passlevel = passlevel + self.reject = reject + + def filter(self, record): + if self.reject: + return record.levelno != self.passlevel + return record.levelno == self.passlevel + + +# h1 = logging.StreamHandler(sys.stdout) +# f1 = SingleLevelFilter(logging.INFO, False) +# h1.addFilter(f1) +# root_logger = logging.getLogger() +# root_logger.addHandler(h1) +# h2 = logging.StreamHandler(sys.stderr) +# f2 = SingleLevelFilter(logging.INFO, True) +# h2.addFilter(f2) +# root_logger.addHandler(h2) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class MyRuntimeError(RuntimeError): + pass + + +class _FluidOpStatesTag: + pass + + +class _FluidGradCVTag: + pass + + +class _FluidGradTempTag: + pass + + +def sponge_func(cv, cv_ref, sigma): + """Apply the sponge.""" + return sigma*(cv_ref - cv) + + +class InitSponge: + r""" + .. automethod:: __init__ + .. automethod:: __call__ + """ + def __init__(self, x_min, x_max, x_thickness, amplitude): + r"""Initialize the sponge parameters. + + Parameters + ---------- + x0: float + sponge starting x location + thickness: float + sponge extent + amplitude: float + sponge strength modifier + """ + + self._x_min = x_min + self._x_max = x_max + self._x_thickness = x_thickness + self._amplitude = amplitude + + def __call__(self, x_vec): + """Create the sponge intensity at locations *x_vec*. + + Parameters + ---------- + x_vec: numpy.ndarray + Coordinates at which solution is desired + """ + xpos = x_vec[0] + actx = xpos.array_context + + sponge = xpos*0.0 + + x0 = self._x_max - self._x_thickness + dx = +(xpos - x0)/self._x_thickness + sponge = sponge + self._amplitude * actx.np.where( + actx.np.greater(xpos, x0), + actx.np.where( + actx.np.greater(xpos, self._x_max), 1., 3.*dx**2 - 2.*dx**3), + 0.) + + x0 = self._x_min + self._x_thickness + dx = -(xpos - x0)/self._x_thickness + sponge = sponge + self._amplitude * actx.np.where( + actx.np.less(xpos, x0), + actx.np.where( + actx.np.less(xpos, self._x_min), 1., 3.*dx**2 - 2.*dx**3), + 0.) + + return sponge + + +@mpi_entry_point +def main(actx_class, use_esdg=False, use_tpe=False, use_overintegration=False, + use_leap=False, casename=None, rst_filename=None): + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = 0 + rank = comm.Get_rank() + nparts = comm.Get_size() + + from mirgecom.simutil import global_reduce as _global_reduce + global_reduce = partial(_global_reduce, comm=comm) + + logmgr = initialize_logmgr(True, filename=(f"{casename}.sqlite"), + mode="wu", mpi_comm=comm) + + from mirgecom.array_context import initialize_actx, actx_class_is_profiling + actx = initialize_actx(actx_class, comm) + queue = getattr(actx, "queue", None) + use_profiling = actx_class_is_profiling(actx_class) + + # ~~~~~~~~~~~~~~~~~~ + + import time + t_start = time.time() + t_shutdown = 720*60 + + rst_path = "restart_data/" + viz_path = "viz_data/" + vizname = viz_path+casename + rst_pattern = rst_path+"{cname}-{step:06d}-{rank:04d}.pkl" + + # default i/o frequencies + ngarbage = 10 + nviz = 1000 + nrestart = 25000 + nhealth = 1 + nstatus = 100 + + mechanism_file = "uiuc_7sp" + + order = 2 + + transport = "power-law" + # transport = "mix-lewis" + # transport = "mix" + + # default timestepping control + integrator = "rk4" + constant_cfl = False + current_cfl = 0.4 + current_dt = 1.0e-9 + niter = 20 + t_final = current_dt * niter + + use_sponge = False + + # use Cantera's 1D flame solution to prescribe the BC and an + # approximated initial condition with hyperbolic tangent profile + use_flame_from_cantera = True + +# ############################################################################ + + dim = 2 + + def _compiled_stepper_wrapper(state, t, dt, rhs): + return compiled_lsrk45_step(actx, state, t, dt, rhs) + + force_eval_stepper = True + timestepper = rk4_step + if integrator == "compiled_lsrk45": + timestepper = _compiled_stepper_wrapper + force_eval_stepper = False + if integrator == "euler": + timestepper = euler_step + + if rank == 0: + print("\n#### Simulation control data: ####") + print(f"\tnviz = {nviz}") + print(f"\tnrestart = {nrestart}") + print(f"\tnhealth = {nhealth}") + print(f"\tnstatus = {nstatus}") + if constant_cfl: + print(f"\tcurrent_cfl = {current_cfl}") + else: + print(f"\tcurrent_dt = {current_dt}") + print(f"\tt_final = {t_final}") + print(f"\tniter = {niter}") + print(f"\torder = {order}") + print(f"\tTime integration = {integrator}") + +# ############################################################################ + + restart_step = 0 + if rst_filename is None: + + import os + path = os.path.dirname(os.path.abspath(__file__)) + xx = np.loadtxt(f"{path}/flame1d_x_050um.dat") + yy = np.loadtxt(f"{path}/flame1d_y_050um.dat") + + from meshmode.mesh import TensorProductElementGroup + grp_cls = TensorProductElementGroup if use_tpe else None + + from meshmode.mesh.generation import generate_box_mesh + generate_mesh = partial(generate_box_mesh, + axis_coords=(xx, yy), + periodic=(False, True), + boundary_tag_to_face={"inlet": ["-x"], + "outlet": ["+x"]}, + group_cls=grp_cls) + + local_mesh, global_nelements = distribute_mesh(comm, generate_mesh) + local_nelements = local_mesh.nelements + + else: + restart_file = f"{rst_filename}-{rank:04d}.pkl" + restart_data = read_restart_data(actx, restart_file) + restart_step = restart_data["step"] + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + restart_order = int(restart_data["order"]) + + assert restart_order == order + assert comm.Get_size() == restart_data["num_parts"] + + from mirgecom.discretization import create_discretization_collection + dcoll = create_discretization_collection(actx, local_mesh, order) + + nodes = actx.thaw(dcoll.nodes()) + zeros = actx.np.zeros_like(nodes[0]) + + from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD + quadrature_tag = DISCR_TAG_QUAD if use_overintegration else DISCR_TAG_BASE + +# ############################################################################ + + # {{{ Set up initial state using Cantera + + # Use Cantera for initialization + if rank == 0: + logging.info("\nUsing Cantera " + cantera.__version__) + + from mirgecom.mechanisms import get_mechanism_input + mech_input = get_mechanism_input(mechanism_file) + + cantera_soln = cantera.Solution(name="gas", yaml=mech_input) + nspecies = cantera_soln.n_species + + cantera_soln.set_equivalence_ratio(phi=1.0, fuel="C2H4:1,H2:1", + oxidizer={"O2": 1.0, "N2": 3.76}) + + # Initial temperature, pressure, and mixture mole fractions are needed to + # set up the initial state in Cantera. + if use_flame_from_cantera: + # Set up flame object + f = cantera.FreeFlame(cantera_soln, width=0.02) + + # Solve with mixture-averaged transport model + f.transport_model = "mixture-averaged" + f.set_refine_criteria(ratio=2, slope=0.15, curve=0.15) + f.solve(loglevel=0, refine_grid=True, auto=True) + + temp_unburned = f.T[0] + temp_burned = f.T[-1] + + y_unburned = f.Y[:, 0] + y_burned = f.Y[:, -1] + + vel_unburned = f.velocity[0] + vel_burned = f.velocity[-1] + + mass_unburned = f.density[0] + mass_burned = f.density[-1] + + else: + temp_unburned = 300.0 + + cantera_soln.TP = temp_unburned, 101325.0 + y_unburned = cantera_soln.Y + mass_unburned = cantera_soln.density + + cantera_soln.equilibrate("HP") + temp_burned = cantera_soln.T + y_burned = cantera_soln.Y + mass_burned = cantera_soln.density + + vel_unburned = 0.5 + vel_burned = vel_unburned*mass_unburned/mass_burned + + pres_unburned = cantera.one_atm # pylint: disable=no-member + pres_burned = cantera.one_atm # pylint: disable=no-member + + # }}} + + # {{{ Create Pyrometheus thermochemistry object & EOS + + # Import Pyrometheus EOS + from mirgecom.thermochemistry import get_pyrometheus_wrapper_class_from_cantera + pyro_mech = get_pyrometheus_wrapper_class_from_cantera(cantera_soln)(actx.np) + eos = PyrometheusMixture(pyro_mech, temperature_guess=1234.56789) + + species_names = pyro_mech.species_names + + # }}} + transport_model = None + if transport == "power-law": + from mirgecom.transport import PowerLawTransport + transport_model = PowerLawTransport(lewis=np.ones((nspecies,)), + beta=4.093e-7) + + if transport == "mix-lewis": + from mirgecom.transport import MixtureAveragedTransport + transport_model = MixtureAveragedTransport(pyro_mech, + lewis=np.ones(nspecies,)) + if transport == "mix": + from mirgecom.transport import MixtureAveragedTransport + transport_model = MixtureAveragedTransport(pyro_mech) + + gas_model = GasModel(eos=eos, transport=transport_model) + + print(f"Pyrometheus mechanism species names {species_names}") + print("Unburned:") + print(f"T = {temp_unburned}") + print(f"D = {mass_unburned}") + print(f"Y = {y_unburned}") + print(f"U = {vel_unburned}\n") + print("Burned:") + print(f"T = {temp_burned}") + print(f"D = {mass_burned}") + print(f"Y = {y_burned}") + print(f"U = {vel_burned}\n") + +# ############################################################################ + + from mirgecom.limiter import bound_preserving_limiter + + def _limit_fluid_cv(cv, temperature_seed, gas_model, dd=None): + + temperature = gas_model.eos.temperature( + cv=cv, temperature_seed=temperature_seed) + pressure = gas_model.eos.pressure(cv=cv, temperature=temperature) + + # limit species + spec_lim = make_obj_array([ + bound_preserving_limiter(dcoll, cv.species_mass_fractions[i], + mmin=0.0, mmax=1.0, modify_average=True, dd=dd) + for i in range(nspecies)]) + + # normalize to ensure sum_Yi = 1.0 + aux = actx.np.zeros_like(cv.mass) + for i in range(0, nspecies): + aux = aux + spec_lim[i] + spec_lim = spec_lim/aux + + # recompute density + mass_lim = eos.get_density(pressure=pressure, + temperature=temperature, species_mass_fractions=spec_lim) + + # recompute energy + energy_lim = mass_lim*( + gas_model.eos.get_internal_energy(temperature, + species_mass_fractions=spec_lim) + + 0.5*np.dot(cv.velocity, cv.velocity)) + + # make a new CV with the limited variables + return make_conserved(dim=dim, mass=mass_lim, energy=energy_lim, + momentum=mass_lim*cv.velocity, species_mass=mass_lim*spec_lim) + + def _get_fluid_state(cv, temp_seed): + return make_fluid_state(cv=cv, gas_model=gas_model, + temperature_seed=temp_seed, limiter_func=_limit_fluid_cv) + + get_fluid_state = actx.compile(_get_fluid_state) + + def get_temperature_update(cv, temperature): + y = cv.species_mass_fractions + e = eos.internal_energy(cv) / cv.mass + return make_obj_array( + [pyro_mech.get_temperature_update_energy(e, temperature, y)] + ) + + compute_temperature_update = actx.compile(get_temperature_update) + +# ############################################################################ + + flame_start_loc = 0.0 + from mirgecom.initializers import PlanarDiscontinuity + bulk_init = PlanarDiscontinuity(dim=dim, disc_location=flame_start_loc, + sigma=0.0005, nspecies=nspecies, + temperature_right=temp_burned, temperature_left=temp_unburned, + pressure_right=pres_burned, pressure_left=pres_unburned, + velocity_right=make_obj_array([vel_burned, 0.0]), + velocity_left=make_obj_array([vel_unburned, 0.0]), + species_mass_right=y_burned, species_mass_left=y_unburned) + + if rst_filename is None: + current_t = 0.0 + current_step = 0 + first_step = 0 + + tseed = 1234.56789 + zeros + + if rank == 0: + logging.info("Initializing soln.") + + current_cv = bulk_init(x_vec=nodes, eos=eos, time=0.) + + else: + current_t = restart_data["t"] + current_step = restart_step + first_step = restart_step + 0 + + current_cv = restart_data["cv"] + tseed = restart_data["temperature_seed"] + + tseed = force_evaluation(actx, tseed) + current_cv = force_evaluation(actx, current_cv) + current_state = get_fluid_state(current_cv, tseed) + +# ############################################################################ + + vis_timer = None + if logmgr: + logmgr_add_cl_device_info(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_set_time(logmgr, current_step, current_t) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("dt.max", "dt: {value:1.6e} s, "), + ("t_sim.max", "sim time: {value:1.6e} s, "), + ("t_step.max", "------- step walltime: {value:6g} s\n") + ]) + + try: + logmgr.add_watches(["memory_usage_python.max", + "memory_usage_gpu.max"]) + except KeyError: + pass + + if use_profiling: + logmgr.add_watches(["pyopencl_array_time.max"]) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) + + gc_timer = IntervalTimer("t_gc", "Time spent garbage collecting") + logmgr.add_quantity(gc_timer) + + # initialize the sponge field + sponge_init = InitSponge(x_max=+0.100, x_min=-0.100, x_thickness=0.065, + amplitude=10000.0) + + sponge_sigma = sponge_init(x_vec=nodes) + + ref_cv = bulk_init(x_vec=nodes, eos=eos, time=0.) + +# ############################################################################ + + # from grudge.dof_desc import DD_VOLUME_ALL + # dd_vol = DD_VOLUME_ALL + + # inflow_cv_cond = op.project(dcoll, dd_vol, dd_vol.trace("inlet"), ref_cv) + + # def inlet_bnd_state_func(dcoll, dd_bdry, gas_model, state_minus, **kwargs): + # return make_fluid_state(cv=inflow_cv_cond, gas_model=gas_model, + # temperature_seed=300.0) + + # from mirgecom.boundary import ( + # PrescribedFluidBoundary, LinearizedOutflow2DBoundary) + # inflow_bnd = PrescribedFluidBoundary(boundary_state_func=inlet_bnd_state_func) + # outflow_bnd = LinearizedOutflow2DBoundary( + # free_stream_density=mass_burned, free_stream_pressure=101325.0, + # free_stream_velocity=make_obj_array([vel_burned, 0.0]), + # free_stream_species_mass_fractions=y_burned) + + from mirgecom.boundary import ( + LinearizedInflowBoundary, PressureOutflowBoundary) + inflow_bnd = LinearizedInflowBoundary( + free_stream_density=mass_unburned, free_stream_pressure=101325.0, + free_stream_velocity=make_obj_array([vel_unburned, 0.0]), + free_stream_species_mass_fractions=y_unburned) + outflow_bnd = PressureOutflowBoundary(boundary_pressure=101325.0) + + boundaries = {BoundaryDomainTag("inlet"): inflow_bnd, + BoundaryDomainTag("outlet"): outflow_bnd} + +# ############################################################################ + + visualizer = make_visualizer(dcoll) + + initname = "flame1D" + eosname = eos.__class__.__name__ + init_message = make_init_message(dim=dim, order=order, + nelements=local_nelements, global_nelements=global_nelements, + t_initial=current_t, dt=current_dt, t_final=t_final, nstatus=nstatus, + nviz=nviz, cfl=current_cfl, constant_cfl=constant_cfl, + initname=initname, eosname=eosname, casename=casename) + + if rank == 0: + logger.info(init_message) + +# ############################################################################ + + # def get_production_rates(cv, temperature): + # return make_obj_array([eos.get_production_rates(cv, temperature)]) + # compute_production_rates = actx.compile(get_production_rates) + + def my_write_viz(step, t, dt, state): + + y = state.cv.species_mass_fractions + # gas_const = gas_model.eos.gas_const(species_mass_fractions=y) + # gamma = eos.gamma(state.cv, state.temperature) + + # reaction_rates, = compute_production_rates(state.cv, state.temperature) + viz_fields = [("CV_rho", state.cv.mass), + ("CV_rhoU", state.cv.momentum), + ("CV_rhoE", state.cv.energy), + ("DV_P", state.pressure), + ("DV_T", state.temperature), + # ("reaction_rates", reaction_rates), + # ("sponge", sponge_sigma), + # ("R", gas_const), + # ("gamma", gamma), + # ("dt", dt), + # ("mu", state.tv.viscosity), + # ("kappa", state.tv.thermal_conductivity), + ] + + # species mass fractions + viz_fields.extend(("Y_"+species_names[i], y[i]) for i in range(nspecies)) + + # species diffusivity + # viz_fields.extend( + # ("diff_"+species_names[i], state.tv.species_diffusivity[i]) + # for i in range(nspecies)) + + if rank == 0: + logger.info("Writing solution file...") + write_visfile(dcoll, viz_fields, visualizer, vizname=vizname, + step=step, t=t, overwrite=True, comm=comm) + + def my_write_restart(step, t, cv, tseed): + restart_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + if restart_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "cv": cv, + "temperature_seed": tseed, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + + write_restart_file(actx, rst_data, restart_fname, comm) + +# ############################################################################ + + def my_health_check(cv, dv): + health_error = False + pressure = force_evaluation(actx, dv.pressure) + temperature = force_evaluation(actx, dv.temperature) + + if check_naninf_local(dcoll, "vol", pressure): + health_error = True + logger.info(f"{rank=}: NANs/Infs in pressure data.") + + if check_naninf_local(dcoll, "vol", temperature): + health_error = True + logger.info(f"{rank=}: NANs/Infs in temperature data.") + + # if check_range_local(dcoll, "vol", pressure, 101250., 101500.): + # health_error = True + # logger.info(f"{rank=}: Pressure range violation.") + + if check_range_local(dcoll, "vol", temperature, 290., 2450.): + health_error = True + logger.info(f"{rank=}: Temperature range violation.") + + # temperature_update is the next temperature update in the + # `get_temperature` Newton solve. The relative size of this + # update is used to gauge convergence of the current temperature + # after a fixed number of Newton iters. + # Note: The local max jig below works around a very long compile + # in lazy mode. + temp_update, = compute_temperature_update(cv, temperature) + temp_resid = force_evaluation(actx, temp_update) / temperature + temp_resid = (actx.to_numpy(op.nodal_max_loc(dcoll, "vol", temp_resid))) + if temp_resid > 1e-8: + health_error = True + logger.info(f"{rank=}: Temperature is not converged {temp_resid=}.") + + return health_error + +# ############################################################################ + + def my_pre_step(step, t, dt, state): + + if logmgr: + logmgr.tick_before() + + cv, tseed = state + cv = force_evaluation(actx, cv) + tseed = force_evaluation(actx, tseed) + + fluid_state = get_fluid_state(cv, tseed) + + if constant_cfl: + dt = get_sim_timestep(dcoll, fluid_state, t, dt, current_cfl, + t_final, constant_cfl) + + try: + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_garbage = check_step(step=step, interval=ngarbage) + + t_elapsed = time.time() - t_start + if t_shutdown - t_elapsed < 300.0: + my_write_restart(step=step, t=t, cv=fluid_state.cv, tseed=tseed) + + if do_garbage: + with gc_timer.start_sub_timer(): + warn("Running gc.collect() to work around memory growth issue ") + gc.collect() + + if do_health: + health_errors = global_reduce( + my_health_check(fluid_state.cv, fluid_state.dv), op="lor") + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise MyRuntimeError("Failed simulation health check.") + + if do_restart: + my_write_restart(step=step, t=t, cv=fluid_state.cv, tseed=tseed) + + if do_viz: + my_write_viz(step=step, t=t, dt=dt, state=fluid_state) + + except MyRuntimeError: + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, dt=dt, state=fluid_state) + raise + + return make_obj_array([fluid_state.cv, fluid_state.temperature]), dt + + def my_rhs(t, state): + cv, tseed = state + + fluid_state = make_fluid_state(cv=cv, gas_model=gas_model, + temperature_seed=tseed, limiter_func=_limit_fluid_cv) + + operator_states_quad = make_operator_fluid_states( + dcoll, fluid_state, gas_model, boundaries, quadrature_tag, + comm_tag=_FluidOpStatesTag, limiter_func=_limit_fluid_cv) + + grad_cv = grad_cv_operator( + dcoll, gas_model, boundaries, fluid_state, time=t, + quadrature_tag=quadrature_tag, comm_tag=_FluidGradCVTag, + limiter_func=_limit_fluid_cv, + operator_states_quad=operator_states_quad) + + grad_t = grad_t_operator( + dcoll, gas_model, boundaries, fluid_state, time=t, + quadrature_tag=quadrature_tag, comm_tag=_FluidGradTempTag, + limiter_func=_limit_fluid_cv, + operator_states_quad=operator_states_quad) + + ns_rhs = ns_operator(dcoll, gas_model, fluid_state, boundaries, time=t, + quadrature_tag=quadrature_tag, grad_cv=grad_cv, grad_t=grad_t, + operator_states_quad=operator_states_quad) + + chem_rhs = eos.get_species_source_terms(fluid_state.cv, + fluid_state.temperature) + + rhs = ns_rhs + chem_rhs + if use_sponge: + sponge_rhs = sponge_func(cv=fluid_state.cv, cv_ref=ref_cv, + sigma=sponge_sigma) + rhs = rhs + sponge_rhs + + return make_obj_array([rhs, zeros]) + + def my_post_step(step, t, dt, state): + if step == first_step + 1: + with gc_timer.start_sub_timer(): + gc.collect() + # Freeze the objects that are still alive so they will not + # be considered in future gc collections. + logger.info("Freezing GC objects to reduce overhead of " + "future GC collections") + gc.freeze() + + if logmgr: + set_dt(logmgr, dt) + logmgr.tick_after() + + return state, dt + +# ############################################################################ + + if constant_cfl: + current_dt = get_sim_timestep(dcoll, current_state, current_t, current_dt, + current_cfl, t_final, constant_cfl) + + if rank == 0: + logging.info("Stepping.") + + current_step, current_t, stepper_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, + istep=current_step, dt=current_dt, t=current_t, + t_final=t_final, force_eval=force_eval_stepper, + state=make_obj_array([current_state.cv, tseed])) + current_cv, tseed = stepper_state + current_state = make_fluid_state(current_cv, gas_model, tseed) + + # Dump the final data + if rank == 0: + logger.info("Checkpointing final state ...") + + my_write_viz(step=current_step, t=current_t, dt=current_dt, + state=current_state) + my_write_restart(step=current_step, t=current_t, cv=current_state.cv, + tseed=tseed) + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) + + sys.exit() + + +if __name__ == "__main__": + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + level=logging.INFO) + + import argparse + parser = argparse.ArgumentParser(description="MIRGE-Com 1D Flame Driver") + parser.add_argument("-r", "--restart_file", type=ascii, + dest="restart_file", nargs="?", action="store", + help="simulation restart file") + parser.add_argument("-i", "--input_file", type=ascii, + dest="input_file", nargs="?", action="store", + help="simulation config file") + parser.add_argument("-c", "--casename", type=ascii, + dest="casename", nargs="?", action="store", + help="simulation case name") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--profiling", action="store_true", default=False, + help="enable kernel profiling [OFF]") +# parser.add_argument("--log", action="store_true", default=True, +# help="enable logging profiling [ON]") + parser.add_argument("--overintegration", action="store_true", default=False, + help="enable overintegration [OFF]") + parser.add_argument("--esdg", action="store_true", + help="use entropy stable DG for inviscid computations.") + parser.add_argument("--lazy", action="store_true", default=False, + help="enable lazy evaluation [OFF]") + parser.add_argument("--numpy", action="store_true", + help="use numpy-based eager actx.") + parser.add_argument("--tpe", action="store_true") + + args = parser.parse_args() + + # for writing output + casename = "flame1D" + if args.casename: + print(f"Custom casename {args.casename}") + casename = (args.casename).replace("'", "") + else: + print(f"Default casename {casename}") + + rst_filename = None + if args.restart_file: + rst_filename = (args.restart_file).replace("'", "") + print(f"Restarting from file: {rst_filename}") + + input_file = None + if args.input_file: + input_file = (args.input_file).replace("'", "") + print(f"Reading user input from {args.input_file}") + else: + print("No user input file, using default values") + + print(f"Running {sys.argv[0]}\n") + + from mirgecom.simutil import ApplicationOptionsError + if args.esdg: + if not args.lazy and not args.numpy: + raise ApplicationOptionsError("ESDG requires lazy or numpy context.") + if not args.overintegration: + warn("ESDG requires overintegration, enabling --overintegration.") + + from mirgecom.array_context import get_reasonable_array_context_class + actx_class = get_reasonable_array_context_class( + lazy=args.lazy, distributed=True, profiling=args.profiling, numpy=args.numpy) + + main(actx_class, use_leap=args.leap, use_esdg=args.esdg, use_tpe=args.tpe, + use_overintegration=args.overintegration or args.esdg, + casename=casename, rst_filename=rst_filename) + +# vim: foldmethod=marker diff --git a/examples/flame1d_x_050um.dat b/examples/flame1d_x_050um.dat new file mode 100644 index 000000000..8e0d09eb5 --- /dev/null +++ b/examples/flame1d_x_050um.dat @@ -0,0 +1,499 @@ +-2.059645113861788002e-01 +-2.000430594485161362e-01 +-1.942940769847659732e-01 +-1.887125406121930027e-01 +-1.832935732601804202e-01 +-1.780324399087118981e-01 +-1.729245434509754653e-01 +-1.679654206764740754e-01 +-1.631507383711329195e-01 +-1.584762895309958786e-01 +-1.539379896862026476e-01 +-1.495318733320344651e-01 +-1.452540904639100239e-01 +-1.411009032133037633e-01 +-1.370686825816471999e-01 +-1.331539052693592740e-01 +-1.293531505972350670e-01 +-1.256630975175028198e-01 +-1.220805217119375202e-01 +-1.186022927744954797e-01 +-1.152253714760080594e-01 +-1.119468071085445438e-01 +-1.087637349071236548e-01 +-1.056733735465208435e-01 +-1.026730227109841354e-01 +-9.976006073473490943e-02 +-9.693194231119196813e-02 +-9.418619626891726326e-02 +-9.152042341233987410e-02 +-8.893229442537153862e-02 +-8.641954783608189750e-02 +-8.397998804065506806e-02 +-8.161148338490085974e-02 +-7.931196430164434708e-02 +-7.707942150236618550e-02 +-7.491190422151360051e-02 +-7.280751851194798563e-02 +-7.076442559003962018e-02 +-6.878084022896353844e-02 +-6.685502919879258987e-02 +-6.498530975202468363e-02 +-6.317004815322088684e-02 +-6.140765825146962675e-02 +-5.969660009442957077e-02 +-5.803537858274019495e-02 +-5.642254216362429381e-02 +-5.485668156254089323e-02 +-5.333642855178030717e-02 +-5.186045475492537249e-02 +-5.042747048613417660e-02 +-4.903622362323010325e-02 +-4.768549851361449793e-02 +-4.637411491204594671e-02 +-4.510092694935803614e-02 +-4.386482213121443291e-02 +-4.266472036602646928e-02 +-4.149957302118378877e-02 +-4.036836200677342001e-02 +-3.927009888598665538e-02 +-3.820382401143639778e-02 +-3.716860568663032244e-02 +-3.616353935186714108e-02 +-3.518774679384463477e-02 +-3.424037537828880651e-02 +-3.332059730493363431e-02 +-3.242760888420045950e-02 +-3.156062983494495322e-02 +-3.071890260265805472e-02 +-2.990169169752514636e-02 +-2.910828305176504233e-02 +-2.833798339568727176e-02 +-2.759011965192244634e-02 +-2.686403834729640325e-02 +-2.615910504183422552e-02 +-2.547470377439521941e-02 +-2.481023652445443761e-02 +-2.416512268956047538e-02 +-2.353879857801293926e-02 +-2.293071691631630091e-02 +-2.234034637097975784e-02 +-2.176717108424525149e-02 +-2.121069022333796350e-02 +-2.067041754284545257e-02 +-2.014588095984301552e-02 +-1.963662214139404866e-02 +-1.914219610406495478e-02 +-1.866217082510467085e-02 +-1.819612686494905679e-02 +-1.774365700072030524e-02 +-1.730436587040112828e-02 +-1.687786962737279975e-02 +-1.646379560501520095e-02 +-1.606178199107578461e-02 +-1.567147751152295210e-02 +-1.529254112360758054e-02 +-1.492464171786450113e-02 +-1.456745782879355008e-02 +-1.422067735396738419e-02 +-1.388399728132062166e-02 +-1.355712342438201642e-02 +-1.323977016521832133e-02 +-1.293166020486521961e-02 +-1.263252432102725600e-02 +-1.234210113283505834e-02 +-1.206013687245428324e-02 +-1.178638516334673536e-02 +-1.152060680498989340e-02 +-1.126256956386674501e-02 +-1.101204797054330074e-02 +-1.076882312265646166e-02 +-1.053268249364011297e-02 +-1.030341974702229935e-02 +-1.008083455613121779e-02 +-9.864732429052497711e-03 +-9.654924538684809057e-03 +-9.451227557745306082e-03 +-9.253463498580740895e-03 +-9.061459557644270177e-03 +-8.875047964502065512e-03 +-8.694065835237789475e-03 +-8.518355030126842017e-03 +-8.347762015456020035e-03 +-8.182137729367842949e-03 +-8.021337451612330777e-03 +-7.865220677092416013e-03 +-7.713650993092499575e-03 +-7.566495960082871182e-03 +-7.423626995995853510e-03 +-7.284919263872535744e-03 +-7.150251562781936333e-03 +-7.019506221917277099e-03 +-6.892568997776831602e-03 +-6.769328974339505582e-03 +-6.649678466147926695e-03 +-6.533512924214355221e-03 +-6.420730844667198525e-03 +-6.311233680058308572e-03 +-6.204925753253561112e-03 +-6.101714173831476393e-03 +-6.001508756916831312e-03 +-5.904221944378340581e-03 +-5.809768728321553118e-03 +-5.718066576810109172e-03 +-5.629035361750455227e-03 +-5.542597288877004584e-03 +-5.458676829776566881e-03 +-5.377200655892647001e-03 +-5.298097574451947692e-03 +-5.221298466257093783e-03 +-5.146736225291216084e-03 +-5.074345700081626422e-03 +-5.004063636771345289e-03 +-4.935828623848742373e-03 +-4.869581038486991802e-03 +-4.805262994446456853e-03 +-4.742818291494481538e-03 +-4.682192366298389141e-03 +-4.623332244748784570e-03 +-4.566178026132012083e-03 +-4.510631069006149942e-03 +-4.456538895530216295e-03 +-4.403699218111612607e-03 +-4.351875361754715434e-03 +-4.300817375196196342e-03 +-4.250284855199419967e-03 +-4.200068512177228090e-03 +-4.150007631655095244e-03 +-4.100000000000002948e-03 +-4.050000000000003250e-03 +-4.000000000000003553e-03 +-3.950000000000003855e-03 +-3.900000000000003724e-03 +-3.850000000000003593e-03 +-3.800000000000003462e-03 +-3.750000000000003331e-03 +-3.700000000000003200e-03 +-3.650000000000003068e-03 +-3.600000000000002937e-03 +-3.550000000000002806e-03 +-3.500000000000002675e-03 +-3.450000000000002544e-03 +-3.400000000000002413e-03 +-3.350000000000002282e-03 +-3.300000000000002150e-03 +-3.250000000000002019e-03 +-3.200000000000001888e-03 +-3.150000000000001757e-03 +-3.100000000000001626e-03 +-3.050000000000001495e-03 +-3.000000000000001363e-03 +-2.950000000000001232e-03 +-2.900000000000001101e-03 +-2.850000000000000970e-03 +-2.800000000000000839e-03 +-2.750000000000000708e-03 +-2.700000000000000577e-03 +-2.650000000000000445e-03 +-2.600000000000000314e-03 +-2.550000000000000183e-03 +-2.500000000000000052e-03 +-2.449999999999999921e-03 +-2.399999999999999790e-03 +-2.349999999999999659e-03 +-2.299999999999999527e-03 +-2.249999999999999396e-03 +-2.199999999999999265e-03 +-2.149999999999999134e-03 +-2.099999999999999003e-03 +-2.049999999999998872e-03 +-1.999999999999998741e-03 +-1.949999999999998609e-03 +-1.899999999999998695e-03 +-1.849999999999998781e-03 +-1.799999999999998867e-03 +-1.749999999999998952e-03 +-1.699999999999999038e-03 +-1.649999999999999124e-03 +-1.599999999999999209e-03 +-1.549999999999999295e-03 +-1.499999999999999381e-03 +-1.449999999999999466e-03 +-1.399999999999999552e-03 +-1.349999999999999638e-03 +-1.299999999999999723e-03 +-1.249999999999999809e-03 +-1.199999999999999895e-03 +-1.149999999999999981e-03 +-1.100000000000000066e-03 +-1.050000000000000152e-03 +-1.000000000000000238e-03 +-9.500000000000003234e-04 +-9.000000000000003006e-04 +-8.500000000000002779e-04 +-8.000000000000002552e-04 +-7.500000000000002325e-04 +-7.000000000000002097e-04 +-6.500000000000001870e-04 +-6.000000000000001643e-04 +-5.500000000000001416e-04 +-5.000000000000001188e-04 +-4.500000000000000961e-04 +-4.000000000000000734e-04 +-3.500000000000000507e-04 +-3.000000000000000279e-04 +-2.500000000000000052e-04 +-2.000000000000000096e-04 +-1.500000000000000140e-04 +-1.000000000000000048e-04 +-5.000000000000000240e-05 +-0.000000000000000000e+00 +5.000000000000000240e-05 +1.000000000000000048e-04 +1.500000000000000140e-04 +2.000000000000000096e-04 +2.500000000000000052e-04 +3.000000000000000279e-04 +3.500000000000000507e-04 +4.000000000000000734e-04 +4.500000000000000961e-04 +5.000000000000001188e-04 +5.500000000000001416e-04 +6.000000000000001643e-04 +6.500000000000001870e-04 +7.000000000000002097e-04 +7.500000000000002325e-04 +8.000000000000002552e-04 +8.500000000000002779e-04 +9.000000000000003006e-04 +9.500000000000003234e-04 +1.000000000000000238e-03 +1.050000000000000152e-03 +1.100000000000000066e-03 +1.149999999999999981e-03 +1.199999999999999895e-03 +1.249999999999999809e-03 +1.299999999999999723e-03 +1.349999999999999638e-03 +1.399999999999999552e-03 +1.449999999999999466e-03 +1.499999999999999381e-03 +1.549999999999999295e-03 +1.599999999999999209e-03 +1.649999999999999124e-03 +1.699999999999999038e-03 +1.749999999999998952e-03 +1.799999999999998867e-03 +1.849999999999998781e-03 +1.899999999999998695e-03 +1.949999999999998609e-03 +1.999999999999998741e-03 +2.049999999999998872e-03 +2.099999999999999003e-03 +2.149999999999999134e-03 +2.199999999999999265e-03 +2.249999999999999396e-03 +2.299999999999999527e-03 +2.349999999999999659e-03 +2.399999999999999790e-03 +2.449999999999999921e-03 +2.500000000000000052e-03 +2.550000000000000183e-03 +2.600000000000000314e-03 +2.650000000000000445e-03 +2.700000000000000577e-03 +2.750000000000000708e-03 +2.800000000000000839e-03 +2.850000000000000970e-03 +2.900000000000001101e-03 +2.950000000000001232e-03 +3.000000000000001363e-03 +3.050000000000001495e-03 +3.100000000000001626e-03 +3.150000000000001757e-03 +3.200000000000001888e-03 +3.250000000000002019e-03 +3.300000000000002150e-03 +3.350000000000002282e-03 +3.400000000000002413e-03 +3.450000000000002544e-03 +3.500000000000002675e-03 +3.550000000000002806e-03 +3.600000000000002937e-03 +3.650000000000003068e-03 +3.700000000000003200e-03 +3.750000000000003331e-03 +3.800000000000003462e-03 +3.850000000000003593e-03 +3.900000000000003724e-03 +3.950000000000003855e-03 +4.000000000000003553e-03 +4.050000000000003250e-03 +4.100000000000002948e-03 +4.150007631655095244e-03 +4.200068512177228090e-03 +4.250284855199419967e-03 +4.300817375196196342e-03 +4.351875361754715434e-03 +4.403699218111612607e-03 +4.456538895530216295e-03 +4.510631069006149942e-03 +4.566178026132012083e-03 +4.623332244748784570e-03 +4.682192366298389141e-03 +4.742818291494481538e-03 +4.805262994446456853e-03 +4.869581038486991802e-03 +4.935828623848742373e-03 +5.004063636771345289e-03 +5.074345700081626422e-03 +5.146736225291216084e-03 +5.221298466257093783e-03 +5.298097574451947692e-03 +5.377200655892647001e-03 +5.458676829776566881e-03 +5.542597288877004584e-03 +5.629035361750455227e-03 +5.718066576810109172e-03 +5.809768728321553118e-03 +5.904221944378340581e-03 +6.001508756916831312e-03 +6.101714173831476393e-03 +6.204925753253561112e-03 +6.311233680058308572e-03 +6.420730844667198525e-03 +6.533512924214355221e-03 +6.649678466147926695e-03 +6.769328974339505582e-03 +6.892568997776831602e-03 +7.019506221917277099e-03 +7.150251562781936333e-03 +7.284919263872535744e-03 +7.423626995995853510e-03 +7.566495960082871182e-03 +7.713650993092499575e-03 +7.865220677092416013e-03 +8.021337451612330777e-03 +8.182137729367842949e-03 +8.347762015456020035e-03 +8.518355030126842017e-03 +8.694065835237789475e-03 +8.875047964502065512e-03 +9.061459557644270177e-03 +9.253463498580740895e-03 +9.451227557745306082e-03 +9.654924538684809057e-03 +9.864732429052497711e-03 +1.008083455613121779e-02 +1.030341974702229935e-02 +1.053268249364011297e-02 +1.076882312265646166e-02 +1.101204797054330074e-02 +1.126256956386674501e-02 +1.152060680498989340e-02 +1.178638516334673536e-02 +1.206013687245428324e-02 +1.234210113283505834e-02 +1.263252432102725600e-02 +1.293166020486521961e-02 +1.323977016521832133e-02 +1.355712342438201642e-02 +1.388399728132062166e-02 +1.422067735396738419e-02 +1.456745782879355008e-02 +1.492464171786450113e-02 +1.529254112360758054e-02 +1.567147751152295210e-02 +1.606178199107578461e-02 +1.646379560501520095e-02 +1.687786962737279975e-02 +1.730436587040112828e-02 +1.774365700072030524e-02 +1.819612686494905679e-02 +1.866217082510467085e-02 +1.914219610406495478e-02 +1.963662214139404866e-02 +2.014588095984301552e-02 +2.067041754284545257e-02 +2.121069022333796350e-02 +2.176717108424525149e-02 +2.234034637097975784e-02 +2.293071691631630091e-02 +2.353879857801293926e-02 +2.416512268956047538e-02 +2.481023652445443761e-02 +2.547470377439521941e-02 +2.615910504183422552e-02 +2.686403834729640325e-02 +2.759011965192244634e-02 +2.833798339568727176e-02 +2.910828305176504233e-02 +2.990169169752514636e-02 +3.071890260265805472e-02 +3.156062983494495322e-02 +3.242760888420045950e-02 +3.332059730493363431e-02 +3.424037537828880651e-02 +3.518774679384463477e-02 +3.616353935186714108e-02 +3.716860568663032244e-02 +3.820382401143639778e-02 +3.927009888598665538e-02 +4.036836200677342001e-02 +4.149957302118378877e-02 +4.266472036602646928e-02 +4.386482213121443291e-02 +4.510092694935803614e-02 +4.637411491204594671e-02 +4.768549851361449793e-02 +4.903622362323010325e-02 +5.042747048613417660e-02 +5.186045475492537249e-02 +5.333642855178030717e-02 +5.485668156254089323e-02 +5.642254216362429381e-02 +5.803537858274019495e-02 +5.969660009442957077e-02 +6.140765825146962675e-02 +6.317004815322088684e-02 +6.498530975202468363e-02 +6.685502919879258987e-02 +6.878084022896353844e-02 +7.076442559003962018e-02 +7.280751851194798563e-02 +7.491190422151360051e-02 +7.707942150236618550e-02 +7.931196430164434708e-02 +8.161148338490085974e-02 +8.397998804065506806e-02 +8.641954783608189750e-02 +8.893229442537153862e-02 +9.152042341233987410e-02 +9.418619626891726326e-02 +9.693194231119196813e-02 +9.976006073473490943e-02 +1.026730227109841354e-01 +1.056733735465208435e-01 +1.087637349071236548e-01 +1.119468071085445438e-01 +1.152253714760080594e-01 +1.186022927744954797e-01 +1.220805217119375202e-01 +1.256630975175028198e-01 +1.293531505972350670e-01 +1.331539052693592740e-01 +1.370686825816471999e-01 +1.411009032133037633e-01 +1.452540904639100239e-01 +1.495318733320344651e-01 +1.539379896862026476e-01 +1.584762895309958786e-01 +1.631507383711329195e-01 +1.679654206764740754e-01 +1.729245434509754653e-01 +1.780324399087118981e-01 +1.832935732601804202e-01 +1.887125406121930027e-01 +1.942940769847659732e-01 +2.000430594485161362e-01 +2.059645113861788002e-01 diff --git a/examples/flame1d_y_050um.dat b/examples/flame1d_y_050um.dat new file mode 100644 index 000000000..df6d1f5d0 --- /dev/null +++ b/examples/flame1d_y_050um.dat @@ -0,0 +1,3 @@ +0.000000000000000000e+00 +5.000000000000000240e-05 +1.000000000000000698e-04 diff --git a/examples/gas-in-annular-cylinder.py b/examples/gas-in-annular-cylinder.py new file mode 100644 index 000000000..19d41dd2c --- /dev/null +++ b/examples/gas-in-annular-cylinder.py @@ -0,0 +1,914 @@ +"""Demonstrate a generic gas example.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import logging +import argparse +import numpy as np +from functools import partial + +from meshmode.mesh import BTAG_ALL +from grudge.shortcuts import make_visualizer +from grudge.dof_desc import DISCR_TAG_QUAD, BoundaryDomainTag +import grudge.op as op + +from logpyle import IntervalTimer, set_dt +from pytools.obj_array import make_obj_array +from mirgecom.mpi import mpi_entry_point +from mirgecom.discretization import create_discretization_collection +from mirgecom.euler import euler_operator +from mirgecom.navierstokes import ns_operator +from mirgecom.simutil import ( + get_sim_timestep, + distribute_mesh +) +from mirgecom.utils import force_evaluation +from mirgecom.io import make_init_message + +from mirgecom.integrators import rk4_step, euler_step +from mirgecom.steppers import advance_state +from mirgecom.boundary import ( + AdiabaticSlipBoundary, + IsothermalWallBoundary, + FarfieldBoundary +) +from mirgecom.initializers import ( + Uniform, + AcousticPulse +) +from mirgecom.eos import ( + IdealSingleGas, + PyrometheusMixture +) +from mirgecom.gas_model import ( + GasModel, + make_fluid_state +) +from mirgecom.inviscid import ( + inviscid_facial_flux_rusanov, + inviscid_facial_flux_central, +) +from mirgecom.transport import ( + SimpleTransport, + MixtureAveragedTransport, + PowerLawTransport, + ArtificialViscosityTransportDiv, + ArtificialViscosityTransportDiv2, + ArtificialViscosityTransportDiv3 +) +from mirgecom.limiter import bound_preserving_limiter +from mirgecom.fluid import make_conserved +from mirgecom.logging_quantities import ( + initialize_logmgr, + # logmgr_add_many_discretization_quantities, + logmgr_add_cl_device_info, + logmgr_add_device_memory_usage +) +import cantera + +logger = logging.getLogger(__name__) + + +class MyRuntimeError(RuntimeError): + """Simple exception to kill the simulation.""" + + pass + + +@mpi_entry_point +def main(actx_class, use_esdg=False, use_tpe=False, + use_overintegration=False, use_leap=False, + casename=None, rst_filename=None, dim=None, + periodic_mesh=False, multiple_boundaries=False, + use_navierstokes=False, use_mixture=False, + use_reactions=False, newton_iters=3, + mech_name="uiuc_7sp", transport_type=0, + use_av=0, use_limiter=False, order=1, + nscale=1, npassive_species=0, map_mesh=False, + rotation_angle=0, add_pulse=False, nsteps=20, + mesh_filename=None, euler_timestepping=False, + geometry_name=None, init_name=None, velocity_field=None, + density_field=None, inviscid_flux=None, + quad_order=None): + """Drive the example.""" + if casename is None: + casename = "gas-in-box" + if geometry_name is None: + geometry_name = "box" + if init_name is None: + init_name = "quiescent" + if inviscid_flux is None: + inviscid_flux = "rusanov" + if quad_order is None: + quad_order = order if use_tpe else order + 3 + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + num_parts = comm.Get_size() + + from mirgecom.simutil import global_reduce as _global_reduce + global_reduce = partial(_global_reduce, comm=comm) + + logmgr = initialize_logmgr(True, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + from mirgecom.array_context import initialize_actx, actx_class_is_profiling + actx = initialize_actx(actx_class, comm, + use_axis_tag_inference_fallback=use_tpe, + use_einsum_inference_fallback=use_tpe) + queue = getattr(actx, "queue", None) + use_profiling = actx_class_is_profiling(actx_class) + + # timestepping control + current_step = 0 + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = euler_step if euler_timestepping else rk4_step + + current_cfl = 1.0 + current_dt = 1e-6 + t_final = current_dt * nsteps + current_t = 0 + constant_cfl = False + temperature_tolerance = 1e-2 + + # some i/o frequencies + nstatus = 1 + nrestart = 100 + nviz = 100 + nhealth = 100 + + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_filename) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["num_parts"] == num_parts + else: # generate the grid from scratch + generate_mesh = None + if mesh_filename is not None: + from meshmode.mesh.io import read_gmsh + mesh_construction_kwargs = { + "force_positive_orientation": True, + "skip_tests": False + } + if dim is None or (dim == 3): + generate_mesh = partial( + read_gmsh, filename=mesh_filename, + mesh_construction_kwargs=mesh_construction_kwargs + ) + else: + generate_mesh = partial( + read_gmsh, filename=mesh_filename, + mesh_construction_kwargs=mesh_construction_kwargs, + force_ambient_dim=dim + ) + else: + if dim is None: + dim = 2 + nscale = max(nscale, 1) + scale_fac = pow(float(nscale), 1.0/dim) + nel_1d = int(scale_fac*24/dim) + from mirgecom.simutil import get_box_mesh + from meshmode.mesh import TensorProductElementGroup + from meshmode.mesh.generation import generate_annular_cylinder_mesh + group_cls = TensorProductElementGroup if use_tpe else None + box_ll = -1 + box_ur = 1 + r0 = .333 + r1 = 8.0 + center = make_obj_array([0, 0, 0]) + if geometry_name == "box": + generate_mesh = partial( + get_box_mesh, dim=dim, a=(box_ll,)*dim, b=(box_ur,)*dim, + n=(nel_1d,)*dim, periodic=(periodic_mesh,)*dim, + tensor_product_elements=use_tpe) + elif geometry_name == "annulus": + generate_mesh = partial( + generate_annular_cylinder_mesh, inner_radius=r0, n=12, + outer_radius=r1, nelements_per_axis=(100, 200, 3), periodic=True, + group_cls=group_cls, center=center, dim=dim) + + local_mesh, global_nelements = distribute_mesh(comm, generate_mesh) + local_nelements = local_mesh.nelements + + if dim is None: + dim = local_mesh.ambient_dim + + def add_wonk(x: np.ndarray) -> np.ndarray: + wonk_field = np.empty_like(x) + if len(x) >= 2: + wonk_field[0] = ( + 1.5*x[0] + np.cos(x[0]) + + 0.1*np.sin(10*x[1])) + wonk_field[1] = ( + 0.05*np.cos(10*x[0]) + + 1.3*x[1] + np.sin(x[1])) + else: + wonk_field[0] = 1.5*x[0] + np.cos(x[0]) + + if len(x) >= 3: + wonk_field[2] = x[2] + np.sin(x[0] / 2) / 2 + return wonk_field + + if map_mesh: + from meshmode.mesh.processing import map_mesh + local_mesh = map_mesh(local_mesh, add_wonk) + + if abs(rotation_angle) > 0: + from meshmode.mesh.processing import rotate_mesh_around_axis + theta = rotation_angle/180.0 * np.pi + local_mesh = rotate_mesh_around_axis(local_mesh, theta=theta) + + print(f"Creating discretization collection: {use_tpe=}{order=}{quad_order=}") + dcoll = create_discretization_collection(actx, local_mesh, order=order, + quadrature_order=quad_order) + nodes = actx.thaw(dcoll.nodes()) + ones = dcoll.zeros(actx) + 1. + + quadrature_tag = DISCR_TAG_QUAD if use_overintegration else None + + vis_timer = None + + if logmgr: + logmgr_add_cl_device_info(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + + rho_init = 1.2039086127319172 + velocity_init = 0*nodes + vinit = make_obj_array([0, 0, 0])[:dim] + + if velocity_field == "rotational": + vmax = 280.0 + velocity_init[0] = -vmax*nodes[1]/r1 + velocity_init[1] = vmax*nodes[0]/r1 + elif geometry_name == "annulus": + vmax = 65.0 + vinit = make_obj_array([vmax, 0]) + velocity_init = 0*nodes + vinit + + if density_field == "gaussian": + r2 = np.dot(nodes, nodes)/(r1*r1) + alpha = np.log(1e-5) # make it 1e-5 at the boundary + rho_init = rho_init*actx.np.exp(-alpha*r2) + + species_diffusivity = None + speedup_factor = 1.0 + pyro_mechanism = None + + if use_mixture: + # {{{ Set up initial state using Cantera + + # Use Cantera for initialization + # -- Pick up the input data for the thermochemistry mechanism + # --- Note: Users may add their own mechanism input file by dropping it into + # --- mirgecom/mechanisms alongside the other mech input files. + from mirgecom.mechanisms import get_mechanism_input + mech_input = get_mechanism_input(mech_name) + + cantera_soln = cantera.Solution(name="gas", yaml=mech_input) + nspecies = cantera_soln.n_species + + species_diffusivity = 1e-5 * np.ones(nspecies) + # Initial temperature, pressure, and mixutre mole fractions are needed to + # set up the initial state in Cantera. + temperature_seed = 1200.0 # Initial temperature hot enough to burn + + # Parameters for calculating the amounts of fuel, oxidizer, and inert species + # which directly sets the species fractions inside cantera + cantera_soln.set_equivalence_ratio(phi=1.0, fuel="C2H4:1", + oxidizer={"O2": 1.0, "N2": 3.76}) + x = cantera_soln.X + + one_atm = cantera.one_atm # pylint: disable=no-member + + # Let the user know about how Cantera is being initilized + print(f"Input state (T,P,X) = ({temperature_seed}, {one_atm}, {x}") + # Set Cantera internal gas temperature, pressure, and mole fractios + cantera_soln.TP = temperature_seed, one_atm + # Pull temperature, total density, mass fractions, and pressure + # from Cantera. We need total density, and mass fractions to initialize + # the fluid/gas state. + can_t, can_rho, can_y = cantera_soln.TDY + can_p = cantera_soln.P + # *can_t*, *can_p* should not differ (significantly) from user's + # initial data, but we want to ensure that we use exactly the same + # starting point as Cantera, so we use Cantera's version of these data. + + # }}} + + # {{{ Create Pyrometheus thermochemistry object & EOS + + # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses + # Cantera and generates a set of methods to calculate chemothermomechanical + # properties and states for this particular mechanism. + from mirgecom.thermochemistry import \ + get_pyrometheus_wrapper_class_from_cantera + pyro_mechanism = \ + get_pyrometheus_wrapper_class_from_cantera( + cantera_soln, temperature_niter=newton_iters)(actx.np) + eos = PyrometheusMixture(pyro_mechanism, temperature_guess=temperature_seed) + initializer = Uniform(dim=dim, pressure=can_p, temperature=can_t, + species_mass_fractions=can_y, velocity=velocity_init) + init_t = can_t + init_p = can_p + else: + use_reactions = False + eos = IdealSingleGas(gamma=1.4) + species_y = None + if npassive_species > 0: + print(f"Initializing with {npassive_species} passive species.") + nspecies = npassive_species + spec_diff = 1e-4 + species_y = np.array([1./float(nspecies) for _ in range(nspecies)]) + species_diffusivity = np.array([spec_diff * 1./float(j+1) + for j in range(nspecies)]) + + init_p = 101325 + initializer = Uniform(velocity=velocity_init, pressure=init_p, + rho=rho_init, + species_mass_fractions=species_y) + init_t = 293.15 + temperature_seed = init_t * ones + temperature_seed = force_evaluation(actx, temperature_seed) + + wall_bc = IsothermalWallBoundary(wall_temperature=init_t) \ + if use_navierstokes else AdiabaticSlipBoundary() + farfield_bc = FarfieldBoundary(free_stream_velocity=vinit, + free_stream_temperature=init_t, + free_stream_pressure=init_p) + + # initialize parameters for transport model + transport = None + thermal_conductivity = 1e-5 + viscosity = 1.0e-5 + transport_alpha = 0.6 + transport_beta = 4.093e-7 + transport_sigma = 2.0 + transport_n = 0.666 + + av2_mu0 = 0.1 + av2_beta0 = 6.0 + av2_kappa0 = 1.0 + av2_d0 = 0.1 + av2_prandtl0 = 0.9 + # av2_mu_s0 = 0. + # av2_kappa_s0 = 0. + # av2_beta_s0 = .01 + # av2_d_s0 = 0. + + # use_av=1 specific parameters + # flow stagnation temperature + static_temp = 2076.43 + # steepness of the smoothed function + theta_sc = 100 + # cutoff, smoothness below this value is ignored + beta_sc = 0.01 + gamma_sc = 1.5 + alpha_sc = 0.3 + kappa_sc = 0.5 + s0_sc = np.log10(1.0e-4 / np.power(order, 4)) + + smoothness_alpha = 0.1 + smoothness_tau = .01 + physical_transport_model = None + + if use_navierstokes: + if transport_type == 2: + if not use_mixture: + error_message = "Invalid transport_type "\ + "{} for single gas.".format(transport_type) + raise RuntimeError(error_message) + if rank == 0: + print("Pyrometheus transport model:") + print("\t temperature/mass fraction dependence") + physical_transport_model = \ + MixtureAveragedTransport(pyro_mechanism, + factor=speedup_factor) + elif transport_type == 0: + if rank == 0: + print("Simple transport model:") + print("\tconstant viscosity, species diffusivity") + print(f"\tmu = {viscosity}") + print(f"\tkappa = {thermal_conductivity}") + print(f"\tspecies diffusivity = {species_diffusivity}") + physical_transport_model = SimpleTransport( + viscosity=viscosity, thermal_conductivity=thermal_conductivity, + species_diffusivity=species_diffusivity) + elif transport_type == 1: + if rank == 0: + print("Power law transport model:") + print("\ttemperature dependent viscosity, species diffusivity") + print(f"\ttransport_alpha = {transport_alpha}") + print(f"\ttransport_beta = {transport_beta}") + print(f"\ttransport_sigma = {transport_sigma}") + print(f"\ttransport_n = {transport_n}") + print(f"\tspecies diffusivity = {species_diffusivity}") + physical_transport_model = PowerLawTransport( + alpha=transport_alpha, beta=transport_beta, + sigma=transport_sigma, n=transport_n, + species_diffusivity=species_diffusivity) + else: + error_message = "Unknown transport_type {}".format(transport_type) + raise RuntimeError(error_message) + + transport = physical_transport_model + if use_av == 1: + transport = ArtificialViscosityTransportDiv( + physical_transport=physical_transport_model, + av_mu=alpha_sc, av_prandtl=0.75) + elif use_av == 2: + transport = ArtificialViscosityTransportDiv2( + physical_transport=physical_transport_model, + av_mu=av2_mu0, av_beta=av2_beta0, av_kappa=av2_kappa0, + av_prandtl=av2_prandtl0) + elif use_av == 3: + transport = ArtificialViscosityTransportDiv3( + physical_transport=physical_transport_model, + av_mu=av2_mu0, av_beta=av2_beta0, + av_kappa=av2_kappa0, av_d=av2_d0, + av_prandtl=av2_prandtl0) + + if rank == 0 and use_navierstokes and use_av > 0: + print(f"Shock capturing parameters: alpha {alpha_sc}, " + f"s0 {s0_sc}, kappa {kappa_sc}") + print(f"Artificial viscosity {smoothness_alpha=}") + print(f"Artificial viscosity {smoothness_tau=}") + + if use_av == 1: + print("Artificial viscosity using modified physical viscosity") + print("Using velocity divergence indicator") + print(f"Shock capturing parameters: alpha {alpha_sc}, " + f"gamma_sc {gamma_sc}" + f"theta_sc {theta_sc}, beta_sc {beta_sc}, Pr 0.75, " + f"stagnation temperature {static_temp}") + elif use_av == 2: + print("Artificial viscosity using modified transport properties") + print("\t mu, beta, kappa") + # MJA update this + print(f"Shock capturing parameters:" + f"\n\tav_mu {av2_mu0}" + f"\n\tav_beta {av2_beta0}" + f"\n\tav_kappa {av2_kappa0}" + f"\n\tav_prantdl {av2_prandtl0}" + f"\nstagnation temperature {static_temp}") + elif use_av == 3: + print("Artificial viscosity using modified transport properties") + print("\t mu, beta, kappa, D") + print(f"Shock capturing parameters:" + f"\tav_mu {av2_mu0}" + f"\tav_beta {av2_beta0}" + f"\tav_kappa {av2_kappa0}" + f"\tav_d {av2_d0}" + f"\tav_prantdl {av2_prandtl0}" + f"stagnation temperature {static_temp}") + else: + error_message = "Unknown artifical viscosity model {}".format(use_av) + raise RuntimeError(error_message) + + inviscid_flux_func = inviscid_facial_flux_rusanov + if inviscid_flux == "central": + inviscid_flux_func = inviscid_facial_flux_central + gas_model = GasModel(eos=eos, transport=transport) + fluid_operator = ns_operator if use_navierstokes else euler_operator + orig = np.zeros(shape=(dim,)) + uniform_cv = initializer(nodes, eos=eos) + + def mixture_mass_fraction_limiter(cv, temperature_seed, gas_model, dd=None): + + temperature = gas_model.eos.temperature( + cv=cv, temperature_seed=temperature_seed) + pressure = gas_model.eos.pressure( + cv=cv, temperature=temperature) + + # limit species + spec_lim = make_obj_array([ + bound_preserving_limiter(dcoll, cv.species_mass_fractions[i], + mmin=0.0, mmax=1.0, modify_average=True, + dd=dd) + for i in range(nspecies) + ]) + + # normalize to ensure sum_Yi = 1.0 + aux = cv.mass*0.0 + for i in range(0, nspecies): + aux = aux + spec_lim[i] + spec_lim = spec_lim/aux + + # recompute density + mass_lim = gas_model.eos.get_density(pressure=pressure, + temperature=temperature, + species_mass_fractions=spec_lim) + + # recompute energy + energy_lim = mass_lim*(gas_model.eos.get_internal_energy( + temperature, species_mass_fractions=spec_lim) + + 0.5*np.dot(cv.velocity, cv.velocity) + ) + + # make a new CV with the limited variables + cv_lim = make_conserved(dim=dim, mass=mass_lim, energy=energy_lim, + momentum=mass_lim*cv.velocity, + species_mass=mass_lim*spec_lim) + + # return make_obj_array([cv_lim, pressure, temperature]) + return cv_lim + + limiter_func = mixture_mass_fraction_limiter if use_limiter else None + + def my_limiter(cv, tseed): + if limiter_func is not None: + return limiter_func(cv, tseed, gas_model=gas_model) + return cv + + limiter_compiled = actx.compile(my_limiter) + + def stepper_state_to_gas_state(stepper_state): + if use_mixture: + cv, tseed = stepper_state + return make_fluid_state(cv=cv, gas_model=gas_model, + limiter_func=limiter_func, + temperature_seed=tseed) + else: + return make_fluid_state(cv=stepper_state, gas_model=gas_model) + + def gas_rhs_to_stepper_rhs(gas_rhs, gas_temperature): + if use_mixture: + return make_obj_array([gas_rhs, 0.*gas_temperature]) + else: + return gas_rhs + + def gas_state_to_stepper_state(gas_state): + if use_mixture: + return make_obj_array([gas_state.cv, gas_state.temperature]) + else: + return gas_state.cv + + boundaries = {} + axis_names = ["r", "theta", "z"] + if periodic_mesh: + if multiple_boundaries: + for idir in range(dim): + if idir == 0: + boundaries[BoundaryDomainTag(f"+{axis_names[idir]}")] = wall_bc + boundaries[BoundaryDomainTag(f"-{axis_names[idir]}")] = wall_bc + else: + boundaries = {BTAG_ALL: wall_bc} + elif geometry_name == "annulus" and dim == 2: + boundaries[BoundaryDomainTag("-r")] = wall_bc + boundaries[BoundaryDomainTag("+r")] = farfield_bc + + def mfs(cv, tseed): + return make_fluid_state(cv, gas_model, limiter_func=limiter_func, + temperature_seed=tseed) + + mfs_compiled = actx.compile(mfs) + + def get_temperature_update(cv, temperature): + if pyro_mechanism is not None: + y = cv.species_mass_fractions + e = gas_model.eos.internal_energy(cv) / cv.mass + return pyro_mechanism.get_temperature_update_energy(e, temperature, y) + else: + return 0*temperature + + gtu_compiled = actx.compile(get_temperature_update) + + if rst_filename: + current_t = restart_data["t"] + current_step = restart_data["step"] + current_cv = restart_data["cv"] + rst_tseed = restart_data["temperature_seed"] + current_cv = force_evaluation(actx, current_cv) + current_gas_state = mfs_compiled(current_cv, rst_tseed) + else: + # Set the current state from time 0 + if add_pulse: + acoustic_pulse = AcousticPulse(dim=dim, amplitude=100., width=.1, + center=orig) + current_cv = acoustic_pulse(x_vec=nodes, cv=uniform_cv, eos=eos, + tseed=temperature_seed) + else: + current_cv = uniform_cv + current_cv = force_evaluation(actx, current_cv) + # Force to use/compile limiter so we can evaluate DAG + if limiter_func is not None: + current_cv = limiter_compiled(current_cv, temperature_seed) + + current_gas_state = mfs_compiled(current_cv, temperature_seed) + + if logmgr: + from mirgecom.logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) + + visualizer = make_visualizer(dcoll) + + initname = casename + eosname = eos.__class__.__name__ + init_message = make_init_message(dim=dim, order=order, + nelements=local_nelements, + global_nelements=global_nelements, + dt=current_dt, t_final=t_final, nstatus=nstatus, + nviz=nviz, cfl=current_cfl, + constant_cfl=constant_cfl, initname=initname, + eosname=eosname, casename=casename) + if rank == 0: + logger.info(init_message) + + def my_write_viz(step, t, gas_state): + dv = gas_state.dv + cv = gas_state.cv + gamma = eos.gamma(cv=cv, temperature=dv.temperature) + entropy = actx.np.sqrt(dv.pressure / cv.mass**gamma) + viz_fields = [("cv", cv), + ("dv", dv), + ("entropy", entropy)] + from mirgecom.simutil import write_visfile + write_visfile(dcoll, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True, vis_timer=vis_timer, + comm=comm) + + def my_write_restart(step, t, gas_state): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "cv": gas_state.cv, + "temperature_seed": gas_state.temperature, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": num_parts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(gas_state): + pressure = gas_state.pressure + temperature = gas_state.temperature + + health_error = False + from mirgecom.simutil import check_naninf_local + if check_naninf_local(dcoll, "vol", pressure): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + if check_naninf_local(dcoll, "vol", temperature): + health_error = True + logger.info(f"{rank=}: Invalid temperature data found.") + + if gas_state.is_mixture: + temper_update = gtu_compiled(gas_state.cv, gas_state.temperature) + temp_relup = temper_update / gas_state.temperature + max_temp_relup = (actx.to_numpy(op.nodal_max_loc(dcoll, "vol", + temp_relup))) + if max_temp_relup > temperature_tolerance: + health_error = True + logger.info(f"{rank=}: Temperature is not " + f"converged {max_temp_relup=}.") + + return health_error + + def my_pre_step(step, t, dt, state): + + if logmgr: + logmgr.tick_before() + + stepper_state = state + gas_state = stepper_state_to_gas_state(stepper_state) + gas_state = force_evaluation(actx, gas_state) + + try: + + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + + if do_health: + health_errors = global_reduce(my_health_check(gas_state), op="lor") + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise MyRuntimeError("Failed simulation health check.") + + if do_restart: + my_write_restart(step=step, t=t, gas_state=gas_state) + + if do_viz: + my_write_viz(step=step, t=t, gas_state=gas_state) + + except MyRuntimeError: + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, gas_state=gas_state) + my_write_restart(step=step, t=t, gas_state=gas_state) + raise + + dt = get_sim_timestep(dcoll, gas_state.cv, t, dt, current_cfl, t_final, + constant_cfl) + return gas_state_to_stepper_state(gas_state), dt + + def my_post_step(step, t, dt, state): + if logmgr: + set_dt(logmgr, dt) + logmgr.tick_after() + return state, dt + + def my_rhs(t, stepper_state): + gas_state = stepper_state_to_gas_state(stepper_state) + # gas_rhs = 0*gas_state.cv + gas_rhs = fluid_operator(dcoll, state=gas_state, time=t, + boundaries=boundaries, + inviscid_numerical_flux_func=inviscid_flux_func, + gas_model=gas_model, use_esdg=use_esdg, + quadrature_tag=quadrature_tag) + if use_reactions: + gas_rhs = \ + gas_rhs + eos.get_species_source_terms(gas_state.cv, + gas_state.temperature) + return gas_rhs_to_stepper_rhs(gas_rhs, gas_state.temperature) + + current_dt = get_sim_timestep(dcoll, current_gas_state.cv, current_t, current_dt, + current_cfl, t_final, constant_cfl) + + current_stepper_state = gas_state_to_stepper_state(current_gas_state) + current_step, current_t, current_stepper_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, dt=current_dt, + state=current_stepper_state, t=current_t, t_final=t_final) + + # Dump the final data + if rank == 0: + logger.info("Checkpointing final state ...") + final_gas_state = stepper_state_to_gas_state(current_stepper_state) + + my_write_viz(step=current_step, t=current_t, gas_state=final_gas_state) + my_write_restart(step=current_step, t=current_t, gas_state=final_gas_state) + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) + + finish_tol = 1e-16 + assert np.abs(current_t - t_final) < finish_tol + + +if __name__ == "__main__": + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + level=logging.INFO) + + example_name = "gas-in-box" + parser = argparse.ArgumentParser( + description=f"MIRGE-Com Example: {example_name}") + parser.add_argument("-a", "--artificial-viscosity", type=int, + choices=[0, 1, 2, 3], + default=0, help="use artificial viscosity") + parser.add_argument("-b", "--boundaries", action="store_true", + help="use multiple (2*ndim) boundaries") + parser.add_argument("-c", "--casename", help="casename to use for i/o") + parser.add_argument("-d", "--dimension", type=int, choices=[1, 2, 3], + help="spatial dimension of simulation") + parser.add_argument("--density-field", type=str, + help="use a named density field initialization") + parser.add_argument("-e", "--limiter", action="store_true", + help="use limiter to limit fluid state") + parser.add_argument("--esdg", action="store_true", + help="use entropy-stable dg for inviscid terms.") + parser.add_argument("--euler-timestepping", action="store_true", + help="use euler timestepping") + parser.add_argument("-f", "--flame", action="store_true", + help="use combustion chemistry") + parser.add_argument("-g", "--rotate", type=float, default=0, + help="rotate mesh by angle (degrees)") + parser.add_argument("--geometry-name", type=str, default="box", + help="preset geometry name (determines mesh shape)") + parser.add_argument("-i", "--iters", type=int, default=1, + help="number of Newton iterations for mixture temperature") + parser.add_argument("--init-name", type=str, default="quiescent", + help="solution initialization name") + parser.add_argument("--inviscid-flux", type=str, default="rusanov", + help="use named inviscid flux function") + parser.add_argument("-k", "--wonky", action="store_true", default=False, + help="make a wonky mesh (adds wonk field)") + parser.add_argument("-l", "--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("-m", "--mixture", action="store_true", + help="use gas mixture EOS") + parser.add_argument("--meshfile", type=str, + help="name of gmsh input file") + parser.add_argument("-n", "--navierstokes", action="store_true", + help="use Navier-Stokes operator", default=False) + parser.add_argument("--nsteps", type=int, default=20, + help="number of timesteps to take") + parser.add_argument("--numpy", action="store_true", + help="use numpy-based eager actx.") + parser.add_argument("-o", "--overintegration", action="store_true", + help="use overintegration in the RHS computations") + parser.add_argument("-p", "--periodic", action="store_true", + help="use periodic boundaries") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("-r", "--restart_file", help="root name of restart file") + parser.add_argument("-s", "--species", type=int, default=0, + help="number of passive species") + parser.add_argument("-t", "--tpe", action="store_true", + help="use tensor-product elements (quads/hexes)") + parser.add_argument("-u", "--pulse", action="store_true", default=False, + help="add an acoustic pulse at the origin") + parser.add_argument("--velocity-field", type=str, + help="use a named velocity field initialization.") + parser.add_argument("-w", "--weak-scale", type=int, default=1, + help="factor by which to scale the number of elements") + parser.add_argument("-x", "--transport", type=int, choices=[0, 1, 2], default=0, + help=("transport model specification\n" + + "(0)Simple\n(1)PowerLaw\n(2)Mix")) + parser.add_argument("-y", "--polynomial-order", type=int, default=1, + help="polynomal order for the discretization") + parser.add_argument("-z", "--mechanism-name", type=str, default="uiuc_7sp", + help="name of thermochemical mechanism yaml file") + args = parser.parse_args() + + from warnings import warn + from mirgecom.simutil import ApplicationOptionsError + if args.esdg: + if not args.lazy and not args.numpy: + raise ApplicationOptionsError("ESDG requires lazy or numpy context.") + if not args.overintegration: + warn("ESDG requires overintegration, enabling --overintegration.") + + from mirgecom.array_context import get_reasonable_array_context_class + actx_class = get_reasonable_array_context_class( + lazy=args.lazy, distributed=True, profiling=args.profiling, numpy=args.numpy) + + logging.basicConfig(format="%(message)s", level=logging.INFO) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(actx_class, use_esdg=args.esdg, dim=args.dimension, + use_overintegration=args.overintegration or args.esdg, + use_leap=args.leap, use_tpe=args.tpe, nsteps=args.nsteps, + casename=args.casename, rst_filename=rst_filename, + periodic_mesh=args.periodic, use_mixture=args.mixture, + multiple_boundaries=args.boundaries, + transport_type=args.transport, order=args.polynomial_order, + use_limiter=args.limiter, use_av=args.artificial_viscosity, + use_reactions=args.flame, newton_iters=args.iters, + use_navierstokes=args.navierstokes, npassive_species=args.species, + nscale=args.weak_scale, mech_name=args.mechanism_name, + map_mesh=args.wonky, rotation_angle=args.rotate, add_pulse=args.pulse, + mesh_filename=args.meshfile, euler_timestepping=args.euler_timestepping, + geometry_name=args.geometry_name, init_name=args.init_name, + velocity_field=args.velocity_field, density_field=args.density_field) + +# vim: foldmethod=marker diff --git a/examples/gas-in-box.py b/examples/gas-in-box.py index bfac79294..3392973f6 100644 --- a/examples/gas-in-box.py +++ b/examples/gas-in-box.py @@ -65,6 +65,10 @@ GasModel, make_fluid_state ) +from mirgecom.inviscid import ( + inviscid_facial_flux_rusanov, + inviscid_facial_flux_central, +) from mirgecom.transport import ( SimpleTransport, MixtureAveragedTransport, @@ -103,16 +107,26 @@ def main(actx_class, use_esdg=False, use_tpe=False, use_av=0, use_limiter=False, order=1, nscale=1, npassive_species=0, map_mesh=False, rotation_angle=0, add_pulse=False, nsteps=20, - mesh_filename=None, euler_timestepping=False): + mesh_filename=None, euler_timestepping=False, quad_order=None, + inviscid_flux=None): """Drive the example.""" if casename is None: casename = "gas-in-box" + if inviscid_flux is None: + inviscid_flux = "rusanov" + + if quad_order is None: + quad_order = 7 + # quad_order = order if use_tpe else order + 3 from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() num_parts = comm.Get_size() + inviscid_numerical_flux_func = inviscid_facial_flux_rusanov + if inviscid_flux == "central": + inviscid_numerical_flux_func = inviscid_facial_flux_central from mirgecom.simutil import global_reduce as _global_reduce global_reduce = partial(_global_reduce, comm=comm) @@ -695,10 +709,11 @@ def my_post_step(step, t, dt, state): def my_rhs(t, stepper_state): gas_state = stepper_state_to_gas_state(stepper_state) - gas_rhs = fluid_operator(dcoll, state=gas_state, time=t, - boundaries=boundaries, - gas_model=gas_model, use_esdg=use_esdg, - quadrature_tag=quadrature_tag) + gas_rhs = fluid_operator( + dcoll, state=gas_state, time=t, + boundaries=boundaries, + inviscid_numerical_flux_func=inviscid_numerical_flux_func, + gas_model=gas_model, use_esdg=use_esdg, quadrature_tag=quadrature_tag) if use_reactions: gas_rhs = \ gas_rhs + eos.get_species_source_terms(gas_state.cv, diff --git a/examples/gas-simulation.py b/examples/gas-simulation.py new file mode 100644 index 000000000..41f95f7bc --- /dev/null +++ b/examples/gas-simulation.py @@ -0,0 +1,1015 @@ +"""Demonstrate a generic gas example.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import logging +import argparse +import numpy as np +from functools import partial + +from meshmode.mesh import BTAG_ALL +from meshmode.mesh import TensorProductElementGroup +from meshmode.mesh.generation import generate_annular_cylinder_mesh +from grudge.shortcuts import make_visualizer +from grudge.dof_desc import ( + DD_VOLUME_ALL, + DISCR_TAG_BASE, + DISCR_TAG_QUAD, + BoundaryDomainTag, +) +from grudge.op import nodal_max +import grudge.op as op + +from logpyle import IntervalTimer, set_dt +from pytools.obj_array import make_obj_array +from mirgecom.mpi import mpi_entry_point +from mirgecom.discretization import create_discretization_collection +from mirgecom.euler import euler_operator +from mirgecom.navierstokes import ns_operator +from mirgecom.simutil import ( + get_sim_timestep, + distribute_mesh, + get_box_mesh, +) +from mirgecom.utils import force_evaluation +from mirgecom.integrators import rk4_step, euler_step +from mirgecom.inviscid import get_inviscid_cfl +from mirgecom.io import make_init_message +from mirgecom.steppers import advance_state +from mirgecom.boundary import ( + AdiabaticSlipBoundary, + IsothermalWallBoundary, + DummyBoundary, + PrescribedFluidBoundary +) +from mirgecom.initializers import ( + Uniform, + AcousticPulse +) +from mirgecom.eos import ( + IdealSingleGas, + PyrometheusMixture +) +from mirgecom.gas_model import ( + GasModel, + make_fluid_state +) +from mirgecom.inviscid import ( + inviscid_facial_flux_rusanov, + inviscid_facial_flux_central, +) +from mirgecom.transport import ( + SimpleTransport, + MixtureAveragedTransport, + PowerLawTransport, + ArtificialViscosityTransportDiv, + ArtificialViscosityTransportDiv2, + ArtificialViscosityTransportDiv3 +) +from mirgecom.limiter import bound_preserving_limiter +from mirgecom.fluid import make_conserved +from mirgecom.logging_quantities import ( + initialize_logmgr, + # logmgr_add_many_discretization_quantities, + logmgr_add_cl_device_info, + logmgr_add_device_memory_usage +) +import cantera + +logger = logging.getLogger(__name__) + + +class MyRuntimeError(RuntimeError): + """Simple exception to kill the simulation.""" + + pass + + +@mpi_entry_point +def main(actx_class, use_esdg=False, use_tpe=False, + use_overintegration=False, use_leap=False, + casename=None, rst_filename=None, dim=None, + periodic_mesh=False, multiple_boundaries=False, + use_navierstokes=False, use_mixture=False, + use_reactions=False, newton_iters=3, + mech_name="uiuc_7sp", transport_type=0, + use_av=0, use_limiter=False, order=1, + nscale=1, npassive_species=0, map_mesh=False, + rotation_angle=0, add_pulse=False, nsteps=20, + mesh_filename=None, euler_timestepping=False, + geometry_name=None, init_name=None, velocity_field=None, + density_field=None, inviscid_flux=None): + """Drive the example.""" + if casename is None: + casename = "gas-simulation" + if geometry_name is None: + geometry_name = "box" + if init_name is None: + init_name = "quiescent" + if inviscid_flux is None: + inviscid_flux = "rusanov" + if dim is None: + dim = 2 + if geometry_name == "annulus": + axis_names = ["r", "theta", "z"] + else: + axis_names = ["1", "2", "3"] + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + num_parts = comm.Get_size() + + from mirgecom.simutil import global_reduce as _global_reduce + global_reduce = partial(_global_reduce, comm=comm) + + logmgr = initialize_logmgr(True, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + from mirgecom.array_context import initialize_actx, actx_class_is_profiling + actx = initialize_actx(actx_class, comm, + use_axis_tag_inference_fallback=use_tpe, + use_einsum_inference_fallback=use_tpe) + queue = getattr(actx, "queue", None) + use_profiling = actx_class_is_profiling(actx_class) + + # timestepping control + current_step = 0 + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = euler_step if euler_timestepping else rk4_step + + current_cfl = 1.0 + current_dt = 1e-6 + t_final = current_dt * nsteps + current_t = 0 + constant_cfl = False + temperature_tolerance = 1e-2 + + # some i/o frequencies + nstatus = 100 + nrestart = 100 + nviz = 100 + nhealth = 1 + + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_filename) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["num_parts"] == num_parts + else: # generate the grid from scratch + generate_mesh = None + if mesh_filename is not None: + from meshmode.mesh.io import read_gmsh + mesh_construction_kwargs = { + "force_positive_orientation": True, + "skip_tests": False + } + if dim is None or (dim == 3): + generate_mesh = partial( + read_gmsh, filename=mesh_filename, + mesh_construction_kwargs=mesh_construction_kwargs + ) + else: + generate_mesh = partial( + read_gmsh, filename=mesh_filename, + mesh_construction_kwargs=mesh_construction_kwargs, + force_ambient_dim=dim + ) + else: + nscale = max(nscale, 1) + scale_fac = pow(float(nscale), 1.0/dim) + nel_1d = int(scale_fac*24/dim) + group_cls = TensorProductElementGroup if use_tpe else None + box_ll = -1 + box_ur = 1 + r0 = .333 + r1 = 1.0 + center = make_obj_array([0, 0, 0]) + if geometry_name == "box": + print(f"MESH RESOLUTION: {nel_1d=}") + generate_mesh = partial( + get_box_mesh, dim=dim, a=(box_ll,)*dim, b=(box_ur,)*dim, + n=(nel_1d,)*dim, periodic=(periodic_mesh,)*dim, + tensor_product_elements=use_tpe) + elif geometry_name == "annulus": + nel_axes = (10, 72, 3) + print(f"MESH RESOLUTION: {nel_axes=}") + generate_mesh = partial( + generate_annular_cylinder_mesh, inner_radius=r0, n=12, + outer_radius=r1, nelements_per_axis=nel_axes, periodic=True, + group_cls=group_cls, center=center) + + local_mesh, global_nelements = distribute_mesh(comm, generate_mesh) + local_nelements = local_mesh.nelements + + if dim is None: + dim = local_mesh.ambient_dim + + def add_wonk(x: np.ndarray) -> np.ndarray: + wonk_field = np.empty_like(x) + if len(x) >= 2: + wonk_field[0] = ( + 1.5*x[0] + np.cos(x[0]) + + 0.1*np.sin(10*x[1])) + wonk_field[1] = ( + 0.05*np.cos(10*x[0]) + + 1.3*x[1] + np.sin(x[1])) + else: + wonk_field[0] = 1.5*x[0] + np.cos(x[0]) + + if len(x) >= 3: + wonk_field[2] = x[2] + np.sin(x[0] / 2) / 2 + return wonk_field + + if map_mesh: + from meshmode.mesh.processing import map_mesh + local_mesh = map_mesh(local_mesh, add_wonk) + + if abs(rotation_angle) > 0: + from meshmode.mesh.processing import rotate_mesh_around_axis + theta = rotation_angle/180.0 * np.pi + local_mesh = rotate_mesh_around_axis(local_mesh, theta=theta) + + dcoll = create_discretization_collection(actx, local_mesh, order=order) + nodes = actx.thaw(dcoll.nodes()) + ones = dcoll.zeros(actx) + 1. + + quadrature_tag = DISCR_TAG_QUAD if use_overintegration else DISCR_TAG_BASE + + vis_timer = None + + if logmgr: + logmgr_add_cl_device_info(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + + rho_base = 1.2039086127319172 + rho_init = rho_base + velocity_init = 0*nodes + v_r = 50.0 + v_z = 0. + + if velocity_field == "rotational": + if rank == 0: + print("Initializing rotational velocity field.") + velocity_init[0] = -v_r*nodes[1]/r1 + velocity_init[1] = v_r*nodes[0]/r1 + if dim > 2: + velocity_init[2] = v_z + + if density_field == "gaussian": + if rank == 0: + print("Initializing Gaussian lump density.") + r2 = np.dot(nodes, nodes)/(r1*r1) + alpha = np.log(1e-5) # make it 1e-5 at the boundary + rho_init = rho_init + rho_init*actx.np.exp(alpha*r2)/5. + + species_diffusivity = None + speedup_factor = 1.0 + pyro_mechanism = None + + if use_mixture: + # {{{ Set up initial state using Cantera + if rank == 0: + print(f"Initializing for gas mixture {mech_name=}.") + + # Use Cantera for initialization + # -- Pick up the input data for the thermochemistry mechanism + # --- Note: Users may add their own mechanism input file by dropping it into + # --- mirgecom/mechanisms alongside the other mech input files. + from mirgecom.mechanisms import get_mechanism_input + mech_input = get_mechanism_input(mech_name) + + cantera_soln = cantera.Solution(name="gas", yaml=mech_input) + nspecies = cantera_soln.n_species + + species_diffusivity = 1e-5 * np.ones(nspecies) + # Initial temperature, pressure, and mixutre mole fractions are needed to + # set up the initial state in Cantera. + temperature_seed = 1200.0 # Initial temperature hot enough to burn + + # Parameters for calculating the amounts of fuel, oxidizer, and inert species + # which directly sets the species fractions inside cantera + cantera_soln.set_equivalence_ratio(phi=1.0, fuel="C2H4:1", + oxidizer={"O2": 1.0, "N2": 3.76}) + x = cantera_soln.X + + one_atm = cantera.one_atm # pylint: disable=no-member + + # Let the user know about how Cantera is being initilized + print(f"Input state (T,P,X) = ({temperature_seed}, {one_atm}, {x}") + # Set Cantera internal gas temperature, pressure, and mole fractios + cantera_soln.TP = temperature_seed, one_atm + # Pull temperature, total density, mass fractions, and pressure + # from Cantera. We need total density, and mass fractions to initialize + # the fluid/gas state. + can_t, can_rho, can_y = cantera_soln.TDY + can_p = cantera_soln.P + # *can_t*, *can_p* should not differ (significantly) from user's + # initial data, but we want to ensure that we use exactly the same + # starting point as Cantera, so we use Cantera's version of these data. + + # }}} + + # {{{ Create Pyrometheus thermochemistry object & EOS + + # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses + # Cantera and generates a set of methods to calculate chemothermomechanical + # properties and states for this particular mechanism. + from mirgecom.thermochemistry import \ + get_pyrometheus_wrapper_class_from_cantera + pyro_mechanism = \ + get_pyrometheus_wrapper_class_from_cantera( + cantera_soln, temperature_niter=newton_iters)(actx.np) + eos = PyrometheusMixture(pyro_mechanism, temperature_guess=temperature_seed) + pressure_init = can_p + temperature_init = can_t + y_init = can_y + rho_init = None + # initializer = Uniform(dim=dim, pressure=can_p, temperature=can_t, + # species_mass_fractions=can_y, velocity=velocity_init) + init_t = can_t + else: + use_reactions = False + eos = IdealSingleGas(gamma=1.4) + pressure_init = 101325 + y_init = None + init_t = 293.15 + temperature_init = None + if npassive_species > 0: + if rank == 0: + print(f"Initializing with {npassive_species} passive species.") + nspecies = npassive_species + spec_diff = 1e-4 + y_init = np.array([1./float(nspecies) for _ in range(nspecies)]) + species_diffusivity = np.array([spec_diff * 1./float(j+1) + for j in range(nspecies)]) + + temperature_seed = init_t * ones + temperature_seed = force_evaluation(actx, temperature_seed) + + initializer = Uniform(dim=dim, pressure=pressure_init, + temperature=temperature_init, rho=rho_init, + velocity=velocity_init, species_mass_fractions=y_init) + + wall_bc = IsothermalWallBoundary(wall_temperature=init_t) \ + if use_navierstokes else AdiabaticSlipBoundary() + if velocity_field == "rotational" and geometry_name == "box": + wall_bc = DummyBoundary() + + # initialize parameters for transport model + transport = None + thermal_conductivity = 1e-5 + viscosity = 1.0e-5 + transport_alpha = 0.6 + transport_beta = 4.093e-7 + transport_sigma = 2.0 + transport_n = 0.666 + + av2_mu0 = 0.1 + av2_beta0 = 6.0 + av2_kappa0 = 1.0 + av2_d0 = 0.1 + av2_prandtl0 = 0.9 + # av2_mu_s0 = 0. + # av2_kappa_s0 = 0. + # av2_beta_s0 = .01 + # av2_d_s0 = 0. + + # use_av=1 specific parameters + # flow stagnation temperature + static_temp = 2076.43 + # steepness of the smoothed function + theta_sc = 100 + # cutoff, smoothness below this value is ignored + beta_sc = 0.01 + gamma_sc = 1.5 + alpha_sc = 0.3 + kappa_sc = 0.5 + s0_sc = np.log10(1.0e-4 / np.power(order, 4)) + + smoothness_alpha = 0.1 + smoothness_tau = .01 + physical_transport_model = None + + if use_navierstokes: + if transport_type == 2: + if not use_mixture: + error_message = "Invalid transport_type "\ + "{} for single gas.".format(transport_type) + raise RuntimeError(error_message) + if rank == 0: + print("Pyrometheus transport model:") + print("\t temperature/mass fraction dependence") + physical_transport_model = \ + MixtureAveragedTransport(pyro_mechanism, + factor=speedup_factor) + elif transport_type == 0: + if rank == 0: + print("Simple transport model:") + print("\tconstant viscosity, species diffusivity") + print(f"\tmu = {viscosity}") + print(f"\tkappa = {thermal_conductivity}") + print(f"\tspecies diffusivity = {species_diffusivity}") + physical_transport_model = SimpleTransport( + viscosity=viscosity, thermal_conductivity=thermal_conductivity, + species_diffusivity=species_diffusivity) + elif transport_type == 1: + if rank == 0: + print("Power law transport model:") + print("\ttemperature dependent viscosity, species diffusivity") + print(f"\ttransport_alpha = {transport_alpha}") + print(f"\ttransport_beta = {transport_beta}") + print(f"\ttransport_sigma = {transport_sigma}") + print(f"\ttransport_n = {transport_n}") + print(f"\tspecies diffusivity = {species_diffusivity}") + physical_transport_model = PowerLawTransport( + alpha=transport_alpha, beta=transport_beta, + sigma=transport_sigma, n=transport_n, + species_diffusivity=species_diffusivity) + else: + error_message = "Unknown transport_type {}".format(transport_type) + raise RuntimeError(error_message) + + transport = physical_transport_model + if use_av == 1: + transport = ArtificialViscosityTransportDiv( + physical_transport=physical_transport_model, + av_mu=alpha_sc, av_prandtl=0.75) + elif use_av == 2: + transport = ArtificialViscosityTransportDiv2( + physical_transport=physical_transport_model, + av_mu=av2_mu0, av_beta=av2_beta0, av_kappa=av2_kappa0, + av_prandtl=av2_prandtl0) + elif use_av == 3: + transport = ArtificialViscosityTransportDiv3( + physical_transport=physical_transport_model, + av_mu=av2_mu0, av_beta=av2_beta0, + av_kappa=av2_kappa0, av_d=av2_d0, + av_prandtl=av2_prandtl0) + + if rank == 0 and use_navierstokes and use_av > 0: + print(f"Shock capturing parameters: alpha {alpha_sc}, " + f"s0 {s0_sc}, kappa {kappa_sc}") + print(f"Artificial viscosity {smoothness_alpha=}") + print(f"Artificial viscosity {smoothness_tau=}") + + if use_av == 1: + print("Artificial viscosity using modified physical viscosity") + print("Using velocity divergence indicator") + print(f"Shock capturing parameters: alpha {alpha_sc}, " + f"gamma_sc {gamma_sc}" + f"theta_sc {theta_sc}, beta_sc {beta_sc}, Pr 0.75, " + f"stagnation temperature {static_temp}") + elif use_av == 2: + print("Artificial viscosity using modified transport properties") + print("\t mu, beta, kappa") + # MJA update this + print(f"Shock capturing parameters:" + f"\n\tav_mu {av2_mu0}" + f"\n\tav_beta {av2_beta0}" + f"\n\tav_kappa {av2_kappa0}" + f"\n\tav_prantdl {av2_prandtl0}" + f"\nstagnation temperature {static_temp}") + elif use_av == 3: + print("Artificial viscosity using modified transport properties") + print("\t mu, beta, kappa, D") + print(f"Shock capturing parameters:" + f"\tav_mu {av2_mu0}" + f"\tav_beta {av2_beta0}" + f"\tav_kappa {av2_kappa0}" + f"\tav_d {av2_d0}" + f"\tav_prantdl {av2_prandtl0}" + f"stagnation temperature {static_temp}") + else: + error_message = "Unknown artifical viscosity model {}".format(use_av) + raise RuntimeError(error_message) + + inviscid_flux_func = inviscid_facial_flux_rusanov + if inviscid_flux == "central": + inviscid_flux_func = inviscid_facial_flux_central + gas_model = GasModel(eos=eos, transport=transport) + fluid_operator = ns_operator if use_navierstokes else euler_operator + orig = np.zeros(shape=(dim,)) + + def rotational_flow_init(dcoll, dd_bdry=None, gas_model=gas_model, + state_minus=None, **kwargs): + + if dd_bdry is None: + dd_bdry = DD_VOLUME_ALL + loc_discr = dcoll.discr_from_dd(dd_bdry) + loc_nodes = actx.thaw(loc_discr.nodes()) + + dim = len(loc_nodes) + velocity_init = 0*loc_nodes + + if velocity_field == "rotational": + if rank == 0: + print("Initializing rotational velocity field.") + velocity_init[0] = -v_r*loc_nodes[1]/r1 + velocity_init[1] = v_r*loc_nodes[0]/r1 + if dim > 2: + velocity_init[2] = v_z + + loc_rho_init = rho_base + if density_field == "gaussian": + if rank == 0: + print("Initializing Gaussian lump density.") + r2 = np.dot(loc_nodes, loc_nodes)/(r1*r1) + alpha = np.log(1e-5) # make it 1e-5 at the boundary + loc_rho_init = rho_base + rho_base*actx.np.exp(alpha*r2)/5. + + init_func = Uniform( + dim=dim, pressure=pressure_init, + temperature=temperature_init, rho=loc_rho_init, + velocity=velocity_init, species_mass_fractions=y_init) + loc_cv = init_func(loc_nodes, eos=eos) + + if add_pulse: + loc_orig = np.zeros(shape=(dim,)) + acoustic_pulse = AcousticPulse(dim=dim, amplitude=100., width=.1, + center=loc_orig) + loc_cv = acoustic_pulse(x_vec=loc_nodes, cv=loc_cv, eos=eos, + tseed=temperature_seed) + + # loc_limiter_func = None + return make_fluid_state(cv=loc_cv, gas_model=gas_model, + temperature_seed=temperature_seed) + + wall_state_func = rotational_flow_init + uniform_cv = initializer(nodes, eos=eos) + + if rank == 0: + if use_navierstokes: + print("Using compressible Navier-Stokes RHS operator.") + else: + print("Using Euler RHS operator.") + print(f"Using inviscid numerical flux: {inviscid_flux}") + + def mixture_mass_fraction_limiter(cv, temperature_seed, gas_model, dd=None): + + temperature = gas_model.eos.temperature( + cv=cv, temperature_seed=temperature_seed) + pressure = gas_model.eos.pressure( + cv=cv, temperature=temperature) + + # limit species + spec_lim = make_obj_array([ + bound_preserving_limiter(dcoll, cv.species_mass_fractions[i], + mmin=0.0, mmax=1.0, modify_average=True, + dd=dd) + for i in range(nspecies) + ]) + + # normalize to ensure sum_Yi = 1.0 + aux = cv.mass*0.0 + for i in range(0, nspecies): + aux = aux + spec_lim[i] + spec_lim = spec_lim/aux + + # recompute density + mass_lim = gas_model.eos.get_density(pressure=pressure, + temperature=temperature, + species_mass_fractions=spec_lim) + + # recompute energy + energy_lim = mass_lim*(gas_model.eos.get_internal_energy( + temperature, species_mass_fractions=spec_lim) + + 0.5*np.dot(cv.velocity, cv.velocity) + ) + + # make a new CV with the limited variables + cv_lim = make_conserved(dim=dim, mass=mass_lim, energy=energy_lim, + momentum=mass_lim*cv.velocity, + species_mass=mass_lim*spec_lim) + + # return make_obj_array([cv_lim, pressure, temperature]) + return cv_lim + + limiter_func = mixture_mass_fraction_limiter if use_limiter else None + + def my_limiter(cv, tseed): + if limiter_func is not None: + return limiter_func(cv, tseed, gas_model=gas_model) + return cv + + limiter_compiled = actx.compile(my_limiter) + + def stepper_state_to_gas_state(stepper_state): + if use_mixture: + cv, tseed = stepper_state + return make_fluid_state(cv=cv, gas_model=gas_model, + limiter_func=limiter_func, + temperature_seed=tseed) + else: + return make_fluid_state(cv=stepper_state, gas_model=gas_model) + + def gas_rhs_to_stepper_rhs(gas_rhs, gas_temperature): + if use_mixture: + return make_obj_array([gas_rhs, 0.*gas_temperature]) + else: + return gas_rhs + + def gas_state_to_stepper_state(gas_state): + if use_mixture: + return make_obj_array([gas_state.cv, gas_state.temperature]) + else: + return gas_state.cv + + wall_bc_type = "prescribed" + wall_bc_type = "" + boundaries = {} + if wall_bc_type == "prescribed": + wall_bc = PrescribedFluidBoundary(boundary_state_func=wall_state_func) + + if geometry_name == "annulus": # Force r-direction to be a wall + boundaries[BoundaryDomainTag(f"+{axis_names[0]}")] = wall_bc + boundaries[BoundaryDomainTag(f"-{axis_names[0]}")] = wall_bc + if not periodic_mesh and geometry_name != "annulus": + if multiple_boundaries: + for idir in range(dim): + boundaries[BoundaryDomainTag(f"+{axis_names[idir]}")] = wall_bc + boundaries[BoundaryDomainTag(f"-{axis_names[idir]}")] = wall_bc + else: + boundaries = {BTAG_ALL: wall_bc} + + def mfs(cv, tseed): + return make_fluid_state(cv, gas_model, limiter_func=limiter_func, + temperature_seed=tseed) + + mfs_compiled = actx.compile(mfs) + + def get_temperature_update(cv, temperature): + if pyro_mechanism is not None: + y = cv.species_mass_fractions + e = gas_model.eos.internal_energy(cv) / cv.mass + return pyro_mechanism.get_temperature_update_energy(e, temperature, y) + else: + return 0*temperature + + gtu_compiled = actx.compile(get_temperature_update) + + if rst_filename: + current_t = restart_data["t"] + current_step = restart_data["step"] + current_cv = restart_data["cv"] + rst_tseed = restart_data["temperature_seed"] + current_cv = force_evaluation(actx, current_cv) + current_gas_state = mfs_compiled(current_cv, rst_tseed) + else: + # Set the current state from time 0 + if add_pulse: + acoustic_pulse = AcousticPulse(dim=dim, amplitude=100., width=.1, + center=orig) + current_cv = acoustic_pulse(x_vec=nodes, cv=uniform_cv, eos=eos, + tseed=temperature_seed) + else: + current_cv = uniform_cv + current_cv = force_evaluation(actx, current_cv) + # Force to use/compile limiter so we can evaluate DAG + if limiter_func is not None: + current_cv = limiter_compiled(current_cv, temperature_seed) + + current_gas_state = mfs_compiled(current_cv, temperature_seed) + + if logmgr: + from mirgecom.logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) + + def get_simulation_cfl(gas_state, dt): + cfl = current_cfl + if not constant_cfl: + cfl = actx.to_numpy( + nodal_max(dcoll, "vol", + get_inviscid_cfl(dcoll, gas_state, dt)))[()] + return cfl + + visualizer = make_visualizer(dcoll) + + initname = "gas-simulation" + eosname = eos.__class__.__name__ + init_message = make_init_message(dim=dim, order=order, + nelements=local_nelements, + global_nelements=global_nelements, + dt=current_dt, t_final=t_final, nstatus=nstatus, + nviz=nviz, cfl=current_cfl, + constant_cfl=constant_cfl, initname=initname, + eosname=eosname, casename=casename) + if rank == 0: + logger.info(init_message) + + exact_func = None + + def my_write_viz(step, t, gas_state): + dv = gas_state.dv + cv = gas_state.cv + ones = actx.np.zeros_like(nodes[0]) + 1 + rank_field = rank*ones + viz_fields = [("cv", cv), + ("dv", dv), + ("vel", cv.velocity), + ("rank", rank_field)] + if exact_func is not None: + exact = exact_func(time=t, state=gas_state) + viz_fields.append(("exact", exact)) + + from mirgecom.simutil import write_visfile + write_visfile(dcoll, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True, vis_timer=vis_timer, + comm=comm) + + def my_write_restart(step, t, gas_state): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "cv": gas_state.cv, + "temperature_seed": gas_state.temperature, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": num_parts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(gas_state): + pressure = gas_state.pressure + temperature = gas_state.temperature + + health_error = False + from mirgecom.simutil import check_naninf_local + if check_naninf_local(dcoll, "vol", pressure): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + if check_naninf_local(dcoll, "vol", temperature): + health_error = True + logger.info(f"{rank=}: Invalid temperature data found.") + + if gas_state.is_mixture: + temper_update = gtu_compiled(gas_state.cv, gas_state.temperature) + temp_relup = temper_update / gas_state.temperature + max_temp_relup = (actx.to_numpy(op.nodal_max_loc(dcoll, "vol", + temp_relup))) + if max_temp_relup > temperature_tolerance: + health_error = True + logger.info(f"{rank=}: Temperature is not " + f"converged {max_temp_relup=}.") + + return health_error + + def my_pre_step(step, t, dt, state): + + if logmgr: + logmgr.tick_before() + + stepper_state = state + gas_state = stepper_state_to_gas_state(stepper_state) + gas_state = force_evaluation(actx, gas_state) + + try: + + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) + + if do_health: + health_errors = global_reduce(my_health_check(gas_state), op="lor") + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise MyRuntimeError("Failed simulation health check.") + + if do_restart: + my_write_restart(step=step, t=t, gas_state=gas_state) + + if do_viz: + my_write_viz(step=step, t=t, gas_state=gas_state) + + dt = get_sim_timestep(dcoll, gas_state.cv, t, dt, current_cfl, + t_final, constant_cfl) + + if do_status: + if rank == 0: + print(f"=== STATUS-Step({step}) ===") + cfl = current_cfl + if not constant_cfl: + cfl = get_simulation_cfl(gas_state, dt) + if rank == 0: + print(f"{dt=}, {cfl=}") + + except MyRuntimeError: + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, gas_state=gas_state) + my_write_restart(step=step, t=t, gas_state=gas_state) + raise + + return gas_state_to_stepper_state(gas_state), dt + + def my_post_step(step, t, dt, state): + if logmgr: + set_dt(logmgr, dt) + logmgr.tick_after() + return state, dt + + def my_rhs(t, stepper_state): + gas_state = stepper_state_to_gas_state(stepper_state) + gas_rhs = fluid_operator(dcoll, state=gas_state, time=t, + boundaries=boundaries, + inviscid_numerical_flux_func=inviscid_flux_func, + gas_model=gas_model, use_esdg=use_esdg, + quadrature_tag=quadrature_tag) + if use_reactions: + gas_rhs = \ + gas_rhs + eos.get_species_source_terms(gas_state.cv, + gas_state.temperature) + return gas_rhs_to_stepper_rhs(gas_rhs, gas_state.temperature) + + current_dt = get_sim_timestep(dcoll, current_gas_state.cv, current_t, current_dt, + current_cfl, t_final, constant_cfl) + + current_stepper_state = gas_state_to_stepper_state(current_gas_state) + current_step, current_t, current_stepper_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, dt=current_dt, + state=current_stepper_state, t=current_t, t_final=t_final) + + # Dump the final data + if rank == 0: + logger.info("Checkpointing final state ...") + final_gas_state = stepper_state_to_gas_state(current_stepper_state) + + my_write_viz(step=current_step, t=current_t, gas_state=final_gas_state) + my_write_restart(step=current_step, t=current_t, gas_state=final_gas_state) + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) + + finish_tol = 1e-16 + assert np.abs(current_t - t_final) < finish_tol + + +if __name__ == "__main__": + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + level=logging.INFO) + + example_name = "gas-simulation" + parser = argparse.ArgumentParser( + description=f"MIRGE-Com Example: {example_name}") + parser.add_argument("-a", "--artificial-viscosity", type=int, + choices=[0, 1, 2, 3], + default=0, help="use artificial viscosity") + parser.add_argument("-b", "--boundaries", action="store_true", + help="use multiple (2*ndim) boundaries") + parser.add_argument("-c", "--casename", help="casename to use for i/o") + parser.add_argument("-d", "--dimension", type=int, choices=[1, 2, 3], + help="spatial dimension of simulation") + parser.add_argument("--density-field", type=str, + help="use a named density field initialization") + parser.add_argument("-e", "--limiter", action="store_true", + help="use limiter to limit fluid state") + parser.add_argument("--esdg", action="store_true", + help="use entropy-stable dg for inviscid terms.") + parser.add_argument("--euler-timestepping", action="store_true", + help="use euler timestepping") + parser.add_argument("-f", "--flame", action="store_true", + help="use combustion chemistry") + parser.add_argument("-g", "--rotate", type=float, default=0, + help="rotate mesh by angle (degrees)") + parser.add_argument("--geometry-name", type=str, default="box", + help="preset geometry name (determines mesh shape)") + parser.add_argument("-i", "--iters", type=int, default=1, + help="number of Newton iterations for mixture temperature") + parser.add_argument("--init-name", type=str, default="quiescent", + help="solution initialization name") + parser.add_argument("--inviscid-flux", type=str, default="rusanov", + help="use named inviscid flux function") + parser.add_argument("-k", "--wonky", action="store_true", default=False, + help="make a wonky mesh (adds wonk field)") + parser.add_argument("-l", "--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("-m", "--mixture", action="store_true", + help="use gas mixture EOS") + parser.add_argument("--meshfile", type=str, + help="name of gmsh input file") + parser.add_argument("-n", "--navierstokes", action="store_true", + help="use Navier-Stokes operator", default=False) + parser.add_argument("--nsteps", type=int, default=20, + help="number of timesteps to take") + parser.add_argument("--numpy", action="store_true", + help="use numpy-based eager actx.") + parser.add_argument("-o", "--overintegration", action="store_true", + help="use overintegration in the RHS computations") + parser.add_argument("-p", "--periodic", action="store_true", + help="use periodic boundaries") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("-r", "--restart_file", help="root name of restart file") + parser.add_argument("-s", "--species", type=int, default=0, + help="number of passive species") + parser.add_argument("-t", "--tpe", action="store_true", + help="use tensor-product elements (quads/hexes)") + parser.add_argument("-u", "--pulse", action="store_true", default=False, + help="add an acoustic pulse at the origin") + parser.add_argument("--velocity-field", type=str, + help="use a named velocity field initialization.") + parser.add_argument("-w", "--weak-scale", type=int, default=1, + help="factor by which to scale the number of elements") + parser.add_argument("-x", "--transport", type=int, choices=[0, 1, 2], default=0, + help=("transport model specification\n" + + "(0)Simple\n(1)PowerLaw\n(2)Mix")) + parser.add_argument("-y", "--polynomial-order", type=int, default=1, + help="polynomal order for the discretization") + parser.add_argument("-z", "--mechanism-name", type=str, default="uiuc_7sp", + help="name of thermochemical mechanism yaml file") + args = parser.parse_args() + + from warnings import warn + from mirgecom.simutil import ApplicationOptionsError + if args.esdg: + if not args.lazy and not args.numpy: + raise ApplicationOptionsError("ESDG requires lazy or numpy context.") + if not args.overintegration: + warn("ESDG requires overintegration, enabling --overintegration.") + + from mirgecom.array_context import get_reasonable_array_context_class + actx_class = get_reasonable_array_context_class( + lazy=args.lazy, distributed=True, profiling=args.profiling, numpy=args.numpy) + + logging.basicConfig(format="%(message)s", level=logging.INFO) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(actx_class, use_esdg=args.esdg, dim=args.dimension, + use_overintegration=args.overintegration or args.esdg, + use_leap=args.leap, use_tpe=args.tpe, nsteps=args.nsteps, + casename=args.casename, rst_filename=rst_filename, + periodic_mesh=args.periodic, use_mixture=args.mixture, + multiple_boundaries=args.boundaries, + transport_type=args.transport, order=args.polynomial_order, + use_limiter=args.limiter, use_av=args.artificial_viscosity, + use_reactions=args.flame, newton_iters=args.iters, + use_navierstokes=args.navierstokes, npassive_species=args.species, + nscale=args.weak_scale, mech_name=args.mechanism_name, + map_mesh=args.wonky, rotation_angle=args.rotate, add_pulse=args.pulse, + mesh_filename=args.meshfile, euler_timestepping=args.euler_timestepping, + geometry_name=args.geometry_name, init_name=args.init_name, + velocity_field=args.velocity_field, density_field=args.density_field, + inviscid_flux=args.inviscid_flux) + +# vim: foldmethod=marker diff --git a/examples/pulse.py b/examples/pulse.py index 74b5bb6aa..8209c4d0a 100644 --- a/examples/pulse.py +++ b/examples/pulse.py @@ -74,12 +74,16 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point -def main(actx_class, use_esdg=False, - use_overintegration=False, use_leap=False, - casename=None, rst_filename=None): +def main(actx_class, use_esdg=False, use_tpe=False, periodic=False, + use_overintegration=False, use_leap=False, order=None, + casename=None, rst_filename=None, quad_order=None): """Drive the example.""" if casename is None: casename = "mirgecom" + if order is None: + order = 1 + if quad_order is None: + quad_order = order if use_tpe else 2*order + 1 from mpi4py import MPI comm = MPI.COMM_WORLD @@ -134,17 +138,21 @@ def main(actx_class, use_esdg=False, box_ll = -1 box_ur = 1 nel_1d = 16 - generate_mesh = partial(generate_regular_rect_mesh, - a=(box_ll,)*dim, b=(box_ur,)*dim, - nelements_per_axis=(nel_1d,)*dim, - # periodic=(True,)*dim - ) + from meshmode.mesh import TensorProductElementGroup + group_cls = TensorProductElementGroup if use_tpe else None + periodic_ax = (periodic,)*dim + generate_mesh = \ + partial(generate_regular_rect_mesh, + a=(box_ll,)*dim, b=(box_ur,)*dim, + nelements_per_axis=(nel_1d,)*dim, + group_cls=group_cls, + periodic=periodic_ax) local_mesh, global_nelements = distribute_mesh(comm, generate_mesh) local_nelements = local_mesh.nelements - order = 1 - dcoll = create_discretization_collection(actx, local_mesh, order=order) + dcoll = create_discretization_collection(actx, local_mesh, order=order, + quadrature_order=quad_order) nodes = actx.thaw(dcoll.nodes()) quadrature_tag = DISCR_TAG_QUAD if use_overintegration else None @@ -176,8 +184,9 @@ def main(actx_class, use_esdg=False, initializer = Uniform(velocity=velocity, pressure=1.0, rho=1.0) uniform_state = initializer(nodes, eos=eos) - boundaries = {BTAG_ALL: AdiabaticSlipBoundary()} - # boundaries = {} + boundaries = {} + if not periodic: + boundaries = {BTAG_ALL: AdiabaticSlipBoundary()} acoustic_pulse = AcousticPulse(dim=dim, amplitude=0.5, width=.1, center=orig) @@ -237,10 +246,15 @@ def my_write_restart(step, t, state): def my_health_check(pressure): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(dcoll, "vol", pressure) \ - or check_range_local(dcoll, "vol", pressure, .8, 1.6): + + if check_naninf_local(dcoll, "vol", pressure): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") + + if check_range_local(dcoll, "vol", pressure, .8, 1.6): + health_error = True + logger.info(f"{rank=}: Out of range pressure data found.") + return health_error def my_pre_step(step, t, dt, state): @@ -330,10 +344,18 @@ def my_rhs(t, state): parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") parser.add_argument("--overintegration", action="store_true", help="use overintegration in the RHS computations") + parser.add_argument("--order", type=int, + help="polynomial degree for element basis") parser.add_argument("--lazy", action="store_true", help="switch to a lazy computation mode") parser.add_argument("--profiling", action="store_true", help="turn on detailed performance profiling") + parser.add_argument("--quad-order", type=int, + help="order for QuadratureGroupFactory") + parser.add_argument("--tpe", action="store_true", + help="use tensor product elements") + parser.add_argument("--periodic", action="store_true", + help="use periodic domain") parser.add_argument("--leap", action="store_true", help="use leap timestepper") parser.add_argument("--esdg", action="store_true", @@ -363,9 +385,10 @@ def my_rhs(t, state): if args.restart_file: rst_filename = args.restart_file - main(actx_class, use_esdg=args.esdg, + main(actx_class, use_esdg=args.esdg, order=args.order, + periodic=args.periodic, use_overintegration=args.overintegration or args.esdg, - use_leap=args.leap, + use_leap=args.leap, use_tpe=args.tpe, quad_order=args.quad_order, casename=casename, rst_filename=rst_filename) # vim: foldmethod=marker diff --git a/examples/vortex.py b/examples/vortex.py index 26782c872..2e7822511 100644 --- a/examples/vortex.py +++ b/examples/vortex.py @@ -96,17 +96,19 @@ def main(actx_class, use_overintegration=False, use_esdg=False, timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step - t_final = 0.01 - current_cfl = 1.0 + + nsteps = 20 current_dt = .001 + t_final = nsteps*current_dt + current_cfl = 1.0 current_t = 0 constant_cfl = False # some i/o frequencies - nrestart = 10 - nstatus = 1 - nviz = 100 - nhealth = 10 + nrestart = 100000 + nstatus = 100 + nviz = 10 + nhealth = 100 dim = 2 if dim != 2: @@ -125,7 +127,7 @@ def main(actx_class, use_overintegration=False, use_esdg=False, global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == num_parts else: # generate the grid from scratch - nel_1d = 16 + nel_1d = 32 box_ll = -5.0 box_ur = 5.0 from meshmode.mesh.generation import generate_regular_rect_mesh @@ -135,8 +137,10 @@ def main(actx_class, use_overintegration=False, use_esdg=False, generate_mesh) local_nelements = local_mesh.nelements - order = 3 - dcoll = create_discretization_collection(actx, local_mesh, order=order) + order = 1 + oorder = 3*order + 1 + dcoll = create_discretization_collection(actx, local_mesh, order=order, + quadrature_order=oorder) nodes = actx.thaw(dcoll.nodes()) from grudge.dof_desc import DISCR_TAG_BASE, DISCR_TAG_QUAD @@ -177,7 +181,7 @@ def main(actx_class, use_overintegration=False, use_esdg=False, eos = IdealSingleGas() vel = np.zeros(shape=(dim,)) orig = np.zeros(shape=(dim,)) - vel[:dim] = 1.0 + # vel[:dim] = 0.0 initializer = Vortex2D(center=orig, velocity=vel) gas_model = GasModel(eos=eos) @@ -268,12 +272,16 @@ def my_write_restart(step, t, state): def my_health_check(pressure, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(dcoll, "vol", pressure) \ - or check_range_local(dcoll, "vol", pressure, .2, 1.02): + if check_naninf_local(dcoll, "vol", pressure): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") - exittol = .1 + if t_final < .02: + if check_range_local(dcoll, "vol", pressure, .2, 1.02): + health_error = True + logger.info(f"{rank=}: Pessure data range violation.") + + exittol = 1000.0 if max(component_errors) > exittol: health_error = True if rank == 0: diff --git a/examples/wave.py b/examples/wave.py index f6c920b83..e138a349f 100644 --- a/examples/wave.py +++ b/examples/wave.py @@ -28,7 +28,7 @@ import numpy as np from grudge.shortcuts import make_visualizer from logpyle import IntervalTimer, set_dt -from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from meshmode.mesh import TensorProductElementGroup from pytools.obj_array import flat_obj_array from mirgecom.array_context import initialize_actx @@ -64,7 +64,8 @@ def bump(actx, nodes, t=0): @mpi_entry_point def main(actx_class, casename="wave", - restart_step=None, use_logmgr: bool = False, mpi: bool = True) -> None: + restart_step=None, use_logmgr: bool = False, mpi: bool = True, + use_tpe: bool = False) -> None: """Drive the example.""" if mpi: @@ -100,9 +101,11 @@ def main(actx_class, casename="wave", from functools import partial from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, - a=(-0.5,)*dim, b=(0.5,)*dim, - nelements_per_axis=(nel_1d,)*dim) + group_cls = TensorProductElementGroup if use_tpe else None + generate_mesh = \ + partial(generate_regular_rect_mesh, + a=(-0.5,)*dim, b=(0.5,)*dim, group_cls=group_cls, + nelements_per_axis=(nel_1d,)*dim) if comm: from mirgecom.simutil import distribute_mesh @@ -127,7 +130,8 @@ def main(actx_class, casename="wave", order = 3 - dcoll = create_discretization_collection(actx, local_mesh, order=order) + dcoll = create_discretization_collection(actx, local_mesh, order=order, + quadrature_order=order) nodes = actx.thaw(dcoll.nodes()) current_cfl = 0.485 diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index 2accbe11e..6dcb364f7 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -45,8 +45,8 @@ """ import numpy as np -from arraycontext import outer from grudge.trace_pair import TracePair +from arraycontext import outer from meshmode.dof_array import DOFArray from meshmode.discretization.connection import FACE_RESTR_ALL from grudge.dof_desc import ( diff --git a/requirements.txt b/requirements.txt index b078bc31e..2bc3cd2a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,14 +13,15 @@ git+https://github.com/pythological/kanren.git#egg=miniKanren # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic #--editable git+https://github.com/inducer/pyopencl.git#egg=pyopencl ---editable git+https://github.com/illinois-ceesd/loopy.git@production#egg=loopy +--editable git+https://github.com/illinois-ceesd/loopy.git@production-pilot#egg=loopy --editable git+https://github.com/inducer/dagrt.git#egg=dagrt --editable git+https://github.com/inducer/leap.git#egg=leap --editable git+https://github.com/inducer/modepy.git#egg=modepy ---editable git+https://github.com/illinois-ceesd/arraycontext.git@production#egg=arraycontext ---editable git+https://github.com/illinois-ceesd/meshmode.git@production#egg=meshmode ---editable git+https://github.com/illinois-ceesd/grudge.git@production#egg=grudge ---editable git+https://github.com/illinois-ceesd/pytato.git@production#egg=pytato +--editable git+https://github.com/inducer/pytools.git#egg=pytools +--editable git+https://github.com/illinois-ceesd/arraycontext.git@production-pilot#egg=arraycontext +--editable git+https://github.com/illinois-ceesd/meshmode.git@production-pilot#egg=meshmode +--editable git+https://github.com/illinois-ceesd/grudge.git@production-pilot#egg=grudge +--editable git+https://github.com/illinois-ceesd/pytato.git@production-pilot#egg=pytato --editable git+https://github.com/pyrometheus/pyrometheus.git@tulio/entropy#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle --editable git+https://github.com/kaushikcfd/feinsum.git#egg=feinsum diff --git a/test/test_filter.py b/test/test_filter.py index 64363b55a..b959cc5ef 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -250,6 +250,7 @@ def poly_func(coeff): # , x_vec): print("Filtered expansions:") print(f"{filtered_element_spectrum=}") print(f"{tot_pow_filtered=}") + nfilt = element_order - mid_cutoff ckfn = partial(xmrfunc, alpha=alpha, cutoff=mid_cutoff, filter_order=filter_order, nfilt=nfilt) diff --git a/testing/test-examples-lassen.sh b/testing/test-examples-lassen.sh new file mode 100755 index 000000000..aecb92277 --- /dev/null +++ b/testing/test-examples-lassen.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +origin=$(pwd) +EXAMPLES_HOME=$2 +# examples_dir=${1-$origin} +BATCH_SCRIPT_NAME="examples-lassen-batch.sh" +examples_dir="${EXAMPLES_HOME}" + +rm -rf ${BATCH_SCRIPT_NAME} +cat < ${BATCH_SCRIPT_NAME} +#!/bin/bash + +#BSUB -nnodes 1 +#BSUB -G uiuc +#BSUB -W 120 +#BSUB -q pdebug + +printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n" + +source "${EMIRGE_HOME}/config/activate_env.sh" +export PYOPENCL_CTX="port:tesla" +export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch" +rm -rf \$XDG_CACHE_HOME +rm -f timing-run-done +which python +conda env list +env +env | grep LSB_MCPU_HOSTS + +serial_spawner_cmd="jsrun -g 1 -a 1 -n 1" +parallel_spawner_cmd="jsrun -g 1 -a 1 -n 2" + +set -o nounset + +rm -f *.vtu *.pvtu + +declare -i numfail=0 +declare -i numsuccess=0 +echo "*** Running examples in $examples_dir ..." +failed_examples="" +succeeded_examples="" + +for example in $examples_dir/*.py +do + if [[ "\$example" == *"-mpi-lazy.py" ]] + then + echo "*** Running parallel lazy example (1 rank): \$example" + \$serial_spawner_cmd python -O -m mpi4py \${example} --lazy + elif [[ "\$example" == *"-mpi.py" ]]; then + echo "*** Running parallel example (2 ranks): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} + elif [[ "\$example" == *"-lazy.py" ]]; then + echo "*** Running serial lazy example: \$example" + python -O \${example} --lazy + else + echo "*** Running serial example: \$example" + python -O \${example} + fi + if [[ \$? -eq 0 ]] + then + ((numsuccess=numsuccess+1)) + echo "*** Example \$example succeeded." + succeeded_examples="\$succeeded_examples \$example" + else + ((numfail=numfail+1)) + echo "*** Example \$example failed." + failed_examples="\$failed_examples \$example" + fi + rm -rf *vtu *sqlite *pkl *-journal restart_data +done +((numtests=numsuccess+numfail)) +echo "*** Done running examples!" +if [[ \$numfail -eq 0 ]] +then + echo "*** No errors." +else + echo "*** Errors detected." + echo "*** Failed tests: (\$numfail/\$numtests): \$failed_examples" +fi +echo "*** Successful tests: (\$numsuccess/\$numtests): \$succeeded_examples" + +rm -rf example-testing-results +printf "\$numfail\n" > example-testing-results +touch example-testing-done +exit \$numfail + +EOF + +rm -f example-testing-done +chmod +x ${BATCH_SCRIPT_NAME} +# ---- Submit the batch script and wait for the job to finish +bsub ${BATCH_SCRIPT_NAME} +# ---- Wait 25 minutes right off the bat +sleep 1500 +iwait=0 +while [ ! -f ./example-testing-done ]; do + iwait=$((iwait+1)) + if [ "$iwait" -gt 89 ]; then # give up after almost 2 hours + printf "Timed out waiting on batch job.\n" + exit 1 # skip the rest of the script + fi + sleep 60 +done +sleep 30 # give the batch system time to spew its junk into the log +cat *.out > example-testing-output +date >> example-testing-output +rm *.out +date diff --git a/testing/test-examples-linux.sh b/testing/test-examples-linux.sh new file mode 100644 index 000000000..f72c9408a --- /dev/null +++ b/testing/test-examples-linux.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +origin=$(pwd) +EXAMPLES_HOME=$2 +BATCH_SCRIPT_NAME="run-examples-linux.sh" +examples_dir="${EXAMPLES_HOME}" + +rm -rf ${BATCH_SCRIPT_NAME} +cat < ${BATCH_SCRIPT_NAME} +#!/bin/bash + +printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n" + +source "${EMIRGE_HOME}/config/activate_env.sh" +# export PYOPENCL_CTX="port:tesla" +export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch" +rm -rf \$XDG_CACHE_HOME +rm -f examples-run-done +which python +conda env list +env + +parallel_spawner_cmd="mpiexec -n 2" + +set -o nounset + +rm -f *.vtu *.pvtu + +declare -i numfail=0 +declare -i numsuccess=0 +echo "*** Running examples in $examples_dir ..." +failed_examples="" +succeeded_examples="" + +for example in $examples_dir/*.py +do + if [[ "\$example" == *"-mpi-lazy.py" ]] + then + echo "*** Running parallel lazy example (2 rank): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} --lazy + elif [[ "\$example" == *"-mpi.py" ]]; then + echo "*** Running parallel example (2 ranks): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} + elif [[ "\$example" == *"-lazy.py" ]]; then + echo "*** Running serial lazy example: \$example" + python -O \${example} --lazy + else + echo "*** Running serial example: \$example" + python -O \${example} + fi + if [[ \$? -eq 0 ]] + then + ((numsuccess=numsuccess+1)) + echo "*** Example \$example succeeded." + succeeded_examples="\$succeeded_examples \$example" + else + ((numfail=numfail+1)) + echo "*** Example \$example failed." + failed_examples="\$failed_examples \$example" + fi + rm -rf *vtu *sqlite *pkl *-journal restart_data +done +((numtests=numsuccess+numfail)) +echo "*** Done running examples!" +if [[ \$numfail -eq 0 ]] +then + echo "*** No errors." +else + echo "*** Errors detected." + echo "*** Failed tests: (\$numfail/\$numtests): \$failed_examples" +fi +echo "*** Successful tests: (\$numsuccess/\$numtests): \$succeeded_examples" + +rm -rf example-testing-results +printf "\$numfail\n" > example-testing-results +touch example-testing-done +exit \$numfail + +EOF + +rm -f example-testing-done +chmod +x ${BATCH_SCRIPT_NAME} +# ---- Submit the batch script and wait for the job to finish +EXAMPLES_RUN_OUTPUT=$(${BATCH_SCRIPT_NAME}) + +# ---- Wait 25 minutes right off the bat +sleep 1500 +iwait=0 +while [ ! -f ./example-testing-done ]; do + iwait=$((iwait+1)) + if [ "$iwait" -gt 89 ]; then # give up after almost 2 hours + printf "Timed out waiting on batch job.\n" + exit 1 # skip the rest of the script + fi + sleep 60 +done +sleep 30 # give the batch system time to spew its junk into the log +printf "${EXAMPLSE_RUN_OUTPUT}\n" > example-testing-output +date >> example-testing-output +date diff --git a/testing/test-examples-quartz.sh b/testing/test-examples-quartz.sh new file mode 100755 index 000000000..9cfd1382a --- /dev/null +++ b/testing/test-examples-quartz.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +origin=$(pwd) +EXAMPLES_HOME=$2 +# examples_dir=${1-$origin} +BATCH_SCRIPT_NAME="examples-quartz-batch.sh" +examples_dir="${EXAMPLES_HOME}" + +rm -rf ${BATCH_SCRIPT_NAME} +cat < ${BATCH_SCRIPT_NAME} +#!/bin/bash + +#SBATCH -N 2 +#SBATCH -J mirgecom-examples-test +#SBATCH -t 120 +#SBATCH -p pbatch +#SBATCH -A uiuc + +printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n" + +source "${EMIRGE_HOME}/config/activate_env.sh" +# export PYOPENCL_CTX="port:tesla" +export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch" +rm -rf \$XDG_CACHE_HOME +rm -f timing-run-done +which python +conda env list +env +env | grep LSB_MCPU_HOSTS + +serial_spawner_cmd="srun -n 1" +parallel_spawner_cmd="srun -n 2" + +set -o nounset + +rm -f *.vtu *.pvtu + +declare -i numfail=0 +declare -i numsuccess=0 +echo "*** Running examples in $examples_dir ..." +failed_examples="" +succeeded_examples="" + +for example in $examples_dir/*.py +do + if [[ "\$example" == *"-mpi-lazy.py" ]] + then + echo "*** Running parallel lazy example (2 rank): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} --lazy + elif [[ "\$example" == *"-mpi.py" ]]; then + echo "*** Running parallel example (2 ranks): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} + elif [[ "\$example" == *"-lazy.py" ]]; then + echo "*** Running serial lazy example: \$example" + python -O \${example} --lazy + else + echo "*** Running serial example: \$example" + python -O \${example} + fi + if [[ \$? -eq 0 ]] + then + ((numsuccess=numsuccess+1)) + echo "*** Example \$example succeeded." + succeeded_examples="\$succeeded_examples \$example" + else + ((numfail=numfail+1)) + echo "*** Example \$example failed." + failed_examples="\$failed_examples \$example" + fi + rm -rf *vtu *sqlite *pkl *-journal restart_data +done +((numtests=numsuccess+numfail)) +echo "*** Done running examples!" +if [[ \$numfail -eq 0 ]] +then + echo "*** No errors." +else + echo "*** Errors detected." + echo "*** Failed tests: (\$numfail/\$numtests): \$failed_examples" +fi +echo "*** Successful tests: (\$numsuccess/\$numtests): \$succeeded_examples" + +rm -rf example-testing-results +printf "\$numfail\n" > example-testing-results +touch example-testing-done +exit \$numfail + +EOF + +rm -f example-testing-done +chmod +x ${BATCH_SCRIPT_NAME} +# ---- Submit the batch script and wait for the job to finish +sbatch ${BATCH_SCRIPT_NAME} +# ---- Wait 60 minutes right off the bat +printf "Waiting for the batch job to finish." +sleep 3600 +printf "." +iwait=0 +while [ ! -f ./example-testing-done ]; do + iwait=$((iwait+1)) + if [ "$iwait" -gt 180 ]; then # give up after 4 hours + printf "\nTimed out waiting on batch job, aborting tests.\n" + exit 1 # skip the rest of the script + fi + sleep 60 + printf "." +done +printf "(finished)\n" +sleep 30 # give the batch system time to spew its junk into the log +cat *.out > example-testing-output +date >> example-testing-output +rm *.out +date diff --git a/testing/test-lassen.sh b/testing/test-lassen.sh new file mode 100755 index 000000000..6e6620c96 --- /dev/null +++ b/testing/test-lassen.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +TESTING_RESULTS_FILE=$2 +TESTING_LOG_FILE=$3 + +printf "Testing examples.\n" +./test-examples-lassen.sh ${EMIRGE_HOME} ../examples +examples_script_result=$? +printf "Examples script result: ${examples_result}" +cat example-testing-output >> ${TESTING_LOG_FILE} +examples_testing_result=$(cat example-testing-results) +printf "mirgecom-examples: ${examples_testing_result}\n" >> ${TESTING_RESULTS_FILE} diff --git a/testing/test-linux.sh b/testing/test-linux.sh new file mode 100644 index 000000000..686561639 --- /dev/null +++ b/testing/test-linux.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +TESTING_RESULTS_FILE=$2 +TESTING_LOG_FILE=$3 + +printf "Testing examples.\n" +./test-examples-linux.sh ${EMIRGE_HOME} ../examples +examples_script_result=$? +printf "Examples script result: ${examples_result}" +cat example-testing-output >> ${TESTING_LOG_FILE} +examples_testing_result=$(cat example-testing-results) +printf "mirgecom-examples: ${examples_testing_result}\n" >> ${TESTING_RESULTS_FILE} diff --git a/testing/test-quartz.sh b/testing/test-quartz.sh new file mode 100755 index 000000000..7c366c4a5 --- /dev/null +++ b/testing/test-quartz.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +TESTING_RESULTS_FILE=$2 +TESTING_LOG_FILE=$3 + +printf "Testing examples.\n" +./test-examples-quartz.sh ${EMIRGE_HOME} ../examples +examples_script_result=$? +printf "Examples script result: ${examples_result}" +cat example-testing-output >> ${TESTING_LOG_FILE} +examples_testing_result=$(cat example-testing-results) +printf "mirgecom-examples: ${examples_testing_result}\n" >> ${TESTING_RESULTS_FILE}