Merge branch 'development' into doc_burn_cell_sdc

Author: zingale

Docs/source/design.rst

@@ -41,7 +41,13 @@ and the generic solvers:
 
 * ``util/``: linear algebra solvers and other routines.
 
-Design Philosophy
+
+.. note::
+
+   All quantities are assumed to be in CGS units, unless otherwise
+   specified.
+
+Design philosophy
 =================
 
 Any application that uses Microphysics will at minimum need to
@@ -67,7 +73,30 @@ solvers (ODE integration for the network and Newton-Raphson root
 finding for the EOS) are separated from the specific implementations of
 the microphysics.
 
-.. note::
 
-   All quantities are assumed to be in CGS units, unless otherwise
-   specified.
+
+GPU considerations
+==================
+
+.. index:: GPUs
+
+All of the Microphysics routines are written to run on GPUs.  This is
+enabled in application codes by using the AMReX lambda-capturing
+mechanism (see the [AMReX GPU
+documentation](https://amrex-codes.github.io/amrex/docs_html/GPU.html)
+for more information).
+
+This means leveraging the AMReX data-structures, macros, and
+functions.  The unit tests (see :ref:`sec:unit_tests`) provide a good
+reference for how to interface the Microphysics solvers and physics
+terms with an AMReX-based code.
+
+There are a few places where Microphysics behaves slightly differently
+when running on a CPU vs. a GPU:
+
+* In the VODE integrator, we disable Jacobian-caching to save memory.
+  See :ref:`ch:networks:integrators`.
+
+* In general we disable printing from GPU kernels, due to register
+  pressure.  Some output can be enabled by compiling with
+  ``USE_GPU_PRINTF=TRUE``.

Docs/source/index.rst

@@ -46,7 +46,6 @@ system.
    getting_started
    design
    data_structures
-   autodiff
    rp_intro
 
 .. toctree::
@@ -85,6 +84,7 @@ system.
    :hidden:
 
    util
+   autodiff
 
 .. toctree::
    :maxdepth: 1

Docs/source/unit_tests.rst

@@ -1,3 +1,5 @@
+.. _sec:unit_tests:
+
 **********************
 Overview of Unit Tests
 **********************

Make.Microphysics

@@ -43,50 +43,18 @@ include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
 MICROPHYSICS_AUTO_SOURCE_DIR := $(TmpBuildDir)/microphysics_sources/$(optionsSuffix).EXE
 
-all: build_status $(executable) starkiller_library
+all: build_status $(executable)
 	@echo SUCCESS
 
 build_status:
 	$(AMREX_HOME)/Tools/C_scripts/describe_sources.py \
           --git_names "Microphysics AMReX" \
           --git_dirs "$(TOP) $(AMREX_HOME)"
 
-starkiller_library: $(executable)
-ifeq ($(USE_COMPILE_WITH_F2PY), TRUE)
-	@echo Wrapping sources with f90wrap ...
-	sh dowrap.sh
-	@echo Linking objects with f2py ...
-	sh dof2py.sh
-endif
-
-# Use a coarse grained OMP approach
-DEFINES += -DCRSEGRNDOMP
-
-
-# OpenACC support
-ifeq ($(USE_ACC), TRUE)
-  DEFINES += -DACC
-endif
-
 ifeq ($(USE_REACT), TRUE)
   DEFINES += -DREACTIONS
 endif
 
-ifeq ($(USE_REACT_SPARSE_JACOBIAN), TRUE)
-  DEFINES += -DREACT_SPARSE_JACOBIAN
-
-  # The following is sometimes useful to turn on for debugging sparse J indices
-  # (if a get/set/scale is called with (row, col) not in the sparse J, stop)
-  # Otherwise, set/scale do nothing, and get returns 0.
-  ifeq ($(USE_SPARSE_STOP_ON_OOB), TRUE)
-    DEFINES += -DSPARSE_STOP_ON_OOB
-  endif
-endif
-
-ifeq ($(USE_COMPILE_WITH_F2PY), TRUE)
-  DEFINES += -DCOMPILE_WITH_F2PY
-endif
-
 
 #------------------------------------------------------------------------------
 # AMReX
@@ -213,20 +181,9 @@ $(objEXETempDir)/AMReX_buildInfo.o: .FORCE
 include $(AMREX_HOME)/Tools/GNUMake/Make.rules
 
 clean::
-	$(SILENT) $(RM) extern.F90
-	$(SILENT) $(RM) network_properties.F90 network_properties.H
-	$(SILENT) $(RM) $(MICROPHYSICS_AUTO_SOURCE_DIR)/*.H $(MICROPHYSICS_AUTO_SOURCE_DIR)/*.[fF]90
-	$(SILENT) $(RM) extern_parameters.cpp extern_parameters_F.H extern_parameters.H
-
-# Older versions of CASTRO generated a "network.f90" file when using
-# the general_null network, which conflicts with our generalized
-# network.f90 file.  So we should delete the local copy if we are
-# trying to rebuild from an old directory.
-# The same is true for "extern.f90".
-
-clean::
-	$(RM) network.f90
-	$(RM) extern.f90
+	$(SILENT) $(RM) network_properties.H
+	$(SILENT) $(RM) $(MICROPHYSICS_AUTO_SOURCE_DIR)/*.H
+	$(SILENT) $(RM) extern_parameters.cpp extern_parameters.H
 
 
 # for debugging.  To see the value of a Makefile variable,

unit_test/test_aprox_rates/aprox_rates_util.cpp

@@ -24,7 +24,8 @@ void aprox_rates_test(const Box& bx,
 
   const int ini56 = network_spec_index("nickel-56");
 
-  AMREX_PARALLEL_FOR_3D(bx, i, j, k,
+  amrex::ParallelFor(bx,
+  [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
   {
 
     eos_extra_t eos_state;
@@ -444,7 +445,8 @@ void aprox_rates_extra_c12ag(const Box& bx,
 
     const int ini56 = network_spec_index("nickel-56");
 
-    AMREX_PARALLEL_FOR_3D(bx, i, j, k,
+    amrex::ParallelFor(bx,
+    [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
     {
 
         eos_extra_t eos_state;

unit_test/test_conductivity/conductivity_util.cpp

@@ -22,7 +22,8 @@ void cond_test_C(const amrex::Box& bx,
   const int ih1 = network_spec_index("hydrogen-1");
   const int ihe4 = network_spec_index("helium-4");
 
-  AMREX_PARALLEL_FOR_3D(bx, i, j, k,
+  amrex::ParallelFor(bx,
+  [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
   {
 
     // set the composition -- approximately solar

unit_test/test_eos/eos_util.cpp

@@ -26,7 +26,8 @@ void eos_test_C(const Box& bx,
   const int ihe4 = network_spec_index("helium-4");
 
 
-  AMREX_PARALLEL_FOR_3D(bx, i, j, k,
+  amrex::ParallelFor(bx,
+  [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
   {
 
     // set the composition -- approximately solar

unit_test/test_jac/main.cpp

@@ -172,7 +172,8 @@ void main_main ()
 
         auto s = state.array(mfi);
 
-        AMREX_PARALLEL_FOR_3D(bx, i, j, k,
+        amrex::ParallelFor(bx,
+        [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
         {
             do_jac(i, j, k, s, vars);
         });

unit_test/test_rhs/main.cpp

@@ -180,7 +180,8 @@ void main_main ()
 
         auto s = state.array(mfi);
 
-        AMREX_PARALLEL_FOR_3D(bx, i, j, k,
+        amrex::ParallelFor(bx,
+        [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
         {
             do_rhs(i, j, k, s, vars);
         });

unit_test/test_sdc/main.cpp

@@ -197,7 +197,8 @@ void main_main ()
         auto s = state.array(mfi);
         auto n_rhs = integrator_n_rhs.array(mfi);
 
-        AMREX_PARALLEL_FOR_3D(bx, i, j, k,
+        amrex::ParallelFor(bx,
+        [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
         {
             bool success = do_react(vars, i, j, k, s, n_rhs);