Python: Clean CUDA Shutdown

Avoid double finalize and segfaults in CUDA debug runs.
BLAST-ImpactX · Jan 28, 2024 · 08ea927 · 08ea927
1 parent ce33709
commit 08ea927
Show file tree

Hide file tree

Showing 42 changed files with 20 additions and 158 deletions.
diff --git a/examples/alignment/run_alignment.py b/examples/alignment/run_alignment.py
@@ -57,7 +57,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/aperture/run_aperture.py b/examples/aperture/run_aperture.py
@@ -62,7 +62,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/apochromatic/run_apochromatic.py b/examples/apochromatic/run_apochromatic.py
@@ -71,7 +71,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/cfbend/run_cfbend.py b/examples/cfbend/run_cfbend.py
@@ -61,7 +61,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/cfbend/run_cfbend_madx.py b/examples/cfbend/run_cfbend_madx.py
@@ -48,7 +48,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/cfchannel/run_cfchannel.py b/examples/cfchannel/run_cfchannel.py
@@ -55,7 +55,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/cfchannel/run_cfchannel_10nC.py b/examples/cfchannel/run_cfchannel_10nC.py
@@ -60,7 +60,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/chicane/run_chicane.py b/examples/chicane/run_chicane.py
@@ -78,7 +78,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/chicane/run_chicane_madx.py b/examples/chicane/run_chicane_madx.py
@@ -48,7 +48,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/compression/run_compression.py b/examples/compression/run_compression.py
@@ -58,7 +58,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/cyclotron/run_cyclotron.py b/examples/cyclotron/run_cyclotron.py
@@ -64,7 +64,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/epac2004_benchmarks/run_bithermal.py b/examples/epac2004_benchmarks/run_bithermal.py
@@ -66,7 +66,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/epac2004_benchmarks/run_fodo_rf_SC.py b/examples/epac2004_benchmarks/run_fodo_rf_SC.py
@@ -197,7 +197,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/epac2004_benchmarks/run_thermal.py b/examples/epac2004_benchmarks/run_thermal.py
@@ -64,7 +64,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/expanding_beam/run_expanding.py b/examples/expanding_beam/run_expanding.py
@@ -59,7 +59,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/fodo/run_fodo.py b/examples/fodo/run_fodo.py
@@ -67,7 +67,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/fodo/run_fodo_madx.py b/examples/fodo/run_fodo_madx.py
@@ -49,7 +49,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/fodo/run_fodo_programmable.py b/examples/fodo/run_fodo_programmable.py
@@ -172,7 +172,3 @@ def my_ref_drift(pge, refpart):
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/fodo_chromatic/run_fodo_chr.py b/examples/fodo_chromatic/run_fodo_chr.py
@@ -67,7 +67,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/fodo_rf/run_fodo_rf.py b/examples/fodo_rf/run_fodo_rf.py
@@ -66,7 +66,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/iota_lattice/run_iotalattice.py b/examples/iota_lattice/run_iotalattice.py
@@ -122,7 +122,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/iota_lattice/run_iotalattice_sdep.py b/examples/iota_lattice/run_iotalattice_sdep.py
@@ -218,7 +218,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/iota_lens/run_iotalens.py b/examples/iota_lens/run_iotalens.py
@@ -56,7 +56,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/iota_lens/run_iotalens_sdep.py b/examples/iota_lens/run_iotalens_sdep.py
@@ -92,7 +92,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/kicker/run_hvkicker_madx.py b/examples/kicker/run_hvkicker_madx.py
@@ -48,7 +48,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/kicker/run_kicker.py b/examples/kicker/run_kicker.py
@@ -56,7 +56,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/kicker/run_kicker_madx.py b/examples/kicker/run_kicker_madx.py
@@ -48,7 +48,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/kurth/run_kurth_10nC_periodic.py b/examples/kurth/run_kurth_10nC_periodic.py
@@ -54,7 +54,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/kurth/run_kurth_periodic.py b/examples/kurth/run_kurth_periodic.py
@@ -52,7 +52,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/multipole/run_multipole.py b/examples/multipole/run_multipole.py
@@ -57,7 +57,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/positron_channel/run_positron.py b/examples/positron_channel/run_positron.py
@@ -69,7 +69,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/pytorch_surrogate_model/run_ml_surrogate.py b/examples/pytorch_surrogate_model/run_ml_surrogate.py
@@ -265,4 +265,3 @@ def surrogate_push(self, pc, step):
 sim.lattice.extend([monitor])
 
 sim.evolve()
-del sim
diff --git a/examples/quadrupole_softedge/run_quadrupole_softedge.py b/examples/quadrupole_softedge/run_quadrupole_softedge.py
@@ -76,7 +76,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/rfcavity/run_rfcavity.py b/examples/rfcavity/run_rfcavity.py
@@ -139,7 +139,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/rotation/run_rotation.py b/examples/rotation/run_rotation.py
@@ -57,7 +57,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/solenoid/run_solenoid.py b/examples/solenoid/run_solenoid.py
@@ -56,7 +56,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/solenoid/run_solenoid_madx.py b/examples/solenoid/run_solenoid_madx.py
@@ -45,7 +45,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/solenoid_softedge/run_solenoid_softedge.py b/examples/solenoid_softedge/run_solenoid_softedge.py
@@ -137,7 +137,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/examples/thin_dipole/run_thin_dipole.py b/examples/thin_dipole/run_thin_dipole.py
@@ -64,7 +64,3 @@
 
 # run simulation
 sim.evolve()
-
-# clean shutdown
-del sim
-amr.finalize()
diff --git a/src/ImpactX.H b/src/ImpactX.H
@@ -40,6 +40,10 @@ namespace impactx
         void operator= (ImpactX const&) = delete;
         void operator= (ImpactX &&) = delete;
 
+        /** Destruct the ImpactX simulation object
+         */
+        ~ImpactX ();
+
         /** Initialize AMReX blocks/grids for domain decomposition & space charge mesh.
          *
          * This must come first, before particle beams and lattice elements are

diff --git a/src/ImpactX.cpp b/src/ImpactX.cpp
@@ -32,16 +32,22 @@
 #include <memory>
 
 
-namespace impactx
-{
-    ImpactX::ImpactX ()
-    {
+namespace impactx {
+    ImpactX::ImpactX() {
         // todo: if amr.n_cells is provided, overwrite/redefine AmrCore object
 
         // todo: if charge deposition and/or space charge are requested, require
         //       amr.n_cells from user inputs
     }
 
+    ImpactX::~ImpactX()
+    {
+        m_lattice.clear()
+
+        // this one last
+        amr_data.release();
+    }
+
     void ImpactX::init_grids ()
     {
         BL_PROFILE("ImpactX::init_grids");

diff --git a/tests/python/test_dataframe.py b/tests/python/test_dataframe.py
@@ -7,13 +7,18 @@
 #
 # -*- coding: utf-8 -*-
 
-from impactx import ImpactX, RefPart, distribution, elements
+from impactx import ImpactX, RefPart, amr, distribution, elements
 
 
 def test_df_pandas():
     """
     This tests using ImpactX and Pandas Dataframes
     """
+    if amr.Config.have_gpu:
+        # pc.to_df copies to/from host memory
+        pp_amrex = amr.ParmParse("amrex")
+        pp_amrex.add("the_arena_is_managed", 1)
+
     sim = ImpactX()
 
     sim.particle_shape = 2
Original file line number	Diff line number	Diff line change
Expand Up		@@ -265,4 +265,3 @@ def surrogate_push(self, pc, step):
		sim.lattice.extend([monitor])

		sim.evolve()
		del sim