G3PythonContext class for handling the Python GIL (#146)

This PR creates a new class that simplifies initialization of python threads, as well as acquiring / releasing the Python global interpreter lock in various contexts.

Use cases include:

1. Ensuring that Py_Initialize() is properly called at the beginning of a program that is expected to interact with the python interpreter, and also that Py_Finalize() is called when the program is finished.
2. Ensuring that the current thread state is saved and the GIL released as necessary, e.g. for IO operations, and then the thread state is restored on completion.
3. Ensuring that the GIL is acquired for one-off interaction with the python interpreter, and released when complete.

A G3PythonContext object is used throughout the library code for cases 2 and 3. Case 1 is handled by a G3PythonInterpreter object constructed at the main program/thread level.  If the python interpreter has not been initialized (i.e. the compiled program is expected to be purely in C++), then the library context objects are essentially no-op. If the python interpreter is initialized (e.g. inside a python program or command-line interface), then these library context objects will handle the GIL appropriately.

See the examples/cppexample.cxx C++ program for a simple implementation of the above behavior in a compiled program.

Closes #145.

---------

Co-authored-by: Cosmin Deaconu <[email protected]>

core/include/core/pybindings.h

@@ -3,6 +3,7 @@
 
 #include <G3.h>
 #include <G3Frame.h>
+#include <G3Logging.h>
 
 #include <boost/preprocessor/cat.hpp>
 #include <boost/preprocessor/facilities/overload.hpp>
@@ -383,4 +384,43 @@ void BOOST_PP_CAT(spt3g_init_module_, name)()
 // If the enclosing package name is not specified, it will default to "spt3g".
 #define SPT3G_PYTHON_MODULE(...) BOOST_PP_OVERLOAD(SPT3G_PYTHON_MODULE_,__VA_ARGS__)(__VA_ARGS__)
 
+// Python runtime context to simplify acquiring or releasing the GIL as necessary.
+// To use, simply construct the context object where necessary, e.g.
+//    G3PythonContext ctx("mycontext", false);
+// The context destructor will clean up after itself (releasing the GIL if acquired, and
+// vice versa).  If hold_gil is true, the context will ensure the GIL is held at construction,
+// and released at destruction.  If hold_gil is false, the context will save the current thread
+// state and release the GIL at construction, and re-acquire it at destruction.
+class G3PythonContext {
+public:
+	G3PythonContext(std::string name, bool hold_gil=false);
+	~G3PythonContext();
+
+private:
+	std::string name_;
+	bool hold_;
+	PyGILState_STATE gil_;
+	PyThreadState *thread_;
+
+	SET_LOGGER("G3PythonContext");
+};
+
+// Convenience class for initializing and finalizing the Python interpreter.  This class
+// will initialize the python interpeter at construction (e.g. at the beginning of a C++
+// compiled program), and immediately initialize the appropriate G3PythonContext depending
+// on the value of hold_gil.  At destruction, it will exit the python context and finalize
+// the interpreter.  The python interpreter should be initialized only once, typically at
+// the beginning of the main program.
+class G3PythonInterpreter {
+public:
+	G3PythonInterpreter(bool hold_gil=false);
+	~G3PythonInterpreter();
+
+private:
+	bool init_;
+	G3PythonContext *ctx_;
+
+	SET_LOGGER("G3PythonInterpreter");
+};
+
 #endif

core/src/G3EventBuilder.cxx

@@ -34,9 +34,7 @@ void G3EventBuilder::AddPolledDataModule(G3ModulePtr mod)
 
 void G3EventBuilder::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 {
-	PyThreadState *_save = nullptr;
-	if (Py_IsInitialized())
-		_save = PyEval_SaveThread();
+	G3PythonContext ctx("G3EventBuilder", false);
 
 	std::unique_lock<std::mutex> lock(out_queue_lock_);
 
@@ -46,9 +44,6 @@ void G3EventBuilder::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	// If dead, out_queue_ will be empty, putting nothing in the queue
 	// and ending processing.
 
-	if (_save != nullptr)
-		PyEval_RestoreThread(_save);
-
 	out.swap(out_queue_);
 }
 

core/src/G3Frame.cxx

@@ -130,6 +130,8 @@ static std::string FrameObjectClassName(G3FrameObjectConstPtr obj)
 {
 	// Try to give python name if possible
 	if (Py_IsInitialized()) {
+		G3PythonContext ctx("G3FrameObjectClassName", true);
+
 		try {
 			boost::python::object pyobj(
 			    boost::const_pointer_cast<G3FrameObject>(obj));
@@ -140,6 +142,8 @@ static std::string FrameObjectClassName(G3FrameObjectConstPtr obj)
 			     "." +
 			     boost::python::extract<std::string>(
 			      pyobj.attr("__class__").attr("__name__"))();
+		} catch (const boost::python::error_already_set& e) {
+			PyErr_Clear();
 		} catch (...) {
 			// Fall through to C++ name
 		}

core/src/G3Reader.cxx

@@ -79,9 +79,7 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	// Python interpreter (pure C++ applications will fail the
 	// Py_IsInitialized test).  Make sure all paths out of this
 	// function reacquire the lock, if it was released.
-	PyThreadState *_save = nullptr;
-	if (Py_IsInitialized())
-		_save = PyEval_SaveThread();
+	G3PythonContext ctx("G3Reader", false);
 
 	while (stream_.peek() == EOF) {
 		if (n_frames_cur_ == 0)
@@ -91,8 +89,6 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 			filename_.pop_front();
 		} else {
 			// Stop processing
-			if (_save != nullptr)
-				PyEval_RestoreThread(_save);
 			return;
 		}
 	}
@@ -102,12 +98,8 @@ void G3Reader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	} catch (...) {
 		log_error("Exception raised while reading file %s",
 		    cur_file_.c_str());
-		if (_save != nullptr)
-			PyEval_RestoreThread(_save);
 		throw;
 	}
-	if (_save != nullptr)
-		PyEval_RestoreThread(_save);
 
 	if (track_filename_)
 		frame->_filename = cur_file_;

core/src/G3Writer.cxx

@@ -48,9 +48,8 @@ void G3Writer::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	// because in some cases serialization requires calling back
 	// out to Python.
 	frame->GenerateBlobs();
-	PyThreadState *_save = nullptr;
-	if (Py_IsInitialized() && PyGILState_Check())
-		_save = PyEval_SaveThread();
+
+	G3PythonContext ctx("G3Writer", false);
 
 	if (frame->type == G3Frame::EndProcessing)
 		stream_.reset();
@@ -60,9 +59,6 @@ void G3Writer::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 		frame->save(stream_);
 
 	out.push_back(frame);
-
-	if (_save != nullptr)
-		PyEval_RestoreThread(_save);
 }
 
 void G3Writer::Flush()

core/src/python.cxx

@@ -32,6 +32,66 @@ void G3ModuleRegistrator::CallRegistrarsFor(const char *mod)
 		(*i)();
 }
 
+G3PythonContext::G3PythonContext(std::string name, bool hold_gil) :
+    name_(name), hold_(false), thread_(nullptr)
+{
+	if (!Py_IsInitialized())
+		return;
+
+	if (hold_gil && !PyGILState_Check()) {
+		log_debug("%s: Ensuring GIL acquired", name_.c_str());
+		gil_ = PyGILState_Ensure();
+		hold_ = true;
+	} else if (!hold_gil && PyGILState_Check()) {
+		log_debug("%s: Saving Python thread state", name_.c_str());
+		thread_ = PyEval_SaveThread();
+	}
+}
+
+G3PythonContext::~G3PythonContext()
+{
+	if (hold_) {
+		log_debug("%s: Releasing GIL", name_.c_str());
+		PyGILState_Release(gil_);
+		hold_ = false;
+	}
+
+	if (!!thread_) {
+		log_debug("%s: Restoring Python thread state", name_.c_str());
+		PyEval_RestoreThread(thread_);
+		thread_ = nullptr;
+	}
+}
+
+G3PythonInterpreter::G3PythonInterpreter(bool hold_gil) :
+    init_(false)
+{
+	if (!Py_IsInitialized()) {
+		log_debug("Initializing");
+		Py_Initialize();
+#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 7)
+		PyEval_InitThreads();
+#endif
+		init_ = true;
+	}
+
+	ctx_ = new G3PythonContext("G3PythonInterpreter", hold_gil);
+}
+
+G3PythonInterpreter::~G3PythonInterpreter()
+{
+	if (!!ctx_) {
+		delete ctx_;
+		ctx_ = nullptr;
+	}
+
+	if (init_) {
+		log_debug("Finalizing");
+		Py_Finalize();
+		init_ = false;
+	}
+}
+
 G3FramePtr
 g3frame_char_constructor(std::string max_4_chars)
 {

examples/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_executable(cppexample cppexample.cxx) 
+target_link_libraries(cppexample core)

examples/cppexample.cxx

@@ -1,6 +1,8 @@
+#include <core/pybindings.h>
 #include <core/G3.h>
 #include <core/G3Pipeline.h>
 #include <core/G3Reader.h>
+#include <core/G3Writer.h>
 
 /*
  * Example of a small C++ program that is the equivalent of the
@@ -27,11 +29,19 @@ main(int argc, const char **argv)
 		return 1;
 	}
 
+	// Initialize the python interpreter, and release the GIL.
+	// Set the argument to true to instead hold the GIL.
+	// Comment this out to disable the interpreter.
+	G3PythonInterpreter interp(false);
+
 	G3Pipeline pipe;
 
 	pipe.Add(G3ModulePtr(new G3Reader(argv[1])));
 	pipe.Add(G3ModulePtr(new Dump));
 
+	if (argc > 2)
+		pipe.Add(G3ModulePtr(new G3Writer(argv[2])));
+
 	pipe.Run();
 
 	return 0;

gcp/src/ARCFileReader.cxx

@@ -799,9 +799,7 @@ void ARCFileReader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 	uint8_t *buffer;
 	off_t off;
 
-	PyThreadState *_save = nullptr;
-	if (Py_IsInitialized() && PyGILState_Check())
-		_save = PyEval_SaveThread();
+	G3PythonContext ctx("ARCFileReader", false);
 
 	try {
 		while (stream_.peek() == EOF) {
@@ -810,8 +808,6 @@ void ARCFileReader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 				filename_.pop_front();
 				StartFile(path);
 			} else {
-				if (_save != nullptr)
-					PyEval_RestoreThread(_save);
 				return;
 			}
 		}
@@ -840,14 +836,10 @@ void ARCFileReader::Process(G3FramePtr frame, std::deque<G3FramePtr> &out)
 		stream_.read((char *)buffer, size);
 
 	} catch (...) {
-		if (_save != nullptr)
-			PyEval_RestoreThread(_save);
+		log_error("Exception raised while reading file %s", cur_file_.c_str());
 		throw;
 	}
 
-	if (_save != nullptr)
-		PyEval_RestoreThread(_save);
-
 	for (auto temp = array_map_.begin(); temp != array_map_.end(); temp++) {
 		G3MapFrameObjectPtr templ(new G3MapFrameObject);
 		for (auto board = temp->second.begin();