Allow python threads and async io

Patch by: thommey

Python threads were not given a chance to run in the background so far because we held onto the GIL.
Now we release it around our select() which should give Python threads or asyncio runners (in a separate thread) enough room to work.

The concern of having multithreaded code call into our Tcl interpreter against its will turns out to be unwarranted - the GIL takes care of this.

configure

@@ -1,5 +1,5 @@
 #! /bin/sh
-# From configure.ac 95f630ee.
+# From configure.ac 9068a673.
 # Guess values for system-dependent variables and create Makefiles.
 # Generated by GNU Autoconf 2.71 for Eggdrop 1.9.5.
 #
@@ -10638,7 +10638,6 @@ printf "%s\n" "#define EGG_TDNS 1" >>confdefs.h
 
 
 # Check for Python
-EGG_PYTHON_ENABLE
 
 
 # Check whether --with-python-config was given.

src/Makefile.in

@@ -176,7 +176,8 @@ modules.o: modules.c main.h ../config.h ../eggint.h ../lush.h lang.h \
 net.o: net.c main.h ../config.h ../eggint.h ../lush.h lang.h eggdrop.h \
  compat/in6.h flags.h proto.h misc_file.h cmdt.h tclegg.h tclhash.h \
  chan.h users.h compat/compat.h compat/base64.h compat/inet_aton.h \
- ../src/main.h compat/snprintf.h compat/explicit_bzero.h compat/strlcpy.h
+ ../src/main.h compat/snprintf.h compat/explicit_bzero.h compat/strlcpy.h \
+ modules.h mod/modvals.h
 rfc1459.o: rfc1459.c main.h ../config.h ../eggint.h ../lush.h lang.h \
  eggdrop.h compat/in6.h flags.h proto.h misc_file.h cmdt.h tclegg.h \
  tclhash.h chan.h users.h compat/compat.h compat/base64.h \

src/main.c

@@ -1023,6 +1023,10 @@ int main(int arg_c, char **arg_v)
   sigaction(SIGILL, &sv, NULL);
   sv.sa_handler = got_alarm;
   sigaction(SIGALRM, &sv, NULL);
+  // Added for python.mod because the _signal handler otherwise overwrites it
+  // see https://discuss.python.org/t/asyncio-skipping-signal-handling-setup-during-import-for-python-embedded-context/37054/6
+  sv.sa_handler = got_term;
+  sigaction(SIGINT, &sv, NULL);
 
   /* Initialize variables and stuff */
   now = time(NULL);

src/mod/compress.mod/configure

@@ -1,5 +1,5 @@
 #! /bin/sh
-# From configure.ac 95f630ee.
+# From configure.ac 9068a673.
 # Guess values for system-dependent variables and create Makefiles.
 # Generated by GNU Autoconf 2.71 for Eggdrop Compress Module 1.9.5.
 #

src/mod/dns.mod/configure

@@ -1,5 +1,5 @@
 #! /bin/sh
-# From configure.ac 95f630ee.
+# From configure.ac 9068a673.
 # Guess values for system-dependent variables and create Makefiles.
 # Generated by GNU Autoconf 2.71 for Eggdrop DNS Module 1.9.5.
 #

src/mod/modvals.h

@@ -39,7 +39,11 @@
 #define HOOK_LOADED              13
 #define HOOK_BACKUP              14
 #define HOOK_DIE                 15
-#define REAL_HOOKS               16
+#define HOOK_PRE_SELECT          16
+#define HOOK_POST_SELECT         17
+
+#define REAL_HOOKS               18
+
 #define HOOK_SHAREOUT           105
 #define HOOK_SHAREIN            106
 #define HOOK_ENCRYPT_PASS       107

src/mod/python.mod/configure

@@ -1,5 +1,5 @@
 #! /bin/sh
-# From configure.ac 95f630ee.
+# From configure.ac 9068a673.
 # Guess values for system-dependent variables and create Makefiles.
 # Generated by GNU Autoconf 2.71 for Eggdrop Python Module 1.10.0.
 #

src/mod/python.mod/python.c

@@ -43,6 +43,7 @@ static PyObject *pirp, *pglobals;
 
 #undef global
 static Function *global = NULL, *irc_funcs = NULL;
+static PyThreadState *_pythreadsave;
 #include "src/mod/python.mod/pycmds.c"
 #include "src/mod/python.mod/tclpython.c"
 
@@ -53,6 +54,16 @@ static int python_expmem()
   return 0; // TODO
 }
 
+static int python_gil_unlock() {
+  _pythreadsave = PyEval_SaveThread();
+  return 0;
+}
+
+static int python_gil_lock() {
+  PyEval_RestoreThread(_pythreadsave);
+  return 0;
+}
+
 // TODO: Do we really have to exit eggdrop on module load failure?
 static void init_python() {
   PyObject *pmodule;
@@ -119,6 +130,8 @@ static void python_report(int idx, int details)
 static char *python_close()
 {
   Context;
+  del_hook(HOOK_PRE_SELECT, (Function)python_gil_unlock);
+  del_hook(HOOK_POST_SELECT, (Function)python_gil_lock);
   kill_python();
   rem_builtins(H_dcc, mydcc);
   rem_tcl_commands(my_tcl_cmds);
@@ -160,5 +173,8 @@ char *python_start(Function *global_funcs)
   /* Add command table to bind list */
   add_builtins(H_dcc, mydcc);
   add_tcl_commands(my_tcl_cmds);
+  add_hook(HOOK_PRE_SELECT, (Function)python_gil_unlock);
+  add_hook(HOOK_POST_SELECT, (Function)python_gil_lock);
+
   return NULL;
 }

src/net.c

@@ -27,6 +27,7 @@
 
 #include <fcntl.h>
 #include "main.h"
+#include "modules.h"
 #include <limits.h>
 #include <string.h>
 #include <netdb.h>
@@ -924,11 +925,13 @@ int sockread(char *s, int *len, sock_list *slist, int slistmax, int tclonly)
   t.tv_sec = td->blocktime.tv_sec;
   t.tv_usec = td->blocktime.tv_usec;
 
+  call_hook(HOOK_PRE_SELECT);
   x = select((SELECT_TYPE_ARG1) maxfd + 1,
              SELECT_TYPE_ARG234 (maxfd_r >= 0 ? &fdr : NULL),
              SELECT_TYPE_ARG234 (maxfd_w >= 0 ? &fdw : NULL),
              SELECT_TYPE_ARG234 (maxfd_e >= 0 ? &fde : NULL),
              SELECT_TYPE_ARG5 &t);
+  call_hook(HOOK_POST_SELECT);
   if (x == -1)
     return -2;                  /* socket error */
   if (x == 0)