gh-104372: Unlock the GIL before exec when using vfork().

Modules/_posixsubprocess.c

@@ -8,6 +8,7 @@
 #if defined(HAVE_PIPE2) && !defined(_GNU_SOURCE)
 # define _GNU_SOURCE
 #endif
+#include <stdbool.h>
 #include <unistd.h>
 #include <fcntl.h>
 #ifdef HAVE_SYS_TYPES_H
@@ -528,6 +529,15 @@ reset_signal_handlers(const sigset_t *child_sigmask)
  *
  * If vfork-unsafe functionality is desired after vfork(), consider using
  * syscall() to obtain it.
+ *
+ * This function knows that vfork() was used rather than fork() if the
+ * vfork_tstate_save_p parameter is non-NULL. When non-NULL that will be used
+ * **for the child process to release the GIL in the parent** before the
+ * potentially blocking exec() system call is made. This is a highly unusual
+ * code pattern! But vfork means the child is running in the parent's
+ * vforking-thread address space while the parent thread is blocked. This
+ * unblocks other parent process threads before the parent vforked-thread
+ * blocks waiting for the child exec or death to unblock the vfork-caller.
  */
 Py_NO_INLINE static void
 child_exec(char *const exec_array[],
@@ -546,7 +556,11 @@ child_exec(char *const exec_array[],
            const void *child_sigmask,
            PyObject *py_fds_to_keep,
            PyObject *preexec_fn,
-           PyObject *preexec_fn_args_tuple)
+           PyObject *preexec_fn_args_tuple
+#ifdef VFORK_USABLE
+           , PyThreadState **vfork_tstate_save_p
+#endif
+          )
 {
     int i, saved_errno, reached_preexec = 0;
     PyObject *result;
@@ -679,6 +693,25 @@ child_exec(char *const exec_array[],
         _close_open_fds(3, py_fds_to_keep);
     }
 
+#ifdef VFORK_USABLE
+    // if vfork() was used rather than fork().
+    if (vfork_tstate_save_p != NULL) {
+        /*
+         * Release the GIL in the vfork()ed child process running within the
+         * parent process's address space. ie: Release the GIL in the parent!
+         *
+         * This prevents other parent threads from blocking on the result of
+         * our vfork'ed child exec() syscall.
+         *
+         * The parent process vfork-caller thread will detect this GIL drop
+         * as we are saving this state in their address space. They will
+         * reacquire the GIL after our exec succeeds or this process dies
+         * which unblocks their execution past the vfork() call.
+         */
+        *vfork_tstate_save_p = PyEval_SaveThread();
+    }
+#endif
+
     /* This loop matches the Lib/os.py _execvpe()'s PATH search when */
     /* given the executable_list generated by Lib/subprocess.py.     */
     saved_errno = 0;
@@ -758,6 +791,19 @@ do_fork_exec(char *const exec_array[],
     pid_t pid;
 
 #ifdef VFORK_USABLE
+    bool vfork_used = false;
+
+    // TODO: Use pycore_atomic.h types and macros for this - it is written to
+    // by the child process running in our address space, before we read it,
+    // but the compiler has no knowledge of that so it could use a register
+    // or avoid re-loading it from memory or just assume NULL due to no use
+    // in the function body so far...
+    //
+    // TODO(gpshead): That this "works" may be Linux specific vfork() semantics.
+    //
+    // GIL state, saved by the child process in our address space.
+    PyThreadState *vfork_tstate_save = NULL;
+
     if (child_sigmask) {
         /* These are checked by our caller; verify them in debug builds. */
         assert(uid == (uid_t)-1);
@@ -771,6 +817,8 @@ do_fork_exec(char *const exec_array[],
              * allowed in a process by the kernel, vfork can return -1
              * with errno EINVAL. https://bugs.python.org/issue47151. */
             pid = fork();
+        } else {
+            vfork_used = true;
         }
     } else
 #endif
@@ -779,6 +827,20 @@ do_fork_exec(char *const exec_array[],
     }
 
     if (pid != 0) {
+        // PARENT process.
+#ifdef VFORK_USABLE
+        /*
+         * Re-acquire the GIL if child_exec acquired it for us in our address
+         * space before it made the exec system call.
+         * As we vfork()ed, this parent thread will only resume execution after
+         * the child process exec succeeds or the child process dies.
+         */
+        if (vfork_tstate_save != NULL) {
+            assert(vfork_used);
+            // Re-acquire the GIL in the parent process after vfork().
+            PyEval_RestoreThread(vfork_tstate_save);
+        }
+#endif
         return pid;
     }
 
@@ -801,7 +863,11 @@ do_fork_exec(char *const exec_array[],
                close_fds, restore_signals, call_setsid, pgid_to_set,
                gid, extra_group_size, extra_groups,
                uid, child_umask, child_sigmask,
-               py_fds_to_keep, preexec_fn, preexec_fn_args_tuple);
+               py_fds_to_keep, preexec_fn, preexec_fn_args_tuple
+#ifdef VFORK_USABLE
+               , vfork_used ? &vfork_tstate_save : NULL
+#endif
+              );
     _exit(255);
     return 0;  /* Dead code to avoid a potential compiler warning. */
 }