From 4e8a976102fbf81a0b16a4c1b9c7d3e1fc799220 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Thu, 20 Jun 2024 14:52:59 -0600
Subject: [PATCH 01/20] Add the option to not reuse the initial tstate.

---
 Include/internal/pycore_interp.h |  4 ++++
 Python/pystate.c                 | 13 +++++++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index 6b5f50b88f7b85..d53c300aa59e09 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -130,6 +130,10 @@ struct _is {
         PyThreadState *head;
         /* The thread currently executing in the __main__ module, if any. */
         PyThreadState *main;
+#ifdef Py_DEBUG
+        /* For testing (_testembed, etc.). */
+        int reuse_init_tstate;
+#endif
         /* Used in Modules/_threadmodule.c. */
         Py_ssize_t count;
         /* Support for runtime thread stack size tuning.
diff --git a/Python/pystate.c b/Python/pystate.c
index e1a95907b57d20..510ad51f7481c1 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -630,6 +630,10 @@ init_interpreter(PyInterpreterState *interp,
     assert(next != NULL || (interp == runtime->interpreters.main));
     interp->next = next;
 
+#ifdef Py_DEBUG
+    interp->threads.reuse_init_tstate = 1;
+#endif
+
     PyStatus status = _PyObject_InitState(interp);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -1555,7 +1559,12 @@ new_threadstate(PyInterpreterState *interp, int whence)
 
     // Allocate the thread state and add it to the interpreter.
     PyThreadState *old_head = interp->threads.head;
-    if (old_head == NULL) {
+    if (old_head == NULL
+#ifdef Py_DEBUG
+        && interp->threads.reuse_init_tstate
+#endif
+        )
+    {
         // It's the interpreter's initial thread state.
         used_newtstate = 0;
         tstate = &interp->_initial_thread;
@@ -1564,7 +1573,7 @@ new_threadstate(PyInterpreterState *interp, int whence)
     else {
         // Every valid interpreter must have at least one thread.
         assert(id > 1);
-        assert(old_head->prev == NULL);
+        assert(old_head == NULL || old_head->prev == NULL);
         used_newtstate = 1;
         tstate = new_tstate;
         // Set to _PyThreadState_INIT.

From 120c2a6930233bd49534df0012f56a225b11d55a Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Thu, 20 Jun 2024 14:56:01 -0600
Subject: [PATCH 02/20] Add a test to verify the initial "main" tstate isn't
 special.

---
 Lib/test/test_embed.py | 13 +++++++++
 Programs/_testembed.c  | 66 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 634513ec7a5812..bc981eeaa95cc1 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -226,6 +226,19 @@ def test_repeated_init_and_inittab(self):
         lines = "\n".join(lines) + "\n"
         self.assertEqual(out, lines)
 
+    def test_replace_main_tstate(self):
+        with self.subTest(exec=False):
+            self.run_embedded_interpreter(
+                'test_replace_main_tstate',
+            )
+
+        with self.subTest(exec=True):
+            out, _ = self.run_embedded_interpreter(
+                'test_replace_main_tstate',
+                'print("spam!")'
+            )
+            self.assertEqual(out.strip(), 'spam!')
+
     def test_forced_io_encoding(self):
         # Checks forced configuration of embedded interpreter IO streams
         env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape")
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index d149b6a0c5cd21..7eb5dbc9ccada0 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -7,6 +7,7 @@
 
 #include <Python.h>
 #include "pycore_initconfig.h"    // _PyConfig_InitCompatConfig()
+#include "pycore_pystate.h"       // _Py_IsMainInterpreter()
 #include "pycore_runtime.h"       // _PyRuntime
 #include "pycore_import.h"        // _PyImport_FrozenBootstrap
 #include <inttypes.h>
@@ -207,6 +208,70 @@ static int test_repeated_simple_init(void)
 }
 
 
+/****************************************************************************
+ * Test discarding the initial main tstate
+ ***************************************************************************/
+
+static int test_replace_main_tstate(void)
+{
+    int err = 0;
+
+    int hascode = 0;
+    for (int i = 2; i < main_argc; i++) {
+        if (strncmp(main_argv[i], "--", 2) == 0) {
+            fprintf(stderr, "ERROR: unsupported arg %s\n", main_argv[i]);
+            err = 1;
+        }
+        else {
+            hascode = 1;
+        }
+        if (err) {
+            fprintf(stderr, "usage: %s test_replace_main_tstate [CODE ...]\n",
+                    PROGRAM);
+            return 1;
+        }
+    }
+
+#ifdef Py_DEBUG
+    _testembed_Py_InitializeFromConfig();
+    PyThreadState *main_tstate = PyThreadState_Get();
+    PyInterpreterState *main_interp = main_tstate->interp;
+    assert(_Py_IsMainInterpreter(main_interp));
+
+    PyThreadState_Clear(main_tstate);
+    PyThreadState_DeleteCurrent();
+    assert(PyInterpreterState_ThreadHead(main_interp) == NULL);
+
+    main_interp->threads.reuse_init_tstate = 0;
+
+    PyThreadState *tstate = PyThreadState_New(main_interp);
+    // The runtime automatically re-uses the initial main tstate.
+    // (It is statically allocated as part of the global runtime state.)
+    assert(_PyThreadState_GET() != main_tstate);
+    (void)PyThreadState_Swap(tstate);
+
+    if (hascode) {
+        for (int i = 2; i < main_argc; i++) {
+            const char *code = main_argv[i];
+            if (PyRun_SimpleString(code) != 0) {
+                err = 1;
+                break;
+            }
+        }
+    }
+
+    assert(PyThreadState_Get() != main_tstate);
+    Py_Finalize();
+
+    return err;
+#else
+    // We want to always force a new "main" tstate, which requires Py_DEBUG.
+    fprintf(stderr, "ERROR: Py_DEBUG is required\n");
+    return 1;
+#endif
+}
+
+
 /*****************************************************
  * Test forcing a particular IO encoding
  *****************************************************/
@@ -2161,6 +2226,7 @@ static struct TestCase TestCases[] = {
     {"test_forced_io_encoding", test_forced_io_encoding},
     {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters},
     {"test_repeated_init_and_inittab", test_repeated_init_and_inittab},
+    {"test_replace_main_tstate", test_replace_main_tstate},
     {"test_pre_initialization_api", test_pre_initialization_api},
     {"test_pre_initialization_sys_options", test_pre_initialization_sys_options},
     {"test_bpo20891", test_bpo20891},

From 608139b0eb211d9590529c265e65b98c04cde32b Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Thu, 20 Jun 2024 15:03:08 -0600
Subject: [PATCH 03/20] Add tests to check different "bad" runtime fini
 situations.

---
 Lib/test/test_embed.py | 13 +++++++
 Programs/_testembed.c  | 86 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index bc981eeaa95cc1..d713bc4071acfa 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -239,6 +239,19 @@ def test_replace_main_tstate(self):
             )
             self.assertEqual(out.strip(), 'spam!')
 
+    def test_fini_in_subthread(self):
+        self.run_embedded_interpreter('test_fini_in_subthread')
+
+    def test_fini_in_main_thread_with_other_tstate(self):
+        self.run_embedded_interpreter(
+            'test_fini_in_main_thread_with_other_tstate',
+        )
+
+    def test_fini_in_main_thread_with_subinterpreter(self):
+        self.run_embedded_interpreter(
+            'test_fini_in_main_thread_with_subinterpreter',
+        )
+
     def test_forced_io_encoding(self):
         # Checks forced configuration of embedded interpreter IO streams
         env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape")
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 7eb5dbc9ccada0..a15c3d6b130a38 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -272,6 +272,87 @@ static int test_replace_main_tstate(void)
 }
 
 
+/****************************************************************************
+ * Test (mostly) unsupported Py_Finalize() scenarios
+ ***************************************************************************/
+
+struct fini_subthread_args {
+    PyThreadState *main_tstate;
+    PyInterpreterState *interp;
+    PyMutex done;
+};
+
+static void fini_with_new_tstate(void *arg)
+{
+    struct fini_subthread_args *args = (struct fini_subthread_args *)arg;
+
+    assert(!_Py_IsMainThread());
+    assert(_PyThreadState_GET() == NULL);
+
+    PyThreadState *tstate = PyThreadState_New(args->interp);
+    assert(tstate != NULL);
+    assert(tstate != args->main_tstate);
+    (void)PyThreadState_Swap(tstate);
+
+    assert(PyThreadState_Get() != args->main_tstate);
+    Py_Finalize();
+
+    PyMutex_Unlock(&args->done);
+}
+
+static int test_fini_in_subthread(void)
+{
+    _testembed_Py_InitializeFromConfig();
+    PyThreadState *main_tstate = PyThreadState_Get();
+
+    struct fini_subthread_args args = {
+        .main_tstate = main_tstate,
+        .interp = main_tstate->interp,
+    };
+    PyMutex_Lock(&args.done);
+    (void)PyThread_start_new_thread(fini_with_new_tstate, &args);
+
+    // Wait for fini to finish.
+    PyMutex_Lock(&args.done);
+    PyMutex_Unlock(&args.done);
+
+    return 0;
+}
+
+static int test_fini_in_main_thread_with_other_tstate(void)
+{
+    _testembed_Py_InitializeFromConfig();
+    PyThreadState *main_tstate = PyThreadState_Get();
+
+    PyThreadState *tstate = PyThreadState_New(main_tstate->interp);
+    (void)PyThreadState_Swap(tstate);
+
+    assert(PyThreadState_Get() != main_tstate);
+    Py_Finalize();
+
+    return 0;
+}
+
+static int test_fini_in_main_thread_with_subinterpreter(void)
+{
+    _testembed_Py_InitializeFromConfig();
+    PyThreadState *main_tstate = PyThreadState_Get();
+
+    PyThreadState *substate = Py_NewInterpreter();
+    assert(substate != main_tstate);
+#ifndef NDEBUG
+    (void)main_tstate;
+    (void)substate;
+#endif
+
+    // The subinterpreter's tstate is still current.
+    assert(PyThreadState_Get() == substate);
+    Py_Finalize();
+
+    return 0;
+}
+
+
 /*****************************************************
  * Test forcing a particular IO encoding
  *****************************************************/
@@ -2227,6 +2308,11 @@ static struct TestCase TestCases[] = {
     {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters},
     {"test_repeated_init_and_inittab", test_repeated_init_and_inittab},
     {"test_replace_main_tstate", test_replace_main_tstate},
+    {"test_fini_in_subthread", test_fini_in_subthread},
+    {"test_fini_in_main_thread_with_other_tstate",
+     test_fini_in_main_thread_with_other_tstate},
+    {"test_fini_in_main_thread_with_subinterpreter",
+     test_fini_in_main_thread_with_subinterpreter},
     {"test_pre_initialization_api", test_pre_initialization_api},
     {"test_pre_initialization_sys_options", test_pre_initialization_sys_options},
     {"test_bpo20891", test_bpo20891},

From 592b2141432e68fac720dc0ab47b2bb8e58bc318 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Thu, 20 Jun 2024 15:20:29 -0600
Subject: [PATCH 04/20] Fix the code in test_audit_subinterpreter.

---
 Programs/_testembed.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index a15c3d6b130a38..ae363a4b49bf4c 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -1525,11 +1525,20 @@ static int test_audit_subinterpreter(void)
     Py_IgnoreEnvironmentFlag = 0;
     PySys_AddAuditHook(_audit_subinterpreter_hook, NULL);
     _testembed_Py_InitializeFromConfig();
+    PyThreadState *save_tstate = PyThreadState_Get();
 
-    Py_NewInterpreter();
-    Py_NewInterpreter();
-    Py_NewInterpreter();
+    PyThreadState *substate1 = Py_NewInterpreter();
+    PyThreadState *substate2 = Py_NewInterpreter();
+    PyThreadState *substate3 = Py_NewInterpreter();
 
+    (void)PyThreadState_Swap(substate3);
+    Py_EndInterpreter(substate3);
+    (void)PyThreadState_Swap(substate2);
+    Py_EndInterpreter(substate2);
+    (void)PyThreadState_Swap(substate1);
+    Py_EndInterpreter(substate1);
+
+    (void)PyThreadState_Swap(save_tstate);
     Py_Finalize();
 
     switch (_audit_subinterpreter_interpreter_count) {

From e6f31ff2b5e133a4d854e1bc264092fe89ce15b3 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Fri, 21 Jun 2024 14:20:50 -0600
Subject: [PATCH 05/20] Always try reusing the main tstate (and drop
 interp.reuse_init_tstate).

---
 Include/internal/pycore_interp.h |  4 ---
 Lib/test/test_embed.py           | 22 +++++++--------
 Programs/_testembed.c            | 47 +++++++++++++++++++++-----------
 Python/pystate.c                 | 13 ++-------
 4 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index d53c300aa59e09..6b5f50b88f7b85 100644
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -130,10 +130,6 @@ struct _is {
         PyThreadState *head;
         /* The thread currently executing in the __main__ module, if any. */
         PyThreadState *main;
-#ifdef Py_DEBUG
-        /* For testing (_testembed, etc.). */
-        int reuse_init_tstate;
-#endif
         /* Used in Modules/_threadmodule.c. */
         Py_ssize_t count;
         /* Support for runtime thread stack size tuning.
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index d713bc4071acfa..5365ff6d29d0e5 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -227,17 +227,17 @@ def test_repeated_init_and_inittab(self):
         self.assertEqual(out, lines)
 
     def test_replace_main_tstate(self):
-        with self.subTest(exec=False):
-            self.run_embedded_interpreter(
-                'test_replace_main_tstate',
-            )
-
-        with self.subTest(exec=True):
-            out, _ = self.run_embedded_interpreter(
-                'test_replace_main_tstate',
-                'print("spam!")'
-            )
-            self.assertEqual(out.strip(), 'spam!')
+        for reuse in (True, False):
+            with self.subTest(reuse=reuse, exec=False):
+                self.run_embedded_interpreter(
+                    'test_replace_main_tstate',
+                )
+            with self.subTest(reuse=reuse, exec=True):
+                out, _ = self.run_embedded_interpreter(
+                    'test_replace_main_tstate',
+                    'print("spam!")'
+                )
+                self.assertEqual(out.strip(), 'spam!')
 
     def test_fini_in_subthread(self):
         self.run_embedded_interpreter('test_fini_in_subthread')
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index ae363a4b49bf4c..732c3e3deb0a9e 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -216,9 +216,13 @@ static int test_replace_main_tstate(void)
 {
     int err = 0;
 
+    int reuse = 0;
     int hascode = 0;
     for (int i = 2; i < main_argc; i++) {
-        if (strncmp(main_argv[i], "--", 2) == 0) {
+        if (strcmp(main_argv[i], "--reuse") == 0) {
+            reuse = 1;
+        }
+        else if (strncmp(main_argv[i], "--", 2) == 0) {
             fprintf(stderr, "ERROR: unsupported arg %s\n", main_argv[i]);
             err = 1;
         }
@@ -226,28 +230,44 @@ static int test_replace_main_tstate(void)
             hascode = 1;
         }
         if (err) {
-            fprintf(stderr, "usage: %s test_replace_main_tstate [CODE ...]\n",
+            fprintf(stderr,
+                    "usage: %s test_replace_main_tstate --reuse [CODE ...]\n",
                     PROGRAM);
             return 1;
         }
     }
 
-#ifdef Py_DEBUG
     _testembed_Py_InitializeFromConfig();
     PyThreadState *main_tstate = PyThreadState_Get();
     PyInterpreterState *main_interp = main_tstate->interp;
     assert(_Py_IsMainInterpreter(main_interp));
 
+    // If the initial tstate is reused then there are slightly
+    // different possible failure paths through the code than if we got
+    // a completely new one.
+
+    PyThreadState *tstate = NULL;
+    if (!reuse) {
+        // The initial thread state is still alive,
+        // so this will create a completely new one,
+        // with its own distinct pointer.
+        tstate = PyThreadState_New(main_interp);
+        assert(tstate != NULL);
+        assert(tstate != main_tstate);
+    }
     PyThreadState_Clear(main_tstate);
     PyThreadState_DeleteCurrent();
-    assert(PyInterpreterState_ThreadHead(main_interp) == NULL);
-
-    main_interp->threads.reuse_init_tstate = 0;
+    assert(reuse == (PyInterpreterState_ThreadHead(main_interp) == NULL));
+    if (reuse) {
+        // The initial thread state has already been "destroyed",
+        // so this will re-use the statically allocated tstate
+        // (along with reinitializing it).
+        tstate = PyThreadState_New(main_interp);
+        assert(tstate != NULL);
+        assert(tstate == main_tstate);
+    }
+    assert(_PyThreadState_GET() == NULL);
 
-    PyThreadState *tstate = PyThreadState_New(main_interp);
-    // The runtime automatically re-uses the initial main tstate.
-    // (It is statically allocated as part of the global runtime state.)
-    assert(_PyThreadState_GET() != main_tstate);
     (void)PyThreadState_Swap(tstate);
 
     if (hascode) {
@@ -260,15 +280,10 @@ static int test_replace_main_tstate(void)
         }
     }
 
-    assert(PyThreadState_Get() != main_tstate);
+    assert(PyThreadState_Get() == tstate);
     Py_Finalize();
 
     return err;
-#else
-    // We want to always force a new "main" tstate, which requires Py_DEBUG.
-    fprintf(stderr, "ERROR: Py_DEBUG is required\n");
-    return 1;
-#endif
 }
 
 
diff --git a/Python/pystate.c b/Python/pystate.c
index 510ad51f7481c1..e1a95907b57d20 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -630,10 +630,6 @@ init_interpreter(PyInterpreterState *interp,
     assert(next != NULL || (interp == runtime->interpreters.main));
     interp->next = next;
 
-#ifdef Py_DEBUG
-    interp->threads.reuse_init_tstate = 1;
-#endif
-
     PyStatus status = _PyObject_InitState(interp);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
@@ -1559,12 +1555,7 @@ new_threadstate(PyInterpreterState *interp, int whence)
 
     // Allocate the thread state and add it to the interpreter.
     PyThreadState *old_head = interp->threads.head;
-    if (old_head == NULL
-#ifdef Py_DEBUG
-        && interp->threads.reuse_init_tstate
-#endif
-        )
-    {
+    if (old_head == NULL) {
         // It's the interpreter's initial thread state.
         used_newtstate = 0;
         tstate = &interp->_initial_thread;
@@ -1573,7 +1564,7 @@ new_threadstate(PyInterpreterState *interp, int whence)
     else {
         // Every valid interpreter must have at least one thread.
         assert(id > 1);
-        assert(old_head == NULL || old_head->prev == NULL);
+        assert(old_head->prev == NULL);
         used_newtstate = 1;
         tstate = new_tstate;
         // Set to _PyThreadState_INIT.

From 05c0cf3347525cd2d09cb483a248ce4470ac69e4 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 25 Jun 2024 11:35:53 -0600
Subject: [PATCH 06/20] Skip test_fini_in_subthread on Windows.

---
 Lib/test/test_embed.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 5365ff6d29d0e5..634efc66913047 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -239,6 +239,7 @@ def test_replace_main_tstate(self):
                 )
                 self.assertEqual(out.strip(), 'spam!')
 
+    @unittest.skipIf(MS_WINDOWS, "this crashes on Windows")
     def test_fini_in_subthread(self):
         self.run_embedded_interpreter('test_fini_in_subthread')
 

From 8b241d288c5955b39e7a60a87f26145dbdab230c Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 25 Jun 2024 11:39:51 -0600
Subject: [PATCH 07/20] Expect a failure on Windows.

---
 Lib/test/test_embed.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 634efc66913047..a37ee8e22d80b4 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -231,26 +231,38 @@ def test_replace_main_tstate(self):
             with self.subTest(reuse=reuse, exec=False):
                 self.run_embedded_interpreter(
                     'test_replace_main_tstate',
+                    # At the moment, this actually succeeds on all platforms.
+                    returncode=0,
                 )
             with self.subTest(reuse=reuse, exec=True):
                 out, _ = self.run_embedded_interpreter(
                     'test_replace_main_tstate',
-                    'print("spam!")'
+                    'print("spam!")',
+                    # At the moment, this actually succeeds on all platforms.
+                    returncode=0,
                 )
                 self.assertEqual(out.strip(), 'spam!')
 
-    @unittest.skipIf(MS_WINDOWS, "this crashes on Windows")
     def test_fini_in_subthread(self):
-        self.run_embedded_interpreter('test_fini_in_subthread')
+        self.run_embedded_interpreter(
+            'test_fini_in_subthread',
+            # At the moment, this actually succeeds on all platforms,
+            # except for Windows.
+            returncode=1 if MS_WINDOWS else 0,
+        )
 
     def test_fini_in_main_thread_with_other_tstate(self):
         self.run_embedded_interpreter(
             'test_fini_in_main_thread_with_other_tstate',
+            # At the moment, this actually succeeds on all platforms.
+            returncode=0,
         )
 
     def test_fini_in_main_thread_with_subinterpreter(self):
         self.run_embedded_interpreter(
             'test_fini_in_main_thread_with_subinterpreter',
+            # At the moment, this actually succeeds on all platforms.
+            returncode=0,
         )
 
     def test_forced_io_encoding(self):

From b66c2b774a736db229f834068a8c08dc4e9c5b86 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 25 Jun 2024 11:52:00 -0600
Subject: [PATCH 08/20] Fail if PyFinalize() fails.

---
 Programs/_testembed.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 732c3e3deb0a9e..d504e2cf093b89 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -281,7 +281,9 @@ static int test_replace_main_tstate(void)
     }
 
     assert(PyThreadState_Get() == tstate);
-    Py_Finalize();
+    if (Py_FinalizeEx() != 0) {
+        err = 1;
+    }
 
     return err;
 }
@@ -295,6 +297,7 @@ struct fini_subthread_args {
     PyThreadState *main_tstate;
     PyInterpreterState *interp;
     PyMutex done;
+    int rc;
 };
 
 static void fini_with_new_tstate(void *arg)
@@ -310,7 +313,7 @@ static void fini_with_new_tstate(void *arg)
     (void)PyThreadState_Swap(tstate);
 
     assert(PyThreadState_Get() != args->main_tstate);
-    Py_Finalize();
+    args->rc = Py_FinalizeEx();
 
     PyMutex_Unlock(&args->done);
 }
@@ -331,7 +334,7 @@ static int test_fini_in_subthread(void)
     PyMutex_Lock(&args.done);
     PyMutex_Unlock(&args.done);
 
-    return 0;
+    return args.rc;
 }
 
 static int test_fini_in_main_thread_with_other_tstate(void)
@@ -343,9 +346,7 @@ static int test_fini_in_main_thread_with_other_tstate(void)
     (void)PyThreadState_Swap(tstate);
 
     assert(PyThreadState_Get() != main_tstate);
-    Py_Finalize();
-
-    return 0;
+    return Py_FinalizeEx();
 }
 
 static int test_fini_in_main_thread_with_subinterpreter(void)
@@ -362,9 +363,7 @@ static int test_fini_in_main_thread_with_subinterpreter(void)
 
     // The subinterpreter's tstate is still current.
     assert(PyThreadState_Get() == substate);
-    Py_Finalize();
-
-    return 0;
+    return Py_FinalizeEx();
 }
 
 

From 88b1679092b313742b8103dd9819fd7e63263577 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Thu, 20 Jun 2024 17:30:42 -0600
Subject: [PATCH 09/20] Add _Py_Finalize() and struct pyfinalize_args.

---
 Include/internal/pycore_pylifecycle.h |  7 +++++++
 Python/pylifecycle.c                  | 26 +++++++++++++++++++++-----
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index f426ae0e103b9c..2f2b93a987cd41 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -114,6 +114,13 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category);
 // Export for special main.c string compiling with source tracebacks
 int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags);
 
+struct pyfinalize_args {
+    const char *caller;
+};
+
+// Export for _testembed
+extern int _Py_Finalize(_PyRuntimeState *, struct pyfinalize_args *);
+
 
 /* interpreter config */
 
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 3639cf6712053e..c066e52a2cd79d 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1910,17 +1910,18 @@ finalize_interp_delete(PyInterpreterState *interp)
 
 
 int
-Py_FinalizeEx(void)
+_Py_Finalize(_PyRuntimeState *runtime, struct pyfinalize_args *args)
 {
     int status = 0;
 
-    _PyRuntimeState *runtime = &_PyRuntime;
+    /* Bail out early if already finalized. */
     if (!runtime->initialized) {
         return status;
     }
 
-    /* Get current thread state and interpreter pointer */
+    /* Get final thread state pointer. */
     PyThreadState *tstate = _PyThreadState_GET();
+    assert(tstate->interp->runtime == runtime);
     // XXX assert(_Py_IsMainInterpreter(tstate->interp));
     // XXX assert(_Py_IsMainThread());
 
@@ -2142,10 +2143,22 @@ Py_FinalizeEx(void)
     return status;
 }
 
+int
+Py_FinalizeEx(void)
+{
+    struct pyfinalize_args args = {
+        .caller = "Py_FinalizeEx",
+    };
+    return _Py_Finalize(&_PyRuntime, &args);
+}
+
 void
 Py_Finalize(void)
 {
-    Py_FinalizeEx();
+    struct pyfinalize_args args = {
+        .caller = "Py_Finalize",
+    };
+    (void)_Py_Finalize(&_PyRuntime, &args);
 }
 
 
@@ -3218,7 +3231,10 @@ Py_Exit(int sts)
     if (tstate != NULL && _PyThreadState_IsRunningMain(tstate)) {
         _PyInterpreterState_SetNotRunningMain(tstate->interp);
     }
-    if (Py_FinalizeEx() < 0) {
+    struct pyfinalize_args args = {
+        .caller = "Py_Exit",
+    };
+    if (_Py_Finalize(&_PyRuntime, &args) < 0) {
         sts = 120;
     }
 

From d9d0e100be2beef33f4447b5015958d009f6092e Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Fri, 21 Jun 2024 15:04:32 -0600
Subject: [PATCH 10/20] Use _Py_Finalize() in the new tests.

---
 Programs/_testembed.c | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index d504e2cf093b89..49ed5f246b6fa1 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -7,6 +7,7 @@
 
 #include <Python.h>
 #include "pycore_initconfig.h"    // _PyConfig_InitCompatConfig()
+#include "pycore_pylifecycle.h"   // _Py_Finalize()
 #include "pycore_pystate.h"       // _Py_IsMainInterpreter()
 #include "pycore_runtime.h"       // _PyRuntime
 #include "pycore_import.h"        // _PyImport_FrozenBootstrap
@@ -281,7 +282,10 @@ static int test_replace_main_tstate(void)
     }
 
     assert(PyThreadState_Get() == tstate);
-    if (Py_FinalizeEx() != 0) {
+    struct pyfinalize_args args = {
+        .caller = "_testembed.test_replace_main_tstate",
+    };
+    if (_Py_Finalize(&_PyRuntime, &args) != 0 && !err) {
         err = 1;
     }
 
@@ -294,6 +298,7 @@ static int test_replace_main_tstate(void)
  ***************************************************************************/
 
 struct fini_subthread_args {
+    struct pyfinalize_args *fini_args;
     PyThreadState *main_tstate;
     PyInterpreterState *interp;
     PyMutex done;
@@ -313,7 +318,9 @@ static void fini_with_new_tstate(void *arg)
     (void)PyThreadState_Swap(tstate);
 
     assert(PyThreadState_Get() != args->main_tstate);
-    args->rc = Py_FinalizeEx();
+    if (_Py_Finalize(&_PyRuntime, args->fini_args) != 0) {
+        args->rc = 1;
+    }
 
     PyMutex_Unlock(&args->done);
 }
@@ -323,7 +330,11 @@ static int test_fini_in_subthread(void)
     _testembed_Py_InitializeFromConfig();
     PyThreadState *main_tstate = PyThreadState_Get();
 
+    struct pyfinalize_args fini_args = {
+        .caller = "_testembed.test_fini_in_subthread",
+    };
     struct fini_subthread_args args = {
+        .fini_args = &fini_args,
         .main_tstate = main_tstate,
         .interp = main_tstate->interp,
     };
@@ -346,7 +357,14 @@ static int test_fini_in_main_thread_with_other_tstate(void)
     (void)PyThreadState_Swap(tstate);
 
     assert(PyThreadState_Get() != main_tstate);
-    return Py_FinalizeEx();
+    struct pyfinalize_args args = {
+        .caller = "_testembed.test_fini_in_main_thread_with_other_tstate",
+    };
+    if (_Py_Finalize(&_PyRuntime, &args) != 0) {
+        return 1;
+    }
+
+    return 0;
 }
 
 static int test_fini_in_main_thread_with_subinterpreter(void)
@@ -363,7 +381,14 @@ static int test_fini_in_main_thread_with_subinterpreter(void)
 
     // The subinterpreter's tstate is still current.
     assert(PyThreadState_Get() == substate);
-    return Py_FinalizeEx();
+    struct pyfinalize_args args = {
+        .caller = "_testembed.test_fini_in_main_thread_with_subinterpreter",
+    };
+    if (_Py_Finalize(&_PyRuntime, &args) != 0) {
+        return 1;
+    }
+
+    return 0;
 }
 
 

From 72e961fc4f0ecda02317ccadb2841f506ccf072e Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Mon, 24 Jun 2024 18:04:30 -0600
Subject: [PATCH 11/20] Factor out resolve_final_tstate().

---
 Python/pylifecycle.c | 52 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 48 insertions(+), 4 deletions(-)

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index c066e52a2cd79d..13814cb2d20021 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1909,6 +1909,53 @@ finalize_interp_delete(PyInterpreterState *interp)
 }
 
 
+/* Conceptually, there isn't a good reason for Py_Finalize()
+   to be called in any other thread than the one where Py_Initialize()
+   was called.  Consequently, it would make sense to fail if the thread
+   or thread state (or interpreter) don't match.  However, such
+   constraints have never been enforced, and, as unlikely as it may be,
+   there may be users relying on the unconstrained behavior.  Thus,
+   we do our best here to accommodate that possibility. */
+
+static PyThreadState *
+resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
+{
+    if (!_Py_IsMainThread()) {
+        fprintf(stderr,
+                "%s: expected to be in the main thread\n", args->caller);
+    }
+
+    PyThreadState *tstate = _PyThreadState_GET();
+    assert(tstate->interp->runtime == runtime);
+    assert(tstate->thread_id == PyThread_get_thread_ident());
+    PyInterpreterState *main_interp = _PyInterpreterState_Main();
+
+    if (tstate->interp != main_interp) {
+        fprintf(stderr,
+                "%s: expected main interpreter to be active\n", args->caller);
+    }
+
+    /* The main tstate is set by Py_Initialize(), but can be unset
+     * or even replaced in unlikely cases. */
+    PyThreadState *main_tstate = runtime->main_tstate;
+    if (main_tstate == NULL) {
+        fprintf(stderr, "%s: main thread state not set\n", args->caller);
+    }
+    else if (tstate != main_tstate) {
+        /* Code running in the main thread could swap out the main tstate,
+           which ends up being a headache. */
+        fprintf(stderr,
+                "%s: using different thread state than Py_Initialize()\n",
+                args->caller);
+    }
+    else {
+        assert(main_tstate->interp != NULL);
+        assert(main_tstate->interp == main_interp);
+    }
+
+    return tstate;
+}
+
 int
 _Py_Finalize(_PyRuntimeState *runtime, struct pyfinalize_args *args)
 {
@@ -1920,10 +1967,7 @@ _Py_Finalize(_PyRuntimeState *runtime, struct pyfinalize_args *args)
     }
 
     /* Get final thread state pointer. */
-    PyThreadState *tstate = _PyThreadState_GET();
-    assert(tstate->interp->runtime == runtime);
-    // XXX assert(_Py_IsMainInterpreter(tstate->interp));
-    // XXX assert(_Py_IsMainThread());
+    PyThreadState *tstate = resolve_final_tstate(runtime, args);
 
     // Block some operations.
     tstate->interp->finalizing = 1;

From e2392c281211029d33f67cc52799c83ecce41ac7 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 25 Jun 2024 12:30:34 -0600
Subject: [PATCH 12/20] Fix the Windows returncode.

---
 Lib/test/test_embed.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index a37ee8e22d80b4..f21a612628dcbc 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -247,8 +247,8 @@ def test_fini_in_subthread(self):
         self.run_embedded_interpreter(
             'test_fini_in_subthread',
             # At the moment, this actually succeeds on all platforms,
-            # except for Windows.
-            returncode=1 if MS_WINDOWS else 0,
+            # except for Windows (STATUS_ACCESS_VIOLATION).
+            returncode=0xC0000005 if MS_WINDOWS else 0,
         )
 
     def test_fini_in_main_thread_with_other_tstate(self):

From 6b1eb57d321d0ac5a559bed117a2d4de70eef6f9 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 25 Jun 2024 12:38:36 -0600
Subject: [PATCH 13/20] Export _Py_Finalize().

---
 Include/internal/pycore_pylifecycle.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index 2f2b93a987cd41..353cd12793ffe3 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -119,7 +119,7 @@ struct pyfinalize_args {
 };
 
 // Export for _testembed
-extern int _Py_Finalize(_PyRuntimeState *, struct pyfinalize_args *);
+PyAPI_FUNC(int) _Py_Finalize(_PyRuntimeState *, struct pyfinalize_args *);
 
 
 /* interpreter config */

From 7b2afb6c3eb2d28bdb97e6bfd278367144e9632f Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 25 Jun 2024 13:34:21 -0600
Subject: [PATCH 14/20] Do not print the error messages with Py_Exit().

---
 Include/internal/pycore_pylifecycle.h |  1 +
 Programs/_testembed.c                 |  4 ++++
 Python/pylifecycle.c                  | 24 ++++++++++++++++--------
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index 353cd12793ffe3..a436adfc1dd822 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -116,6 +116,7 @@ int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCo
 
 struct pyfinalize_args {
     const char *caller;
+    int verbose;
 };
 
 // Export for _testembed
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 49ed5f246b6fa1..51ca1c8f435b3e 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -284,6 +284,7 @@ static int test_replace_main_tstate(void)
     assert(PyThreadState_Get() == tstate);
     struct pyfinalize_args args = {
         .caller = "_testembed.test_replace_main_tstate",
+        .verbose = 1,
     };
     if (_Py_Finalize(&_PyRuntime, &args) != 0 && !err) {
         err = 1;
@@ -332,6 +333,7 @@ static int test_fini_in_subthread(void)
 
     struct pyfinalize_args fini_args = {
         .caller = "_testembed.test_fini_in_subthread",
+        .verbose = 1,
     };
     struct fini_subthread_args args = {
         .fini_args = &fini_args,
@@ -359,6 +361,7 @@ static int test_fini_in_main_thread_with_other_tstate(void)
     assert(PyThreadState_Get() != main_tstate);
     struct pyfinalize_args args = {
         .caller = "_testembed.test_fini_in_main_thread_with_other_tstate",
+        .verbose = 1,
     };
     if (_Py_Finalize(&_PyRuntime, &args) != 0) {
         return 1;
@@ -383,6 +386,7 @@ static int test_fini_in_main_thread_with_subinterpreter(void)
     assert(PyThreadState_Get() == substate);
     struct pyfinalize_args args = {
         .caller = "_testembed.test_fini_in_main_thread_with_subinterpreter",
+        .verbose = 1,
     };
     if (_Py_Finalize(&_PyRuntime, &args) != 0) {
         return 1;
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 13814cb2d20021..97daa45c1c5df9 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1920,9 +1920,13 @@ finalize_interp_delete(PyInterpreterState *interp)
 static PyThreadState *
 resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
 {
+#define ERROR(msg) \
+    if (args->verbose) { \
+        fprintf(stderr, "%s: %s\n", args->caller, msg); \
+    }
+
     if (!_Py_IsMainThread()) {
-        fprintf(stderr,
-                "%s: expected to be in the main thread\n", args->caller);
+        ERROR("expected to be in the main thread");
     }
 
     PyThreadState *tstate = _PyThreadState_GET();
@@ -1931,22 +1935,19 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
     PyInterpreterState *main_interp = _PyInterpreterState_Main();
 
     if (tstate->interp != main_interp) {
-        fprintf(stderr,
-                "%s: expected main interpreter to be active\n", args->caller);
+        ERROR("expected main interpreter to be active");
     }
 
     /* The main tstate is set by Py_Initialize(), but can be unset
      * or even replaced in unlikely cases. */
     PyThreadState *main_tstate = runtime->main_tstate;
     if (main_tstate == NULL) {
-        fprintf(stderr, "%s: main thread state not set\n", args->caller);
+        ERROR("main thread state not set");
     }
     else if (tstate != main_tstate) {
         /* Code running in the main thread could swap out the main tstate,
            which ends up being a headache. */
-        fprintf(stderr,
-                "%s: using different thread state than Py_Initialize()\n",
-                args->caller);
+        ERROR("using different thread state than Py_Initialize()");
     }
     else {
         assert(main_tstate->interp != NULL);
@@ -1954,6 +1955,8 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
     }
 
     return tstate;
+
+#undef ERROR
 }
 
 int
@@ -2192,6 +2195,7 @@ Py_FinalizeEx(void)
 {
     struct pyfinalize_args args = {
         .caller = "Py_FinalizeEx",
+        .verbose = 1,
     };
     return _Py_Finalize(&_PyRuntime, &args);
 }
@@ -2201,6 +2205,7 @@ Py_Finalize(void)
 {
     struct pyfinalize_args args = {
         .caller = "Py_Finalize",
+        .verbose = 1,
     };
     (void)_Py_Finalize(&_PyRuntime, &args);
 }
@@ -3277,6 +3282,9 @@ Py_Exit(int sts)
     }
     struct pyfinalize_args args = {
         .caller = "Py_Exit",
+#ifdef Py_DEBUG
+        .verbose = 1,
+#endif
     };
     if (_Py_Finalize(&_PyRuntime, &args) < 0) {
         sts = 120;

From 155dddc6a23348fe8468ae0a5736894ab92efd58 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 25 Jun 2024 13:43:00 -0600
Subject: [PATCH 15/20] Rename the macro.

---
 Python/pylifecycle.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 97daa45c1c5df9..8680306b713384 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1920,13 +1920,13 @@ finalize_interp_delete(PyInterpreterState *interp)
 static PyThreadState *
 resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
 {
-#define ERROR(msg) \
-    if (args->verbose) { \
+#define HANDLE_ERROR(msg)                               \
+    if (args->verbose) {                                \
         fprintf(stderr, "%s: %s\n", args->caller, msg); \
     }
 
     if (!_Py_IsMainThread()) {
-        ERROR("expected to be in the main thread");
+        HANDLE_ERROR("expected to be in the main thread");
     }
 
     PyThreadState *tstate = _PyThreadState_GET();
@@ -1935,19 +1935,19 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
     PyInterpreterState *main_interp = _PyInterpreterState_Main();
 
     if (tstate->interp != main_interp) {
-        ERROR("expected main interpreter to be active");
+        HANDLE_ERROR("expected main interpreter to be active");
     }
 
     /* The main tstate is set by Py_Initialize(), but can be unset
      * or even replaced in unlikely cases. */
     PyThreadState *main_tstate = runtime->main_tstate;
     if (main_tstate == NULL) {
-        ERROR("main thread state not set");
+        HANDLE_ERROR("main thread state not set");
     }
     else if (tstate != main_tstate) {
         /* Code running in the main thread could swap out the main tstate,
            which ends up being a headache. */
-        ERROR("using different thread state than Py_Initialize()");
+        HANDLE_ERROR("using different thread state than Py_Initialize()");
     }
     else {
         assert(main_tstate->interp != NULL);

From f37bcc76405e349ee6ea580d4c70e7756351cb1f Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Tue, 25 Jun 2024 14:04:45 -0600
Subject: [PATCH 16/20] Fix the macro.

---
 Python/pylifecycle.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 8680306b713384..6f17763dbcb78c 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1920,13 +1920,13 @@ finalize_interp_delete(PyInterpreterState *interp)
 static PyThreadState *
 resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
 {
-#define HANDLE_ERROR(msg)                               \
+#define PRINT_ERROR(msg)                                \
     if (args->verbose) {                                \
         fprintf(stderr, "%s: %s\n", args->caller, msg); \
     }
 
     if (!_Py_IsMainThread()) {
-        HANDLE_ERROR("expected to be in the main thread");
+        PRINT_ERROR("expected to be in the main thread");
     }
 
     PyThreadState *tstate = _PyThreadState_GET();
@@ -1935,19 +1935,19 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
     PyInterpreterState *main_interp = _PyInterpreterState_Main();
 
     if (tstate->interp != main_interp) {
-        HANDLE_ERROR("expected main interpreter to be active");
+        PRINT_ERROR("expected main interpreter to be active");
     }
 
     /* The main tstate is set by Py_Initialize(), but can be unset
      * or even replaced in unlikely cases. */
     PyThreadState *main_tstate = runtime->main_tstate;
     if (main_tstate == NULL) {
-        HANDLE_ERROR("main thread state not set");
+        PRINT_ERROR("main thread state not set");
     }
     else if (tstate != main_tstate) {
         /* Code running in the main thread could swap out the main tstate,
            which ends up being a headache. */
-        HANDLE_ERROR("using different thread state than Py_Initialize()");
+        PRINT_ERROR("using different thread state than Py_Initialize()");
     }
     else {
         assert(main_tstate->interp != NULL);
@@ -1956,7 +1956,7 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
 
     return tstate;
 
-#undef ERROR
+#undef PRINT_ERROR
 }
 
 int

From 361d4771ee030fa4d924b3acb24a54534e32c79d Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Wed, 26 Jun 2024 16:46:55 -0600
Subject: [PATCH 17/20] Fix tests.

---
 Lib/test/test_embed.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index f21a612628dcbc..2dc8eaac95d1f1 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -117,6 +117,8 @@ def run_embedded_interpreter(self, *args, env=None,
             p.terminate()
             p.wait()
             raise
+        if returncode is None:
+            returncode = 1 if p.returncode == 0 else p.returncode
         if p.returncode != returncode and support.verbose:
             print(f"--- {cmd} failed ---")
             print(f"stdout:\n{out}")
@@ -231,8 +233,8 @@ def test_replace_main_tstate(self):
             with self.subTest(reuse=reuse, exec=False):
                 self.run_embedded_interpreter(
                     'test_replace_main_tstate',
-                    # At the moment, this actually succeeds on all platforms.
-                    returncode=0,
+                    # At the moment, this fails because main_tstate gets broken.
+                    returncode=None,
                 )
             with self.subTest(reuse=reuse, exec=True):
                 out, _ = self.run_embedded_interpreter(
@@ -248,7 +250,7 @@ def test_fini_in_subthread(self):
             'test_fini_in_subthread',
             # At the moment, this actually succeeds on all platforms,
             # except for Windows (STATUS_ACCESS_VIOLATION).
-            returncode=0xC0000005 if MS_WINDOWS else 0,
+            returncode=None if MS_WINDOWS else 0,
         )
 
     def test_fini_in_main_thread_with_other_tstate(self):

From d3beea4b22a212c80232c87c36cb2be587dac60d Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Thu, 27 Jun 2024 13:28:48 -0600
Subject: [PATCH 18/20] Tweaks to resolve_final_tstate().

---
 Python/pylifecycle.c | 56 ++++++++++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 24d638eca1db08..c8a731cc179363 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1923,6 +1923,9 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
     assert(tstate->interp->runtime == runtime);
     assert(tstate->thread_id == PyThread_get_thread_ident());
     PyInterpreterState *main_interp = _PyInterpreterState_Main();
+    PyThreadState *main_tstate = runtime->main_tstate;
+
+    /* First we report unexpected Py_Finalize() usage. */
 
 #define PRINT_ERROR(msg)                                \
     if (args->verbose) {                                \
@@ -1931,9 +1934,10 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
 
     /* The main tstate is set by Py_Initialize(), but can be unset
      * or even replaced in unlikely cases. */
-    PyThreadState *main_tstate = runtime->main_tstate;
     if (main_tstate == NULL) {
         PRINT_ERROR("main thread state not set");
+        /* Ideally, we would make sure a main tstate is set.
+           For now we leave it unset. */
     }
     else {
         assert(main_tstate->thread_id == runtime->main_thread);
@@ -1949,48 +1953,51 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
         }
     }
 
-    if (_Py_IsMainThread()) {
+    if (!_Py_IsMainThread()) {
+        PRINT_ERROR("expected to be in the main thread");
+    }
+
+    if (tstate->interp != main_interp) {
+        PRINT_ERROR("expected main interpreter to be active");
+    }
+
+#undef PRINT_ERROR
+
+    /* Then we decide the thread state we should use for finalization. */
+
+    PyThreadState *final_tstate = tstate;
+    if (_Py_IsMainThread() && main_tstate != NULL) {
         if (tstate != main_tstate) {
             /* This implies that Py_Finalize() was called while
                a non-main interpreter was active or while the main
                tstate was temporarily swapped out with another.
                Neither case should be allowed, but, until we get around
                to fixing that (and Py_Exit()), we're letting it go. */
-            if (tstate->interp != main_interp) {
-                PRINT_ERROR("expected main interpreter to be active");
-            }
-            (void)PyThreadState_Swap(main_tstate);
+            final_tstate = main_tstate;
         }
     }
     else {
-        PRINT_ERROR("expected to be in the main thread");
         /* This is another unfortunate case where Py_Finalize() was
            called when it shouldn't have been.  We can't simply switch
            over to the main thread.  At the least, however, we can make
            sure the main interpreter is active. */
-        if (!_Py_IsMainInterpreter(tstate->interp)) {
-            PRINT_ERROR("expected main interpreter to be active");
+        if (tstate->interp != main_interp) {
             /* We don't go to the trouble of updating runtime->main_tstate
                since it will be dead soon anyway. */
-            main_tstate =
+            final_tstate =
                 _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI);
-            if (main_tstate != NULL) {
-                _PyThreadState_Bind(main_tstate);
-                (void)PyThreadState_Swap(main_tstate);
-            }
-            else {
-                /* Fall back to the current tstate.  It's better than nothing. */
-                main_tstate = tstate;
+            if (final_tstate != NULL) {
+                _PyThreadState_Bind(final_tstate);
             }
+            /* Otherwise we fall back to the current tstate.
+               It's better than nothing. */
         }
     }
-    assert(main_tstate != NULL);
+    assert(final_tstate != NULL);
 
-    /* We might want to warn if main_tstate->current_frame != NULL. */
+    /* We might want to warn if final_tstate->current_frame != NULL. */
 
-    return main_tstate;
-
-#undef PRINT_ERROR
+    return final_tstate;
 }
 
 int
@@ -2003,8 +2010,11 @@ _Py_Finalize(_PyRuntimeState *runtime, struct pyfinalize_args *args)
         return status;
     }
 
-    /* Get final thread state pointer. */
+    /* Get/attach the final thread state pointer. */
     PyThreadState *tstate = resolve_final_tstate(runtime, args);
+    if (tstate != _PyThreadState_GET()) {
+        (void)PyThreadState_Swap(tstate);
+    }
 
     // Block some operations.
     tstate->interp->finalizing = 1;

From d78f65541bf1fe67d7dce469c576988dce46ae45 Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Thu, 27 Jun 2024 13:40:24 -0600
Subject: [PATCH 19/20] Fix the test failures.

---
 Lib/test/test_embed.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 2dc8eaac95d1f1..a8d0ebfa4b4c0d 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -93,6 +93,8 @@ def setUp(self):
     def tearDown(self):
         os.chdir(self.oldcwd)
 
+    ANY_FAILURE = object()
+
     def run_embedded_interpreter(self, *args, env=None,
                                  timeout=None, returncode=0, input=None,
                                  cwd=None):
@@ -117,7 +119,7 @@ def run_embedded_interpreter(self, *args, env=None,
             p.terminate()
             p.wait()
             raise
-        if returncode is None:
+        if returncode is self.ANY_FAILURE:
             returncode = 1 if p.returncode == 0 else p.returncode
         if p.returncode != returncode and support.verbose:
             print(f"--- {cmd} failed ---")
@@ -234,23 +236,24 @@ def test_replace_main_tstate(self):
                 self.run_embedded_interpreter(
                     'test_replace_main_tstate',
                     # At the moment, this fails because main_tstate gets broken.
-                    returncode=None,
+                    returncode=self.ANY_FAILURE,
                 )
             with self.subTest(reuse=reuse, exec=True):
-                out, _ = self.run_embedded_interpreter(
+                out, rc = self.run_embedded_interpreter(
                     'test_replace_main_tstate',
                     'print("spam!")',
-                    # At the moment, this actually succeeds on all platforms.
-                    returncode=0,
+                    # At the moment, this fails because main_tstate gets broken.
+                    returncode=self.ANY_FAILURE,
                 )
-                self.assertEqual(out.strip(), 'spam!')
+                if rc == 0:
+                    self.assertEqual(out.strip(), 'spam!')
 
     def test_fini_in_subthread(self):
         self.run_embedded_interpreter(
             'test_fini_in_subthread',
             # At the moment, this actually succeeds on all platforms,
             # except for Windows (STATUS_ACCESS_VIOLATION).
-            returncode=None if MS_WINDOWS else 0,
+            returncode=self.ANY_FAILURE if MS_WINDOWS else 0,
         )
 
     def test_fini_in_main_thread_with_other_tstate(self):

From e5c5a5f703088580ea89b81484ec89fdce19f23a Mon Sep 17 00:00:00 2001
From: Eric Snow <ericsnowcurrently@gmail.com>
Date: Thu, 27 Jun 2024 14:29:37 -0600
Subject: [PATCH 20/20] More tweaking resolve_final_tstate().

---
 Python/pylifecycle.c | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index c8a731cc179363..bd4f164272780b 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1966,8 +1966,14 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
     /* Then we decide the thread state we should use for finalization. */
 
     PyThreadState *final_tstate = tstate;
-    if (_Py_IsMainThread() && main_tstate != NULL) {
-        if (tstate != main_tstate) {
+    if (_Py_IsMainThread()) {
+        if (main_tstate == NULL) {
+            if (tstate->interp != main_interp) {
+                /* We will swap in a tstate for the main interpreter. */
+                final_tstate = NULL;
+            }
+        }
+        else if (tstate != main_tstate) {
             /* This implies that Py_Finalize() was called while
                a non-main interpreter was active or while the main
                tstate was temporarily swapped out with another.
@@ -1982,18 +1988,23 @@ resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args)
            over to the main thread.  At the least, however, we can make
            sure the main interpreter is active. */
         if (tstate->interp != main_interp) {
-            /* We don't go to the trouble of updating runtime->main_tstate
-               since it will be dead soon anyway. */
-            final_tstate =
-                _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI);
-            if (final_tstate != NULL) {
-                _PyThreadState_Bind(final_tstate);
-            }
-            /* Otherwise we fall back to the current tstate.
-               It's better than nothing. */
+            final_tstate = NULL;
         }
     }
-    assert(final_tstate != NULL);
+    if (final_tstate == NULL) {
+        /* We don't go to the trouble of updating runtime->main_tstate
+           since it will be dead soon anyway. */
+        final_tstate =
+            _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI);
+        if (final_tstate == NULL) {
+            /* Fall back to the current tstate.  It's better than nothing. */
+            final_tstate = tstate;
+        }
+        else {
+            _PyThreadState_Bind(final_tstate);
+        }
+    }
+    assert(final_tstate->interp == main_interp);
 
     /* We might want to warn if final_tstate->current_frame != NULL. */