gh-123880: Allow recursive import of single-phase-init modules

Lib/test/test_import/__init__.py

@@ -1894,6 +1894,30 @@ def test_absolute_circular_submodule(self):
             str(cm.exception),
         )
 
+    def test_singlephase_circular(self):
+        """Regression test for gh-123950
+
+        Import a single-phase-init module that imports itself
+        from the PyInit_* function (before it's added to sys.modules).
+        Manages its own cache (which is `static`, and so incompatible
+        with multiple interpreters or interpreter reset).
+        """
+        name = '_testsinglephase_circular'
+        filename = _testsinglephase.__file__
+        loader = importlib.machinery.ExtensionFileLoader(name, filename)
+        spec = importlib.util.spec_from_file_location(name, filename,
+                                                      loader=loader)
+        mod = importlib._bootstrap._load(spec)
+
+        self.assertIn(name, sys.modules)
+        self.assertIn(mod.helper_mod_name, sys.modules)
+
+        del sys.modules[mod.helper_mod_name]
+        del sys.modules[name]
+        self.assertIs(mod.clear_static_var(), mod)
+        _testinternalcapi.clear_extension('_testsinglephase_circular',
+                                          mod.__spec__.origin)
+
     def test_unwritable_module(self):
         self.addCleanup(unload, "test.test_import.data.unwritable")
         self.addCleanup(unload, "test.test_import.data.unwritable.x")

Lib/test/test_import/data/circular_imports/singlephase.py

@@ -0,0 +1,14 @@
+"""Cilcular import involving a single-phase-init extension.
+
+This module is imported from the _testsinglephase_circular module from
+_testsinglephase, and imports that module again.
+"""
+
+import importlib
+import _testsinglephase
+
+name = '_testsinglephase_circular'
+filename = _testsinglephase.__file__
+loader = importlib.machinery.ExtensionFileLoader(name, filename)
+spec = importlib.util.spec_from_file_location(name, filename, loader=loader)
+mod = importlib._bootstrap._load(spec)

Misc/NEWS.d/next/C_API/2024-09-12-16-16-24.gh-issue-123880.2-8vcj.rst

@@ -0,0 +1,2 @@
+Fixed a bug that prevented circular imports of extension modules that use
+single-phase initialization.

Modules/_testsinglephase.c

@@ -1,7 +1,7 @@
 
 /* Testing module for single-phase initialization of extension modules
 
-This file contains 8 distinct modules, meaning each as its own name
+This file contains several distinct modules, meaning each as its own name
 and its own init function (PyInit_...).  The default import system will
 only find the one matching the filename: _testsinglephase.  To load the
 others you must do so manually.  For example:
@@ -14,7 +14,7 @@ spec = importlib.util.spec_from_file_location(name, filename, loader=loader)
 mod = importlib._bootstrap._load(spec)
 ```
 
-Here are the 8 modules:
+Here are the modules:
 
 * _testsinglephase
    * def: _testsinglephase_basic,
@@ -163,6 +163,11 @@ Here are the 8 modules:
    * functions: none
    * import system: same as _testsinglephase_with_state
 
+* _testsinglephase_circular
+   Regression test for gh-123880.
+   Does not have the common attributes & methods.
+   See test_singlephase_circular test.test_import.SinglephaseInitTests.
+
 Module state:
 
 * fields
@@ -740,3 +745,53 @@ PyInit__testsinglephase_with_state_check_cache_first(void)
     }
     return PyModule_Create(&_testsinglephase_with_state_check_cache_first);
 }
+
+
+/****************************************/
+/* the _testsinglephase_circular module */
+/****************************************/
+
+static PyObject *static_module_circular;
+
+static PyObject *
+circularmod_clear_static_var(PyObject *self, PyObject *arg)
+{
+    PyObject *result = static_module_circular;
+    static_module_circular = NULL;
+    return result;
+}
+
+static struct PyModuleDef _testsinglephase_circular = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "_testsinglephase_circular",
+    .m_doc = PyDoc_STR("Test module _testsinglephase_circular"),
+    .m_methods = (PyMethodDef[]) {
+        {"clear_static_var", circularmod_clear_static_var, METH_NOARGS,
+         "Clear the static variable and return its previous value."},
+        {NULL, NULL}           /* sentinel */
+    }
+};
+
+PyMODINIT_FUNC
+PyInit__testsinglephase_circular(void)
+{
+    if (!static_module_circular) {
+        static_module_circular = PyModule_Create(&_testsinglephase_circular);
+        if (!static_module_circular) {
+            return NULL;
+        }
+    }
+    static const char helper_mod_name[] = (
+        "test.test_import.data.circular_imports.singlephase");
+    PyObject *helper_mod = PyImport_ImportModule(helper_mod_name);
+    Py_XDECREF(helper_mod);
+    if (!helper_mod) {
+        return NULL;
+    }
+    if(PyModule_AddStringConstant(static_module_circular,
+                                  "helper_mod_name",
+                                  helper_mod_name) < 0) {
+        return NULL;
+    }
+    return Py_NewRef(static_module_circular);
+}

Python/import.c

@@ -815,6 +815,8 @@ static int clear_singlephase_extension(PyInterpreterState *interp,
 
 // Currently, this is only used for testing.
 // (See _testinternalcapi.clear_extension().)
+// If adding another use uses, careful about modules that import themselves
+// recursively (see gh-123880)
 int
 _PyImport_ClearExtension(PyObject *name, PyObject *filename)
 {
@@ -1322,12 +1324,16 @@ _extensions_cache_set(PyObject *path, PyObject *name,
     value = entry == NULL
         ? NULL
         : (struct extensions_cache_value *)entry->value;
-    /* We should never be updating an existing cache value. */
-    assert(value == NULL);
     if (value != NULL) {
-        PyErr_Format(PyExc_SystemError,
-                     "extension module %R is already cached", name);
-        goto finally;
+        /* gh-123880: If there's an existing cache value, it means a module is
+         * being imported recursively from its PyInit_* or Py_mod_* function.
+         * (That function presumably handles returning a partially
+         *  constructed module in such a case.)
+         * We can reuse the existing cache value; it is owned by the cache.
+         * (Entries get removed from it in exceptional circumstances,
+         *  after interpreter shutdown, and in runtime shutdown.)
+         */
+        goto finally_oldvalue;
     }
     newvalue = alloc_extensions_cache_value();
     if (newvalue == NULL) {
@@ -1392,6 +1398,7 @@ _extensions_cache_set(PyObject *path, PyObject *name,
         cleanup_old_cached_def(&olddefbase);
     }
 
+finally_oldvalue:
     extensions_lock_release();
     if (key != NULL) {
         hashtable_destroy_str(key);
@@ -2128,6 +2135,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0,
 }
 
 
+// Used in _PyImport_ClearExtension; see notes there
 static int
 clear_singlephase_extension(PyInterpreterState *interp,
                             PyObject *name, PyObject *path)

Tools/c-analyzer/cpython/ignored.tsv

@@ -590,6 +590,7 @@ Modules/_testmultiphase.c	-	slots_nonmodule_with_exec_slots	-
 Modules/_testmultiphase.c	-	testexport_methods	-
 Modules/_testmultiphase.c	-	uninitialized_def	-
 Modules/_testsinglephase.c	-	global_state	-
+Modules/_testsinglephase.c	-	static_module_circular	-
 Modules/_xxtestfuzz/_xxtestfuzz.c	-	_fuzzmodule	-
 Modules/_xxtestfuzz/_xxtestfuzz.c	-	module_methods	-
 Modules/_xxtestfuzz/fuzzer.c	-	RE_FLAG_DEBUG	-