From ac7ff4fd251a63b23adf2e192527e6b4829f7089 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 09:41:50 -0600 Subject: [PATCH 01/52] Add the -X frozen_modules CLI option and PyConfig.use_frozen_modules. --- Doc/using/cmdline.rst | 8 ++++ Include/cpython/initconfig.h | 3 ++ Lib/test/test_embed.py | 1 + Python/initconfig.c | 77 ++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 7ebc34f47fc39b..9c24cb42d17d80 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -480,6 +480,11 @@ Miscellaneous options objects and pyc files are desired as well as supressing the extra visual location indicators when the interpreter displays tracebacks. See also :envvar:`PYTHONNODEBUGRANGES`. + * ``-X frozen_modules`` determines whether or not frozen modules are + ignored by the import machinery. A value of "on" means they get + imported and "off" means they are ignored. The default is "on". + Note that the "importlib_bootstrap" and "importlib_bootstrap_external" + frozen modules are always used, even if this flag is set to "off". It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -518,6 +523,9 @@ Miscellaneous options .. versionadded:: 3.11 The ``-X no_debug_ranges`` option. + .. versionadded:: 3.11 + The ``-X frozen_modules`` option. + Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 22ad0f14e58004..fc0c40feb85c71 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -2,6 +2,8 @@ #define Py_PYCORECONFIG_H #ifndef Py_LIMITED_API +#include + /* --- PyStatus ----------------------------------------------- */ typedef struct { @@ -172,6 +174,7 @@ typedef struct PyConfig { int legacy_windows_stdio; #endif wchar_t *check_hash_pycs_mode; + bool use_frozen_modules; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 8e3dd50c1f8be0..6fabaaf6448c92 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -426,6 +426,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, + 'use_frozen_modules': GET_DEFAULT_CONFIG, } if MS_WINDOWS: CONFIG_COMPAT.update({ diff --git a/Python/initconfig.c b/Python/initconfig.c index 61cd0e6213ed17..55f1e7fc50b570 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -100,6 +100,8 @@ static const char usage_3[] = "\ instruction in code objects. This is useful when smaller code objects and pyc \n\ files are desired as well as supressing the extra visual location indicators \n\ when the interpreter displays tracebacks.\n\ + -X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ + The default is \"on\" (or \"off\" if you are running a local build).\n\ \n\ --check-hash-based-pycs always|default|never:\n\ control how Python invalidates hash-based .pyc files\n\ @@ -949,6 +951,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_ATTR(pathconfig_warnings); COPY_ATTR(_init_main); COPY_ATTR(_isolated_interpreter); + COPY_ATTR(use_frozen_modules); COPY_WSTRLIST(orig_argv); #undef COPY_ATTR @@ -982,6 +985,10 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR)) #define SET_ITEM_UINT(ATTR) \ SET_ITEM(#ATTR, PyLong_FromUnsignedLong(config->ATTR)) +#define SET_ITEM_BOOL(ATTR) \ + SET_ITEM(#ATTR, ( \ + Py_INCREF(config->ATTR ? Py_True : Py_False), \ + (config->ATTR ? Py_True : Py_False))) #define FROM_WSTRING(STR) \ ((STR != NULL) ? \ PyUnicode_FromWideChar(STR, -1) \ @@ -1052,6 +1059,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_INT(_init_main); SET_ITEM_INT(_isolated_interpreter); SET_ITEM_WSTRLIST(orig_argv); + SET_ITEM_BOOL(use_frozen_modules); return dict; @@ -1063,6 +1071,7 @@ _PyConfig_AsDict(const PyConfig *config) #undef SET_ITEM #undef SET_ITEM_INT #undef SET_ITEM_UINT +#undef SET_ITEM_BOOL #undef SET_ITEM_WSTR #undef SET_ITEM_WSTRLIST } @@ -1138,6 +1147,23 @@ config_dict_get_ulong(PyObject *dict, const char *name, unsigned long *result) } +static int +config_dict_get_bool(PyObject *dict, const char *name, bool *result) +{ + PyObject *item = config_dict_get(dict, name); + if (item == NULL) { + return -1; + } + int value = PyObject_IsTrue(item); + if (value < 0) { + config_dict_invalid_value(name); + return -1; + } + *result = value == 0 ? false : true; + return 0; +} + + static int config_dict_get_wstr(PyObject *dict, const char *name, PyConfig *config, wchar_t **result) @@ -1241,6 +1267,12 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) } \ CHECK_VALUE(#KEY, config->KEY >= 0); \ } while (0) +#define GET_BOOL(KEY) \ + do { \ + if (config_dict_get_bool(dict, #KEY, &config->KEY) < 0) { \ + return -1; \ + } \ + } while (0) #define GET_WSTR(KEY) \ do { \ if (config_dict_get_wstr(dict, #KEY, config, &config->KEY) < 0) { \ @@ -1334,9 +1366,11 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(_install_importlib); GET_UINT(_init_main); GET_UINT(_isolated_interpreter); + GET_BOOL(use_frozen_modules); #undef CHECK_VALUE #undef GET_UINT +#undef GET_BOOL #undef GET_WSTR #undef GET_WSTR_OPT return 0; @@ -1590,6 +1624,17 @@ config_get_xoption(const PyConfig *config, wchar_t *name) return _Py_get_xoption(&config->xoptions, name); } +static const wchar_t* +config_get_xoption_value(const PyConfig *config, wchar_t *name) +{ + const wchar_t *xoption = config_get_xoption(config, name); + if (xoption == NULL) { + return NULL; + } + const wchar_t *sep = wcschr(xoption, L'='); + return sep ? sep + 1 : L""; +} + static PyStatus config_init_home(PyConfig *config) @@ -2065,6 +2110,34 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) } +static PyStatus +config_init_imports(PyConfig *config) +{ + /* -X frozen_modules=[on|off] */ + const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); + if (value == NULL) { + // XXX Set default to false if in development. + config->use_frozen_modules = true; + } + else if (wcscmp(value, L"on") == 0) { + config->use_frozen_modules = true; + } + else if (wcscmp(value, L"off") == 0) { + config->use_frozen_modules = false; + } + else if (wcslen(value) == 0) { + // "-X frozen_modules" and "-X frozen_modules=" both imply "on". + config->use_frozen_modules = true; + } + else { + return PyStatus_Error("bad value for option -X frozen_modules " + "(expected \"on\" or \"off\")"); + } + + return _PyStatus_OK(); +} + + static PyStatus config_read(PyConfig *config, int compute_path_config) { @@ -2115,6 +2188,10 @@ config_read(PyConfig *config, int compute_path_config) if (_PyStatus_EXCEPTION(status)) { return status; } + status = config_init_imports(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } } /* default values */ From 7d2a4707627195db9c4125ea467bbe0bb524a5f3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 12:41:03 -0600 Subject: [PATCH 02/52] Ignore frozen modules depending on "-X frozen_modules". --- Python/import.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Python/import.c b/Python/import.c index d896ff476e1796..fa7301dd645d10 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1057,10 +1057,19 @@ list_frozen_module_names(bool force) if (names == NULL) { return NULL; } + const PyConfig *config = _Py_GetConfig(); for (const struct _frozen *p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) { break; } + if (!config->use_frozen_modules) { + /* These modules are necessary to bootstrap the import system. */ + if (!strcmp(p->name, "_frozen_importlib") && + !strcmp(p->name, "_frozen_importlib_external")) + { + continue; + } + } PyObject *name = PyUnicode_FromString(p->name); if (name == NULL) { Py_DECREF(names); @@ -1084,6 +1093,16 @@ find_frozen(PyObject *name) if (name == NULL) return NULL; + const PyConfig *config = _Py_GetConfig(); + if (!config->use_frozen_modules) { + /* These modules are necessary to bootstrap the import system. */ + if (!_PyUnicode_EqualToASCIIString(name, "_frozen_importlib") && + !_PyUnicode_EqualToASCIIString(name, "_frozen_importlib_external")) + { + return NULL; + } + } + for (p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) return NULL; From aacaf24fe8622aec29e1478f84b0b7a8082e31bb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 1 Sep 2021 17:01:56 -0600 Subject: [PATCH 03/52] Explicitly control PyConfig.use_frozen_modules during tests. --- Lib/test/support/import_helper.py | 18 +- Lib/test/test_frozen.py | 7 +- Lib/test/test_importlib/frozen/test_finder.py | 8 +- Lib/test/test_importlib/frozen/test_loader.py | 211 ++++++++++-------- Python/import.c | 74 +++++- 5 files changed, 205 insertions(+), 113 deletions(-) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 5d1e9406879cc6..0d352eba78551e 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -6,7 +6,7 @@ import unittest import warnings -from .os_helper import unlink +from .os_helper import unlink, EnvironmentVarGuard @contextlib.contextmanager @@ -109,7 +109,18 @@ def _save_and_block_module(name, orig_modules): return saved -def import_fresh_module(name, fresh=(), blocked=(), deprecated=False): +@contextlib.contextmanager +def frozen_modules(*, disabled=False): + with EnvironmentVarGuard() as env: + os.environ = env + os.environ['_PYTHONTESTFROZENMODULES'] = '' if disabled else '1' + yield + + +def import_fresh_module(name, fresh=(), blocked=(), *, + deprecated=False, + usefrozen=False, + ): """Import and return a module, deliberately bypassing sys.modules. This function imports and returns a fresh copy of the named Python module @@ -148,7 +159,8 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False): for blocked_name in blocked: if not _save_and_block_module(blocked_name, orig_modules): names_to_remove.append(blocked_name) - fresh_module = importlib.import_module(name) + with frozen_modules(disabled=not usefrozen): + fresh_module = importlib.import_module(name) except ImportError: fresh_module = None finally: diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py index 142f17d518e783..52d8f7ced96803 100644 --- a/Lib/test/test_frozen.py +++ b/Lib/test/test_frozen.py @@ -12,7 +12,7 @@ import sys import unittest -from test.support import captured_stdout +from test.support import captured_stdout, import_helper class TestFrozen(unittest.TestCase): @@ -20,8 +20,9 @@ def test_frozen(self): name = '__hello__' if name in sys.modules: del sys.modules[name] - with captured_stdout() as out: - import __hello__ + with import_helper.frozen_modules(): + with captured_stdout() as out: + import __hello__ self.assertEqual(out.getvalue(), 'Hello world!\n') diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index eb7a4d275cfd26..fbc3fc0d547ff7 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -6,6 +6,8 @@ import unittest import warnings +from test.support import import_helper + class FindSpecTests(abc.FinderTests): @@ -13,7 +15,8 @@ class FindSpecTests(abc.FinderTests): def find(self, name, path=None): finder = self.machinery.FrozenImporter - return finder.find_spec(name, path) + with import_helper.frozen_modules(): + return finder.find_spec(name, path) def test_module(self): name = '__hello__' @@ -52,7 +55,8 @@ def find(self, name, path=None): finder = self.machinery.FrozenImporter with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - return finder.find_module(name, path) + with import_helper.frozen_modules(): + return finder.find_module(name, path) def test_module(self): name = '__hello__' diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index f0cf17990b372d..1b0a56f7e8afa9 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -3,27 +3,54 @@ machinery = util.import_importlib('importlib.machinery') -from test.support import captured_stdout +from test.support import captured_stdout, import_helper +import contextlib import types import unittest import warnings +@contextlib.contextmanager +def deprecated(): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + yield + + +@contextlib.contextmanager +def fresh(name, *, oldapi=False): + with util.uncache(name): + with import_helper.frozen_modules(): + with captured_stdout() as stdout: + if oldapi: + with deprecated(): + yield stdout + else: + yield stdout + + class ExecModuleTests(abc.LoaderTests): def exec_module(self, name): - with util.uncache(name), captured_stdout() as stdout: - spec = self.machinery.ModuleSpec( - name, self.machinery.FrozenImporter, origin='frozen', - is_package=self.machinery.FrozenImporter.is_package(name)) - module = types.ModuleType(name) - module.__spec__ = spec - assert not hasattr(module, 'initialized') + with import_helper.frozen_modules(): + is_package = self.machinery.FrozenImporter.is_package(name) + spec = self.machinery.ModuleSpec( + name, + self.machinery.FrozenImporter, + origin='frozen', + is_package=is_package, + ) + module = types.ModuleType(name) + module.__spec__ = spec + assert not hasattr(module, 'initialized') + + with fresh(name) as stdout: self.machinery.FrozenImporter.exec_module(module) - self.assertTrue(module.initialized) - self.assertTrue(hasattr(module, '__spec__')) - self.assertEqual(module.__spec__.origin, 'frozen') - return module, stdout.getvalue() + + self.assertTrue(module.initialized) + self.assertTrue(hasattr(module, '__spec__')) + self.assertEqual(module.__spec__.origin, 'frozen') + return module, stdout.getvalue() def test_module(self): name = '__hello__' @@ -50,20 +77,19 @@ def test_lacking_parent(self): name = '__phello__.spam' with util.uncache('__phello__'): module, output = self.exec_module(name) - check = {'__name__': name} - for attr, value in check.items(): - attr_value = getattr(module, attr) - self.assertEqual(attr_value, value, - 'for {name}.{attr}, {given} != {expected!r}'.format( - name=name, attr=attr, given=attr_value, - expected=value)) - self.assertEqual(output, 'Hello world!\n') + check = {'__name__': name} + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + 'for {name}.{attr}, {given} != {expected!r}'.format( + name=name, attr=attr, given=attr_value, + expected=value)) + self.assertEqual(output, 'Hello world!\n') def test_module_repr(self): name = '__hello__' module, output = self.exec_module(name) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) + with deprecated(): repr_str = self.machinery.FrozenImporter.module_repr(module) self.assertEqual(repr_str, "") @@ -78,7 +104,8 @@ def test_module_repr_indirect(self): test_state_after_failure = None def test_unloadable(self): - assert self.machinery.FrozenImporter.find_spec('_not_real') is None + with import_helper.frozen_modules(): + assert self.machinery.FrozenImporter.find_spec('_not_real') is None with self.assertRaises(ImportError) as cm: self.exec_module('_not_real') self.assertEqual(cm.exception.name, '_not_real') @@ -91,84 +118,76 @@ def test_unloadable(self): class LoaderTests(abc.LoaderTests): + def load_module(self, name): + with fresh(name, oldapi=True) as stdout: + module = self.machinery.FrozenImporter.load_module(name) + return module, stdout + def test_module(self): - with util.uncache('__hello__'), captured_stdout() as stdout: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.machinery.FrozenImporter.load_module('__hello__') - check = {'__name__': '__hello__', - '__package__': '', - '__loader__': self.machinery.FrozenImporter, - } - for attr, value in check.items(): - self.assertEqual(getattr(module, attr), value) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - self.assertFalse(hasattr(module, '__file__')) + module, stdout = self.load_module('__hello__') + check = {'__name__': '__hello__', + '__package__': '', + '__loader__': self.machinery.FrozenImporter, + } + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) def test_package(self): - with util.uncache('__phello__'), captured_stdout() as stdout: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.machinery.FrozenImporter.load_module('__phello__') - check = {'__name__': '__phello__', - '__package__': '__phello__', - '__path__': [], - '__loader__': self.machinery.FrozenImporter, - } - for attr, value in check.items(): - attr_value = getattr(module, attr) - self.assertEqual(attr_value, value, - "for __phello__.%s, %r != %r" % - (attr, attr_value, value)) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - self.assertFalse(hasattr(module, '__file__')) + module, stdout = self.load_module('__phello__') + check = {'__name__': '__phello__', + '__package__': '__phello__', + '__path__': [], + '__loader__': self.machinery.FrozenImporter, + } + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + "for __phello__.%s, %r != %r" % + (attr, attr_value, value)) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) def test_lacking_parent(self): - with util.uncache('__phello__', '__phello__.spam'), \ - captured_stdout() as stdout: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.machinery.FrozenImporter.load_module('__phello__.spam') - check = {'__name__': '__phello__.spam', - '__package__': '__phello__', - '__loader__': self.machinery.FrozenImporter, - } - for attr, value in check.items(): - attr_value = getattr(module, attr) - self.assertEqual(attr_value, value, - "for __phello__.spam.%s, %r != %r" % - (attr, attr_value, value)) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - self.assertFalse(hasattr(module, '__file__')) + with util.uncache('__phello__'): + module, stdout = self.load_module('__phello__.spam') + check = {'__name__': '__phello__.spam', + '__package__': '__phello__', + '__loader__': self.machinery.FrozenImporter, + } + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + "for __phello__.spam.%s, %r != %r" % + (attr, attr_value, value)) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) def test_module_reuse(self): - with util.uncache('__hello__'), captured_stdout() as stdout: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module1 = self.machinery.FrozenImporter.load_module('__hello__') - module2 = self.machinery.FrozenImporter.load_module('__hello__') - self.assertIs(module1, module2) - self.assertEqual(stdout.getvalue(), - 'Hello world!\nHello world!\n') + with fresh('__hello__', oldapi=True) as stdout: + module1 = self.machinery.FrozenImporter.load_module('__hello__') + module2 = self.machinery.FrozenImporter.load_module('__hello__') + self.assertIs(module1, module2) + self.assertEqual(stdout.getvalue(), + 'Hello world!\nHello world!\n') def test_module_repr(self): - with util.uncache('__hello__'), captured_stdout(): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.machinery.FrozenImporter.load_module('__hello__') - repr_str = self.machinery.FrozenImporter.module_repr(module) - self.assertEqual(repr_str, - "") + with fresh('__hello__', oldapi=True) as stdout: + module = self.machinery.FrozenImporter.load_module('__hello__') + repr_str = self.machinery.FrozenImporter.module_repr(module) + self.assertEqual(repr_str, + "") # No way to trigger an error in a frozen module. test_state_after_failure = None def test_unloadable(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - assert self.machinery.FrozenImporter.find_module('_not_real') is None + with import_helper.frozen_modules(): + with deprecated(): + assert self.machinery.FrozenImporter.find_module('_not_real') is None with self.assertRaises(ImportError) as cm: - self.machinery.FrozenImporter.load_module('_not_real') + self.load_module('_not_real') self.assertEqual(cm.exception.name, '_not_real') @@ -185,15 +204,17 @@ def test_get_code(self): # Make sure that the code object is good. name = '__hello__' with captured_stdout() as stdout: - code = self.machinery.FrozenImporter.get_code(name) - mod = types.ModuleType(name) - exec(code, mod.__dict__) - self.assertTrue(hasattr(mod, 'initialized')) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') + with import_helper.frozen_modules(): + code = self.machinery.FrozenImporter.get_code(name) + mod = types.ModuleType(name) + exec(code, mod.__dict__) + self.assertTrue(hasattr(mod, 'initialized')) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') def test_get_source(self): # Should always return None. - result = self.machinery.FrozenImporter.get_source('__hello__') + with import_helper.frozen_modules(): + result = self.machinery.FrozenImporter.get_source('__hello__') self.assertIsNone(result) def test_is_package(self): @@ -201,7 +222,8 @@ def test_is_package(self): test_for = (('__hello__', False), ('__phello__', True), ('__phello__.spam', False)) for name, is_package in test_for: - result = self.machinery.FrozenImporter.is_package(name) + with import_helper.frozen_modules(): + result = self.machinery.FrozenImporter.is_package(name) self.assertEqual(bool(result), is_package) def test_failure(self): @@ -209,7 +231,8 @@ def test_failure(self): for meth_name in ('get_code', 'get_source', 'is_package'): method = getattr(self.machinery.FrozenImporter, meth_name) with self.assertRaises(ImportError) as cm: - method('importlib') + with import_helper.frozen_modules(): + method('importlib') self.assertEqual(cm.exception.name, 'importlib') (Frozen_ILTests, diff --git a/Python/import.c b/Python/import.c index fa7301dd645d10..c1528bed2ba252 100644 --- a/Python/import.c +++ b/Python/import.c @@ -886,7 +886,7 @@ _imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, /* Forward */ -static const struct _frozen * find_frozen(PyObject *); +static const struct _frozen * find_frozen(PyObject *, bool); /* Helper to test for built-in module */ @@ -1086,7 +1086,7 @@ list_frozen_module_names(bool force) } static const struct _frozen * -find_frozen(PyObject *name) +find_frozen(PyObject *name, bool force) { const struct _frozen *p; @@ -1094,7 +1094,7 @@ find_frozen(PyObject *name) return NULL; const PyConfig *config = _Py_GetConfig(); - if (!config->use_frozen_modules) { + if (!config->use_frozen_modules && !force) { /* These modules are necessary to bootstrap the import system. */ if (!_PyUnicode_EqualToASCIIString(name, "_frozen_importlib") && !_PyUnicode_EqualToASCIIString(name, "_frozen_importlib_external")) @@ -1113,9 +1113,9 @@ find_frozen(PyObject *name) } static PyObject * -get_frozen_object(PyObject *name) +get_frozen_object(PyObject *name, bool force) { - const struct _frozen *p = find_frozen(name); + const struct _frozen *p = find_frozen(name, force); int size; if (p == NULL) { @@ -1137,9 +1137,9 @@ get_frozen_object(PyObject *name) } static PyObject * -is_frozen_package(PyObject *name) +is_frozen_package(PyObject *name, bool force) { - const struct _frozen *p = find_frozen(name); + const struct _frozen *p = find_frozen(name, force); int size; if (p == NULL) { @@ -1172,7 +1172,7 @@ PyImport_ImportFrozenModuleObject(PyObject *name) int ispackage; int size; - p = find_frozen(name); + p = find_frozen(name, false); if (p == NULL) return 0; @@ -1933,6 +1933,55 @@ _imp_init_frozen_impl(PyObject *module, PyObject *name) return import_add_module(tstate, name); } +static const char * +get_env_var(const char *name) +{ + /* Try os.environ first. */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *modules = interp->modules; + PyObject *osmod; + if (PyDict_CheckExact(modules)) { + osmod = PyDict_GetItemString(modules, "os"); /* borrowed */ + Py_XINCREF(osmod); + } + else { + osmod = PyMapping_GetItemString(modules, "os"); + } + if (osmod != NULL) { + PyObject *env = PyObject_GetAttrString(osmod, "environ"); + Py_DECREF(osmod); + if (env != NULL) { + const char *value = NULL; + PyObject *obj = PyMapping_GetItemString(env, name); + Py_DECREF(env); + if (obj != NULL) { + value = PyUnicode_AsUTF8(obj); + Py_DECREF(obj); + } + PyErr_Clear(); + return value; + } + } + PyErr_Clear(); + + /* Fall back to the process env. */ + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + assert(config != NULL); + return _Py_GetEnv(config->use_environment, name); +} + +static bool +get_env_var_flag(const char *name) +{ + const char *env = get_env_var(name); + /* It isn't set or it is set to an empty string. */ + if (env == NULL || strlen(env) == 0) { + return false; + } + /* For now we treat all non-empty strings as true. */ + return true; +} + /*[clinic input] _imp.get_frozen_object @@ -1946,7 +1995,8 @@ static PyObject * _imp_get_frozen_object_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=2568cc5b7aa0da63 input=ed689bc05358fdbd]*/ { - return get_frozen_object(name); + bool force = get_env_var_flag("_PYTHONTESTFROZENMODULES"); + return get_frozen_object(name, force); } /*[clinic input] @@ -1962,7 +2012,8 @@ static PyObject * _imp_is_frozen_package_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=e70cbdb45784a1c9 input=81b6cdecd080fbb8]*/ { - return is_frozen_package(name); + bool force = get_env_var_flag("_PYTHONTESTFROZENMODULES"); + return is_frozen_package(name, force); } /*[clinic input] @@ -1994,9 +2045,10 @@ static PyObject * _imp_is_frozen_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=01f408f5ec0f2577 input=7301dbca1897d66b]*/ { + bool force = get_env_var_flag("_PYTHONTESTFROZENMODULES"); const struct _frozen *p; - p = find_frozen(name); + p = find_frozen(name, force); return PyBool_FromLong((long) (p == NULL ? 0 : p->size)); } From 5a6e40c54ed4ab08c75a541dbf929759f88f7d92 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Sep 2021 13:11:51 -0600 Subject: [PATCH 04/52] Factor out is_essential_frozen_module(). --- Python/import.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/Python/import.c b/Python/import.c index c1528bed2ba252..afce9d0e7a9209 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1050,6 +1050,19 @@ _imp_create_builtin(PyObject *module, PyObject *spec) /* Frozen modules */ +static bool +is_essential_frozen_module(PyObject *name) +{ + /* These modules are necessary to bootstrap the import system. */ + if (_PyUnicode_EqualToASCIIString(name, "_frozen_importlib")) { + return true; + } + if (_PyUnicode_EqualToASCIIString(name, "_frozen_importlib_external")) { + return true; + } + return false; +} + static PyObject * list_frozen_module_names(bool force) { @@ -1062,19 +1075,17 @@ list_frozen_module_names(bool force) if (p->name == NULL) { break; } - if (!config->use_frozen_modules) { - /* These modules are necessary to bootstrap the import system. */ - if (!strcmp(p->name, "_frozen_importlib") && - !strcmp(p->name, "_frozen_importlib_external")) - { - continue; - } - } PyObject *name = PyUnicode_FromString(p->name); if (name == NULL) { Py_DECREF(names); return NULL; } + if (!config->use_frozen_modules && !force) { + if (!is_essential_frozen_module(name)) { + Py_DECREF(name); + continue; + } + } int res = PyList_Append(names, name); Py_DECREF(name); if (res != 0) { @@ -1095,10 +1106,7 @@ find_frozen(PyObject *name, bool force) const PyConfig *config = _Py_GetConfig(); if (!config->use_frozen_modules && !force) { - /* These modules are necessary to bootstrap the import system. */ - if (!_PyUnicode_EqualToASCIIString(name, "_frozen_importlib") && - !_PyUnicode_EqualToASCIIString(name, "_frozen_importlib_external")) - { + if (!is_essential_frozen_module(name)) { return NULL; } } From 4da6302ec0bb1d6c45417f1fa9a651cb1006ed70 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 10:24:43 -0600 Subject: [PATCH 05/52] Treat __main__ as an essential frozen module. --- Python/import.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Python/import.c b/Python/import.c index afce9d0e7a9209..e664ae1272968c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1060,6 +1060,11 @@ is_essential_frozen_module(PyObject *name) if (_PyUnicode_EqualToASCIIString(name, "_frozen_importlib_external")) { return true; } + /* This doesn't otherwise have anywhere to find the module. + See frozenmain.c. */ + if (_PyUnicode_EqualToASCIIString(name, "__main__")) { + return true; + } return false; } From 26469e538ec30cadbd840f8989298fc1ca9205d8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 13:33:50 -0600 Subject: [PATCH 06/52] Look up $_PYTHONTESTFROZENMODULES using getenv() instead of in os.environ. --- Lib/test/support/import_helper.py | 10 +++--- Python/import.c | 51 +++++-------------------------- 2 files changed, 14 insertions(+), 47 deletions(-) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 0d352eba78551e..72d500044999c3 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -6,7 +6,7 @@ import unittest import warnings -from .os_helper import unlink, EnvironmentVarGuard +from .os_helper import unlink @contextlib.contextmanager @@ -111,10 +111,12 @@ def _save_and_block_module(name, orig_modules): @contextlib.contextmanager def frozen_modules(*, disabled=False): - with EnvironmentVarGuard() as env: - os.environ = env - os.environ['_PYTHONTESTFROZENMODULES'] = '' if disabled else '1' + # FYI: the env var will never show up in os.environ. + os.putenv('_PYTHONTESTFROZENMODULES', '' if disabled else '1') + try: yield + finally: + os.unsetenv('_PYTHONTESTFROZENMODULES') def import_fresh_module(name, fresh=(), blocked=(), *, diff --git a/Python/import.c b/Python/import.c index e664ae1272968c..a0d7cc803c77cc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1946,49 +1946,11 @@ _imp_init_frozen_impl(PyObject *module, PyObject *name) return import_add_module(tstate, name); } -static const char * -get_env_var(const char *name) -{ - /* Try os.environ first. */ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *modules = interp->modules; - PyObject *osmod; - if (PyDict_CheckExact(modules)) { - osmod = PyDict_GetItemString(modules, "os"); /* borrowed */ - Py_XINCREF(osmod); - } - else { - osmod = PyMapping_GetItemString(modules, "os"); - } - if (osmod != NULL) { - PyObject *env = PyObject_GetAttrString(osmod, "environ"); - Py_DECREF(osmod); - if (env != NULL) { - const char *value = NULL; - PyObject *obj = PyMapping_GetItemString(env, name); - Py_DECREF(env); - if (obj != NULL) { - value = PyUnicode_AsUTF8(obj); - Py_DECREF(obj); - } - PyErr_Clear(); - return value; - } - } - PyErr_Clear(); - - /* Fall back to the process env. */ - const PyConfig *config = _PyInterpreterState_GetConfig(interp); - assert(config != NULL); - return _Py_GetEnv(config->use_environment, name); -} - static bool -get_env_var_flag(const char *name) +parse_env_var_flag(const char *value) { - const char *env = get_env_var(name); /* It isn't set or it is set to an empty string. */ - if (env == NULL || strlen(env) == 0) { + if (value == NULL || strlen(value) == 0) { return false; } /* For now we treat all non-empty strings as true. */ @@ -2008,7 +1970,8 @@ static PyObject * _imp_get_frozen_object_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=2568cc5b7aa0da63 input=ed689bc05358fdbd]*/ { - bool force = get_env_var_flag("_PYTHONTESTFROZENMODULES"); + /* Note that we don't bother with os.environ. */ + bool force = parse_env_var_flag(getenv("_PYTHONTESTFROZENMODULES")); return get_frozen_object(name, force); } @@ -2025,7 +1988,8 @@ static PyObject * _imp_is_frozen_package_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=e70cbdb45784a1c9 input=81b6cdecd080fbb8]*/ { - bool force = get_env_var_flag("_PYTHONTESTFROZENMODULES"); + /* Note that we don't bother with os.environ. */ + bool force = parse_env_var_flag(getenv("_PYTHONTESTFROZENMODULES")); return is_frozen_package(name, force); } @@ -2058,7 +2022,8 @@ static PyObject * _imp_is_frozen_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=01f408f5ec0f2577 input=7301dbca1897d66b]*/ { - bool force = get_env_var_flag("_PYTHONTESTFROZENMODULES"); + /* Note that we don't bother with os.environ. */ + bool force = parse_env_var_flag(getenv("_PYTHONTESTFROZENMODULES")); const struct _frozen *p; p = find_frozen(name, force); From f274b1ce590e85f334bd532c2a0f51a5b8d957fa Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 14:41:39 -0600 Subject: [PATCH 07/52] Add zipimport to the list of essential frozen modules. --- Python/import.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/import.c b/Python/import.c index a0d7cc803c77cc..536326ee46ea01 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1060,6 +1060,9 @@ is_essential_frozen_module(PyObject *name) if (_PyUnicode_EqualToASCIIString(name, "_frozen_importlib_external")) { return true; } + if (_PyUnicode_EqualToASCIIString(name, "zipimport")) { + return true; + } /* This doesn't otherwise have anywhere to find the module. See frozenmain.c. */ if (_PyUnicode_EqualToASCIIString(name, "__main__")) { From 6881cb88c4f0521bda259d6f9e771e409ffd4a75 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Sep 2021 12:34:03 -0600 Subject: [PATCH 08/52] Flip around the arg to the "frozen_modules" test helper. --- Lib/test/support/import_helper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 72d500044999c3..d0805462b59a83 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -110,9 +110,11 @@ def _save_and_block_module(name, orig_modules): @contextlib.contextmanager -def frozen_modules(*, disabled=False): +def frozen_modules(enabled=True): + # No other code should be setting this env var. + assert os.getenv('_PYTHONTESTFROZENMODULES') is None # FYI: the env var will never show up in os.environ. - os.putenv('_PYTHONTESTFROZENMODULES', '' if disabled else '1') + os.putenv('_PYTHONTESTFROZENMODULES', '1' if enabled else '') try: yield finally: @@ -161,7 +163,7 @@ def import_fresh_module(name, fresh=(), blocked=(), *, for blocked_name in blocked: if not _save_and_block_module(blocked_name, orig_modules): names_to_remove.append(blocked_name) - with frozen_modules(disabled=not usefrozen): + with frozen_modules(usefrozen): fresh_module = importlib.import_module(name) except ImportError: fresh_module = None From 785addf39770bd47dde881287902f5aed27c64ec Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Sep 2021 14:35:17 -0600 Subject: [PATCH 09/52] Disable frozen modules if $_PYTHONTESTFROZENMODULES is 0. --- Lib/test/support/import_helper.py | 4 +- Python/import.c | 120 +++++++++++++++++------------- 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index d0805462b59a83..0429b6a5fdc30e 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -111,10 +111,8 @@ def _save_and_block_module(name, orig_modules): @contextlib.contextmanager def frozen_modules(enabled=True): - # No other code should be setting this env var. - assert os.getenv('_PYTHONTESTFROZENMODULES') is None # FYI: the env var will never show up in os.environ. - os.putenv('_PYTHONTESTFROZENMODULES', '1' if enabled else '') + os.putenv('_PYTHONTESTFROZENMODULES', '1' if enabled else '0') try: yield finally: diff --git a/Python/import.c b/Python/import.c index 536326ee46ea01..c83320694a2e32 100644 --- a/Python/import.c +++ b/Python/import.c @@ -886,7 +886,7 @@ _imp__fix_co_filename_impl(PyObject *module, PyCodeObject *code, /* Forward */ -static const struct _frozen * find_frozen(PyObject *, bool); +static const struct _frozen * find_frozen(PyObject *); /* Helper to test for built-in module */ @@ -1051,49 +1051,82 @@ _imp_create_builtin(PyObject *module, PyObject *spec) /* Frozen modules */ static bool -is_essential_frozen_module(PyObject *name) +parse_env_var_flag(const char *rawvalue, bool *isset) +{ + /* It isn't set or it is set to an empty string. */ + if (rawvalue == NULL || strlen(rawvalue) == 0) { + if (isset != NULL) { + *isset = false; + } + return false; + } + if (isset != NULL) { + *isset = true; + } + if (strcmp(rawvalue, "0") == 0) { + return false; + } + /* For now we treat all other non-empty strings as true. */ + return true; +} + +static bool +is_essential_frozen_module(const char *name) { /* These modules are necessary to bootstrap the import system. */ - if (_PyUnicode_EqualToASCIIString(name, "_frozen_importlib")) { + if (strcmp(name, "_frozen_importlib") == 0) { return true; } - if (_PyUnicode_EqualToASCIIString(name, "_frozen_importlib_external")) { + if (strcmp(name, "_frozen_importlib_external") == 0) { return true; } - if (_PyUnicode_EqualToASCIIString(name, "zipimport")) { + if (strcmp(name, "zipimport") == 0) { return true; } /* This doesn't otherwise have anywhere to find the module. See frozenmain.c. */ - if (_PyUnicode_EqualToASCIIString(name, "__main__")) { + if (strcmp(name, "__main__") == 0) { return true; } return false; } +static bool +use_frozen(const char *modname) +{ + bool isset; + /* Note that we don't bother with os.environ. */ + bool use = parse_env_var_flag(getenv("_PYTHONTESTFROZENMODULES"), &isset); + if (!isset) { + const PyConfig *config = _Py_GetConfig(); + use = config->use_frozen_modules; + } + if (!use && modname != NULL) { + use = is_essential_frozen_module(modname); + } + return use; +} + static PyObject * -list_frozen_module_names(bool force) +list_frozen_module_names() { PyObject *names = PyList_New(0); if (names == NULL) { return NULL; } - const PyConfig *config = _Py_GetConfig(); + bool enabled = use_frozen(NULL); for (const struct _frozen *p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) { break; } + if (!enabled && !is_essential_frozen_module(p->name)) { + continue; + } PyObject *name = PyUnicode_FromString(p->name); if (name == NULL) { Py_DECREF(names); return NULL; } - if (!config->use_frozen_modules && !force) { - if (!is_essential_frozen_module(name)) { - Py_DECREF(name); - continue; - } - } int res = PyList_Append(names, name); Py_DECREF(name); if (res != 0) { @@ -1105,33 +1138,31 @@ list_frozen_module_names(bool force) } static const struct _frozen * -find_frozen(PyObject *name, bool force) +find_frozen(PyObject *modname) { - const struct _frozen *p; - - if (name == NULL) + if (modname == NULL) { return NULL; - - const PyConfig *config = _Py_GetConfig(); - if (!config->use_frozen_modules && !force) { - if (!is_essential_frozen_module(name)) { - return NULL; - } } - + const char *name = PyUnicode_AsUTF8(modname); + if (!use_frozen(name)) { + return NULL; + } + const struct _frozen *p; for (p = PyImport_FrozenModules; ; p++) { - if (p->name == NULL) + if (p->name == NULL) { return NULL; - if (_PyUnicode_EqualToASCIIString(name, p->name)) + } + if (strcmp(name, p->name) == 0) { break; + } } return p; } static PyObject * -get_frozen_object(PyObject *name, bool force) +get_frozen_object(PyObject *name) { - const struct _frozen *p = find_frozen(name, force); + const struct _frozen *p = find_frozen(name); int size; if (p == NULL) { @@ -1153,9 +1184,9 @@ get_frozen_object(PyObject *name, bool force) } static PyObject * -is_frozen_package(PyObject *name, bool force) +is_frozen_package(PyObject *name) { - const struct _frozen *p = find_frozen(name, force); + const struct _frozen *p = find_frozen(name); int size; if (p == NULL) { @@ -1188,7 +1219,7 @@ PyImport_ImportFrozenModuleObject(PyObject *name) int ispackage; int size; - p = find_frozen(name, false); + p = find_frozen(name); if (p == NULL) return 0; @@ -1949,17 +1980,6 @@ _imp_init_frozen_impl(PyObject *module, PyObject *name) return import_add_module(tstate, name); } -static bool -parse_env_var_flag(const char *value) -{ - /* It isn't set or it is set to an empty string. */ - if (value == NULL || strlen(value) == 0) { - return false; - } - /* For now we treat all non-empty strings as true. */ - return true; -} - /*[clinic input] _imp.get_frozen_object @@ -1973,9 +1993,7 @@ static PyObject * _imp_get_frozen_object_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=2568cc5b7aa0da63 input=ed689bc05358fdbd]*/ { - /* Note that we don't bother with os.environ. */ - bool force = parse_env_var_flag(getenv("_PYTHONTESTFROZENMODULES")); - return get_frozen_object(name, force); + return get_frozen_object(name); } /*[clinic input] @@ -1991,9 +2009,7 @@ static PyObject * _imp_is_frozen_package_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=e70cbdb45784a1c9 input=81b6cdecd080fbb8]*/ { - /* Note that we don't bother with os.environ. */ - bool force = parse_env_var_flag(getenv("_PYTHONTESTFROZENMODULES")); - return is_frozen_package(name, force); + return is_frozen_package(name); } /*[clinic input] @@ -2025,11 +2041,9 @@ static PyObject * _imp_is_frozen_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=01f408f5ec0f2577 input=7301dbca1897d66b]*/ { - /* Note that we don't bother with os.environ. */ - bool force = parse_env_var_flag(getenv("_PYTHONTESTFROZENMODULES")); const struct _frozen *p; - p = find_frozen(name, force); + p = find_frozen(name); return PyBool_FromLong((long) (p == NULL ? 0 : p->size)); } @@ -2043,7 +2057,7 @@ static PyObject * _imp__frozen_module_names_impl(PyObject *module) /*[clinic end generated code: output=80609ef6256310a8 input=76237fbfa94460d2]*/ { - return list_frozen_module_names(true); + return list_frozen_module_names(); } /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ From a8cb984f0f91b3d07f616c17e8d0764a03fc54ea Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Sep 2021 15:03:36 -0600 Subject: [PATCH 10/52] Add the "usefrozen" arg to CleanImport.__init__(). --- Lib/test/support/import_helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 0429b6a5fdc30e..612fdc284800b6 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -185,7 +185,7 @@ class CleanImport(object): importlib.import_module("foo") # new reference """ - def __init__(self, *module_names): + def __init__(self, *module_names, usefrozen=False): self.original_modules = sys.modules.copy() for module_name in module_names: if module_name in sys.modules: @@ -197,12 +197,15 @@ def __init__(self, *module_names): if module.__name__ != module_name: del sys.modules[module.__name__] del sys.modules[module_name] + self._frozen_modules = frozen_modules(usefrozen) def __enter__(self): + self._frozen_modules.__enter__() return self def __exit__(self, *ignore_exc): sys.modules.update(self.original_modules) + self._frozen_modules.__exit__(*ignore_exc) class DirsOnSysPath(object): From 6d9d46945e7098457acbafc304a79b1caf24a822 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Sep 2021 16:48:17 -0600 Subject: [PATCH 11/52] Ignore decode errors in find_frozen(). --- Python/import.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/import.c b/Python/import.c index c83320694a2e32..c7d78a3ffd23de 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1144,6 +1144,10 @@ find_frozen(PyObject *modname) return NULL; } const char *name = PyUnicode_AsUTF8(modname); + if (name == NULL) { + PyErr_Clear(); + return NULL; + } if (!use_frozen(name)) { return NULL; } From df19f79627966c5ef13f18f67fbc433b4fcfafd3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Sep 2021 17:05:19 -0600 Subject: [PATCH 12/52] Fix test_ctypes. --- Lib/ctypes/test/test_values.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py index aa31d44184145b..90730851f2aaf5 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -79,7 +79,8 @@ class struct_frozen(Structure): else: self.assertIsNone(spec.submodule_search_locations) - expected = imp._frozen_module_names() + with import_helper.frozen_modules(): + expected = imp._frozen_module_names() self.maxDiff = None self.assertEqual(modules, expected, "PyImport_FrozenModules example " "in Doc/library/ctypes.rst may be out of date") From abfef0b8671d51fafd7610da561649a684c5ef1e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Sep 2021 15:51:40 -0600 Subject: [PATCH 13/52] Ensure we get the frozen module. --- Lib/ctypes/test/test_values.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py index 90730851f2aaf5..03bc9bba317571 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -72,7 +72,8 @@ class struct_frozen(Structure): self.assertGreater(abs(entry.size), 10) self.assertTrue([entry.code[i] for i in range(abs(entry.size))]) # Check the module's package-ness. - spec = importlib.util.find_spec(modname) + with import_helper.frozen_modules(): + spec = importlib.util.find_spec(modname) if entry.size < 0: # It's a package. self.assertIsNotNone(spec.submodule_search_locations) From e0258cf8054e99966fce8191e25deffa0e3925a4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Sep 2021 16:29:47 -0600 Subject: [PATCH 14/52] Fix docstrings for the import test helpers. --- Lib/test/support/import_helper.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 612fdc284800b6..b15ae77ef86ae2 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -111,6 +111,11 @@ def _save_and_block_module(name, orig_modules): @contextlib.contextmanager def frozen_modules(enabled=True): + """Force frozen modules to be used (or not). + + This only applies to modules that haven't been imported yet. + Also, some essential modules will always be imported frozen. + """ # FYI: the env var will never show up in os.environ. os.putenv('_PYTHONTESTFROZENMODULES', '1' if enabled else '0') try: @@ -146,6 +151,11 @@ def import_fresh_module(name, fresh=(), blocked=(), *, This function will raise ImportError if the named module cannot be imported. + + If "usefrozen" is False (the default), any stdlib source module will + always be imported from its .py file, even if the module has been + frozen. The only exception is essential modules (like + importlib._bootstrap). """ # NOTE: test_heapq, test_json and test_warnings include extra sanity checks # to make sure that this utility function is working as expected @@ -183,6 +193,11 @@ class CleanImport(object): with CleanImport("foo"): importlib.import_module("foo") # new reference + + If "usefrozen" is False (the default), any stdlib source module will + always be imported from its .py file, even if the module has been + frozen. The only exception is essential modules (like + importlib._bootstrap). """ def __init__(self, *module_names, usefrozen=False): From 0fd66d4bc1642c16469b4f28822b45e742db3e2d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 17:45:02 -0600 Subject: [PATCH 15/52] Add a NEWS entry. --- .../Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst new file mode 100644 index 00000000000000..c88a8deb829533 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst @@ -0,0 +1,2 @@ +Add a new command line option, "-X frozen_modules=[on|off]" to opt out +of (or into) using optional frozen modules. This defaults to "on". From 30e8cd15e0c0e956e77a36365a54f7d73eb37be7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 11 Jun 2021 18:46:18 -0600 Subject: [PATCH 16/52] Remember the stdlib dir during startup (adding PyConfig.stdlib_dir). --- Include/cpython/initconfig.h | 1 + Include/internal/pycore_pathconfig.h | 1 + Include/internal/pycore_pylifecycle.h | 1 + Lib/test/test_embed.py | 2 ++ Modules/getpath.c | 9 +++++++++ Python/initconfig.c | 5 +++++ Python/pathconfig.c | 17 +++++++++++++++++ 7 files changed, 36 insertions(+) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index fc0c40feb85c71..ccbd74b1e0381d 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -186,6 +186,7 @@ typedef struct PyConfig { /* --- Path configuration outputs ----------- */ int module_search_paths_set; PyWideStringList module_search_paths; + wchar_t *stdlib_dir; wchar_t *executable; wchar_t *base_executable; wchar_t *prefix; diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 15447f54490fb4..a258aab2397660 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -13,6 +13,7 @@ typedef struct _PyPathConfig { wchar_t *program_full_path; wchar_t *prefix; wchar_t *exec_prefix; + wchar_t *stdlib_dir; /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */ wchar_t *module_search_path; /* Python program name */ diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 524be9d4cbb940..4f12fef8d65466 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -122,6 +122,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const PyConfig *config, const struct _PyArgv *args); +PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 6fabaaf6448c92..419d6754beb168 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -398,6 +398,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'module_search_paths': GET_DEFAULT_CONFIG, 'module_search_paths_set': 1, 'platlibdir': sys.platlibdir, + 'stdlib_dir': GET_DEFAULT_CONFIG, 'site_import': 1, 'bytes_warning': 0, @@ -507,6 +508,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix', 'program_name', 'home', + 'stdlib_dir', # program_full_path and module_search_path are copied indirectly from # the core configuration in check_path_config(). ] diff --git a/Modules/getpath.c b/Modules/getpath.c index 363d62a0657ebd..2aed7fd5ebf046 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -1540,6 +1540,15 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) } } + if (pathconfig->stdlib_dir == NULL) { + if (calculate->prefix_found) { + pathconfig->stdlib_dir = _PyMem_RawWcsdup(calculate->prefix); + if (pathconfig->stdlib_dir == NULL) { + return _PyStatus_NO_MEMORY(); + } + } + } + if (pathconfig->prefix == NULL) { status = calculate_set_prefix(calculate, pathconfig); if (_PyStatus_EXCEPTION(status)) { diff --git a/Python/initconfig.c b/Python/initconfig.c index 55f1e7fc50b570..f3592e1371c8ee 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -674,6 +674,7 @@ PyConfig_Clear(PyConfig *config) _PyWideStringList_Clear(&config->xoptions); _PyWideStringList_Clear(&config->module_search_paths); config->module_search_paths_set = 0; + CLEAR(config->stdlib_dir); CLEAR(config->executable); CLEAR(config->base_executable); @@ -914,6 +915,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_WSTRLIST(xoptions); COPY_WSTRLIST(module_search_paths); COPY_ATTR(module_search_paths_set); + COPY_WSTR_ATTR(stdlib_dir); COPY_WSTR_ATTR(executable); COPY_WSTR_ATTR(base_executable); @@ -1024,6 +1026,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_WSTR(home); SET_ITEM_INT(module_search_paths_set); SET_ITEM_WSTRLIST(module_search_paths); + SET_ITEM_WSTR(stdlib_dir); SET_ITEM_WSTR(executable); SET_ITEM_WSTR(base_executable); SET_ITEM_WSTR(prefix); @@ -1351,6 +1354,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) // Path configuration output GET_UINT(module_search_paths_set); GET_WSTRLIST(module_search_paths); + GET_WSTR_OPT(stdlib_dir); GET_WSTR_OPT(executable); GET_WSTR_OPT(base_executable); GET_WSTR_OPT(prefix); @@ -3118,6 +3122,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) PySys_WriteStderr(" environment = %i\n", config->use_environment); PySys_WriteStderr(" user site = %i\n", config->user_site_directory); PySys_WriteStderr(" import site = %i\n", config->site_import); + DUMP_CONFIG("stdlib dir", stdlib_dir); #undef DUMP_CONFIG #define DUMP_SYS(NAME) \ diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 470aba75bea969..af73d6ed45f4b2 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -54,6 +54,7 @@ pathconfig_clear(_PyPathConfig *config) CLEAR(config->program_full_path); CLEAR(config->prefix); CLEAR(config->exec_prefix); + CLEAR(config->stdlib_dir); CLEAR(config->module_search_path); CLEAR(config->program_name); CLEAR(config->home); @@ -83,6 +84,7 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2) COPY_ATTR(prefix); COPY_ATTR(exec_prefix); COPY_ATTR(module_search_path); + COPY_ATTR(stdlib_dir); COPY_ATTR(program_name); COPY_ATTR(home); #ifdef MS_WINDOWS @@ -167,6 +169,7 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) COPY_CONFIG(program_full_path, executable); COPY_CONFIG(prefix, prefix); COPY_CONFIG(exec_prefix, exec_prefix); + COPY_CONFIG(stdlib_dir, stdlib_dir); COPY_CONFIG(program_name, program_name); COPY_CONFIG(home, home); #ifdef MS_WINDOWS @@ -218,6 +221,7 @@ _PyPathConfig_AsDict(void) SET_ITEM_STR(prefix); SET_ITEM_STR(exec_prefix); SET_ITEM_STR(module_search_path); + SET_ITEM_STR(stdlib_dir); SET_ITEM_STR(program_name); SET_ITEM_STR(home); #ifdef MS_WINDOWS @@ -311,6 +315,7 @@ config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig) - exec_prefix - module_search_path + - stdlib_dir - prefix - program_full_path @@ -401,6 +406,7 @@ config_init_pathconfig(PyConfig *config, int compute_path_config) COPY_ATTR(program_full_path, executable); COPY_ATTR(prefix, prefix); COPY_ATTR(exec_prefix, exec_prefix); + COPY_ATTR(stdlib_dir, stdlib_dir); #undef COPY_ATTR @@ -486,16 +492,20 @@ Py_SetPath(const wchar_t *path) PyMem_RawFree(_Py_path_config.prefix); PyMem_RawFree(_Py_path_config.exec_prefix); + PyMem_RawFree(_Py_path_config.stdlib_dir); PyMem_RawFree(_Py_path_config.module_search_path); _Py_path_config.prefix = _PyMem_RawWcsdup(L""); _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L""); + // XXX Copy this from the new module_search_path? + _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L""); _Py_path_config.module_search_path = _PyMem_RawWcsdup(path); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); if (_Py_path_config.prefix == NULL || _Py_path_config.exec_prefix == NULL + || _Py_path_config.stdlib_dir == NULL || _Py_path_config.module_search_path == NULL) { path_out_of_memory(__func__); @@ -572,6 +582,13 @@ Py_GetPath(void) } +wchar_t * +_Py_GetStdlibDir(void) +{ + return _Py_path_config.stdlib_dir; +} + + wchar_t * Py_GetPrefix(void) { From 1a83ef6f90f7d4d78cf4b20b4468635488a3a46a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 15:54:15 -0600 Subject: [PATCH 17/52] Add _Py_GetMainConfig(). --- Include/internal/pycore_pystate.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 4b894f3eff4967..aef318989aa6e5 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -30,6 +30,17 @@ _Py_IsMainInterpreter(PyInterpreterState *interp) } +static inline const PyConfig * +_Py_GetMainConfig(void) +{ + PyInterpreterState *interp = _PyRuntime.interpreters.main; + if (interp == NULL) { + return NULL; + } + return _PyInterpreterState_GetConfig(interp); +} + + /* Only handle signals on the main thread of the main interpreter. */ static inline int _Py_ThreadCanHandleSignals(PyInterpreterState *interp) From 3b57bca7b5e721b51856613f6a214f7340ab3fc4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 16:08:44 -0600 Subject: [PATCH 18/52] Let _Py_GetStdlibDir() fall back to a config. --- Include/internal/pycore_pylifecycle.h | 2 +- Python/pathconfig.c | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 4f12fef8d65466..df24103eb7b6ae 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -122,7 +122,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const PyConfig *config, const struct _PyArgv *args); -PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void); +PyAPI_FUNC(const wchar_t *) _Py_GetStdlibDir(const PyConfig *); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Python/pathconfig.c b/Python/pathconfig.c index af73d6ed45f4b2..e9ee54278aa634 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -6,6 +6,7 @@ #include "pycore_fileutils.h" #include "pycore_pathconfig.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() +#include "pycore_pystate.h" // _Py_GetMainConfig() #include #ifdef MS_WINDOWS # include // GetFullPathNameW(), MAX_PATH @@ -582,10 +583,19 @@ Py_GetPath(void) } -wchar_t * -_Py_GetStdlibDir(void) +const wchar_t * +_Py_GetStdlibDir(const PyConfig *fallback) { - return _Py_path_config.stdlib_dir; + if (_Py_path_config.stdlib_dir != NULL) { + return _Py_path_config.stdlib_dir; + } + if (fallback == NULL) { + fallback = _Py_GetMainConfig(); + if (fallback == NULL) { + return NULL; + } + } + return fallback->stdlib_dir; } From b45e5f748966d8a66cb28a996501b0a0a975d099 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 17:18:03 -0600 Subject: [PATCH 19/52] Add _PyConfig_InitImportConfig(). --- Include/internal/pycore_initconfig.h | 1 + Python/initconfig.c | 20 ++++++++++++++------ Python/pylifecycle.c | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 4b009e816b4927..9014fcd41d8686 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -155,6 +155,7 @@ extern PyStatus _PyConfig_Copy( extern PyStatus _PyConfig_InitPathConfig( PyConfig *config, int compute_path_config); +extern PyStatus _PyConfig_InitImportConfig(PyConfig *config); extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config); extern PyStatus _PyConfig_Write(const PyConfig *config, struct pyruntimestate *runtime); diff --git a/Python/initconfig.c b/Python/initconfig.c index f3592e1371c8ee..7165cfac55bfa1 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2114,9 +2114,16 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) } -static PyStatus -config_init_imports(PyConfig *config) +PyStatus +_PyConfig_InitImportConfig(PyConfig *config) { + PyStatus status; + + status = _PyConfig_InitPathConfig(config, 1); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + /* -X frozen_modules=[on|off] */ const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { @@ -2188,11 +2195,12 @@ config_read(PyConfig *config, int compute_path_config) } if (config->_install_importlib) { - status = _PyConfig_InitPathConfig(config, compute_path_config); - if (_PyStatus_EXCEPTION(status)) { - return status; + if (compute_path_config) { + status = _PyConfig_InitImportConfig(config); + } + else { + status = _PyConfig_InitPathConfig(config, 0); } - status = config_init_imports(config); if (_PyStatus_EXCEPTION(status)) { return status; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f3b6b0ac68a1ea..3d4c63cb3325c3 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1079,7 +1079,7 @@ init_interp_main(PyThreadState *tstate) } // Compute the path configuration - status = _PyConfig_InitPathConfig(&interp->config, 1); + status = _PyConfig_InitImportConfig(&interp->config); if (_PyStatus_EXCEPTION(status)) { return status; } From 2298dc0500899bc50dabebbc493be6c15a803090 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 18:31:39 -0600 Subject: [PATCH 20/52] Identify whether or not the executable is running installed (adding _Py_IsInstalled()). --- Include/internal/pycore_pylifecycle.h | 2 ++ Python/pathconfig.c | 43 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index df24103eb7b6ae..8cafa602f8a91c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -12,6 +12,7 @@ extern "C" { #include #endif +#include #include "pycore_runtime.h" // _PyRuntimeState #ifndef NSIG @@ -123,6 +124,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const struct _PyArgv *args); PyAPI_FUNC(const wchar_t *) _Py_GetStdlibDir(const PyConfig *); +PyAPI_FUNC(bool) _Py_IsInstalled(const PyConfig *); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Python/pathconfig.c b/Python/pathconfig.c index e9ee54278aa634..554367ad07d3a8 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -8,6 +8,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _Py_GetMainConfig() #include +#include #ifdef MS_WINDOWS # include // GetFullPathNameW(), MAX_PATH #endif @@ -633,6 +634,48 @@ Py_GetProgramName(void) return _Py_path_config.program_name; } + +static const wchar_t * +get_executable(const PyConfig *fallback) +{ + const wchar_t *executable = Py_GetProgramFullPath(); + if (executable != NULL) { + return executable; + } + if (fallback == NULL) { + fallback = _Py_GetMainConfig(); + if (fallback == NULL) { + return NULL; + } + } + return fallback->executable; +} + +bool +_Py_IsInstalled(const PyConfig *fallback) +{ + /* If the stdlib dir is *not* in the same dir as the executable + then we consider it an installed Python. */ + const wchar_t *stdlib = _Py_GetStdlibDir(fallback); + if (stdlib != NULL) { + size_t i = wcslen(stdlib); + while (i > 0 && stdlib[i] != SEP) { + --i; + } + const wchar_t *executable = get_executable(fallback); + if (executable != NULL) { + if (wcslen(executable) > i) { + /* If they share a parent dir then it is a local (dev) + build and therefore not installed. */ + return wcsncmp(stdlib, executable, i + 1) != 0; + } + } + } + /* We fall back to assuming this is an installed Python. */ + return true; +} + + /* Compute module search path from argv[0] or the current working directory ("-m module" case) which will be prepended to sys.argv: sys.path[0]. From 2b67c449be21b30ba69b3d26c8c0a2440576e22f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 09:43:56 -0600 Subject: [PATCH 21/52] Default to "off" if in development. --- Doc/using/cmdline.rst | 4 +++- Lib/test/test_embed.py | 6 ++++-- Python/initconfig.c | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 9c24cb42d17d80..05c659bea328a9 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -482,7 +482,9 @@ Miscellaneous options :envvar:`PYTHONNODEBUGRANGES`. * ``-X frozen_modules`` determines whether or not frozen modules are ignored by the import machinery. A value of "on" means they get - imported and "off" means they are ignored. The default is "on". + imported and "off" means they are ignored. The default is "on" + if this is an installed Python (the normal case). If it's under + development (running from the build dir) then the default is "off". Note that the "importlib_bootstrap" and "importlib_bootstrap_external" frozen modules are always used, even if this flag is set to "off". diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 419d6754beb168..b1e38eb20ec678 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -427,7 +427,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': GET_DEFAULT_CONFIG, + 'use_frozen_modules': True, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -1247,7 +1247,9 @@ def test_init_setpythonhome(self): 'pythonpath_env': paths_str, } self.default_program_name(config) - env = {'TESTHOME': home, 'PYTHONPATH': paths_str} + env = {'TESTHOME': home, + 'PYTHONPATH': paths_str, + '_PYTHONTESTFROZENMODULES': '1'} self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) diff --git a/Python/initconfig.c b/Python/initconfig.c index 7165cfac55bfa1..a54053d90a52cb 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2127,8 +2127,7 @@ _PyConfig_InitImportConfig(PyConfig *config) /* -X frozen_modules=[on|off] */ const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { - // XXX Set default to false if in development. - config->use_frozen_modules = true; + config->use_frozen_modules = _Py_IsInstalled(config); } else if (wcscmp(value, L"on") == 0) { config->use_frozen_modules = true; From 0933e4dc5611707084fdf8efc3bfbd9a1d8cd080 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 31 Aug 2021 17:45:02 -0600 Subject: [PATCH 22/52] Add a NEWS entry. --- .../2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst index c88a8deb829533..b9ed3cf69b2b26 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst @@ -1,2 +1,4 @@ Add a new command line option, "-X frozen_modules=[on|off]" to opt out -of (or into) using optional frozen modules. This defaults to "on". +of (or into) using optional frozen modules. This defaults to "on" (or +"off" if running in a local development environment, saving contributors +some frustration). From 4823ad6f188fa2904c3a1397b4b9e8717c3820ae Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 09:23:14 -0600 Subject: [PATCH 23/52] Fix test_embed for PyConfig.stdlib_dir. --- Lib/test/test_embed.py | 13 +++++++++++++ Modules/getpath.c | 1 + Python/pathconfig.c | 12 ++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index b1e38eb20ec678..e5c99182ca0ebf 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1136,6 +1136,9 @@ def test_init_setpath(self): 'base_prefix': '', 'exec_prefix': '', 'base_exec_prefix': '', + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': '', } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1156,6 +1159,9 @@ def test_init_setpath_config(self): 'base_prefix': '', 'exec_prefix': '', 'base_exec_prefix': '', + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': '', # overriden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', @@ -1245,6 +1251,7 @@ def test_init_setpythonhome(self): 'exec_prefix': exec_prefix, 'base_exec_prefix': exec_prefix, 'pythonpath_env': paths_str, + 'stdlib_dir': home, } self.default_program_name(config) env = {'TESTHOME': home, @@ -1284,6 +1291,9 @@ def test_init_pybuilddir(self): 'base_executable': executable, 'executable': executable, 'module_search_paths': module_search_paths, + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': None, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1333,6 +1343,9 @@ def test_init_pyvenv_cfg(self): 'base_executable': executable, 'executable': executable, 'module_search_paths': paths, + # The current getpath.c doesn't determine the stdlib dir + # in this case. + 'stdlib_dir': None, } path_config = {} if MS_WINDOWS: diff --git a/Modules/getpath.c b/Modules/getpath.c index 2aed7fd5ebf046..906978b3857dd1 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -1542,6 +1542,7 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) if (pathconfig->stdlib_dir == NULL) { if (calculate->prefix_found) { + /* This must be done *before* calculate_set_prefix() is called. */ pathconfig->stdlib_dir = _PyMem_RawWcsdup(calculate->prefix); if (pathconfig->stdlib_dir == NULL) { return _PyStatus_NO_MEMORY(); diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 554367ad07d3a8..d95ff51cba4657 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -500,7 +500,12 @@ Py_SetPath(const wchar_t *path) _Py_path_config.prefix = _PyMem_RawWcsdup(L""); _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L""); // XXX Copy this from the new module_search_path? - _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L""); + if (_Py_path_config.home != NULL) { + _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(_Py_path_config.home); + } + else { + _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L""); + } _Py_path_config.module_search_path = _PyMem_RawWcsdup(path); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -527,10 +532,13 @@ Py_SetPythonHome(const wchar_t *home) PyMem_RawFree(_Py_path_config.home); _Py_path_config.home = _PyMem_RawWcsdup(home); + if (_Py_path_config.home != NULL) { + _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(home); + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.home == NULL) { + if (_Py_path_config.home == NULL || _Py_path_config.stdlib_dir == NULL) { path_out_of_memory(__func__); } } From 0e5c617c54f7907d055e7ff45d37120bef3c255f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 09:54:20 -0600 Subject: [PATCH 24/52] In _Py_GetStdlibDir(), treat "" as though it were NULL. --- Python/pathconfig.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index d95ff51cba4657..387826f4ca1844 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -595,8 +595,9 @@ Py_GetPath(void) const wchar_t * _Py_GetStdlibDir(const PyConfig *fallback) { - if (_Py_path_config.stdlib_dir != NULL) { - return _Py_path_config.stdlib_dir; + wchar_t *stdlib_dir = _Py_path_config.stdlib_dir; + if (stdlib_dir != NULL && stdlib_dir[0] != L'\0') { + return stdlib_dir; } if (fallback == NULL) { fallback = _Py_GetMainConfig(); @@ -604,7 +605,11 @@ _Py_GetStdlibDir(const PyConfig *fallback) return NULL; } } - return fallback->stdlib_dir; + stdlib_dir = fallback->stdlib_dir; + if (stdlib_dir != NULL && stdlib_dir[0] != L'\0') { + return stdlib_dir; + } + return NULL; } From d427baec80ad6058e75ee0594b20260470027f7d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 10:13:51 -0600 Subject: [PATCH 25/52] Fix test_embed for "Default to "off" if in development.". --- Lib/test/test_embed.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e5c99182ca0ebf..84b3c8afcb99f0 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -427,7 +427,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': True, + 'use_frozen_modules': False, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -1028,6 +1028,7 @@ def test_init_read_set(self): config = { 'program_name': './init_read_set', 'executable': 'my_executable', + 'use_frozen_modules': True, } def modify_path(path): path.insert(1, "test_path_insert1") @@ -1139,6 +1140,7 @@ def test_init_setpath(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', + 'use_frozen_modules': True, } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1162,6 +1164,7 @@ def test_init_setpath_config(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', + 'use_frozen_modules': True, # overriden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', @@ -1252,6 +1255,7 @@ def test_init_setpythonhome(self): 'base_exec_prefix': exec_prefix, 'pythonpath_env': paths_str, 'stdlib_dir': home, + 'use_frozen_modules': True, } self.default_program_name(config) env = {'TESTHOME': home, @@ -1294,6 +1298,7 @@ def test_init_pybuilddir(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': None, + 'use_frozen_modules': True, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1346,6 +1351,7 @@ def test_init_pyvenv_cfg(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': None, + 'use_frozen_modules': True, } path_config = {} if MS_WINDOWS: From f6bd2fd6b8c8f7848ac822eef86ee942d47b86ec Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 10:40:32 -0600 Subject: [PATCH 26/52] Fix test_cmd_line_script for "Default to "off" if in development.". --- Lib/test/support/os_helper.py | 4 ++++ Lib/test/test_cmd_line_script.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index d9807a1e114b65..ce01417ed07d88 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -599,6 +599,10 @@ def set(self, envvar, value): def unset(self, envvar): del self[envvar] + def copy(self): + # We do what os.environ.copy() does. + return dict(self) + def __enter__(self): return self diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e50c9925799917..2f96e492ebd957 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -322,14 +322,14 @@ def test_module_in_package(self): cwd=script_dir) def test_module_in_package_in_zipfile(self): - with os_helper.temp_dir() as script_dir: + with import_helper.frozen_modules(), os_helper.temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') self._check_script(["-m", "test_pkg.script"], run_name, run_name, script_dir, 'test_pkg', zipimport.zipimporter, PYTHONPATH=zip_name, cwd=script_dir) def test_module_in_subpackage_in_zipfile(self): - with os_helper.temp_dir() as script_dir: + with import_helper.frozen_modules(), os_helper.temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) self._check_script(["-m", "test_pkg.test_pkg.script"], run_name, run_name, script_dir, 'test_pkg.test_pkg', From 965c7fee50ef83d4f62d53e5de0c6d9a415bce0f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 13:11:20 -0600 Subject: [PATCH 27/52] _Py_IsInstalled -> _Py_IsDevelopmentEnv. --- Include/internal/pycore_pylifecycle.h | 2 +- Lib/test/test_embed.py | 8 +---- Python/initconfig.c | 2 +- Python/pathconfig.c | 46 +++++++++++++++++---------- 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 8cafa602f8a91c..77014fb97a75bb 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -124,7 +124,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const struct _PyArgv *args); PyAPI_FUNC(const wchar_t *) _Py_GetStdlibDir(const PyConfig *); -PyAPI_FUNC(bool) _Py_IsInstalled(const PyConfig *); +PyAPI_FUNC(bool) _Py_IsDevelopmentEnv(const PyConfig *); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 84b3c8afcb99f0..e5c99182ca0ebf 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -427,7 +427,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': False, + 'use_frozen_modules': True, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -1028,7 +1028,6 @@ def test_init_read_set(self): config = { 'program_name': './init_read_set', 'executable': 'my_executable', - 'use_frozen_modules': True, } def modify_path(path): path.insert(1, "test_path_insert1") @@ -1140,7 +1139,6 @@ def test_init_setpath(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', - 'use_frozen_modules': True, } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1164,7 +1162,6 @@ def test_init_setpath_config(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', - 'use_frozen_modules': True, # overriden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', @@ -1255,7 +1252,6 @@ def test_init_setpythonhome(self): 'base_exec_prefix': exec_prefix, 'pythonpath_env': paths_str, 'stdlib_dir': home, - 'use_frozen_modules': True, } self.default_program_name(config) env = {'TESTHOME': home, @@ -1298,7 +1294,6 @@ def test_init_pybuilddir(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': None, - 'use_frozen_modules': True, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1351,7 +1346,6 @@ def test_init_pyvenv_cfg(self): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': None, - 'use_frozen_modules': True, } path_config = {} if MS_WINDOWS: diff --git a/Python/initconfig.c b/Python/initconfig.c index a54053d90a52cb..3279f6b69e6db5 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2127,7 +2127,7 @@ _PyConfig_InitImportConfig(PyConfig *config) /* -X frozen_modules=[on|off] */ const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { - config->use_frozen_modules = _Py_IsInstalled(config); + config->use_frozen_modules = !_Py_IsDevelopmentEnv(config); } else if (wcscmp(value, L"on") == 0) { config->use_frozen_modules = true; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 387826f4ca1844..a84512d70ca0b2 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -37,6 +37,17 @@ copy_wstr(wchar_t **dst, const wchar_t *src) return 0; } +static size_t +find_basename(const wchar_t *filename) +{ + for (size_t i = wcslen(filename); i > 0; --i) { + if (filename[i] == SEP) { + return i + 1; + } + } + return 0; +} + static void pathconfig_clear(_PyPathConfig *config) @@ -665,26 +676,27 @@ get_executable(const PyConfig *fallback) } bool -_Py_IsInstalled(const PyConfig *fallback) +_Py_IsDevelopmentEnv(const PyConfig *fallback) { - /* If the stdlib dir is *not* in the same dir as the executable - then we consider it an installed Python. */ + const wchar_t *executable = get_executable(fallback); + if (executable == NULL) { + return false; + } + size_t len = find_basename(executable); + if (wcscmp(executable + len, L"python") != 0) { + return false; + } + /* If dirname() is the same for both then it is a local (dev) build. */ const wchar_t *stdlib = _Py_GetStdlibDir(fallback); - if (stdlib != NULL) { - size_t i = wcslen(stdlib); - while (i > 0 && stdlib[i] != SEP) { - --i; - } - const wchar_t *executable = get_executable(fallback); - if (executable != NULL) { - if (wcslen(executable) > i) { - /* If they share a parent dir then it is a local (dev) - build and therefore not installed. */ - return wcsncmp(stdlib, executable, i + 1) != 0; - } - } + if (stdlib == NULL) { + return false; + } + if (len != find_basename(stdlib)) { + return false; + } + if (wcsncmp(stdlib, executable, len) != 0) { + return false; } - /* We fall back to assuming this is an installed Python. */ return true; } From 4f49adcc37b1acdae9dfbd5b965bf5054981af8e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 Sep 2021 14:53:00 -0600 Subject: [PATCH 28/52] Stop using the frozen helper in test_cmd_line_script. --- Lib/test/test_cmd_line_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 2f96e492ebd957..e50c9925799917 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -322,14 +322,14 @@ def test_module_in_package(self): cwd=script_dir) def test_module_in_package_in_zipfile(self): - with import_helper.frozen_modules(), os_helper.temp_dir() as script_dir: + with os_helper.temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') self._check_script(["-m", "test_pkg.script"], run_name, run_name, script_dir, 'test_pkg', zipimport.zipimporter, PYTHONPATH=zip_name, cwd=script_dir) def test_module_in_subpackage_in_zipfile(self): - with import_helper.frozen_modules(), os_helper.temp_dir() as script_dir: + with os_helper.temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) self._check_script(["-m", "test_pkg.test_pkg.script"], run_name, run_name, script_dir, 'test_pkg.test_pkg', From 5297edcad0d2fafd6bdf2934f41aa2504ee9a45f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Sep 2021 10:49:30 -0600 Subject: [PATCH 29/52] Allow for a .exe suffix on the executable. --- Python/pathconfig.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index a84512d70ca0b2..37866147251808 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -684,7 +684,9 @@ _Py_IsDevelopmentEnv(const PyConfig *fallback) } size_t len = find_basename(executable); if (wcscmp(executable + len, L"python") != 0) { - return false; + if (wcscmp(executable + len, L"python.exe") != 0) { + return false; + } } /* If dirname() is the same for both then it is a local (dev) build. */ const wchar_t *stdlib = _Py_GetStdlibDir(fallback); From 2b6f978730429078fcd69c7ccdacd7002b04e68d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 10 Sep 2021 10:23:59 -0600 Subject: [PATCH 30/52] Drop _Py_IsDevelopmentEnv(). --- Include/internal/pycore_pylifecycle.h | 1 - .../2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst | 4 +- Python/initconfig.c | 4 +- Python/pathconfig.c | 56 ------------------- 4 files changed, 4 insertions(+), 61 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 77014fb97a75bb..f3fa25038aa1a9 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -124,7 +124,6 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const struct _PyArgv *args); PyAPI_FUNC(const wchar_t *) _Py_GetStdlibDir(const PyConfig *); -PyAPI_FUNC(bool) _Py_IsDevelopmentEnv(const PyConfig *); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst index b9ed3cf69b2b26..c88a8deb829533 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst @@ -1,4 +1,2 @@ Add a new command line option, "-X frozen_modules=[on|off]" to opt out -of (or into) using optional frozen modules. This defaults to "on" (or -"off" if running in a local development environment, saving contributors -some frustration). +of (or into) using optional frozen modules. This defaults to "on". diff --git a/Python/initconfig.c b/Python/initconfig.c index 3279f6b69e6db5..873698b27a9e3d 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2127,7 +2127,9 @@ _PyConfig_InitImportConfig(PyConfig *config) /* -X frozen_modules=[on|off] */ const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { - config->use_frozen_modules = !_Py_IsDevelopmentEnv(config); + // Use a meaningful default. + // XXX Add the logic. + config->use_frozen_modules = false; } else if (wcscmp(value, L"on") == 0) { config->use_frozen_modules = true; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 37866147251808..ec34272c211483 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -8,7 +8,6 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _Py_GetMainConfig() #include -#include #ifdef MS_WINDOWS # include // GetFullPathNameW(), MAX_PATH #endif @@ -37,17 +36,6 @@ copy_wstr(wchar_t **dst, const wchar_t *src) return 0; } -static size_t -find_basename(const wchar_t *filename) -{ - for (size_t i = wcslen(filename); i > 0; --i) { - if (filename[i] == SEP) { - return i + 1; - } - } - return 0; -} - static void pathconfig_clear(_PyPathConfig *config) @@ -659,50 +647,6 @@ Py_GetProgramName(void) } -static const wchar_t * -get_executable(const PyConfig *fallback) -{ - const wchar_t *executable = Py_GetProgramFullPath(); - if (executable != NULL) { - return executable; - } - if (fallback == NULL) { - fallback = _Py_GetMainConfig(); - if (fallback == NULL) { - return NULL; - } - } - return fallback->executable; -} - -bool -_Py_IsDevelopmentEnv(const PyConfig *fallback) -{ - const wchar_t *executable = get_executable(fallback); - if (executable == NULL) { - return false; - } - size_t len = find_basename(executable); - if (wcscmp(executable + len, L"python") != 0) { - if (wcscmp(executable + len, L"python.exe") != 0) { - return false; - } - } - /* If dirname() is the same for both then it is a local (dev) build. */ - const wchar_t *stdlib = _Py_GetStdlibDir(fallback); - if (stdlib == NULL) { - return false; - } - if (len != find_basename(stdlib)) { - return false; - } - if (wcsncmp(stdlib, executable, len) != 0) { - return false; - } - return true; -} - - /* Compute module search path from argv[0] or the current working directory ("-m module" case) which will be prepended to sys.argv: sys.path[0]. From dc2aa340ab5e37604f38ccda877cd4c50bd0042c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 10 Sep 2021 10:25:25 -0600 Subject: [PATCH 31/52] Drop -_Py_GetStdlibDir() and PyConfig.stdlib_dir. --- Include/cpython/initconfig.h | 1 - Include/internal/pycore_pathconfig.h | 1 - Include/internal/pycore_pylifecycle.h | 2 -- Modules/getpath.c | 10 ------- Python/initconfig.c | 5 ---- Python/pathconfig.c | 41 +-------------------------- 6 files changed, 1 insertion(+), 59 deletions(-) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index ccbd74b1e0381d..fc0c40feb85c71 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -186,7 +186,6 @@ typedef struct PyConfig { /* --- Path configuration outputs ----------- */ int module_search_paths_set; PyWideStringList module_search_paths; - wchar_t *stdlib_dir; wchar_t *executable; wchar_t *base_executable; wchar_t *prefix; diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index a258aab2397660..15447f54490fb4 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -13,7 +13,6 @@ typedef struct _PyPathConfig { wchar_t *program_full_path; wchar_t *prefix; wchar_t *exec_prefix; - wchar_t *stdlib_dir; /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */ wchar_t *module_search_path; /* Python program name */ diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index f3fa25038aa1a9..bd26f0d6a0dd01 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -123,8 +123,6 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const PyConfig *config, const struct _PyArgv *args); -PyAPI_FUNC(const wchar_t *) _Py_GetStdlibDir(const PyConfig *); - PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable); diff --git a/Modules/getpath.c b/Modules/getpath.c index 906978b3857dd1..363d62a0657ebd 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -1540,16 +1540,6 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) } } - if (pathconfig->stdlib_dir == NULL) { - if (calculate->prefix_found) { - /* This must be done *before* calculate_set_prefix() is called. */ - pathconfig->stdlib_dir = _PyMem_RawWcsdup(calculate->prefix); - if (pathconfig->stdlib_dir == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - } - if (pathconfig->prefix == NULL) { status = calculate_set_prefix(calculate, pathconfig); if (_PyStatus_EXCEPTION(status)) { diff --git a/Python/initconfig.c b/Python/initconfig.c index 873698b27a9e3d..63315dbbfd9502 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -674,7 +674,6 @@ PyConfig_Clear(PyConfig *config) _PyWideStringList_Clear(&config->xoptions); _PyWideStringList_Clear(&config->module_search_paths); config->module_search_paths_set = 0; - CLEAR(config->stdlib_dir); CLEAR(config->executable); CLEAR(config->base_executable); @@ -915,7 +914,6 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_WSTRLIST(xoptions); COPY_WSTRLIST(module_search_paths); COPY_ATTR(module_search_paths_set); - COPY_WSTR_ATTR(stdlib_dir); COPY_WSTR_ATTR(executable); COPY_WSTR_ATTR(base_executable); @@ -1026,7 +1024,6 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_WSTR(home); SET_ITEM_INT(module_search_paths_set); SET_ITEM_WSTRLIST(module_search_paths); - SET_ITEM_WSTR(stdlib_dir); SET_ITEM_WSTR(executable); SET_ITEM_WSTR(base_executable); SET_ITEM_WSTR(prefix); @@ -1354,7 +1351,6 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) // Path configuration output GET_UINT(module_search_paths_set); GET_WSTRLIST(module_search_paths); - GET_WSTR_OPT(stdlib_dir); GET_WSTR_OPT(executable); GET_WSTR_OPT(base_executable); GET_WSTR_OPT(prefix); @@ -3131,7 +3127,6 @@ _Py_DumpPathConfig(PyThreadState *tstate) PySys_WriteStderr(" environment = %i\n", config->use_environment); PySys_WriteStderr(" user site = %i\n", config->user_site_directory); PySys_WriteStderr(" import site = %i\n", config->site_import); - DUMP_CONFIG("stdlib dir", stdlib_dir); #undef DUMP_CONFIG #define DUMP_SYS(NAME) \ diff --git a/Python/pathconfig.c b/Python/pathconfig.c index ec34272c211483..7f6bb045b8f392 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -55,7 +55,6 @@ pathconfig_clear(_PyPathConfig *config) CLEAR(config->program_full_path); CLEAR(config->prefix); CLEAR(config->exec_prefix); - CLEAR(config->stdlib_dir); CLEAR(config->module_search_path); CLEAR(config->program_name); CLEAR(config->home); @@ -85,7 +84,6 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2) COPY_ATTR(prefix); COPY_ATTR(exec_prefix); COPY_ATTR(module_search_path); - COPY_ATTR(stdlib_dir); COPY_ATTR(program_name); COPY_ATTR(home); #ifdef MS_WINDOWS @@ -170,7 +168,6 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) COPY_CONFIG(program_full_path, executable); COPY_CONFIG(prefix, prefix); COPY_CONFIG(exec_prefix, exec_prefix); - COPY_CONFIG(stdlib_dir, stdlib_dir); COPY_CONFIG(program_name, program_name); COPY_CONFIG(home, home); #ifdef MS_WINDOWS @@ -222,7 +219,6 @@ _PyPathConfig_AsDict(void) SET_ITEM_STR(prefix); SET_ITEM_STR(exec_prefix); SET_ITEM_STR(module_search_path); - SET_ITEM_STR(stdlib_dir); SET_ITEM_STR(program_name); SET_ITEM_STR(home); #ifdef MS_WINDOWS @@ -316,7 +312,6 @@ config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig) - exec_prefix - module_search_path - - stdlib_dir - prefix - program_full_path @@ -407,7 +402,6 @@ config_init_pathconfig(PyConfig *config, int compute_path_config) COPY_ATTR(program_full_path, executable); COPY_ATTR(prefix, prefix); COPY_ATTR(exec_prefix, exec_prefix); - COPY_ATTR(stdlib_dir, stdlib_dir); #undef COPY_ATTR @@ -493,25 +487,16 @@ Py_SetPath(const wchar_t *path) PyMem_RawFree(_Py_path_config.prefix); PyMem_RawFree(_Py_path_config.exec_prefix); - PyMem_RawFree(_Py_path_config.stdlib_dir); PyMem_RawFree(_Py_path_config.module_search_path); _Py_path_config.prefix = _PyMem_RawWcsdup(L""); _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L""); - // XXX Copy this from the new module_search_path? - if (_Py_path_config.home != NULL) { - _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(_Py_path_config.home); - } - else { - _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L""); - } _Py_path_config.module_search_path = _PyMem_RawWcsdup(path); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); if (_Py_path_config.prefix == NULL || _Py_path_config.exec_prefix == NULL - || _Py_path_config.stdlib_dir == NULL || _Py_path_config.module_search_path == NULL) { path_out_of_memory(__func__); @@ -531,13 +516,10 @@ Py_SetPythonHome(const wchar_t *home) PyMem_RawFree(_Py_path_config.home); _Py_path_config.home = _PyMem_RawWcsdup(home); - if (_Py_path_config.home != NULL) { - _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(home); - } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.home == NULL || _Py_path_config.stdlib_dir == NULL) { + if (_Py_path_config.home == NULL) { path_out_of_memory(__func__); } } @@ -591,27 +573,6 @@ Py_GetPath(void) } -const wchar_t * -_Py_GetStdlibDir(const PyConfig *fallback) -{ - wchar_t *stdlib_dir = _Py_path_config.stdlib_dir; - if (stdlib_dir != NULL && stdlib_dir[0] != L'\0') { - return stdlib_dir; - } - if (fallback == NULL) { - fallback = _Py_GetMainConfig(); - if (fallback == NULL) { - return NULL; - } - } - stdlib_dir = fallback->stdlib_dir; - if (stdlib_dir != NULL && stdlib_dir[0] != L'\0') { - return stdlib_dir; - } - return NULL; -} - - wchar_t * Py_GetPrefix(void) { From 218ba84730224d9d0c3d21153aceb7a56a01984d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 10 Sep 2021 10:41:12 -0600 Subject: [PATCH 32/52] Default to "-X frozen_modules=off" if built with --with-debug. --- .../2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst | 3 ++- Python/initconfig.c | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst index c88a8deb829533..f6dffa0831c54a 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-31-17-44-51.bpo-45020.ZPI_3L.rst @@ -1,2 +1,3 @@ Add a new command line option, "-X frozen_modules=[on|off]" to opt out -of (or into) using optional frozen modules. This defaults to "on". +of (or into) using optional frozen modules. This defaults to "on" (or +"off" if it's a debug build). diff --git a/Python/initconfig.c b/Python/initconfig.c index 63315dbbfd9502..fb88a20dcf8cea 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2123,9 +2123,15 @@ _PyConfig_InitImportConfig(PyConfig *config) /* -X frozen_modules=[on|off] */ const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { - // Use a meaningful default. - // XXX Add the logic. + // Use a meaningful default: + // * "off" for core development (running in a local repo) + // * "on" otherwise (e.g. for release builds) +#ifdef Py_DEBUG + // For now, Py_DEBUG is an adequate approximation of core development. config->use_frozen_modules = false; +#else + config->use_frozen_modules = true; +#endif } else if (wcscmp(value, L"on") == 0) { config->use_frozen_modules = true; From 81b66c4f03789fda31949a1f1414c9b156631124 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 10 Sep 2021 13:16:47 -0600 Subject: [PATCH 33/52] Fix test_embed. --- Lib/test/test_embed.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e5c99182ca0ebf..ff5fe64c615d12 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -398,7 +398,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'module_search_paths': GET_DEFAULT_CONFIG, 'module_search_paths_set': 1, 'platlibdir': sys.platlibdir, - 'stdlib_dir': GET_DEFAULT_CONFIG, 'site_import': 1, 'bytes_warning': 0, @@ -427,7 +426,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - 'use_frozen_modules': True, + 'use_frozen_modules': False, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -508,7 +507,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix', 'program_name', 'home', - 'stdlib_dir', # program_full_path and module_search_path are copied indirectly from # the core configuration in check_path_config(). ] @@ -1136,9 +1134,6 @@ def test_init_setpath(self): 'base_prefix': '', 'exec_prefix': '', 'base_exec_prefix': '', - # The current getpath.c doesn't determine the stdlib dir - # in this case. - 'stdlib_dir': '', } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1159,9 +1154,6 @@ def test_init_setpath_config(self): 'base_prefix': '', 'exec_prefix': '', 'base_exec_prefix': '', - # The current getpath.c doesn't determine the stdlib dir - # in this case. - 'stdlib_dir': '', # overriden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', @@ -1251,7 +1243,6 @@ def test_init_setpythonhome(self): 'exec_prefix': exec_prefix, 'base_exec_prefix': exec_prefix, 'pythonpath_env': paths_str, - 'stdlib_dir': home, } self.default_program_name(config) env = {'TESTHOME': home, @@ -1291,9 +1282,6 @@ def test_init_pybuilddir(self): 'base_executable': executable, 'executable': executable, 'module_search_paths': module_search_paths, - # The current getpath.c doesn't determine the stdlib dir - # in this case. - 'stdlib_dir': None, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, @@ -1343,9 +1331,6 @@ def test_init_pyvenv_cfg(self): 'base_executable': executable, 'executable': executable, 'module_search_paths': paths, - # The current getpath.c doesn't determine the stdlib dir - # in this case. - 'stdlib_dir': None, } path_config = {} if MS_WINDOWS: From 8a0e2d6a12cd19593297fef442db483b513a13fb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 10 Sep 2021 18:28:27 -0600 Subject: [PATCH 34/52] Fix the ASAN job. --- Lib/test/test_embed.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index ff5fe64c615d12..a79fbb60694a3d 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -12,6 +12,7 @@ import shutil import subprocess import sys +import sysconfig import tempfile import textwrap @@ -432,6 +433,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, }) + else: + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + if '--with-address-sanitizer' in config_args: + CONFIG_COMPAT['use_frozen_modules'] = True CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, From 015035dd14198278ff865063ef421c281d4d6709 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Sep 2021 16:55:34 -0600 Subject: [PATCH 35/52] Fix the docs. --- Doc/using/cmdline.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 05c659bea328a9..2398ab0d0cf262 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -483,8 +483,7 @@ Miscellaneous options * ``-X frozen_modules`` determines whether or not frozen modules are ignored by the import machinery. A value of "on" means they get imported and "off" means they are ignored. The default is "on" - if this is an installed Python (the normal case). If it's under - development (running from the build dir) then the default is "off". + for non-debug builds (the normal case) and "off" for debug builds. Note that the "importlib_bootstrap" and "importlib_bootstrap_external" frozen modules are always used, even if this flag is set to "off". From 7e2ce410a8d27b31ebd1823e06c01f7b005fbebe Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Sep 2021 16:57:22 -0600 Subject: [PATCH 36/52] Drop unused changes. --- Include/internal/pycore_pylifecycle.h | 2 +- Python/pathconfig.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index bd26f0d6a0dd01..524be9d4cbb940 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -12,7 +12,6 @@ extern "C" { #include #endif -#include #include "pycore_runtime.h" // _PyRuntimeState #ifndef NSIG @@ -123,6 +122,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const PyConfig *config, const struct _PyArgv *args); + PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable); diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 7f6bb045b8f392..470aba75bea969 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -6,7 +6,6 @@ #include "pycore_fileutils.h" #include "pycore_pathconfig.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() -#include "pycore_pystate.h" // _Py_GetMainConfig() #include #ifdef MS_WINDOWS # include // GetFullPathNameW(), MAX_PATH @@ -607,7 +606,6 @@ Py_GetProgramName(void) return _Py_path_config.program_name; } - /* Compute module search path from argv[0] or the current working directory ("-m module" case) which will be prepended to sys.argv: sys.path[0]. From b73486aae013a7612afbc6904cfbcdb1be39dc38 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Sep 2021 17:13:17 -0600 Subject: [PATCH 37/52] Fix the Windows CI jobs. --- Lib/test/test_embed.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index a79fbb60694a3d..4ea06e1ce51247 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -433,6 +433,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, }) + # On Windows we run tests with a non-debug build. + CONFIG_COMPAT['use_frozen_modules'] = True else: config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' if '--with-address-sanitizer' in config_args: From fc2123beb24b477b1535b504ed52ba394b67639f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Sep 2021 17:52:45 -0600 Subject: [PATCH 38/52] Fix test_embed on non-debug builds. --- Lib/test/test_embed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4ea06e1ce51247..9237a5ff60c9a6 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -437,7 +437,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT['use_frozen_modules'] = True else: config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' - if '--with-address-sanitizer' in config_args: + if '--with-pydebug' not in config_args: CONFIG_COMPAT['use_frozen_modules'] = True CONFIG_PYTHON = dict(CONFIG_COMPAT, From c3ed1047e965755babed61ba4db6142e23cbd24f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 13 Sep 2021 18:02:49 -0600 Subject: [PATCH 39/52] Fix test_embed on non-debug builds (on Windows). --- Lib/test/test_embed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 9237a5ff60c9a6..48da599a35f0a9 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -433,8 +433,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, }) - # On Windows we run tests with a non-debug build. - CONFIG_COMPAT['use_frozen_modules'] = True + if not sys.executable.endswith('_d.exe'): + # It's not a debug build. + CONFIG_COMPAT['use_frozen_modules'] = True else: config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' if '--with-pydebug' not in config_args: From 9b110fd6d278dd9e5b1846d7640cbd7e61af0b8e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 08:30:56 -0600 Subject: [PATCH 40/52] Drop parse_env_var_flag(). --- Python/import.c | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/Python/import.c b/Python/import.c index c7d78a3ffd23de..91fe7ac7db6c01 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1050,26 +1050,6 @@ _imp_create_builtin(PyObject *module, PyObject *spec) /* Frozen modules */ -static bool -parse_env_var_flag(const char *rawvalue, bool *isset) -{ - /* It isn't set or it is set to an empty string. */ - if (rawvalue == NULL || strlen(rawvalue) == 0) { - if (isset != NULL) { - *isset = false; - } - return false; - } - if (isset != NULL) { - *isset = true; - } - if (strcmp(rawvalue, "0") == 0) { - return false; - } - /* For now we treat all other non-empty strings as true. */ - return true; -} - static bool is_essential_frozen_module(const char *name) { @@ -1094,13 +1074,23 @@ is_essential_frozen_module(const char *name) static bool use_frozen(const char *modname) { - bool isset; - /* Note that we don't bother with os.environ. */ - bool use = parse_env_var_flag(getenv("_PYTHONTESTFROZENMODULES"), &isset); - if (!isset) { + bool use; + + /* Note that we don't bother with os.environ nor do we use Py_GETENV(). */ + const char *env = getenv("_PYTHONTESTFROZENMODULES"); + /* We would use "bool _Py_ParseEnvVarFlag(const char *)" if it existed. */ + if (env == NULL || strlen(env) == 0) { const PyConfig *config = _Py_GetConfig(); use = config->use_frozen_modules; } + else { + use = true; + if (strcmp(env, "0") == 0) { + use = false; + } + /* For now we treat all other non-empty strings as true. */ + } + if (!use && modname != NULL) { use = is_essential_frozen_module(modname); } From 9df64c179061e42655f595c69c9b00ee626ee84e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 08:43:47 -0600 Subject: [PATCH 41/52] Don't use env var _PYTHONTESTFROZENMODULES when running _testembed. --- Lib/test/test_embed.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 48da599a35f0a9..f3e3bf3296722b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1253,9 +1253,7 @@ def test_init_setpythonhome(self): 'pythonpath_env': paths_str, } self.default_program_name(config) - env = {'TESTHOME': home, - 'PYTHONPATH': paths_str, - '_PYTHONTESTFROZENMODULES': '1'} + env = {'TESTHOME': home, 'PYTHONPATH': paths_str} self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) From ccd468e8a524f547a8a9dde9fda194ea1a7297bb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 13:32:02 -0600 Subject: [PATCH 42/52] Do not use an env var to override PyConfig.use_frozen_modules. --- Include/internal/pycore_interp.h | 3 +++ Lib/test/support/import_helper.py | 6 ++--- Python/clinic/import.c.h | 33 ++++++++++++++++++++++++- Python/import.c | 41 ++++++++++++++++++++++--------- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index bfd082b588256b..0e6edf4ec26702 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -246,6 +246,9 @@ struct _is { PyObject *builtins; // importlib module PyObject *importlib; + // override for config->use_frozen_modules (for tests) + // (-1: "off", 1: "on", 0: no override) + int override_frozen_modules; /* Used in Modules/_threadmodule.c. */ long num_threads; diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index b15ae77ef86ae2..ffe2e3506b6998 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -1,4 +1,5 @@ import contextlib +import _imp import importlib import importlib.util import os @@ -116,12 +117,11 @@ def frozen_modules(enabled=True): This only applies to modules that haven't been imported yet. Also, some essential modules will always be imported frozen. """ - # FYI: the env var will never show up in os.environ. - os.putenv('_PYTHONTESTFROZENMODULES', '1' if enabled else '0') + _imp._override_frozen_modules_for_tests(1 if enabled else -1) try: yield finally: - os.unsetenv('_PYTHONTESTFROZENMODULES') + _imp._override_frozen_modules_for_tests(0) def import_fresh_module(name, fresh=(), blocked=(), *, diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index ec4ebca36d9430..438a348fa097fb 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -315,6 +315,37 @@ _imp__frozen_module_names(PyObject *module, PyObject *Py_UNUSED(ignored)) return _imp__frozen_module_names_impl(module); } +PyDoc_STRVAR(_imp__override_frozen_modules_for_tests__doc__, +"_override_frozen_modules_for_tests($module, override, /)\n" +"--\n" +"\n" +"(internal-only) Override PyConfig.use_frozen_modules.\n" +"\n" +"(-1: \"off\", 1: \"on\", 0: no override)\n" +"See frozen_modules() in Lib/test/support/import_helper.py."); + +#define _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF \ + {"_override_frozen_modules_for_tests", (PyCFunction)_imp__override_frozen_modules_for_tests, METH_O, _imp__override_frozen_modules_for_tests__doc__}, + +static PyObject * +_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override); + +static PyObject * +_imp__override_frozen_modules_for_tests(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int override; + + override = _PyLong_AsInt(arg); + if (override == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _imp__override_frozen_modules_for_tests_impl(module, override); + +exit: + return return_value; +} + #if defined(HAVE_DYNAMIC_LOADING) PyDoc_STRVAR(_imp_create_dynamic__doc__, @@ -467,4 +498,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=0ab3fa7c5808bba4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=96038c277119d6e3 input=a9049054013a1b77]*/ diff --git a/Python/import.c b/Python/import.c index 91fe7ac7db6c01..d28e9b525fdab8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1076,19 +1076,16 @@ use_frozen(const char *modname) { bool use; - /* Note that we don't bother with os.environ nor do we use Py_GETENV(). */ - const char *env = getenv("_PYTHONTESTFROZENMODULES"); - /* We would use "bool _Py_ParseEnvVarFlag(const char *)" if it existed. */ - if (env == NULL || strlen(env) == 0) { - const PyConfig *config = _Py_GetConfig(); - use = config->use_frozen_modules; + PyInterpreterState *interp = _PyInterpreterState_GET(); + int override = interp->override_frozen_modules; + if (override > 0) { + use = true; + } + else if (override > 0) { + use = false; } else { - use = true; - if (strcmp(env, "0") == 0) { - use = false; - } - /* For now we treat all other non-empty strings as true. */ + use = interp->config.use_frozen_modules; } if (!use && modname != NULL) { @@ -2054,6 +2051,27 @@ _imp__frozen_module_names_impl(PyObject *module) return list_frozen_module_names(); } +/*[clinic input] +_imp._override_frozen_modules_for_tests + + override: int + / + +(internal-only) Override PyConfig.use_frozen_modules. + +(-1: "off", 1: "on", 0: no override) +See frozen_modules() in Lib/test/support/import_helper.py. +[clinic start generated code]*/ + +static PyObject * +_imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) +/*[clinic end generated code: output=36d5cb1594160811 input=8f1f95a3ef21aec3]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + interp->override_frozen_modules = override; + Py_RETURN_NONE; +} + /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */ static int exec_builtin_or_dynamic(PyObject *mod) { @@ -2215,6 +2233,7 @@ static PyMethodDef imp_methods[] = { _IMP_IS_BUILTIN_METHODDEF _IMP_IS_FROZEN_METHODDEF _IMP__FROZEN_MODULE_NAMES_METHODDEF + _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF _IMP_CREATE_DYNAMIC_METHODDEF _IMP_EXEC_DYNAMIC_METHODDEF _IMP_EXEC_BUILTIN_METHODDEF From 5bc6730c544b106196f2bbb3484dcbad3a992c12 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 13:43:10 -0600 Subject: [PATCH 43/52] For now, always default to "-X frozen_modules=off". --- Lib/test/test_embed.py | 8 +------- Python/initconfig.c | 10 ++-------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index f3e3bf3296722b..620c671cd46500 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -427,19 +427,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, + # Currently we default to using source modules instead of frozen. 'use_frozen_modules': False, } if MS_WINDOWS: CONFIG_COMPAT.update({ 'legacy_windows_stdio': 0, }) - if not sys.executable.endswith('_d.exe'): - # It's not a debug build. - CONFIG_COMPAT['use_frozen_modules'] = True - else: - config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' - if '--with-pydebug' not in config_args: - CONFIG_COMPAT['use_frozen_modules'] = True CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, diff --git a/Python/initconfig.c b/Python/initconfig.c index fb88a20dcf8cea..61cc70038b10d0 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2123,15 +2123,9 @@ _PyConfig_InitImportConfig(PyConfig *config) /* -X frozen_modules=[on|off] */ const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { - // Use a meaningful default: - // * "off" for core development (running in a local repo) - // * "on" otherwise (e.g. for release builds) -#ifdef Py_DEBUG - // For now, Py_DEBUG is an adequate approximation of core development. + // For now we always default to "off". + // In the near future we will be factoring in PGO and in-development. config->use_frozen_modules = false; -#else - config->use_frozen_modules = true; -#endif } else if (wcscmp(value, L"on") == 0) { config->use_frozen_modules = true; From f50476c6df7e9479e7a7e81db2b5769863f12154 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 13:50:59 -0600 Subject: [PATCH 44/52] Change PyConfig.use_frozen_modules from bool to int. --- Include/cpython/initconfig.h | 4 +--- Python/initconfig.c | 41 ++++++------------------------------ 2 files changed, 7 insertions(+), 38 deletions(-) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index fc0c40feb85c71..65d52c45783f18 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -2,8 +2,6 @@ #define Py_PYCORECONFIG_H #ifndef Py_LIMITED_API -#include - /* --- PyStatus ----------------------------------------------- */ typedef struct { @@ -174,7 +172,7 @@ typedef struct PyConfig { int legacy_windows_stdio; #endif wchar_t *check_hash_pycs_mode; - bool use_frozen_modules; + int use_frozen_modules; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Python/initconfig.c b/Python/initconfig.c index 61cc70038b10d0..c9109eeabbe9a7 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -985,10 +985,6 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR)) #define SET_ITEM_UINT(ATTR) \ SET_ITEM(#ATTR, PyLong_FromUnsignedLong(config->ATTR)) -#define SET_ITEM_BOOL(ATTR) \ - SET_ITEM(#ATTR, ( \ - Py_INCREF(config->ATTR ? Py_True : Py_False), \ - (config->ATTR ? Py_True : Py_False))) #define FROM_WSTRING(STR) \ ((STR != NULL) ? \ PyUnicode_FromWideChar(STR, -1) \ @@ -1059,7 +1055,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_INT(_init_main); SET_ITEM_INT(_isolated_interpreter); SET_ITEM_WSTRLIST(orig_argv); - SET_ITEM_BOOL(use_frozen_modules); + SET_ITEM_INT(use_frozen_modules); return dict; @@ -1071,7 +1067,6 @@ _PyConfig_AsDict(const PyConfig *config) #undef SET_ITEM #undef SET_ITEM_INT #undef SET_ITEM_UINT -#undef SET_ITEM_BOOL #undef SET_ITEM_WSTR #undef SET_ITEM_WSTRLIST } @@ -1147,23 +1142,6 @@ config_dict_get_ulong(PyObject *dict, const char *name, unsigned long *result) } -static int -config_dict_get_bool(PyObject *dict, const char *name, bool *result) -{ - PyObject *item = config_dict_get(dict, name); - if (item == NULL) { - return -1; - } - int value = PyObject_IsTrue(item); - if (value < 0) { - config_dict_invalid_value(name); - return -1; - } - *result = value == 0 ? false : true; - return 0; -} - - static int config_dict_get_wstr(PyObject *dict, const char *name, PyConfig *config, wchar_t **result) @@ -1267,12 +1245,6 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) } \ CHECK_VALUE(#KEY, config->KEY >= 0); \ } while (0) -#define GET_BOOL(KEY) \ - do { \ - if (config_dict_get_bool(dict, #KEY, &config->KEY) < 0) { \ - return -1; \ - } \ - } while (0) #define GET_WSTR(KEY) \ do { \ if (config_dict_get_wstr(dict, #KEY, config, &config->KEY) < 0) { \ @@ -1366,11 +1338,10 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(_install_importlib); GET_UINT(_init_main); GET_UINT(_isolated_interpreter); - GET_BOOL(use_frozen_modules); + GET_UINT(use_frozen_modules); #undef CHECK_VALUE #undef GET_UINT -#undef GET_BOOL #undef GET_WSTR #undef GET_WSTR_OPT return 0; @@ -2125,17 +2096,17 @@ _PyConfig_InitImportConfig(PyConfig *config) if (value == NULL) { // For now we always default to "off". // In the near future we will be factoring in PGO and in-development. - config->use_frozen_modules = false; + config->use_frozen_modules = 0; } else if (wcscmp(value, L"on") == 0) { - config->use_frozen_modules = true; + config->use_frozen_modules = 1; } else if (wcscmp(value, L"off") == 0) { - config->use_frozen_modules = false; + config->use_frozen_modules = 0; } else if (wcslen(value) == 0) { // "-X frozen_modules" and "-X frozen_modules=" both imply "on". - config->use_frozen_modules = true; + config->use_frozen_modules = 1; } else { return PyStatus_Error("bad value for option -X frozen_modules " From 0dcdb449a51ce4fa958ebf604ecb3eecc7fc671a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 14:07:46 -0600 Subject: [PATCH 45/52] Fix use of _PyConfig_InitImportConfig(). --- Python/initconfig.c | 19 ++++++++++--------- Python/pylifecycle.c | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index c9109eeabbe9a7..8740cc1cf7a2b3 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2081,12 +2081,12 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) } -PyStatus -_PyConfig_InitImportConfig(PyConfig *config) +static PyStatus +config_init_import(PyConfig *config, int compute_path_config) { PyStatus status; - status = _PyConfig_InitPathConfig(config, 1); + status = _PyConfig_InitPathConfig(config, compute_path_config); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -2116,6 +2116,12 @@ _PyConfig_InitImportConfig(PyConfig *config) return _PyStatus_OK(); } +PyStatus +_PyConfig_InitImportConfig(PyConfig *config) +{ + return config_init_import(config, 1); +} + static PyStatus config_read(PyConfig *config, int compute_path_config) @@ -2163,12 +2169,7 @@ config_read(PyConfig *config, int compute_path_config) } if (config->_install_importlib) { - if (compute_path_config) { - status = _PyConfig_InitImportConfig(config); - } - else { - status = _PyConfig_InitPathConfig(config, 0); - } + status = config_init_import(config, compute_path_config); if (_PyStatus_EXCEPTION(status)) { return status; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 3d4c63cb3325c3..b10a19c0d8d5f2 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1078,7 +1078,7 @@ init_interp_main(PyThreadState *tstate) return _PyStatus_OK(); } - // Compute the path configuration + // Initialize the import-related configuration. status = _PyConfig_InitImportConfig(&interp->config); if (_PyStatus_EXCEPTION(status)) { return status; From 2ec99090b6fd532ad479afe1d3807beb103a1ae6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 15:02:19 -0600 Subject: [PATCH 46/52] Drop a superfluous comment. --- Lib/test/test_embed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 620c671cd46500..e5e7c833e10d34 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -427,7 +427,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pathconfig_warnings': 1, '_init_main': 1, '_isolated_interpreter': 0, - # Currently we default to using source modules instead of frozen. 'use_frozen_modules': False, } if MS_WINDOWS: From 80d8fcd3f6a724e3b27b8c45288924166cc5773c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 15:03:06 -0600 Subject: [PATCH 47/52] Fix a typo. --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index d28e9b525fdab8..74cb80a98910c5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1081,7 +1081,7 @@ use_frozen(const char *modname) if (override > 0) { use = true; } - else if (override > 0) { + else if (override < 0) { use = false; } else { From aff01e6f02add9a7e92e5d3201a19288728f24fd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 15:05:43 -0600 Subject: [PATCH 48/52] Simplify a comment. --- Lib/test/support/import_helper.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index ffe2e3506b6998..86564337358f7a 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -152,10 +152,8 @@ def import_fresh_module(name, fresh=(), blocked=(), *, This function will raise ImportError if the named module cannot be imported. - If "usefrozen" is False (the default), any stdlib source module will - always be imported from its .py file, even if the module has been - frozen. The only exception is essential modules (like - importlib._bootstrap). + If "usefrozen" is False (the default) then the frozen importer is + disabled (except for essential modules (like importlib._bootstrap). """ # NOTE: test_heapq, test_json and test_warnings include extra sanity checks # to make sure that this utility function is working as expected @@ -194,10 +192,8 @@ class CleanImport(object): with CleanImport("foo"): importlib.import_module("foo") # new reference - If "usefrozen" is False (the default), any stdlib source module will - always be imported from its .py file, even if the module has been - frozen. The only exception is essential modules (like - importlib._bootstrap). + If "usefrozen" is False (the default) then the frozen importer is + disabled (except for essential modules (like importlib._bootstrap). """ def __init__(self, *module_names, usefrozen=False): From 7476d65275475c37048233152c6fb025a49cb1dc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 15:09:32 -0600 Subject: [PATCH 49/52] Simplify use_frozen(). --- Python/import.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/Python/import.c b/Python/import.c index 74cb80a98910c5..3d6172225c20cb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1072,26 +1072,19 @@ is_essential_frozen_module(const char *name) } static bool -use_frozen(const char *modname) +use_frozen(void) { - bool use; - PyInterpreterState *interp = _PyInterpreterState_GET(); int override = interp->override_frozen_modules; if (override > 0) { - use = true; + return true; } else if (override < 0) { - use = false; + return false; } else { - use = interp->config.use_frozen_modules; - } - - if (!use && modname != NULL) { - use = is_essential_frozen_module(modname); + return interp->config.use_frozen_modules; } - return use; } static PyObject * @@ -1101,7 +1094,7 @@ list_frozen_module_names() if (names == NULL) { return NULL; } - bool enabled = use_frozen(NULL); + bool enabled = use_frozen(); for (const struct _frozen *p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) { break; @@ -1135,8 +1128,10 @@ find_frozen(PyObject *modname) PyErr_Clear(); return NULL; } - if (!use_frozen(name)) { - return NULL; + if (!use_frozen()) { + if (!is_essential_frozen_module(name)) { + return NULL; + } } const struct _frozen *p; for (p = PyImport_FrozenModules; ; p++) { From e3e57442c79b9fd9eda5d670d27416133792c527 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 16:09:22 -0600 Subject: [PATCH 50/52] Fix a comment. --- Lib/test/support/import_helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 86564337358f7a..10f745aa6b1448 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -153,7 +153,7 @@ def import_fresh_module(name, fresh=(), blocked=(), *, imported. If "usefrozen" is False (the default) then the frozen importer is - disabled (except for essential modules (like importlib._bootstrap). + disabled (except for essential modules like importlib._bootstrap). """ # NOTE: test_heapq, test_json and test_warnings include extra sanity checks # to make sure that this utility function is working as expected @@ -193,7 +193,7 @@ class CleanImport(object): importlib.import_module("foo") # new reference If "usefrozen" is False (the default) then the frozen importer is - disabled (except for essential modules (like importlib._bootstrap). + disabled (except for essential modules like importlib._bootstrap). """ def __init__(self, *module_names, usefrozen=False): From 669beac50155b2427ecabaf57d7cfbc51895c0a1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 16:10:20 -0600 Subject: [PATCH 51/52] Collapse some lines. --- Python/import.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Python/import.c b/Python/import.c index 3d6172225c20cb..317a836617c51d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1128,10 +1128,8 @@ find_frozen(PyObject *modname) PyErr_Clear(); return NULL; } - if (!use_frozen()) { - if (!is_essential_frozen_module(name)) { - return NULL; - } + if (!use_frozen() && !is_essential_frozen_module(name)) { + return NULL; } const struct _frozen *p; for (p = PyImport_FrozenModules; ; p++) { From c644012ccab1f60fac16245b6b497967b8970767 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 14 Sep 2021 17:08:24 -0600 Subject: [PATCH 52/52] Drop _Py_GetMainConfig(). --- Include/internal/pycore_pystate.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index aef318989aa6e5..4b894f3eff4967 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -30,17 +30,6 @@ _Py_IsMainInterpreter(PyInterpreterState *interp) } -static inline const PyConfig * -_Py_GetMainConfig(void) -{ - PyInterpreterState *interp = _PyRuntime.interpreters.main; - if (interp == NULL) { - return NULL; - } - return _PyInterpreterState_GetConfig(interp); -} - - /* Only handle signals on the main thread of the main interpreter. */ static inline int _Py_ThreadCanHandleSignals(PyInterpreterState *interp)