From a10471a0df156a7e4ca4610653ee664edf26f887 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 10 Jun 2021 17:35:07 -0600 Subject: [PATCH 01/41] Clean up _freeze_importlib.c. --- Programs/_freeze_importlib.c | 218 +++++++++++++++++++++-------------- Python/frozen.c | 10 +- Python/importlib.h | 2 +- Python/importlib_external.h | 2 +- 4 files changed, 142 insertions(+), 90 deletions(-) diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 2e4ccbb154a414..d5d3b5d20cda79 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -30,52 +30,9 @@ const struct _frozen *PyImport_FrozenModules; static const char header[] = "/* Auto-generated by Programs/_freeze_importlib.c */"; -int -main(int argc, char *argv[]) +static void +runtime_init(void) { - const char *name, *inpath, *outpath; - char buf[100]; - FILE *infile = NULL, *outfile = NULL; - struct _Py_stat_struct stat; - size_t text_size, data_size, i, n; - char *text = NULL; - unsigned char *data; - PyObject *code = NULL, *marshalled = NULL; - - PyImport_FrozenModules = _PyImport_FrozenModules; - - if (argc != 4) { - fprintf(stderr, "need to specify the name, input and output paths\n"); - return 2; - } - name = argv[1]; - inpath = argv[2]; - outpath = argv[3]; - infile = fopen(inpath, "rb"); - if (infile == NULL) { - fprintf(stderr, "cannot open '%s' for reading\n", inpath); - goto error; - } - if (_Py_fstat_noraise(fileno(infile), &stat)) { - fprintf(stderr, "cannot fstat '%s'\n", inpath); - goto error; - } - text_size = (size_t)stat.st_size; - text = (char *) malloc(text_size + 1); - if (text == NULL) { - fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size); - goto error; - } - n = fread(text, 1, text_size, infile); - fclose(infile); - infile = NULL; - if (n < text_size) { - fprintf(stderr, "read too short: got %ld instead of %ld bytes\n", - (long) n, (long) text_size); - goto error; - } - text[text_size] = '\0'; - PyConfig config; PyConfig_InitIsolatedConfig(&config); @@ -98,39 +55,89 @@ main(int argc, char *argv[]) if (PyStatus_Exception(status)) { Py_ExitStatusException(status); } +} + +static const char * +read_text(const char *inpath) +{ + FILE *infile = fopen(inpath, "rb"); + if (infile == NULL) { + fprintf(stderr, "cannot open '%s' for reading\n", inpath); + return NULL; + } + + struct _Py_stat_struct stat; + if (_Py_fstat_noraise(fileno(infile), &stat)) { + fprintf(stderr, "cannot fstat '%s'\n", inpath); + fclose(infile); + return NULL; + } + size_t text_size = (size_t)stat.st_size; + + char *text = (char *) malloc(text_size + 1); + if (text == NULL) { + fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size); + fclose(infile); + return NULL; + } + size_t n = fread(text, 1, text_size, infile); + fclose(infile); + + if (n < text_size) { + fprintf(stderr, "read too short: got %ld instead of %ld bytes\n", + (long) n, (long) text_size); + free(text); + return NULL; + } + + text[text_size] = '\0'; + return (const char *)text; +} +static PyObject * +compile_and_marshal(const char *name, const char *text) +{ + char buf[100]; sprintf(buf, "", name); - code = Py_CompileStringExFlags(text, buf, Py_file_input, NULL, 0); - if (code == NULL) - goto error; - free(text); - text = NULL; + PyObject *code = Py_CompileStringExFlags(text, buf, Py_file_input, NULL, 0); + if (code == NULL) { + return NULL; + } - marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION); + PyObject *marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION); Py_CLEAR(code); - if (marshalled == NULL) - goto error; - + if (marshalled == NULL) { + return NULL; + } assert(PyBytes_CheckExact(marshalled)); - data = (unsigned char *) PyBytes_AS_STRING(marshalled); - data_size = PyBytes_GET_SIZE(marshalled); - /* Open the file in text mode. The hg checkout should be using the eol extension, - which in turn should cause the EOL style match the C library's text mode */ - outfile = fopen(outpath, "w"); - if (outfile == NULL) { - fprintf(stderr, "cannot open '%s' for writing\n", outpath); - goto error; - } - fprintf(outfile, "%s\n", header); - for (i = n = 0; name[i] != '\0'; i++) { - if (name[i] != '.') { + return marshalled; +} + +static void +get_varname(const char *name, char *buf) +{ + (void)strcpy(buf, "_Py_M__"); + size_t n = 7; + for (size_t i = 0; name[i] != '\0'; i++) { + if (name[i] == '.') { + buf[n++] = '_'; + } + else { buf[n++] = name[i]; } } buf[n] = '\0'; - fprintf(outfile, "const unsigned char _Py_M__%s[] = {\n", buf); - for (n = 0; n < data_size; n += 16) { +} + +static void +write_code(FILE *outfile, PyObject *marshalled, const char *varname) +{ + unsigned char *data = (unsigned char *) PyBytes_AS_STRING(marshalled); + size_t data_size = PyBytes_GET_SIZE(marshalled); + + fprintf(outfile, "const unsigned char %s[] = {\n", varname); + for (size_t n = 0; n < data_size; n += 16) { size_t i, end = Py_MIN(n + 16, data_size); fprintf(outfile, " "); for (i = n; i < end; i++) { @@ -139,29 +146,72 @@ main(int argc, char *argv[]) fprintf(outfile, "\n"); } fprintf(outfile, "};\n"); +} + +static int +write_frozen(const char *outpath, const char *inpath, const char *name, + PyObject *marshalled) +{ + /* Open the file in text mode. The hg checkout should be using the eol extension, + which in turn should cause the EOL style match the C library's text mode */ + FILE *outfile = fopen(outpath, "w"); + if (outfile == NULL) { + fprintf(stderr, "cannot open '%s' for writing\n", outpath); + return -1; + } - Py_CLEAR(marshalled); + char buf[100]; + fprintf(outfile, "%s\n", header); + get_varname(name, buf); + write_code(outfile, marshalled, buf); - Py_Finalize(); - if (outfile) { - if (ferror(outfile)) { - fprintf(stderr, "error when writing to '%s'\n", outpath); - goto error; - } - fclose(outfile); + if (ferror(outfile)) { + fprintf(stderr, "error when writing to '%s'\n", outpath); + return -1; } + fclose(outfile); + return 0; +} + +int +main(int argc, char *argv[]) +{ + const char *name, *inpath, *outpath; + + PyImport_FrozenModules = _PyImport_FrozenModules; + + if (argc != 4) { + fprintf(stderr, "need to specify the name, input and output paths\n"); + return 2; + } + name = argv[1]; + inpath = argv[2]; + outpath = argv[3]; + + runtime_init(); + + const char *text = read_text(inpath); + if (text == NULL) { + goto error; + } + + PyObject *marshalled = compile_and_marshal(name, text); + free((char *)text); + if (marshalled == NULL) { + goto error; + } + + int res = write_frozen(outpath, inpath, name, marshalled); + Py_DECREF(marshalled); + if (res != 0) { + goto error; + } + + Py_Finalize(); return 0; error: PyErr_Print(); Py_Finalize(); - if (infile) - fclose(infile); - if (outfile) - fclose(outfile); - if (text) - free(text); - if (marshalled) - Py_DECREF(marshalled); return 1; } diff --git a/Python/frozen.c b/Python/frozen.c index 7f433ff80ca129..0540a482569ea4 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -19,17 +19,19 @@ static const struct _frozen _PyImport_FrozenModules[] = { /* importlib */ - {"_frozen_importlib", _Py_M__importlib_bootstrap, - (int)sizeof(_Py_M__importlib_bootstrap)}, - {"_frozen_importlib_external", _Py_M__importlib_bootstrap_external, - (int)sizeof(_Py_M__importlib_bootstrap_external)}, + {"_frozen_importlib", _Py_M__importlib__bootstrap, + (int)sizeof(_Py_M__importlib__bootstrap)}, + {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, + (int)sizeof(_Py_M__importlib__bootstrap_external)}, {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, + /* Test module */ {"__hello__", _Py_M__hello, SIZE}, /* Test package (negative size indicates package-ness) */ {"__phello__", _Py_M__hello, -SIZE}, {"__phello__.spam", _Py_M__hello, SIZE}, + {0, 0, 0} /* sentinel */ }; diff --git a/Python/importlib.h b/Python/importlib.h index 69bd9727237f17..8487f1d50663a3 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1,5 +1,5 @@ /* Auto-generated by Programs/_freeze_importlib.c */ -const unsigned char _Py_M__importlib_bootstrap[] = { +const unsigned char _Py_M__importlib__bootstrap[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0, 0,0,0,0,0,115,130,1,0,0,100,0,90,0,100,1, 132,0,90,1,100,2,90,2,100,2,90,3,100,2,90,4, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index c49fa5516eb26c..cfe3cc542adf1b 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -1,5 +1,5 @@ /* Auto-generated by Programs/_freeze_importlib.c */ -const unsigned char _Py_M__importlib_bootstrap_external[] = { +const unsigned char _Py_M__importlib__bootstrap_external[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0, 0,0,0,0,0,115,158,2,0,0,100,0,90,0,100,1, 97,1,100,2,100,1,108,2,90,2,100,2,100,1,108,3, From 6252a37071af8a061d7b1c8f811ef1a13b1fa25d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 10 Aug 2021 13:20:29 -0600 Subject: [PATCH 02/41] _freeze_importlib -> _freeze_module --- .gitignore | 2 +- Doc/c-api/init.rst | 2 +- Makefile.pre.in | 26 +++++++++---------- .../{_freeze_importlib.c => _freeze_module.c} | 4 +-- Python/frozen_hello.h | 2 +- Python/importlib.h | 2 +- Python/importlib_external.h | 2 +- Python/importlib_zipimport.h | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) rename Programs/{_freeze_importlib.c => _freeze_module.c} (97%) diff --git a/.gitignore b/.gitignore index a96be67962217e..0ed4c8bdd0ccff 100644 --- a/.gitignore +++ b/.gitignore @@ -68,7 +68,7 @@ Modules/Setup.config Modules/Setup.local Modules/config.c Modules/ld_so_aix -Programs/_freeze_importlib +Programs/_freeze_module Programs/_testembed PC/python_nt*.h PC/pythonnt_rc*.h diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 60564241bfd61e..2fcbcc8d77be45 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -110,7 +110,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Suppress error messages when calculating the module search path in :c:func:`Py_GetPath`. - Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. + Private flag used by ``_freeze_module`` and ``frozenmain`` programs. .. c:var:: int Py_HashRandomizationFlag diff --git a/Makefile.pre.in b/Makefile.pre.in index f503ac4d876726..027df899fc6025 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -735,28 +735,28 @@ Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS) ############################################################################ # Importlib -Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile +Programs/_freeze_module.o: Programs/_freeze_module.c Makefile -Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) - $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) + $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) .PHONY: regen-importlib -regen-importlib: Programs/_freeze_importlib +regen-importlib: Programs/_freeze_module # Regenerate Python/importlib_external.h - # from Lib/importlib/_bootstrap_external.py using _freeze_importlib - ./Programs/_freeze_importlib importlib._bootstrap_external \ + # from Lib/importlib/_bootstrap_external.py using _freeze_module + ./Programs/_freeze_module importlib._bootstrap_external \ $(srcdir)/Lib/importlib/_bootstrap_external.py \ $(srcdir)/Python/importlib_external.h.new $(UPDATE_FILE) $(srcdir)/Python/importlib_external.h $(srcdir)/Python/importlib_external.h.new # Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py - # using _freeze_importlib - ./Programs/_freeze_importlib importlib._bootstrap \ + # using _freeze_module + ./Programs/_freeze_module importlib._bootstrap \ $(srcdir)/Lib/importlib/_bootstrap.py \ $(srcdir)/Python/importlib.h.new $(UPDATE_FILE) $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib.h.new # Regenerate Python/importlib_zipimport.h from Lib/zipimport.py - # using _freeze_importlib - ./Programs/_freeze_importlib zipimport \ + # using _freeze_module + ./Programs/_freeze_module zipimport \ $(srcdir)/Lib/zipimport.py \ $(srcdir)/Python/importlib_zipimport.h.new $(UPDATE_FILE) $(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/importlib_zipimport.h.new @@ -884,9 +884,9 @@ regen-opcode: $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new .PHONY: regen-frozen -regen-frozen: Programs/_freeze_importlib +regen-frozen: Programs/_freeze_module # Regenerate code for frozen module "__hello__". - ./Programs/_freeze_importlib hello \ + ./Programs/_freeze_module hello \ $(srcdir)/Tools/freeze/flag.py \ $(srcdir)/Python/frozen_hello.h.new $(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h \ @@ -1917,7 +1917,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f Lib/lib2to3/*Grammar*.pickle - -rm -f Programs/_testembed Programs/_freeze_importlib + -rm -f Programs/_testembed Programs/_freeze_module -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_module.c similarity index 97% rename from Programs/_freeze_importlib.c rename to Programs/_freeze_module.c index d5d3b5d20cda79..dc941fc49e0500 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_module.c @@ -28,7 +28,7 @@ const struct _frozen *PyImport_FrozenModules; #endif static const char header[] = - "/* Auto-generated by Programs/_freeze_importlib.c */"; + "/* Auto-generated by Programs/_freeze_module.c */"; static void runtime_init(void) @@ -40,7 +40,7 @@ runtime_init(void) PyStatus status; status = PyConfig_SetString(&config, &config.program_name, - L"./_freeze_importlib"); + L"./_freeze_module"); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); Py_ExitStatusException(status); diff --git a/Python/frozen_hello.h b/Python/frozen_hello.h index c65c661e9bfb67..2658c05886a6db 100644 --- a/Python/frozen_hello.h +++ b/Python/frozen_hello.h @@ -1,4 +1,4 @@ -/* Auto-generated by Programs/_freeze_importlib.c */ +/* Auto-generated by Programs/_freeze_module.c */ const unsigned char _Py_M__hello[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, 0,0,0,0,0,115,16,0,0,0,100,0,90,0,101,1, diff --git a/Python/importlib.h b/Python/importlib.h index 8487f1d50663a3..2716896c21f4a5 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1,4 +1,4 @@ -/* Auto-generated by Programs/_freeze_importlib.c */ +/* Auto-generated by Programs/_freeze_module.c */ const unsigned char _Py_M__importlib__bootstrap[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0, 0,0,0,0,0,115,130,1,0,0,100,0,90,0,100,1, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index cfe3cc542adf1b..7a3410067d4a80 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -1,4 +1,4 @@ -/* Auto-generated by Programs/_freeze_importlib.c */ +/* Auto-generated by Programs/_freeze_module.c */ const unsigned char _Py_M__importlib__bootstrap_external[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0, 0,0,0,0,0,115,158,2,0,0,100,0,90,0,100,1, diff --git a/Python/importlib_zipimport.h b/Python/importlib_zipimport.h index c12ed5215b3f88..b4e2e85283cf49 100644 --- a/Python/importlib_zipimport.h +++ b/Python/importlib_zipimport.h @@ -1,4 +1,4 @@ -/* Auto-generated by Programs/_freeze_importlib.c */ +/* Auto-generated by Programs/_freeze_module.c */ const unsigned char _Py_M__zipimport[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0, 0,0,0,0,0,115,48,1,0,0,100,0,90,0,100,1, From b230af80f33440991586afef3593639ec2ca7cfa Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 8 Jun 2021 15:55:44 -0600 Subject: [PATCH 03/41] Add tooling to freeze stdlib modules. --- Makefile.pre.in | 10 ++++++-- Tools/scripts/freeze_stdlib.py | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 Tools/scripts/freeze_stdlib.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 027df899fc6025..781b348da4a2e8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -765,13 +765,19 @@ regen-importlib: Programs/_freeze_module regen-limited-abi: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt +############################################################################ +# Frozen stdlib + +.PHONY: regen-frozen-stdlib +regen-frozen-stdlib: Programs/_freeze_importlib + $(PYTHON_FOR_REGEN) ./Tools/scripts/freeze_stdlib.py ############################################################################ # Regenerate all generated files regen-all: regen-opcode regen-opcode-targets regen-typeslots \ - regen-token regen-ast regen-keyword regen-importlib clinic \ - regen-pegen-metaparser regen-pegen regen-frozen regen-test-frozenmain + regen-token regen-ast regen-keyword regen-importlib regen-frozen-stdlib \ + clinic regen-pegen-metaparser regen-pegen regen-frozen regen-test-frozenmain @echo @echo "Note: make regen-stdlib-module-names and autoconf should be run manually" diff --git a/Tools/scripts/freeze_stdlib.py b/Tools/scripts/freeze_stdlib.py new file mode 100644 index 00000000000000..941fdfdedafce7 --- /dev/null +++ b/Tools/scripts/freeze_stdlib.py @@ -0,0 +1,45 @@ +import os +import os.path +import subprocess + + +SCRIPTS_DIR = os.path.dirname(__file__) +TOOLS_DIR = os.path.dirname(SCRIPTS_DIR) +ROOT_DIR = os.path.dirname(TOOLS_DIR) +STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') +MODULES_DIR = os.path.join(ROOT_DIR, 'Python') +#MODULES_DIR = os.path.join(ROOT_DIR, 'Modules') +TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') + +STARTUP_MODULES_WITHOUT_SITE = [ +] +STARTUP_MODULES_WITH_SITE = [ +] + + +def freeze_module(modname, destdir): + if modname.startswith('<'): + modname = modname[1:-1] + pkgdir = os.path.join(STDLIB_DIR, *modname.split('.')) + pyfile = os.path.join(pkgdir, '__init__.py') + else: + pyfile = os.path.join(STDLIB_DIR, *modname.split('.')) + '.py' + destfile = os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' + tmpfile = destfile + '.new' + + argv = [TOOL, modname, pyfile, tmpfile] + print('#', ' '.join(argv)) + subprocess.run(argv, check=True) + + os.replace(tmpfile, destfile) + + +def main(): + for modname in STARTUP_MODULES_WITHOUT_SITE: + freeze_module(modname, MODULES_DIR) + for modname in STARTUP_MODULES_WITH_SITE: + freeze_module(modname, MODULES_DIR) + + +if __name__ == '__main__': + main() From e4ef8fc5dc92b8b03d2ea355ab1fa1cea437da6f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 10 Aug 2021 13:38:51 -0600 Subject: [PATCH 04/41] Support freezing a package's submodules. --- Tools/scripts/freeze_stdlib.py | 36 +++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/Tools/scripts/freeze_stdlib.py b/Tools/scripts/freeze_stdlib.py index 941fdfdedafce7..38bddc3a401ab5 100644 --- a/Tools/scripts/freeze_stdlib.py +++ b/Tools/scripts/freeze_stdlib.py @@ -19,9 +19,21 @@ def freeze_module(modname, destdir): if modname.startswith('<'): - modname = modname[1:-1] - pkgdir = os.path.join(STDLIB_DIR, *modname.split('.')) - pyfile = os.path.join(pkgdir, '__init__.py') + pkgname = modname[1:-1] + pkgname, sep, match= pkgname.partition('.') + pyfile, _ = _freeze_module(pkgname, MODULES_DIR, ispkg=True) + + if sep: + pkgdir = os.path.dirname(pyfile) + for modname, ispkg in _iter_submodules(pkgname, pkgdir, match): + _freeze_module(modname, MODULES_DIR, ispkg) + else: + _freeze_module(modname, MODULES_DIR) + + +def _freeze_module(modname, destdir, ispkg=False): + if ispkg: + pyfile = os.path.join(STDLIB_DIR, *modname.split('.'), '__init__.py') else: pyfile = os.path.join(STDLIB_DIR, *modname.split('.')) + '.py' destfile = os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' @@ -33,6 +45,24 @@ def freeze_module(modname, destdir): os.replace(tmpfile, destfile) + return pyfile, destfile + + +def _iter_submodules(pkgname, pkgdir, match='*'): + if not match: + match = '*' + if match != '*': + raise NotImplementedError(match) + + for entry in os.scandir(pkgdir): + modname = f'{pkgname}.{entry.name}' + if modname.endswith('.py'): + yield modname[:-3], False + elif entry.is_dir(): + if os.path.exists(os.path.join(entry.path, '__init__.py')): + yield modname, True + yield from _iter_submodules(modname, entry.path) + def main(): for modname in STARTUP_MODULES_WITHOUT_SITE: From 6bcc8c9f361ed29189d8448d88f4505a57ad7c71 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 10 Aug 2021 20:01:31 -0600 Subject: [PATCH 05/41] Use freeze_modules.py for all frozen modules. --- Makefile.pre.in | 62 ++++------ Tools/scripts/freeze_modules.py | 199 ++++++++++++++++++++++++++++++++ Tools/scripts/freeze_stdlib.py | 75 ------------ 3 files changed, 223 insertions(+), 113 deletions(-) create mode 100644 Tools/scripts/freeze_modules.py delete mode 100644 Tools/scripts/freeze_stdlib.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 781b348da4a2e8..d15ea8df8f8744 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -574,7 +574,7 @@ coverage-lcov: @echo # Force regeneration of parser and importlib -coverage-report: regen-token regen-importlib +coverage-report: regen-token regen-importlib regen-frozen-stdlib @ # build with coverage info $(MAKE) coverage @ # run tests, ignore failures @@ -733,51 +733,46 @@ Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ -# Importlib +# frozen modules (including importlib) Programs/_freeze_module.o: Programs/_freeze_module.c Makefile Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +Tools/scripts/freeze_modules.py: Programs/_freeze_module + .PHONY: regen-importlib -regen-importlib: Programs/_freeze_module - # Regenerate Python/importlib_external.h - # from Lib/importlib/_bootstrap_external.py using _freeze_module - ./Programs/_freeze_module importlib._bootstrap_external \ - $(srcdir)/Lib/importlib/_bootstrap_external.py \ - $(srcdir)/Python/importlib_external.h.new - $(UPDATE_FILE) $(srcdir)/Python/importlib_external.h $(srcdir)/Python/importlib_external.h.new - # Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py - # using _freeze_module - ./Programs/_freeze_module importlib._bootstrap \ - $(srcdir)/Lib/importlib/_bootstrap.py \ - $(srcdir)/Python/importlib.h.new - $(UPDATE_FILE) $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib.h.new - # Regenerate Python/importlib_zipimport.h from Lib/zipimport.py - # using _freeze_module - ./Programs/_freeze_module zipimport \ - $(srcdir)/Lib/zipimport.py \ - $(srcdir)/Python/importlib_zipimport.h.new - $(UPDATE_FILE) $(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/importlib_zipimport.h.new +regen-importlib: Tools/scripts/freeze_modules.py + # Regenerate Python/importlib_external.h, importlib.h, and zipimport.h. + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py importlib +.PHONY: regen-frozen-stdlib +regen-frozen-stdlib: Tools/scripts/freeze_modules.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py stdlib -regen-limited-abi: all - $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt +.PHONY: regen-frozen +regen-frozen: Tools/scripts/freeze_modules.py + # Regenerate code for frozen module "__hello__". + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py test + +.PHONY: regen-frozen-all +regen-frozen-all: Tools/scripts/freeze_modules.py + # Regenerate code for frozen module "__hello__". + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py all ############################################################################ -# Frozen stdlib +# ABI -.PHONY: regen-frozen-stdlib -regen-frozen-stdlib: Programs/_freeze_importlib - $(PYTHON_FOR_REGEN) ./Tools/scripts/freeze_stdlib.py +regen-limited-abi: all + $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt ############################################################################ # Regenerate all generated files regen-all: regen-opcode regen-opcode-targets regen-typeslots \ - regen-token regen-ast regen-keyword regen-importlib regen-frozen-stdlib \ - clinic regen-pegen-metaparser regen-pegen regen-frozen regen-test-frozenmain + regen-token regen-ast regen-keyword regen-frozen-all clinic \ + regen-pegen-metaparser regen-pegen regen-test-frozenmain @echo @echo "Note: make regen-stdlib-module-names and autoconf should be run manually" @@ -889,15 +884,6 @@ regen-opcode: $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new -.PHONY: regen-frozen -regen-frozen: Programs/_freeze_module - # Regenerate code for frozen module "__hello__". - ./Programs/_freeze_module hello \ - $(srcdir)/Tools/freeze/flag.py \ - $(srcdir)/Python/frozen_hello.h.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h \ - $(srcdir)/Python/frozen_hello.h.new - .PHONY: regen-token regen-token: # Regenerate Doc/library/token-list.inc from Grammar/Tokens diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py new file mode 100644 index 00000000000000..c9934c8e61998b --- /dev/null +++ b/Tools/scripts/freeze_modules.py @@ -0,0 +1,199 @@ +import os +import os.path +import subprocess +import sys + + +SCRIPTS_DIR = os.path.abspath(os.path.dirname(__file__)) +TOOLS_DIR = os.path.dirname(SCRIPTS_DIR) +ROOT_DIR = os.path.dirname(TOOLS_DIR) +STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') +MODULES_DIR = os.path.join(ROOT_DIR, 'Python') +#MODULES_DIR = os.path.join(ROOT_DIR, 'Modules') +TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') + +# These are modules that get frozen. +FROZEN = { + # frozen_id: (pyfile, frozenfile) + # frozen_id: pyfile + + # importlib + 'importlib._bootstrap': (None, 'importlib.h'), + 'importlib._bootstrap_external': (None, 'importlib_external.h'), + 'zipimport': (None, 'importlib_zipimport.h'), + # stdlib + # ... + # test + 'hello': os.path.join(TOOLS_DIR, 'freeze', 'flag.py'), +} +FROZEN_GROUPS = { + 'importlib': [ + 'importlib._bootstrap', + 'importlib._bootstrap_external', + 'zipimport', + ], + 'stdlib': [ + # without site (python -S): + + # with site: + + ], + 'test': [ + 'hello', + ], +} + + +def expand_frozen(destdir=MODULES_DIR): + # First, expand FROZEN. + frozen = {} + packages = {} + for frozenid, info in FROZEN.items(): + if frozenid.startswith('<'): + assert info is None, (frozenid, info) + modids = packages[frozenid] = [] + pkgname = frozenid[1:-1] + _pkgname, sep, match = pkgname.rpartition('.') + if not sep or match.isidentifier(): + frozen[pkgname] = (None, None, True) + modids.append(pkgname) + else: + pkgname = _pkgname + frozen[pkgname] = (None, None, True) + modids.append(okgname) + for subname, ispkg in _iter_submodules(pkgname, match=match): + frozen[subname] = (None, None, ispkg) + modids.append(subname) + else: + if info is None or isinstance(info, str): + info = (info, None, False) + else: + pyfile, frozenfile = info + info = (pyfile, frozenfile, False) + frozen[frozenid] = info + for frozenid, info in frozen.items(): + pyfile, frozenfile, ispkg = info + + if not pyfile: + pyfile = _resolve_module(frozenid, ispkg) + info = (pyfile, frozenfile, ispkg) + + if not frozenfile: + frozenfile = _resolve_frozen(frozenid, destdir) +# frozenfile = f'frozen_{frozenid}' +# frozenfile = os.path.join(destdir, frozenfile) + info = (pyfile, frozenfile, ispkg) + elif not os.path.isabs(frozenfile): + frozenfile = os.path.join(destdir, frozenfile) + info = (pyfile, frozenfile, ispkg) + + frozen[frozenid] = info + + # Then, expand FROZEN_GROUPS. + groups = {} + for groupname, frozenids in FROZEN_GROUPS.items(): + group = groups[groupname] = [] + for frozenid in frozenids: + if frozenid in packages: + group.extend(packages[frozenid]) + else: + group.append(frozenid) + + return groups, frozen + + +#def freeze_stdlib_module(modname, destdir=MODULES_DIR): +# if modname.startswith('<'): +# pkgname = modname[1:-1] +# pkgname, sep, match = pkgname.rpartition('.') +# if not sep or match.isidentifier(): +# pyfile = _resolve_module(pkgname, ispkg=True) +# frozenfile = _resolve_frozen(pkgname, destdir) +# _freeze_module(pkgname, pyfile, frozenfile) +# else: +# pkgdir = os.path.dirname(pyfile) +# for subname, ispkg in _iter_submodules(pkgname, pkgdir, match): +# pyfile = _resolve_module(subname, ispkg) +# frozenfile = _resolve_frozen(subname, destdir) +# _freeze_module(subname, pyfile, frozenfile) +# else: +# pyfile = _resolve_module(modname, ispkg=False) +# frozenfile = _resolve_frozen(modname, destdir) +# _freeze_module(modname, pyfile, frozenfile) + + +def _freeze_module(frozenid, pyfile, frozenfile): + tmpfile = frozenfile + '.new' + + argv = [TOOL, frozenid, pyfile, tmpfile] + print('#', ' '.join(argv)) + subprocess.run(argv, check=True) + + os.replace(tmpfile, frozenfile) + + +def _resolve_module(modname, ispkg=False): + if ispkg: + return os.path.join(STDLIB_DIR, *modname.split('.'), '__init__.py') + return os.path.join(STDLIB_DIR, *modname.split('.')) + '.py' + + +def _resolve_frozen(modname, destdir): + return os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' + + +def _iter_submodules(pkgname, pkgdir=None, match='*'): + if not pkgdir: + pkgdir = os.path.join(STDLIB_DIR, *pkgname.split('.')) + if not match: + match = '*' + if match != '*': + raise NotImplementedError(match) + + for entry in os.scandir(pkgdir): + modname = f'{pkgname}.{entry.name}' + if modname.endswith('.py'): + yield modname[:-3], False + elif entry.is_dir(): + if os.path.exists(os.path.join(entry.path, '__init__.py')): + yield modname, True + yield from _iter_submodules(modname, entry.path) + + +####################################### +# the script + +def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): + KINDS = ['all', *sorted(FROZEN_GROUPS)] + usage = f'usage: {prog} [-h] [{"|".join(KINDS)}] ...' + if '-h' in argv or '--help' in argv: + print(usage) + sys.exit(0) + else: + kinds = set() + for arg in argv: + if arg not in KINDS: + print(usage, file=sys.stderr) + sys.exit(f'ERROR: unsupported kind {arg!r}') + kinds.add(arg) + return {'kinds': kinds if kinds and 'all' not in kinds else None} + + +def main(kinds=None): + groups, frozen = expand_frozen(MODULES_DIR) + + # First, freeze the modules. + # (We use a consistent order:) + ordered = ['importlib', 'stdlib', 'test'] + assert (set(ordered) == set(groups)) + for kind in ordered: + if not kinds or kind in kinds: + for frozenid in groups[kind]: + #freeze_module(frozenid, frozen) + pyfile, frozenfile, _ = frozen[frozenid] + _freeze_module(frozenid, pyfile, frozenfile) + + +if __name__ == '__main__': + kwargs = parse_args() + main(**kwargs) diff --git a/Tools/scripts/freeze_stdlib.py b/Tools/scripts/freeze_stdlib.py deleted file mode 100644 index 38bddc3a401ab5..00000000000000 --- a/Tools/scripts/freeze_stdlib.py +++ /dev/null @@ -1,75 +0,0 @@ -import os -import os.path -import subprocess - - -SCRIPTS_DIR = os.path.dirname(__file__) -TOOLS_DIR = os.path.dirname(SCRIPTS_DIR) -ROOT_DIR = os.path.dirname(TOOLS_DIR) -STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') -MODULES_DIR = os.path.join(ROOT_DIR, 'Python') -#MODULES_DIR = os.path.join(ROOT_DIR, 'Modules') -TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') - -STARTUP_MODULES_WITHOUT_SITE = [ -] -STARTUP_MODULES_WITH_SITE = [ -] - - -def freeze_module(modname, destdir): - if modname.startswith('<'): - pkgname = modname[1:-1] - pkgname, sep, match= pkgname.partition('.') - pyfile, _ = _freeze_module(pkgname, MODULES_DIR, ispkg=True) - - if sep: - pkgdir = os.path.dirname(pyfile) - for modname, ispkg in _iter_submodules(pkgname, pkgdir, match): - _freeze_module(modname, MODULES_DIR, ispkg) - else: - _freeze_module(modname, MODULES_DIR) - - -def _freeze_module(modname, destdir, ispkg=False): - if ispkg: - pyfile = os.path.join(STDLIB_DIR, *modname.split('.'), '__init__.py') - else: - pyfile = os.path.join(STDLIB_DIR, *modname.split('.')) + '.py' - destfile = os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' - tmpfile = destfile + '.new' - - argv = [TOOL, modname, pyfile, tmpfile] - print('#', ' '.join(argv)) - subprocess.run(argv, check=True) - - os.replace(tmpfile, destfile) - - return pyfile, destfile - - -def _iter_submodules(pkgname, pkgdir, match='*'): - if not match: - match = '*' - if match != '*': - raise NotImplementedError(match) - - for entry in os.scandir(pkgdir): - modname = f'{pkgname}.{entry.name}' - if modname.endswith('.py'): - yield modname[:-3], False - elif entry.is_dir(): - if os.path.exists(os.path.join(entry.path, '__init__.py')): - yield modname, True - yield from _iter_submodules(modname, entry.path) - - -def main(): - for modname in STARTUP_MODULES_WITHOUT_SITE: - freeze_module(modname, MODULES_DIR) - for modname in STARTUP_MODULES_WITH_SITE: - freeze_module(modname, MODULES_DIR) - - -if __name__ == '__main__': - main() From 1cae77177cb8887cfd301cdaee3d4a98fae8dad5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 10:45:21 -0600 Subject: [PATCH 06/41] Regen frozen.c and Makefile.pre.in. --- Makefile.pre.in | 10 +- Python/frozen.c | 18 +-- Tools/scripts/freeze_modules.py | 193 +++++++++++++++++++++++++++++++- 3 files changed, 210 insertions(+), 11 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index d15ea8df8f8744..19fcddd8335f96 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -986,8 +986,14 @@ regen-opcode-targets: Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \ $(srcdir)/Python/condvar.h -Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h \ - $(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/frozen_hello.h +# FROZEN_FILES is auto-generated by Tools/scripts/freeze_modules.py. +FROZEN_FILES = \ + $(srcdir)/Python/importlib.h \ + $(srcdir)/Python/importlib_external.h \ + $(srcdir)/Python/importlib_zipimport.h \ + $(srcdir)/Python/frozen_hello.h + +Python/frozen.o: $(FROZEN_FILES) # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to # follow our naming conventions. dtrace(1) uses the output filename to generate diff --git a/Python/frozen.c b/Python/frozen.c index 0540a482569ea4..bd512eaea2c89c 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -1,10 +1,13 @@ +/* Auto-generated by Tools/scripts/freeze_modules.py */"; /* Frozen modules initializer */ #include "Python.h" +/* importlib */ #include "importlib.h" #include "importlib_external.h" #include "importlib_zipimport.h" +/* stdlib */ /* In order to test the support for frozen modules, by default we define a single frozen module, __hello__. Loading it will print @@ -15,22 +18,23 @@ */ #include "frozen_hello.h" -#define SIZE (int)sizeof(_Py_M__hello) - static const struct _frozen _PyImport_FrozenModules[] = { /* importlib */ {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)}, - {"zipimport", _Py_M__zipimport, - (int)sizeof(_Py_M__zipimport)}, + {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, + + /* stdlib */ + /* without site (python -S) */ + /* with site */ /* Test module */ - {"__hello__", _Py_M__hello, SIZE}, + {"__hello__", _Py_M__hello, (int)sizeof(_Py_M__hello)}, /* Test package (negative size indicates package-ness) */ - {"__phello__", _Py_M__hello, -SIZE}, - {"__phello__.spam", _Py_M__hello, SIZE}, + {"__phello__", _Py_M__hello, -(int)sizeof(_Py_M__hello)}, + {"__phello__.spam", _Py_M__hello, (int)sizeof(_Py_M__hello)}, {0, 0, 0} /* sentinel */ }; diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index c9934c8e61998b..3908c2164f60ec 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -2,16 +2,21 @@ import os.path import subprocess import sys +import textwrap SCRIPTS_DIR = os.path.abspath(os.path.dirname(__file__)) TOOLS_DIR = os.path.dirname(SCRIPTS_DIR) ROOT_DIR = os.path.dirname(TOOLS_DIR) + STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') MODULES_DIR = os.path.join(ROOT_DIR, 'Python') #MODULES_DIR = os.path.join(ROOT_DIR, 'Modules') TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') +FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') +MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') + # These are modules that get frozen. FROZEN = { # frozen_id: (pyfile, frozenfile) @@ -42,6 +47,32 @@ 'hello', ], } +# These are the modules defined in frozen.c, in order. +MODULES = [ + # frozen_id + # (module, frozen_id) + # (, frozen_id) + + # importlib + '# [importlib]', + ('_frozen_importlib', 'importlib._bootstrap'), + ('_frozen_importlib_external', 'importlib._bootstrap_external'), + 'zipimport', + + # stdlib + '# [stdlib]', + '# without site (python -S)', + # ... + '# with site', + # ... + + # test + '# [Test module]', + ('__hello__', 'hello'), + '# Test package (negative size indicates package-ness)', + ('<__phello__>', 'hello'), + ('__phello__.spam', 'hello'), +] def expand_frozen(destdir=MODULES_DIR): @@ -99,7 +130,44 @@ def expand_frozen(destdir=MODULES_DIR): else: group.append(frozenid) - return groups, frozen + # Finally, expand MODULES. + rows = [] + for row in MODULES: + if isinstance(row, str) and not row.startswith('#'): + if row in packages: + for modname in packages[row]: + rows.append((modname, modname)) + continue + row = (row, row) + rows.append(row) + headers = [] + specs = [] + for row in rows: + if isinstance(row, str): + assert row.startswith('# ') + comment = row[2:] + if comment.startswith('['): + line = f'/* {comment[1:-1]} */' + headers.append(line) + specs.extend(['', line]) + else: + specs.append(f'/* {comment} */') + continue + modname, frozenid = row + _, frozenfile, ispkg = frozen[frozenid] + + assert frozenfile.startswith(destdir), (frozenfile, destdir) + header = os.path.relpath(frozenfile, destdir) + if header not in headers: + headers.append(header) + + if modname.startswith('<'): + modname = modname[1:-1] + ispkg = True + spec = (modname, frozenid, ispkg) + specs.append(spec) + + return groups, frozen, headers, specs #def freeze_stdlib_module(modname, destdir=MODULES_DIR): @@ -122,6 +190,123 @@ def expand_frozen(destdir=MODULES_DIR): # _freeze_module(modname, pyfile, frozenfile) +def regen_frozen(headers, specs): + headerlines = [] + for row in headers: + assert isinstance(row, str), row + if row == '/* Test module */': + comment = textwrap.dedent(''' + /* In order to test the support for frozen modules, by default we + define a single frozen module, __hello__. Loading it will print + some famous words... */ + + /* Run "make regen-frozen" to regen the file below (e.g. after a bytecode + * format change). The include file defines _Py_M__hello as an array of bytes. + */ + '''.rstrip()) + headerlines.extend(comment.splitlines()) + continue + elif not row.startswith('/*'): + assert row.endswith('.h') + assert os.path.basename(row) == row, row + row = f'#include "{row}"' + headerlines.append(row) + + speclines = [] + indent = ' ' + for spec in specs: + if isinstance(spec, str): + speclines.append(spec) + continue + modname, frozenid, ispkg = spec + # This matches what we do in Programs/_freeze_module.c: + symbol = '_Py_M__' + frozenid.replace('.', '_') + pkg = '-' if ispkg else '' + line = '{"%s", %s, %s(int)sizeof(%s)},' % (modname, symbol, pkg, symbol) + if len(line) < 80: + speclines.append(line) + else: + line1, _, line2 = line.rpartition(' ') + speclines.append(line1) + speclines.append(indent + line2) + if not speclines[0]: + del speclines[0] + for i, line in enumerate(speclines): + if line: + speclines[i] = indent + line + + text = textwrap.dedent('''\ + /* Auto-generated by Tools/scripts/freeze_modules.py */"; + + /* Frozen modules initializer */ + + #include "Python.h" + %s + + static const struct _frozen _PyImport_FrozenModules[] = { + %s + + {0, 0, 0} /* sentinel */ + }; + + /* Embedding apps may change this pointer to point to their favorite + collection of frozen modules: */ + + const struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules; + ''' + ) % ( + os.linesep.join(headerlines), + os.linesep.join(speclines), + ) + with open(FROZEN_FILE, 'w') as outfile: + outfile.write(text) + + +def regen_makefile(headers, destdir=MODULES_DIR): + reldir = os.path.relpath(destdir, ROOT_DIR) + assert destdir.endswith(reldir), destdir + frozenfiles = [] + for row in headers: + if not row.startswith('/*'): + header = row + assert header.endswith('.h'), header + assert os.path.basename(header) == header, header + relfile = f'{reldir}/{header}'.replace('\\', '/') + frozenfiles.append(f'$(srcdir)/{relfile}') + + tmpfile = MAKEFILE + '.tmp' + try: + with open(tmpfile, 'w') as outfile: + with open(MAKEFILE) as infile: + lines = iter(infile) + # Get to $FROZEN_FILES. + for line in lines: + outfile.write(line) + if line.rstrip() == 'FROZEN_FILES = \\': + break + else: + raise Exception("did you delete a line you shouldn't have?") + # Pop off the existing values. + for line in lines: + if line.rstrip() == 'Python/frozen.o: $(FROZEN_FILES)': + break + else: + raise Exception("did you delete a line you shouldn't have?") + # Add the regen'ed values. + for header in frozenfiles[:-1]: + outfile.write(f'\t\t{header} \\{os.linesep}') + outfile.write(f'\t\t{frozenfiles[-1]}{os.linesep}') + # Keep the rest of the file. + outfile.write(os.linesep) + outfile.write('Python/frozen.o: $(FROZEN_FILES)' + os.linesep) + for line in lines: + outfile.write(line) + os.rename(tmpfile, MAKEFILE) + finally: + if os.path.exists(tmpfile): + os.remove(tmpfile) + + def _freeze_module(frozenid, pyfile, frozenfile): tmpfile = frozenfile + '.new' @@ -180,7 +365,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): def main(kinds=None): - groups, frozen = expand_frozen(MODULES_DIR) + groups, frozen, headers, specs = expand_frozen(MODULES_DIR) # First, freeze the modules. # (We use a consistent order:) @@ -193,6 +378,10 @@ def main(kinds=None): pyfile, frozenfile, _ = frozen[frozenid] _freeze_module(frozenid, pyfile, frozenfile) + # Then regen frozen.c and Makefile.pre.in. + regen_frozen(headers, specs) + regen_makefile(headers, MODULES_DIR) + if __name__ == '__main__': kwargs = parse_args() From ba9417cf169e15facca67578f01c0e0a618b47ba Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 12:54:47 -0600 Subject: [PATCH 07/41] Add the updating_file_with_tmpfile() "tool". --- Tools/scripts/update_file.py | 40 ++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/Tools/scripts/update_file.py b/Tools/scripts/update_file.py index 224585c69bbaeb..ec4cfa354086c2 100644 --- a/Tools/scripts/update_file.py +++ b/Tools/scripts/update_file.py @@ -6,23 +6,51 @@ actually change the in-tree generated code. """ +import contextlib import os +import os.path import sys -def main(old_path, new_path): - with open(old_path, 'rb') as f: +@contextlib.contextmanager +def updating_file_with_tmpfile(filename, tmpfile=None): + """A context manager for updating a file via a temp file. + + The context manager provides two open files: the source file open + for reading, and the temp file, open for writing. + + Upon exiting: both files are closed, and the source file is replaced + with the temp file. + """ + # XXX Optionally use tempfile.TemporaryFile? + if not tmpfile: + tmpfile = filename + '.tmp' + elif os.path.isdir(tmpfile): + tmpfile = os.path.join(tmpfile, filename + '.tmp') + + try: + with open(tmpfile, 'w') as outfile: + with open(filename) as infile: + yield infile, outfile + os.replace(tmpfile, filename) + finally: + if os.path.exists(tmpfile): + os.remove(tmpfile) + + +def update_file_with_tmpfile(filename, tmpfile): + with open(filename, 'rb') as f: old_contents = f.read() - with open(new_path, 'rb') as f: + with open(tmpfile, 'rb') as f: new_contents = f.read() if old_contents != new_contents: - os.replace(new_path, old_path) + os.replace(tmpfile, filename) else: - os.unlink(new_path) + os.unlink(tmpfile) if __name__ == '__main__': if len(sys.argv) != 3: print("Usage: %s " % (sys.argv[0],)) sys.exit(1) - main(sys.argv[1], sys.argv[2]) + update_file_with_tmpfile(sys.argv[1], sys.argv[2]) From 6dcb926ea24617e413fc8aa44bf9a16b244413fe Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 12:59:48 -0600 Subject: [PATCH 08/41] Regen frozen.c in place. --- Programs/_freeze_module.c | 7 +- Python/frozen.c | 49 ++++++++--- Tools/scripts/freeze_modules.py | 145 +++++++++++++++++--------------- 3 files changed, 121 insertions(+), 80 deletions(-) diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index dc941fc49e0500..87004da0dc6bf3 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -1,5 +1,10 @@ /* This is built as a stand-alone executable by the Makefile, and helps turn - Lib/importlib/_bootstrap.py into a frozen module in Python/importlib.h + modules into frozen modules (like Lib/importlib/_bootstrap.py + into Python/importlib.h) + + This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen-*". + + See Python/frozen.c for more info. */ #include diff --git a/Python/frozen.c b/Python/frozen.c index bd512eaea2c89c..343fc114308db0 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -1,21 +1,50 @@ -/* Auto-generated by Tools/scripts/freeze_modules.py */"; -/* Frozen modules initializer */ +/* Frozen modules initializer + * + * Frozen modules are written to header files by Programs/_freeze_module. + * These files are typically put in Python/. Each holds an array of bytes + * named "_Py_M__", which is used below. + * + * These files must be regenerated any time * the corresponding .pyc + * file would change (e.g. compiler, bytecode format, marshal format). + * This can be done with "make regen-frozen-all". + * + * Additionally, specific groups of frozen modules can be regenerated: + * + * import-related make regen-importlib + * stdlib (partial) make regen-frozen-stdlib + * test modules make regen-frozen + * + * Those make targets simply run Tools/scripts/freeze_modules.py, which + * does the following: + * + * 1. run Programs/_freeze_module on the target modules + * 2. update the includes and _PyImport_FrozenModules[] in this file + * 3. update the FROZEN_FILES variable in Makefile.pre.in + * + * (Note that most of the date in this file is auto-generated by the script.) + * + * Additional modules can be frozen by adding them to freeze_modules.py + * and then re-running "make regen-frozen-all". This can also be done + * manually by following those steps, though this is not recommended. + * Expect such manual changes to be removed the next time + * freeze_modules.py runs. + * */ + +/* In order to test the support for frozen modules, by default we + define some simple frozen modules: __hello__, __phello__ (a package), + and __phello__.spam. Loading any will print some famous words... */ #include "Python.h" + +/* Includes for frozen modules: */ + /* importlib */ #include "importlib.h" #include "importlib_external.h" #include "importlib_zipimport.h" /* stdlib */ - -/* In order to test the support for frozen modules, by default we - define a single frozen module, __hello__. Loading it will print - some famous words... */ - -/* Run "make regen-frozen" to regen the file below (e.g. after a bytecode - * format change). The include file defines _Py_M__hello as an array of bytes. - */ +/* Test module */ #include "frozen_hello.h" static const struct _frozen _PyImport_FrozenModules[] = { diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 3908c2164f60ec..36dd7bbf31c065 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -1,9 +1,16 @@ +"""Freeze modules and regen related files (e.g. Python/frozen.c). + +See the notes at the top of Python/frozen.c for more info. +""" + import os import os.path import subprocess import sys import textwrap +from update_file import updating_file_with_tmpfile + SCRIPTS_DIR = os.path.abspath(os.path.dirname(__file__)) TOOLS_DIR = os.path.dirname(SCRIPTS_DIR) @@ -194,25 +201,14 @@ def regen_frozen(headers, specs): headerlines = [] for row in headers: assert isinstance(row, str), row - if row == '/* Test module */': - comment = textwrap.dedent(''' - /* In order to test the support for frozen modules, by default we - define a single frozen module, __hello__. Loading it will print - some famous words... */ - - /* Run "make regen-frozen" to regen the file below (e.g. after a bytecode - * format change). The include file defines _Py_M__hello as an array of bytes. - */ - '''.rstrip()) - headerlines.extend(comment.splitlines()) - continue - elif not row.startswith('/*'): + if not row.startswith('/*'): assert row.endswith('.h') assert os.path.basename(row) == row, row row = f'#include "{row}"' headerlines.append(row) speclines = [] + sentinel = '{0, 0, 0} /* sentinel */' indent = ' ' for spec in specs: if isinstance(spec, str): @@ -231,35 +227,53 @@ def regen_frozen(headers, specs): speclines.append(indent + line2) if not speclines[0]: del speclines[0] + speclines.extend(['', sentinel]) for i, line in enumerate(speclines): if line: speclines[i] = indent + line - text = textwrap.dedent('''\ - /* Auto-generated by Tools/scripts/freeze_modules.py */"; - - /* Frozen modules initializer */ - - #include "Python.h" - %s - - static const struct _frozen _PyImport_FrozenModules[] = { - %s - - {0, 0, 0} /* sentinel */ - }; - - /* Embedding apps may change this pointer to point to their favorite - collection of frozen modules: */ - - const struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules; - ''' - ) % ( - os.linesep.join(headerlines), - os.linesep.join(speclines), - ) - with open(FROZEN_FILE, 'w') as outfile: - outfile.write(text) + with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): + lines = iter(infile) + # Get to the frozen includes. + for line in lines: + outfile.write(line) + if line.rstrip() == '/* Includes for frozen modules: */': + break + else: + raise Exception("did you delete a line you shouldn't have?") + # Pop off the existing values. + blank_before = next(lines) + assert blank_before.strip() == '' + for line in lines: + if not line.strip(): + blank_after = line + break + else: + raise Exception("did you delete a line you shouldn't have?") + # Add the frozen includes. + outfile.write(blank_before) + for line in headerlines: + outfile.write(line + os.linesep) + outfile.write(blank_after) + # Get to the modules array. + for line in lines: + outfile.write(line) + if line.rstrip() == 'static const struct _frozen _PyImport_FrozenModules[] = {': + break + else: + raise Exception("did you delete a line you shouldn't have?") + # Pop off the existing values. + for line in lines: + if line.strip() == sentinel: + break + else: + raise Exception("did you delete a line you shouldn't have?") + # Add the modules. + for line in speclines: + outfile.write(line + os.linesep) + # Keep the rest of the file. + for line in lines: + outfile.write(line) def regen_makefile(headers, destdir=MODULES_DIR): @@ -274,37 +288,30 @@ def regen_makefile(headers, destdir=MODULES_DIR): relfile = f'{reldir}/{header}'.replace('\\', '/') frozenfiles.append(f'$(srcdir)/{relfile}') - tmpfile = MAKEFILE + '.tmp' - try: - with open(tmpfile, 'w') as outfile: - with open(MAKEFILE) as infile: - lines = iter(infile) - # Get to $FROZEN_FILES. - for line in lines: - outfile.write(line) - if line.rstrip() == 'FROZEN_FILES = \\': - break - else: - raise Exception("did you delete a line you shouldn't have?") - # Pop off the existing values. - for line in lines: - if line.rstrip() == 'Python/frozen.o: $(FROZEN_FILES)': - break - else: - raise Exception("did you delete a line you shouldn't have?") - # Add the regen'ed values. - for header in frozenfiles[:-1]: - outfile.write(f'\t\t{header} \\{os.linesep}') - outfile.write(f'\t\t{frozenfiles[-1]}{os.linesep}') - # Keep the rest of the file. - outfile.write(os.linesep) - outfile.write('Python/frozen.o: $(FROZEN_FILES)' + os.linesep) - for line in lines: - outfile.write(line) - os.rename(tmpfile, MAKEFILE) - finally: - if os.path.exists(tmpfile): - os.remove(tmpfile) + with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): + lines = iter(infile) + # Get to $FROZEN_FILES. + for line in lines: + outfile.write(line) + if line.rstrip() == 'FROZEN_FILES = \\': + break + else: + raise Exception("did you delete a line you shouldn't have?") + # Pop off the existing values. + for line in lines: + if line.rstrip() == 'Python/frozen.o: $(FROZEN_FILES)': + break + else: + raise Exception("did you delete a line you shouldn't have?") + # Add the regen'ed values. + for header in frozenfiles[:-1]: + outfile.write(f'\t\t{header} \\{os.linesep}') + outfile.write(f'\t\t{frozenfiles[-1]}{os.linesep}') + # Keep the rest of the file. + outfile.write(os.linesep) + outfile.write('Python/frozen.o: $(FROZEN_FILES)' + os.linesep) + for line in lines: + outfile.write(line) def _freeze_module(frozenid, pyfile, frozenfile): From 8a2b4b73beb93f30a1d74033b1e89d420a8f68a5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 16:03:34 -0600 Subject: [PATCH 09/41] Clean up. --- Tools/scripts/freeze_modules.py | 231 ++++++++++++++++++++------------ 1 file changed, 148 insertions(+), 83 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 36dd7bbf31c065..d25cf6a464f5a2 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -17,8 +17,8 @@ ROOT_DIR = os.path.dirname(TOOLS_DIR) STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') +# XXX It may make more sense to put frozen modules in Modules/ or Lib/... MODULES_DIR = os.path.join(ROOT_DIR, 'Python') -#MODULES_DIR = os.path.join(ROOT_DIR, 'Modules') TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') @@ -87,39 +87,34 @@ def expand_frozen(destdir=MODULES_DIR): frozen = {} packages = {} for frozenid, info in FROZEN.items(): - if frozenid.startswith('<'): - assert info is None, (frozenid, info) - modids = packages[frozenid] = [] - pkgname = frozenid[1:-1] - _pkgname, sep, match = pkgname.rpartition('.') - if not sep or match.isidentifier(): - frozen[pkgname] = (None, None, True) - modids.append(pkgname) - else: - pkgname = _pkgname - frozen[pkgname] = (None, None, True) - modids.append(okgname) - for subname, ispkg in _iter_submodules(pkgname, match=match): - frozen[subname] = (None, None, ispkg) - modids.append(subname) + if info is None or isinstance(info, str): + pyfile = info + frozenfile = None else: - if info is None or isinstance(info, str): - info = (info, None, False) - else: - pyfile, frozenfile = info - info = (pyfile, frozenfile, False) - frozen[frozenid] = info + pyfile, frozenfile = info + + resolved = iter(resolve_modules(frozenid, pyfile)) + modname, _pyfile, ispkg = next(resolved) + if not pyfile: + pyfile = _pyfile + frozen[modname] = (pyfile, frozenfile, ispkg) + if ispkg: + assert info is None + modids = packages[frozenid] = [modname] + for subname, subpyfile, ispkg in resolved: + frozen[subname] = (subpyfile, None, ispkg) + modids.append(subname) + else: + assert not list(resolved) for frozenid, info in frozen.items(): pyfile, frozenfile, ispkg = info if not pyfile: - pyfile = _resolve_module(frozenid, ispkg) + pyfile = _resolve_module(frozenid, ispkg=ispkg) info = (pyfile, frozenfile, ispkg) if not frozenfile: frozenfile = _resolve_frozen(frozenid, destdir) -# frozenfile = f'frozen_{frozenid}' -# frozenfile = os.path.join(destdir, frozenfile) info = (pyfile, frozenfile, ispkg) elif not os.path.isabs(frozenfile): frozenfile = os.path.join(destdir, frozenfile) @@ -135,6 +130,7 @@ def expand_frozen(destdir=MODULES_DIR): if frozenid in packages: group.extend(packages[frozenid]) else: + assert frozenid in frozen, (frozenid, frozen) group.append(frozenid) # Finally, expand MODULES. @@ -144,8 +140,8 @@ def expand_frozen(destdir=MODULES_DIR): if row in packages: for modname in packages[row]: rows.append((modname, modname)) - continue - row = (row, row) + else: + row = (row, row) rows.append(row) headers = [] specs = [] @@ -170,6 +166,7 @@ def expand_frozen(destdir=MODULES_DIR): if modname.startswith('<'): modname = modname[1:-1] + assert check_modname(modname), modname ispkg = True spec = (modname, frozenid, ispkg) specs.append(spec) @@ -177,26 +174,132 @@ def expand_frozen(destdir=MODULES_DIR): return groups, frozen, headers, specs -#def freeze_stdlib_module(modname, destdir=MODULES_DIR): -# if modname.startswith('<'): -# pkgname = modname[1:-1] -# pkgname, sep, match = pkgname.rpartition('.') -# if not sep or match.isidentifier(): -# pyfile = _resolve_module(pkgname, ispkg=True) -# frozenfile = _resolve_frozen(pkgname, destdir) -# _freeze_module(pkgname, pyfile, frozenfile) -# else: -# pkgdir = os.path.dirname(pyfile) -# for subname, ispkg in _iter_submodules(pkgname, pkgdir, match): -# pyfile = _resolve_module(subname, ispkg) -# frozenfile = _resolve_frozen(subname, destdir) -# _freeze_module(subname, pyfile, frozenfile) -# else: -# pyfile = _resolve_module(modname, ispkg=False) -# frozenfile = _resolve_frozen(modname, destdir) -# _freeze_module(modname, pyfile, frozenfile) +def resolve_modules(modname, pyfile=None): + if modname.startswith('<') and modname.endswith('>'): + if pyfile: + assert os.path.isdir(pyfile) or os.path.basename(pyfile) == '__init__.py', pyfile + ispkg = True + modname = modname[1:-1] + rawname = modname + # For now, we only expect match patterns at the end of the name. + _modname, sep, match = pkgname.rpartition('.') + if sep: + if _modname.endswith('.**'): + modname = _modname[:-3] + match = f'**.{match}' + elif match and not match.isidentifier(): + modname = _modname + # Otherwise it's a plain name so we leave it alone. + else: + match = None + else: + ispkg = False + rawname = modname + match = None + + if not check_modname(modname): + raise ValueError(f'not a valid module name ({rawname})') + + if not pyfile: + pyfile = _resolve_module(modname, ispkg=ispkg) + elif os.path.isdir(pyfile): + pyfile = _resolve_module(modname, pyfile, ispkg) + yield modname, pyfile, ispkg + + if match: + pkgdir = os.path.dirname(pyfile) + yield from iter_submodules(modname, pkgdir, match) + + +def check_modname(modname): + return all(n.isidentifier() for n in modname.split('.')) + + +def iter_submodules(pkgname, pkgdir=None, match='*'): + if not pkgdir: + pkgdir = os.path.join(STDLIB_DIR, *pkgname.split('.')) + if not match: + match = '**.*' + match_modname = _resolve_modname_matcher(match, pkgdir) + + def _iter_submodules(pkgname, pkgdir): + for entry in os.scandir(pkgdir): + matched, recursive = match_modname(entry.name) + if not matched: + continue + modname = f'{pkgname}.{entry.name}' + if modname.endswith('.py'): + yield modname[:-3], entry.path, False + elif entry.is_dir(): + pyfile = os.path.join(entry.path, '__init__.py') + # We ignore namespace packages. + if os.path.exists(pyfile): + yield modname, pyfile, True + if recursive: + yield from _iter_submodules(modname, entry.path) + + return _iter_submodules(pkgname, pkgdir) + + +def _resolve_modname_matcher(match, rootdir=None): + if isinstance(match, str): + if match.startswith('**.'): + recursive = True + pat = match[3:] + assert match + else: + recursive = False + pat = match + + if pat == '*': + def match_modname(modname): + return True, recursive + else: + raise NotImplementedError(match) + elif callable(match): + match_modname = match(rootdir) + else: + raise ValueError(f'unsupported matcher {match!r}') + return match_modname + + +def _resolve_module(modname, pathentry=STDLIB_DIR, ispkg=False): + assert pathentry, pathentry + pathentry = os.path.normpath(pathentry) + assert os.path.isabs(pathentry) + if ispkg: + return os.path.join(pathentry, *modname.split('.'), '__init__.py') + return os.path.join(pathentry, *modname.split('.')) + '.py' + + +def _resolve_frozen(modname, destdir): + # We use a consistent naming convention for frozen modules. + return os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' + + +####################################### +# freezing modules + +def freeze_module(modname, pyfile=None, destdir=MODULES_DIR): + """Generate the frozen module .h file for the given module.""" + for modname, pyfile, ispkg in resolve_modules(modname, pyfile): + frozenfile = _resolve_frozen(modname, destdir) + _freeze_module(modname, pyfile, frozenfile) + + +def _freeze_module(frozenid, pyfile, frozenfile): + tmpfile = frozenfile + '.new' + + argv = [TOOL, frozenid, pyfile, tmpfile] + print('#', ' '.join(argv)) + subprocess.run(argv, check=True) + + os.replace(tmpfile, frozenfile) +####################################### +# regenerating dependent files + def regen_frozen(headers, specs): headerlines = [] for row in headers: @@ -314,44 +417,6 @@ def regen_makefile(headers, destdir=MODULES_DIR): outfile.write(line) -def _freeze_module(frozenid, pyfile, frozenfile): - tmpfile = frozenfile + '.new' - - argv = [TOOL, frozenid, pyfile, tmpfile] - print('#', ' '.join(argv)) - subprocess.run(argv, check=True) - - os.replace(tmpfile, frozenfile) - - -def _resolve_module(modname, ispkg=False): - if ispkg: - return os.path.join(STDLIB_DIR, *modname.split('.'), '__init__.py') - return os.path.join(STDLIB_DIR, *modname.split('.')) + '.py' - - -def _resolve_frozen(modname, destdir): - return os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' - - -def _iter_submodules(pkgname, pkgdir=None, match='*'): - if not pkgdir: - pkgdir = os.path.join(STDLIB_DIR, *pkgname.split('.')) - if not match: - match = '*' - if match != '*': - raise NotImplementedError(match) - - for entry in os.scandir(pkgdir): - modname = f'{pkgname}.{entry.name}' - if modname.endswith('.py'): - yield modname[:-3], False - elif entry.is_dir(): - if os.path.exists(os.path.join(entry.path, '__init__.py')): - yield modname, True - yield from _iter_submodules(modname, entry.path) - - ####################################### # the script From 8ad3c1d9ae7d93304c1638ef45247ba3a0a3d8be Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 16:03:47 -0600 Subject: [PATCH 10/41] Add --no-regen. --- Tools/scripts/freeze_modules.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index d25cf6a464f5a2..b4f88f1bde7e2f 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -422,21 +422,21 @@ def regen_makefile(headers, destdir=MODULES_DIR): def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): KINDS = ['all', *sorted(FROZEN_GROUPS)] - usage = f'usage: {prog} [-h] [{"|".join(KINDS)}] ...' - if '-h' in argv or '--help' in argv: - print(usage) - sys.exit(0) - else: - kinds = set() - for arg in argv: - if arg not in KINDS: - print(usage, file=sys.stderr) - sys.exit(f'ERROR: unsupported kind {arg!r}') - kinds.add(arg) - return {'kinds': kinds if kinds and 'all' not in kinds else None} + import argparse + parser = argparse.ArgumentParser(prog=prog) + parser.add_argument('--no-regen', dest='regen', action='store_false') + parser.add_argument('kinds', nargs='*', choices=KINDS) + + args = parser.parse_args(argv) + ns = vars(args) + + if not args.kinds or 'all' in args.kinds: + args.kinds = None + + return ns -def main(kinds=None): +def main(kinds=None, *, regen=True): groups, frozen, headers, specs = expand_frozen(MODULES_DIR) # First, freeze the modules. @@ -450,9 +450,10 @@ def main(kinds=None): pyfile, frozenfile, _ = frozen[frozenid] _freeze_module(frozenid, pyfile, frozenfile) - # Then regen frozen.c and Makefile.pre.in. - regen_frozen(headers, specs) - regen_makefile(headers, MODULES_DIR) + if regen: + # Then regen frozen.c and Makefile.pre.in. + regen_frozen(headers, specs) + regen_makefile(headers, MODULES_DIR) if __name__ == '__main__': From 1e88d34ac3844163f9d257b48fa86f6c31ca7995 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 18:31:36 -0600 Subject: [PATCH 11/41] Be consistent about the frozen module filename (frozen_*.h). --- Makefile.pre.in | 6 +-- Python/frozen.c | 6 +-- ...ortlib.h => frozen_importlib__bootstrap.h} | 0 ...=> frozen_importlib__bootstrap_external.h} | 0 ...portlib_zipimport.h => frozen_zipimport.h} | 0 Tools/scripts/freeze_modules.py | 41 ++++++------------- 6 files changed, 19 insertions(+), 34 deletions(-) rename Python/{importlib.h => frozen_importlib__bootstrap.h} (100%) rename Python/{importlib_external.h => frozen_importlib__bootstrap_external.h} (100%) rename Python/{importlib_zipimport.h => frozen_zipimport.h} (100%) diff --git a/Makefile.pre.in b/Makefile.pre.in index 19fcddd8335f96..221befbfc407e8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -988,9 +988,9 @@ Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \ # FROZEN_FILES is auto-generated by Tools/scripts/freeze_modules.py. FROZEN_FILES = \ - $(srcdir)/Python/importlib.h \ - $(srcdir)/Python/importlib_external.h \ - $(srcdir)/Python/importlib_zipimport.h \ + $(srcdir)/Python/frozen_importlib__bootstrap.h \ + $(srcdir)/Python/frozen_importlib__bootstrap_external.h \ + $(srcdir)/Python/frozen_zipimport.h \ $(srcdir)/Python/frozen_hello.h Python/frozen.o: $(FROZEN_FILES) diff --git a/Python/frozen.c b/Python/frozen.c index 343fc114308db0..1da980c37e009c 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -40,9 +40,9 @@ /* Includes for frozen modules: */ /* importlib */ -#include "importlib.h" -#include "importlib_external.h" -#include "importlib_zipimport.h" +#include "frozen_importlib__bootstrap.h" +#include "frozen_importlib__bootstrap_external.h" +#include "frozen_zipimport.h" /* stdlib */ /* Test module */ #include "frozen_hello.h" diff --git a/Python/importlib.h b/Python/frozen_importlib__bootstrap.h similarity index 100% rename from Python/importlib.h rename to Python/frozen_importlib__bootstrap.h diff --git a/Python/importlib_external.h b/Python/frozen_importlib__bootstrap_external.h similarity index 100% rename from Python/importlib_external.h rename to Python/frozen_importlib__bootstrap_external.h diff --git a/Python/importlib_zipimport.h b/Python/frozen_zipimport.h similarity index 100% rename from Python/importlib_zipimport.h rename to Python/frozen_zipimport.h diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index b4f88f1bde7e2f..8c2956fcb650d1 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -30,9 +30,9 @@ # frozen_id: pyfile # importlib - 'importlib._bootstrap': (None, 'importlib.h'), - 'importlib._bootstrap_external': (None, 'importlib_external.h'), - 'zipimport': (None, 'importlib_zipimport.h'), + 'importlib._bootstrap': None, + 'importlib._bootstrap_external': None, + 'zipimport': None, # stdlib # ... # test @@ -84,43 +84,28 @@ def expand_frozen(destdir=MODULES_DIR): # First, expand FROZEN. - frozen = {} + _frozen = [] packages = {} - for frozenid, info in FROZEN.items(): - if info is None or isinstance(info, str): - pyfile = info - frozenfile = None - else: - pyfile, frozenfile = info - + for frozenid, pyfile in FROZEN.items(): resolved = iter(resolve_modules(frozenid, pyfile)) modname, _pyfile, ispkg = next(resolved) if not pyfile: pyfile = _pyfile - frozen[modname] = (pyfile, frozenfile, ispkg) + _frozen.append((modname, pyfile, ispkg)) if ispkg: - assert info is None modids = packages[frozenid] = [modname] for subname, subpyfile, ispkg in resolved: - frozen[subname] = (subpyfile, None, ispkg) + frozen[subname] = (subpyfile, ispkg) + _frozen.append((subname, subpyfile, ispkg)) modids.append(subname) else: assert not list(resolved) - for frozenid, info in frozen.items(): - pyfile, frozenfile, ispkg = info - + frozen = {} + for frozenid, pyfile, ispkg in _frozen: if not pyfile: pyfile = _resolve_module(frozenid, ispkg=ispkg) - info = (pyfile, frozenfile, ispkg) - - if not frozenfile: - frozenfile = _resolve_frozen(frozenid, destdir) - info = (pyfile, frozenfile, ispkg) - elif not os.path.isabs(frozenfile): - frozenfile = os.path.join(destdir, frozenfile) - info = (pyfile, frozenfile, ispkg) - - frozen[frozenid] = info + frozenfile = _resolve_frozen(frozenid, destdir) + frozen[frozenid] = (pyfile, frozenfile, ispkg) # Then, expand FROZEN_GROUPS. groups = {} @@ -425,7 +410,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): import argparse parser = argparse.ArgumentParser(prog=prog) parser.add_argument('--no-regen', dest='regen', action='store_false') - parser.add_argument('kinds', nargs='*', choices=KINDS) + parser.add_argument('kinds', nargs='*', choices=KINDS, default='all') args = parser.parse_args(argv) ns = vars(args) From c4874f2bab39f2ed577e0373ff8daf324b770f3e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 18:36:41 -0600 Subject: [PATCH 12/41] Use consistently named make targets for regen-frozen-*. --- Makefile.pre.in | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 221befbfc407e8..e9c1e11565fe75 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -742,8 +742,8 @@ Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Tools/scripts/freeze_modules.py: Programs/_freeze_module -.PHONY: regen-importlib -regen-importlib: Tools/scripts/freeze_modules.py +.PHONY: regen-frozen-importlib +regen-frozen-importlib: Tools/scripts/freeze_modules.py # Regenerate Python/importlib_external.h, importlib.h, and zipimport.h. $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py importlib @@ -751,8 +751,8 @@ regen-importlib: Tools/scripts/freeze_modules.py regen-frozen-stdlib: Tools/scripts/freeze_modules.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py stdlib -.PHONY: regen-frozen -regen-frozen: Tools/scripts/freeze_modules.py +.PHONY: regen-frozen-test +regen-frozen-test: Tools/scripts/freeze_modules.py # Regenerate code for frozen module "__hello__". $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py test @@ -761,6 +761,14 @@ regen-frozen-all: Tools/scripts/freeze_modules.py # Regenerate code for frozen module "__hello__". $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py all +# We keep some renamed targets around for folks with muscle memory. + +.PHONY: regen-importlib +regen-importlib: regen-frozen-importlib + +.PHONY: regen-frozen +regen-frozen: regen-frozen-test + ############################################################################ # ABI From 06af093d7a2b9b7fcf2fc317abfb6b16026074bb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 18:41:26 -0600 Subject: [PATCH 13/41] Drop the stdlib placeholders. --- Makefile.pre.in | 4 ---- Python/frozen.c | 5 ----- Tools/scripts/freeze_modules.py | 21 +++------------------ 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index e9c1e11565fe75..98e4494510ba78 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -747,10 +747,6 @@ regen-frozen-importlib: Tools/scripts/freeze_modules.py # Regenerate Python/importlib_external.h, importlib.h, and zipimport.h. $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py importlib -.PHONY: regen-frozen-stdlib -regen-frozen-stdlib: Tools/scripts/freeze_modules.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py stdlib - .PHONY: regen-frozen-test regen-frozen-test: Tools/scripts/freeze_modules.py # Regenerate code for frozen module "__hello__". diff --git a/Python/frozen.c b/Python/frozen.c index 1da980c37e009c..ccfd40f8918937 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -43,7 +43,6 @@ #include "frozen_importlib__bootstrap.h" #include "frozen_importlib__bootstrap_external.h" #include "frozen_zipimport.h" -/* stdlib */ /* Test module */ #include "frozen_hello.h" @@ -55,10 +54,6 @@ static const struct _frozen _PyImport_FrozenModules[] = { (int)sizeof(_Py_M__importlib__bootstrap_external)}, {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, - /* stdlib */ - /* without site (python -S) */ - /* with site */ - /* Test module */ {"__hello__", _Py_M__hello, (int)sizeof(_Py_M__hello)}, /* Test package (negative size indicates package-ness) */ diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 8c2956fcb650d1..259c7c56c85664 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -26,15 +26,13 @@ # These are modules that get frozen. FROZEN = { - # frozen_id: (pyfile, frozenfile) - # frozen_id: pyfile + # frozenid: pyfile + # : pyfile # importlib 'importlib._bootstrap': None, 'importlib._bootstrap_external': None, 'zipimport': None, - # stdlib - # ... # test 'hello': os.path.join(TOOLS_DIR, 'freeze', 'flag.py'), } @@ -44,12 +42,6 @@ 'importlib._bootstrap_external', 'zipimport', ], - 'stdlib': [ - # without site (python -S): - - # with site: - - ], 'test': [ 'hello', ], @@ -66,13 +58,6 @@ ('_frozen_importlib_external', 'importlib._bootstrap_external'), 'zipimport', - # stdlib - '# [stdlib]', - '# without site (python -S)', - # ... - '# with site', - # ... - # test '# [Test module]', ('__hello__', 'hello'), @@ -426,7 +411,7 @@ def main(kinds=None, *, regen=True): # First, freeze the modules. # (We use a consistent order:) - ordered = ['importlib', 'stdlib', 'test'] + ordered = ['importlib', 'test'] assert (set(ordered) == set(groups)) for kind in ordered: if not kinds or kind in kinds: From 3a036b7c3943fbf7cfcb9fdd54e473b710516181 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 19:01:21 -0600 Subject: [PATCH 14/41] Simplify FROZEN a little. --- Tools/scripts/freeze_modules.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 259c7c56c85664..1fa97e174e239d 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -25,18 +25,21 @@ MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') # These are modules that get frozen. -FROZEN = { - # frozenid: pyfile - # : pyfile +FROZEN = [ + # frozenid + # + # (frozenid, pyfile) # importlib - 'importlib._bootstrap': None, - 'importlib._bootstrap_external': None, - 'zipimport': None, + 'importlib._bootstrap', + 'importlib._bootstrap_external', + 'zipimport', # test - 'hello': os.path.join(TOOLS_DIR, 'freeze', 'flag.py'), -} + ('hello', os.path.join(TOOLS_DIR, 'freeze', 'flag.py')), +] FROZEN_GROUPS = { + # group: [frozenid] + # group: [] 'importlib': [ 'importlib._bootstrap', 'importlib._bootstrap_external', @@ -48,9 +51,10 @@ } # These are the modules defined in frozen.c, in order. MODULES = [ - # frozen_id - # (module, frozen_id) - # (, frozen_id) + # frozenid + # + # (module, frozenid) + # (, frozenid) # importlib '# [importlib]', @@ -71,7 +75,8 @@ def expand_frozen(destdir=MODULES_DIR): # First, expand FROZEN. _frozen = [] packages = {} - for frozenid, pyfile in FROZEN.items(): + for row in FROZEN: + frozenid, pyfile = (row, None) if isinstance(row, str) else row resolved = iter(resolve_modules(frozenid, pyfile)) modname, _pyfile, ispkg = next(resolved) if not pyfile: From 6a27a1d2b271348b3f560fb28660249db90a8470 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 21:15:19 -0600 Subject: [PATCH 15/41] Combine FROZEN and MODULES. --- Tools/scripts/freeze_modules.py | 238 +++++++++++++++++--------------- 1 file changed, 124 insertions(+), 114 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 1fa97e174e239d..ed76db8ff58c56 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -26,16 +26,20 @@ # These are modules that get frozen. FROZEN = [ - # frozenid - # - # (frozenid, pyfile) + # See parse_frozen_spec() for the format. + # The comments are added to Python/frozen.c at the relative position. + # In cases where the frozenid is duplicated, the first one is re-used. - # importlib - 'importlib._bootstrap', - 'importlib._bootstrap_external', + '# [importlib]', + 'importlib._bootstrap : _frozen_importlib', + 'importlib._bootstrap_external : _frozen_importlib_external', 'zipimport', - # test - ('hello', os.path.join(TOOLS_DIR, 'freeze', 'flag.py')), + + '# [Test module]', + 'hello : __hello__ = ' + os.path.join(TOOLS_DIR, 'freeze', 'flag.py'), + '# Test package (negative size indicates package-ness)', + 'hello : <__phello__>', + 'hello : __phello__.spam', ] FROZEN_GROUPS = { # group: [frozenid] @@ -49,104 +53,110 @@ 'hello', ], } -# These are the modules defined in frozen.c, in order. -MODULES = [ - # frozenid - # - # (module, frozenid) - # (, frozenid) - - # importlib - '# [importlib]', - ('_frozen_importlib', 'importlib._bootstrap'), - ('_frozen_importlib_external', 'importlib._bootstrap_external'), - 'zipimport', - # test - '# [Test module]', - ('__hello__', 'hello'), - '# Test package (negative size indicates package-ness)', - ('<__phello__>', 'hello'), - ('__phello__.spam', 'hello'), -] +def parse_frozen_spec(spec, known=None): + """Yield (frozenid, pyfile, modname, ispkg) for the corresponding modules. -def expand_frozen(destdir=MODULES_DIR): - # First, expand FROZEN. - _frozen = [] - packages = {} - for row in FROZEN: - frozenid, pyfile = (row, None) if isinstance(row, str) else row - resolved = iter(resolve_modules(frozenid, pyfile)) - modname, _pyfile, ispkg = next(resolved) - if not pyfile: - pyfile = _pyfile - _frozen.append((modname, pyfile, ispkg)) - if ispkg: - modids = packages[frozenid] = [modname] - for subname, subpyfile, ispkg in resolved: - frozen[subname] = (subpyfile, ispkg) - _frozen.append((subname, subpyfile, ispkg)) - modids.append(subname) + Supported formats: + + frozenid + frozenid : modname + frozenid : modname = pyfile + + "frozenid" and "modname" must be valid module names (dot-separated + identifiers). If "modname" is not provided then "frozenid" is used. + If "pyfile" is not provided then the filename of the module + corresponding to "frozenid" is used. + + Angle brackets around a frozenid (e.g. '") indicate + it is a package. This also means it must be an actual module + (i.e. "pyfile" cannot have been provided). Such values can have + patterns to expand submodules: + + - also freeze all direct submodules + - also freeze the full submodule tree + + As with "frozenid", angle brackets around "modname" indicate + it is a package. However, in this case "pyfile" should not + have been provided and patterns in "modname" are not supported. + Also, if "modname" has brackets then "frozenid" should not, + and "pyfile" should have been provided.. + """ + frozenid, _, remainder = spec.partition(':') + modname, _, pyfile = remainder.partition('=') + frozenid = frozenid.strip() + modname = modname.strip() + pyfile = pyfile.strip() + + if modname.startswith('<') and modname.endswith('>'): + assert check_modname(frozenid), spec + modname = modname[1:-1] + assert check_modname(modname), spec + if known and frozenid in known: + assert not pyfile, spec + elif pyfile: + assert not os.path.isdir(pyfile), spec else: - assert not list(resolved) + pyfile = _resolve_module(frozenid, ispkg=False) + yield frozenid, pyfile or None, modname, True + elif pyfile: + assert check_modname(frozenid), spec + assert not known or frozenid not in known, spec + assert check_modname(modname), spec + assert not os.path.isdir(pyfile), spec + yield frozenid, pyfile, modname, False + elif known and frozenid in known: + assert check_modname(frozenid), spec + assert check_modname(modname), spec + yield frozenid, None, modname, False + else: + assert not modname or check_modname(modname), spec + resolved = iter(resolve_modules(frozenid)) + frozenid, pyfile, ispkg = next(resolved) + yield frozenid, pyfile, modname or frozenid, ispkg + if ispkg: + pkgid = frozenid + pkgname = modname + for frozenid, pyfile, ispkg in resolved: + assert not known or frozenid not in known, (frozenid, spec) + if pkgname: + modname = frozenid.replace(pkgid, pkgname, 1) + else: + modname = frozenid + yield frozenid, pyfile, modname, ispkg + + +def expand_frozen(destdir=MODULES_DIR): frozen = {} - for frozenid, pyfile, ispkg in _frozen: - if not pyfile: - pyfile = _resolve_module(frozenid, ispkg=ispkg) - frozenfile = _resolve_frozen(frozenid, destdir) - frozen[frozenid] = (pyfile, frozenfile, ispkg) - - # Then, expand FROZEN_GROUPS. - groups = {} - for groupname, frozenids in FROZEN_GROUPS.items(): - group = groups[groupname] = [] - for frozenid in frozenids: - if frozenid in packages: - group.extend(packages[frozenid]) - else: - assert frozenid in frozen, (frozenid, frozen) - group.append(frozenid) - - # Finally, expand MODULES. - rows = [] - for row in MODULES: - if isinstance(row, str) and not row.startswith('#'): - if row in packages: - for modname in packages[row]: - rows.append((modname, modname)) - else: - row = (row, row) - rows.append(row) headers = [] - specs = [] - for row in rows: - if isinstance(row, str): - assert row.startswith('# ') - comment = row[2:] + definitions = [] + for spec in FROZEN: + if spec.startswith('#'): + comment = spec[2:] if comment.startswith('['): line = f'/* {comment[1:-1]} */' headers.append(line) - specs.extend(['', line]) + definitions.extend(['', line]) else: - specs.append(f'/* {comment} */') - continue - modname, frozenid = row - _, frozenfile, ispkg = frozen[frozenid] - - assert frozenfile.startswith(destdir), (frozenfile, destdir) - header = os.path.relpath(frozenfile, destdir) - if header not in headers: - headers.append(header) + definitions.append(f'/* {comment} */') + else: + for frozenid, pyfile, modname, ispkg in parse_frozen_spec(spec, frozen): + if frozenid not in frozen: + assert pyfile, spec + frozenfile = _resolve_frozen(frozenid, destdir) +# assert frozenfile.startswith(destdir), (frozenfile, destdir) + frozen[frozenid] = (pyfile, frozenfile) + else: + assert not pyfile, spec - if modname.startswith('<'): - modname = modname[1:-1] - assert check_modname(modname), modname - ispkg = True - spec = (modname, frozenid, ispkg) - specs.append(spec) + header = os.path.relpath(frozenfile, destdir) + if header not in headers: + headers.append(header) - return groups, frozen, headers, specs + definition = (modname, frozenid, ispkg) + definitions.append(definition) + return frozen, headers, definitions def resolve_modules(modname, pyfile=None): @@ -275,7 +285,7 @@ def _freeze_module(frozenid, pyfile, frozenfile): ####################################### # regenerating dependent files -def regen_frozen(headers, specs): +def regen_frozen(headers, definitions): headerlines = [] for row in headers: assert isinstance(row, str), row @@ -285,30 +295,30 @@ def regen_frozen(headers, specs): row = f'#include "{row}"' headerlines.append(row) - speclines = [] + deflines = [] sentinel = '{0, 0, 0} /* sentinel */' indent = ' ' - for spec in specs: - if isinstance(spec, str): - speclines.append(spec) + for definition in definitions: + if isinstance(definition, str): + deflines.append(definition) continue - modname, frozenid, ispkg = spec + modname, frozenid, ispkg = definition # This matches what we do in Programs/_freeze_module.c: symbol = '_Py_M__' + frozenid.replace('.', '_') pkg = '-' if ispkg else '' line = '{"%s", %s, %s(int)sizeof(%s)},' % (modname, symbol, pkg, symbol) if len(line) < 80: - speclines.append(line) + deflines.append(line) else: line1, _, line2 = line.rpartition(' ') - speclines.append(line1) - speclines.append(indent + line2) - if not speclines[0]: - del speclines[0] - speclines.extend(['', sentinel]) - for i, line in enumerate(speclines): + deflines.append(line1) + deflines.append(indent + line2) + if not deflines[0]: + del deflines[0] + deflines.extend(['', sentinel]) + for i, line in enumerate(deflines): if line: - speclines[i] = indent + line + deflines[i] = indent + line with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): lines = iter(infile) @@ -347,7 +357,7 @@ def regen_frozen(headers, specs): else: raise Exception("did you delete a line you shouldn't have?") # Add the modules. - for line in speclines: + for line in deflines: outfile.write(line + os.linesep) # Keep the rest of the file. for line in lines: @@ -412,22 +422,22 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): def main(kinds=None, *, regen=True): - groups, frozen, headers, specs = expand_frozen(MODULES_DIR) + frozen, headers, definitions = expand_frozen(MODULES_DIR) # First, freeze the modules. # (We use a consistent order:) ordered = ['importlib', 'test'] - assert (set(ordered) == set(groups)) + assert (set(ordered) == set(FROZEN_GROUPS)) for kind in ordered: if not kinds or kind in kinds: - for frozenid in groups[kind]: + for frozenid in FROZEN_GROUPS[kind]: #freeze_module(frozenid, frozen) - pyfile, frozenfile, _ = frozen[frozenid] + pyfile, frozenfile = frozen[frozenid] _freeze_module(frozenid, pyfile, frozenfile) if regen: # Then regen frozen.c and Makefile.pre.in. - regen_frozen(headers, specs) + regen_frozen(headers, definitions) regen_makefile(headers, MODULES_DIR) From e3e07a9f5ea43240117fd891c6d6b3af3e5bcf72 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 21:27:17 -0600 Subject: [PATCH 16/41] Drop the frozen module groups. --- Makefile.pre.in | 25 +++++----------------- Python/frozen.c | 10 ++------- Tools/scripts/freeze_modules.py | 38 ++++++++------------------------- 3 files changed, 16 insertions(+), 57 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 98e4494510ba78..a23d033760b725 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -742,28 +742,13 @@ Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Tools/scripts/freeze_modules.py: Programs/_freeze_module -.PHONY: regen-frozen-importlib -regen-frozen-importlib: Tools/scripts/freeze_modules.py - # Regenerate Python/importlib_external.h, importlib.h, and zipimport.h. - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py importlib - -.PHONY: regen-frozen-test -regen-frozen-test: Tools/scripts/freeze_modules.py - # Regenerate code for frozen module "__hello__". - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py test - -.PHONY: regen-frozen-all -regen-frozen-all: Tools/scripts/freeze_modules.py - # Regenerate code for frozen module "__hello__". - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py all - -# We keep some renamed targets around for folks with muscle memory. +.PHONY: regen-frozen +regen-frozen: Tools/scripts/freeze_modules.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py +# We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib -regen-importlib: regen-frozen-importlib - -.PHONY: regen-frozen -regen-frozen: regen-frozen-test +regen-importlib: regen-frozen ############################################################################ # ABI diff --git a/Python/frozen.c b/Python/frozen.c index ccfd40f8918937..b7a54793b75dc5 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -7,15 +7,9 @@ * * These files must be regenerated any time * the corresponding .pyc * file would change (e.g. compiler, bytecode format, marshal format). - * This can be done with "make regen-frozen-all". + * This can be done with "make regen-frozen". * - * Additionally, specific groups of frozen modules can be regenerated: - * - * import-related make regen-importlib - * stdlib (partial) make regen-frozen-stdlib - * test modules make regen-frozen - * - * Those make targets simply run Tools/scripts/freeze_modules.py, which + * That make target simply runs Tools/scripts/freeze_modules.py, which * does the following: * * 1. run Programs/_freeze_module on the target modules diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index ed76db8ff58c56..63d150d70a6ded 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -41,18 +41,6 @@ 'hello : <__phello__>', 'hello : __phello__.spam', ] -FROZEN_GROUPS = { - # group: [frozenid] - # group: [] - 'importlib': [ - 'importlib._bootstrap', - 'importlib._bootstrap_external', - 'zipimport', - ], - 'test': [ - 'hello', - ], -} def parse_frozen_spec(spec, known=None): @@ -129,6 +117,7 @@ def parse_frozen_spec(spec, known=None): def expand_frozen(destdir=MODULES_DIR): frozen = {} + frozenids = [] headers = [] definitions = [] for spec in FROZEN: @@ -147,6 +136,7 @@ def expand_frozen(destdir=MODULES_DIR): frozenfile = _resolve_frozen(frozenid, destdir) # assert frozenfile.startswith(destdir), (frozenfile, destdir) frozen[frozenid] = (pyfile, frozenfile) + frozenids.append(frozenid) else: assert not pyfile, spec @@ -156,7 +146,7 @@ def expand_frozen(destdir=MODULES_DIR): definition = (modname, frozenid, ispkg) definitions.append(definition) - return frozen, headers, definitions + return frozen, frozenids, headers, definitions def resolve_modules(modname, pyfile=None): @@ -406,34 +396,24 @@ def regen_makefile(headers, destdir=MODULES_DIR): # the script def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): - KINDS = ['all', *sorted(FROZEN_GROUPS)] import argparse parser = argparse.ArgumentParser(prog=prog) parser.add_argument('--no-regen', dest='regen', action='store_false') - parser.add_argument('kinds', nargs='*', choices=KINDS, default='all') args = parser.parse_args(argv) ns = vars(args) - if not args.kinds or 'all' in args.kinds: - args.kinds = None - return ns -def main(kinds=None, *, regen=True): - frozen, headers, definitions = expand_frozen(MODULES_DIR) +def main(*, regen=True): + frozen, frozenids, headers, definitions = expand_frozen(MODULES_DIR) # First, freeze the modules. - # (We use a consistent order:) - ordered = ['importlib', 'test'] - assert (set(ordered) == set(FROZEN_GROUPS)) - for kind in ordered: - if not kinds or kind in kinds: - for frozenid in FROZEN_GROUPS[kind]: - #freeze_module(frozenid, frozen) - pyfile, frozenfile = frozen[frozenid] - _freeze_module(frozenid, pyfile, frozenfile) + # (We use a consistent order: that of FROZEN above.) + for frozenid in frozenids: + pyfile, frozenfile = frozen[frozenid] + _freeze_module(frozenid, pyfile, frozenfile) if regen: # Then regen frozen.c and Makefile.pre.in. From 3d2ebdbb875d6b90ea93d5f2336db9fcaf117b38 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 11 Aug 2021 21:31:43 -0600 Subject: [PATCH 17/41] Fix a typo. --- Tools/scripts/freeze_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 63d150d70a6ded..a1ec20577116c3 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -157,7 +157,7 @@ def resolve_modules(modname, pyfile=None): modname = modname[1:-1] rawname = modname # For now, we only expect match patterns at the end of the name. - _modname, sep, match = pkgname.rpartition('.') + _modname, sep, match = modname.rpartition('.') if sep: if _modname.endswith('.**'): modname = _modname[:-3] From 9a5141f4c514fa97e1940b40cacc21dcd8844c9d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 24 Aug 2021 08:37:21 -0600 Subject: [PATCH 18/41] Fix the Makefile. --- Makefile.pre.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a23d033760b725..707b672e715354 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -573,8 +573,8 @@ coverage-lcov: @echo "lcov report at $(COVERAGE_REPORT)/index.html" @echo -# Force regeneration of parser and importlib -coverage-report: regen-token regen-importlib regen-frozen-stdlib +# Force regeneration of parser and frozen modules +coverage-report: regen-token regen-frozen @ # build with coverage info $(MAKE) coverage @ # run tests, ignore failures From 3687fdab62dbdc6f03b538b97503d61dbd35e3f4 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 24 Aug 2021 15:36:32 -0700 Subject: [PATCH 19/41] Make freeze_modules.py less fragile. --- Makefile.pre.in | 1 + Python/frozen.c | 3 +- Tools/scripts/freeze_modules.py | 127 ++++++++++++++------------------ 3 files changed, 59 insertions(+), 72 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 707b672e715354..5f4afede02ab50 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -981,6 +981,7 @@ FROZEN_FILES = \ $(srcdir)/Python/frozen_importlib__bootstrap_external.h \ $(srcdir)/Python/frozen_zipimport.h \ $(srcdir)/Python/frozen_hello.h +# End FROZEN_FILES Python/frozen.o: $(FROZEN_FILES) diff --git a/Python/frozen.c b/Python/frozen.c index b7a54793b75dc5..679dd4d4cc0c46 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -32,13 +32,13 @@ #include "Python.h" /* Includes for frozen modules: */ - /* importlib */ #include "frozen_importlib__bootstrap.h" #include "frozen_importlib__bootstrap_external.h" #include "frozen_zipimport.h" /* Test module */ #include "frozen_hello.h" +/* End includes */ static const struct _frozen _PyImport_FrozenModules[] = { /* importlib */ @@ -53,7 +53,6 @@ static const struct _frozen _PyImport_FrozenModules[] = { /* Test package (negative size indicates package-ness) */ {"__phello__", _Py_M__hello, -(int)sizeof(_Py_M__hello)}, {"__phello__.spam", _Py_M__hello, (int)sizeof(_Py_M__hello)}, - {0, 0, 0} /* sentinel */ }; diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index a1ec20577116c3..b740f395ce0935 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -198,7 +198,7 @@ def iter_submodules(pkgname, pkgdir=None, match='*'): match_modname = _resolve_modname_matcher(match, pkgdir) def _iter_submodules(pkgname, pkgdir): - for entry in os.scandir(pkgdir): + for entry in sorted(os.scandir(pkgdir), key=lambda e: e.name): matched, recursive = match_modname(entry.name) if not matched: continue @@ -275,6 +275,24 @@ def _freeze_module(frozenid, pyfile, frozenfile): ####################################### # regenerating dependent files +def find_marker(lines, marker, file): + for pos, line in enumerate(lines): + if marker in line: + return pos + raise Exception(f"Can't find {marker!r} in file {file}") + + +def replace_block(lines, start_marker, end_marker, replacements, file): + start_pos = find_marker(lines, start_marker, file) + end_pos = find_marker(lines, end_marker, file) + if end_pos <= start_pos: + raise Exception(f"End marker {end_marker!r} " + f"occurs before start marker {start_marker!r} " + f"in file {file}") + replacements = [line.rstrip() + os.linesep for line in replacements] + return lines[:start_pos + 1] + replacements + lines[end_pos:] + + def regen_frozen(headers, definitions): headerlines = [] for row in headers: @@ -286,7 +304,6 @@ def regen_frozen(headers, definitions): headerlines.append(row) deflines = [] - sentinel = '{0, 0, 0} /* sentinel */' indent = ' ' for definition in definitions: if isinstance(definition, str): @@ -294,69 +311,50 @@ def regen_frozen(headers, definitions): continue modname, frozenid, ispkg = definition # This matches what we do in Programs/_freeze_module.c: - symbol = '_Py_M__' + frozenid.replace('.', '_') + name = frozenid.replace('.', '_') + symbol = '_Py_M__' + name pkg = '-' if ispkg else '' - line = '{"%s", %s, %s(int)sizeof(%s)},' % (modname, symbol, pkg, symbol) + line = ('{"%s", %s, %s(int)sizeof(%s)},' + % (modname, symbol, pkg, symbol)) + # TODO: Consider not folding lines if len(line) < 80: deflines.append(line) else: line1, _, line2 = line.rpartition(' ') deflines.append(line1) deflines.append(indent + line2) + if not deflines[0]: del deflines[0] - deflines.extend(['', sentinel]) for i, line in enumerate(deflines): if line: deflines[i] = indent + line with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): - lines = iter(infile) - # Get to the frozen includes. - for line in lines: - outfile.write(line) - if line.rstrip() == '/* Includes for frozen modules: */': - break - else: - raise Exception("did you delete a line you shouldn't have?") - # Pop off the existing values. - blank_before = next(lines) - assert blank_before.strip() == '' - for line in lines: - if not line.strip(): - blank_after = line - break - else: - raise Exception("did you delete a line you shouldn't have?") - # Add the frozen includes. - outfile.write(blank_before) - for line in headerlines: - outfile.write(line + os.linesep) - outfile.write(blank_after) - # Get to the modules array. - for line in lines: - outfile.write(line) - if line.rstrip() == 'static const struct _frozen _PyImport_FrozenModules[] = {': - break - else: - raise Exception("did you delete a line you shouldn't have?") - # Pop off the existing values. - for line in lines: - if line.strip() == sentinel: - break - else: - raise Exception("did you delete a line you shouldn't have?") - # Add the modules. - for line in deflines: - outfile.write(line + os.linesep) - # Keep the rest of the file. - for line in lines: - outfile.write(line) + lines = infile.readlines() + # TODO: Use more obvious markers, e.g. + # $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$ + lines = replace_block( + lines, + "/* Includes for frozen modules: */", + "/* End includes */", + headerlines, + FROZEN_FILE, + ) + lines = replace_block( + lines, + "static const struct _frozen _PyImport_FrozenModules[] =", + "/* sentinel */", + deflines, + FROZEN_FILE, + ) + outfile.writelines(lines) def regen_makefile(headers, destdir=MODULES_DIR): reldir = os.path.relpath(destdir, ROOT_DIR) assert destdir.endswith(reldir), destdir + frozenfiles = [] for row in headers: if not row.startswith('/*'): @@ -364,32 +362,21 @@ def regen_makefile(headers, destdir=MODULES_DIR): assert header.endswith('.h'), header assert os.path.basename(header) == header, header relfile = f'{reldir}/{header}'.replace('\\', '/') - frozenfiles.append(f'$(srcdir)/{relfile}') + frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') + cgfile = relfile.replace("/frozen", "/codegen") + cgfile = cgfile[:-2] + ".o" + frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): - lines = iter(infile) - # Get to $FROZEN_FILES. - for line in lines: - outfile.write(line) - if line.rstrip() == 'FROZEN_FILES = \\': - break - else: - raise Exception("did you delete a line you shouldn't have?") - # Pop off the existing values. - for line in lines: - if line.rstrip() == 'Python/frozen.o: $(FROZEN_FILES)': - break - else: - raise Exception("did you delete a line you shouldn't have?") - # Add the regen'ed values. - for header in frozenfiles[:-1]: - outfile.write(f'\t\t{header} \\{os.linesep}') - outfile.write(f'\t\t{frozenfiles[-1]}{os.linesep}') - # Keep the rest of the file. - outfile.write(os.linesep) - outfile.write('Python/frozen.o: $(FROZEN_FILES)' + os.linesep) - for line in lines: - outfile.write(line) + lines = infile.readlines() + lines = replace_block( + lines, + "FROZEN_FILES =", + "# End FROZEN_FILES", + frozenfiles, + MAKEFILE, + ) + outfile.writelines(lines) ####################################### From 1331a77b839588af2832527184671fd30ce0b6f3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 25 Aug 2021 15:04:41 -0600 Subject: [PATCH 20/41] Simplify specs. --- Python/frozen.c | 5 +- Tools/scripts/freeze_modules.py | 258 +++++++++++++++++++------------- 2 files changed, 156 insertions(+), 107 deletions(-) diff --git a/Python/frozen.c b/Python/frozen.c index 679dd4d4cc0c46..db258ddef32c3e 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -32,14 +32,14 @@ #include "Python.h" /* Includes for frozen modules: */ -/* importlib */ #include "frozen_importlib__bootstrap.h" #include "frozen_importlib__bootstrap_external.h" #include "frozen_zipimport.h" -/* Test module */ #include "frozen_hello.h" /* End includes */ +/* Note that a negative size indicates a package. */ + static const struct _frozen _PyImport_FrozenModules[] = { /* importlib */ {"_frozen_importlib", _Py_M__importlib__bootstrap, @@ -50,7 +50,6 @@ static const struct _frozen _PyImport_FrozenModules[] = { /* Test module */ {"__hello__", _Py_M__hello, (int)sizeof(_Py_M__hello)}, - /* Test package (negative size indicates package-ness) */ {"__phello__", _Py_M__hello, -(int)sizeof(_Py_M__hello)}, {"__phello__.spam", _Py_M__hello, (int)sizeof(_Py_M__hello)}, {0, 0, 0} /* sentinel */ diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index b740f395ce0935..e32a787a57ab39 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -27,23 +27,24 @@ # These are modules that get frozen. FROZEN = [ # See parse_frozen_spec() for the format. - # The comments are added to Python/frozen.c at the relative position. # In cases where the frozenid is duplicated, the first one is re-used. - - '# [importlib]', - 'importlib._bootstrap : _frozen_importlib', - 'importlib._bootstrap_external : _frozen_importlib_external', - 'zipimport', - - '# [Test module]', - 'hello : __hello__ = ' + os.path.join(TOOLS_DIR, 'freeze', 'flag.py'), - '# Test package (negative size indicates package-ness)', - 'hello : <__phello__>', - 'hello : __phello__.spam', + ('importlib', [ + 'importlib._bootstrap : _frozen_importlib', + 'importlib._bootstrap_external : _frozen_importlib_external', + 'zipimport', + ]), + ('Test module', [ + 'hello : __hello__ = ' + os.path.join(TOOLS_DIR, 'freeze', 'flag.py'), + 'hello : <__phello__>', + 'hello : __phello__.spam', + ]), ] -def parse_frozen_spec(spec, known=None): +####################################### +# specs + +def parse_frozen_spec(rawspec, knownids=None, section=None): """Yield (frozenid, pyfile, modname, ispkg) for the corresponding modules. Supported formats: @@ -71,83 +72,131 @@ def parse_frozen_spec(spec, known=None): Also, if "modname" has brackets then "frozenid" should not, and "pyfile" should have been provided.. """ - frozenid, _, remainder = spec.partition(':') + frozenid, _, remainder = rawspec.partition(':') modname, _, pyfile = remainder.partition('=') frozenid = frozenid.strip() modname = modname.strip() pyfile = pyfile.strip() + submodules = None if modname.startswith('<') and modname.endswith('>'): - assert check_modname(frozenid), spec + assert check_modname(frozenid), rawspec modname = modname[1:-1] - assert check_modname(modname), spec - if known and frozenid in known: - assert not pyfile, spec + assert check_modname(modname), rawspec + if frozenid in knownids: + pass elif pyfile: - assert not os.path.isdir(pyfile), spec + assert not os.path.isdir(pyfile), rawspec else: pyfile = _resolve_module(frozenid, ispkg=False) - yield frozenid, pyfile or None, modname, True + ispkg = True elif pyfile: - assert check_modname(frozenid), spec - assert not known or frozenid not in known, spec - assert check_modname(modname), spec - assert not os.path.isdir(pyfile), spec - yield frozenid, pyfile, modname, False - elif known and frozenid in known: - assert check_modname(frozenid), spec - assert check_modname(modname), spec - yield frozenid, None, modname, False + assert check_modname(frozenid), rawspec + assert not knownids or frozenid not in knownids, rawspec + assert check_modname(modname), rawspec + assert not os.path.isdir(pyfile), rawspec + ispkg = False + elif knownids and frozenid in knownids: + assert check_modname(frozenid), rawspec + assert check_modname(modname), rawspec + ispkg = False else: - assert not modname or check_modname(modname), spec + assert not modname or check_modname(modname), rawspec resolved = iter(resolve_modules(frozenid)) frozenid, pyfile, ispkg = next(resolved) - yield frozenid, pyfile, modname or frozenid, ispkg + if not modname: + modname = frozenid if ispkg: - pkgid = frozenid - pkgname = modname - for frozenid, pyfile, ispkg in resolved: - assert not known or frozenid not in known, (frozenid, spec) - if pkgname: - modname = frozenid.replace(pkgid, pkgname, 1) - else: - modname = frozenid - yield frozenid, pyfile, modname, ispkg - - -def expand_frozen(destdir=MODULES_DIR): + def iter_subs(): + pkgid = frozenid + pkgname = modname + for frozenid, pyfile, ispkg in resolved: + assert not knownids or frozenid not in knownids, (frozenid, rawspec) + if pkgname: + modname = frozenid.replace(pkgid, pkgname, 1) + else: + modname = frozenid + yield frozenid, section, pyfile, modname, ispkg + submodules = iter_subs() + + spec = (frozenid, pyfile or None, modname, ispkg, section) + return spec, submodules + + +def parse_frozen_specs(rawspecs=FROZEN): + seen = set() + for section, _specs in rawspecs: + for spec in _parse_frozen_specs(_specs, section, seen): + frozenid = spec[0] + yield spec + seen.add(frozenid) + + +def _parse_frozen_specs(rawspecs, section, seen): + for rawspec in rawspecs: + spec, subs = parse_frozen_spec(rawspec, seen, section) + yield spec + for spec in subs or (): + yield spec + + +def resolve_frozen_file(spec, destdir=MODULES_DIR): + if isinstance(spec, str): + modname = spec + else: + _, frozenid, _, _, _= spec + modname = frozenid + # We use a consistent naming convention for all frozen modules. + return os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' + + +def resolve_frozen_files(specs, destdir=MODULES_DIR): frozen = {} frozenids = [] - headers = [] - definitions = [] - for spec in FROZEN: - if spec.startswith('#'): - comment = spec[2:] - if comment.startswith('['): - line = f'/* {comment[1:-1]} */' - headers.append(line) - definitions.extend(['', line]) + lastsection = None + for spec in specs: + frozenid, pyfile, *_, section = spec + if frozenid in frozen: + if section is None: + lastsection = None else: - definitions.append(f'/* {comment} */') - else: - for frozenid, pyfile, modname, ispkg in parse_frozen_spec(spec, frozen): - if frozenid not in frozen: - assert pyfile, spec - frozenfile = _resolve_frozen(frozenid, destdir) -# assert frozenfile.startswith(destdir), (frozenfile, destdir) - frozen[frozenid] = (pyfile, frozenfile) - frozenids.append(frozenid) - else: - assert not pyfile, spec - - header = os.path.relpath(frozenfile, destdir) - if header not in headers: - headers.append(header) + assert section == lastsection + continue + lastsection = section + frozenfile = resolve_frozen_file(frozenid, destdir) + frozen[frozenid] = (pyfile, frozenfile) + frozenids.append(frozenid) + return frozen, frozenids + + +#def expand_frozen(destdir=MODULES_DIR): +# frozen = {} +# frozenids = [] +# headers = [] +# definitions = [] +# for section, specs in FROZEN: +# for spec in specs: +# for frozenid, pyfile, modname, ispkg in parse_frozen_spec(spec, frozen): +# if frozenid not in frozen: +# assert pyfile, spec +# frozenfile = _resolve_frozen(frozenid, destdir) +## assert frozenfile.startswith(destdir), (frozenfile, destdir) +# frozen[frozenid] = (pyfile, frozenfile) +# frozenids.append(frozenid) +# else: +# assert not pyfile, spec +# +# header = os.path.relpath(frozenfile, destdir) +# if header not in headers: +# headers.append((section, header)) +# +# definition = (section, modname, frozenid, ispkg) +# definitions.append(definition) +# return frozen, frozenids, headers, definitions - definition = (modname, frozenid, ispkg) - definitions.append(definition) - return frozen, frozenids, headers, definitions +####################################### +# generic helpers def resolve_modules(modname, pyfile=None): if modname.startswith('<') and modname.endswith('>'): @@ -247,11 +296,6 @@ def _resolve_module(modname, pathentry=STDLIB_DIR, ispkg=False): return os.path.join(pathentry, *modname.split('.')) + '.py' -def _resolve_frozen(modname, destdir): - # We use a consistent naming convention for frozen modules. - return os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' - - ####################################### # freezing modules @@ -293,23 +337,32 @@ def replace_block(lines, start_marker, end_marker, replacements, file): return lines[:start_pos + 1] + replacements + lines[end_pos:] -def regen_frozen(headers, definitions): +def regen_frozen(specs, dest=MODULES_DIR): + if isinstance(dest, str): + frozen, frozenids = resolve_frozen_files(specs, destdir) + else: + frozenids, frozen = dest + headerlines = [] - for row in headers: - assert isinstance(row, str), row - if not row.startswith('/*'): - assert row.endswith('.h') - assert os.path.basename(row) == row, row - row = f'#include "{row}"' - headerlines.append(row) + parentdir = os.path.dirname(FROZEN_FILE) + for frozenid in frozenids: + # Adding a comment to separate sections here doesn't add much, + # so we don't. + _, frozenfile = frozen[frozenid] + header = os.path.relpath(frozenfile, parentdir) + headerlines.append(f'#include "{header}"') deflines = [] indent = ' ' - for definition in definitions: - if isinstance(definition, str): - deflines.append(definition) - continue - modname, frozenid, ispkg = definition + lastsection = None + for spec in specs: + frozenid, _, modname, ispkg, section = spec + if section != lastsection: + if lastsection is not None: + deflines.append('') + deflines.append(f'/* {section} */') + lastsection = section + # This matches what we do in Programs/_freeze_module.c: name = frozenid.replace('.', '_') symbol = '_Py_M__' + name @@ -351,20 +404,15 @@ def regen_frozen(headers, definitions): outfile.writelines(lines) -def regen_makefile(headers, destdir=MODULES_DIR): - reldir = os.path.relpath(destdir, ROOT_DIR) - assert destdir.endswith(reldir), destdir - +def regen_makefile(frozenids, frozen): frozenfiles = [] - for row in headers: - if not row.startswith('/*'): - header = row - assert header.endswith('.h'), header - assert os.path.basename(header) == header, header - relfile = f'{reldir}/{header}'.replace('\\', '/') - frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') - cgfile = relfile.replace("/frozen", "/codegen") - cgfile = cgfile[:-2] + ".o" + for frozenid in frozenids: + _pyfile, frozenfile = frozen[frozenid] + header = os.path.relpath(frozenfile, ROOT_DIR) + # Adding a comment to separate sections here doesn't add much, + # so we don't. + relfile = header.replace('\\', '/') + frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): @@ -394,7 +442,9 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): def main(*, regen=True): - frozen, frozenids, headers, definitions = expand_frozen(MODULES_DIR) + # Expand the raw specs, preserving order. + specs = list(parse_frozen_specs()) + frozen, frozenids = resolve_frozen_files(specs, MODULES_DIR) # First, freeze the modules. # (We use a consistent order: that of FROZEN above.) @@ -404,8 +454,8 @@ def main(*, regen=True): if regen: # Then regen frozen.c and Makefile.pre.in. - regen_frozen(headers, definitions) - regen_makefile(headers, MODULES_DIR) + regen_frozen(specs, (frozenids, frozen)) + regen_makefile(frozenids, frozen) if __name__ == '__main__': From 30972b2030b5c3c9b56518d8c9ddfa3ff0a600d7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 25 Aug 2021 15:41:26 -0600 Subject: [PATCH 21/41] Explicitly freeze modules in Makefile. --- Makefile.pre.in | 32 ++++++++++++++- Tools/scripts/freeze_modules.py | 73 ++++++++++++++++++++------------- 2 files changed, 75 insertions(+), 30 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 5f4afede02ab50..7a32914c906459 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -743,8 +743,36 @@ Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Tools/scripts/freeze_modules.py: Programs/_freeze_module .PHONY: regen-frozen -regen-frozen: Tools/scripts/freeze_modules.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py +regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py --no-freeze + +# BEGIN: freezing modules + +Python/frozen_importlib__bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py + $(srcdir)/Programs/_freeze_module importlib._bootstrap \ + $(srcdir)/Lib/importlib/_bootstrap.py \ + $(srcdir)/Lib/importlib/_bootstrap.py.new + $(UPDATE_FILE) $(srcdir)/Python/frozen_importlib__bootstrap.h Lib/importlib/_bootstrap.py.new + +Python/frozen_importlib__bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py + $(srcdir)/Programs/_freeze_module importlib._bootstrap_external \ + $(srcdir)/Lib/importlib/_bootstrap_external.py \ + $(srcdir)/Lib/importlib/_bootstrap_external.py.new + $(UPDATE_FILE) $(srcdir)/Python/frozen_importlib__bootstrap_external.h Lib/importlib/_bootstrap_external.py.new + +Python/frozen_zipimport.h: Programs/_freeze_module Lib/zipimport.py + $(srcdir)/Programs/_freeze_module zipimport \ + $(srcdir)/Lib/zipimport.py \ + $(srcdir)/Lib/zipimport.py.new + $(UPDATE_FILE) $(srcdir)/Python/frozen_zipimport.h Lib/zipimport.py.new + +Python/frozen_hello.h: Programs/_freeze_module Tools/freeze/flag.py + $(srcdir)/Programs/_freeze_module hello \ + $(srcdir)/Tools/freeze/flag.py \ + $(srcdir)/Tools/freeze/flag.py.new + $(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h Tools/freeze/flag.py.new + +# END: freezing modules # We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index e32a787a57ab39..d9abf9b6dd2f1e 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -296,26 +296,6 @@ def _resolve_module(modname, pathentry=STDLIB_DIR, ispkg=False): return os.path.join(pathentry, *modname.split('.')) + '.py' -####################################### -# freezing modules - -def freeze_module(modname, pyfile=None, destdir=MODULES_DIR): - """Generate the frozen module .h file for the given module.""" - for modname, pyfile, ispkg in resolve_modules(modname, pyfile): - frozenfile = _resolve_frozen(modname, destdir) - _freeze_module(modname, pyfile, frozenfile) - - -def _freeze_module(frozenid, pyfile, frozenfile): - tmpfile = frozenfile + '.new' - - argv = [TOOL, frozenid, pyfile, tmpfile] - print('#', ' '.join(argv)) - subprocess.run(argv, check=True) - - os.replace(tmpfile, frozenfile) - - ####################################### # regenerating dependent files @@ -406,13 +386,24 @@ def regen_frozen(specs, dest=MODULES_DIR): def regen_makefile(frozenids, frozen): frozenfiles = [] + rules = [''] for frozenid in frozenids: - _pyfile, frozenfile = frozen[frozenid] + pyfile, frozenfile = frozen[frozenid] header = os.path.relpath(frozenfile, ROOT_DIR) # Adding a comment to separate sections here doesn't add much, # so we don't. relfile = header.replace('\\', '/') frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') + + _pyfile = os.path.relpath(pyfile, ROOT_DIR) + tmpfile = f'{_pyfile}.new' + rules.append(f'{header}: Programs/_freeze_module {_pyfile}') + rules.append(f'\t$(srcdir)/Programs/_freeze_module {frozenid} \\') + rules.append(f'\t\t$(srcdir)/{_pyfile} \\') + rules.append(f'\t\t$(srcdir)/{tmpfile}') + rules.append(f'\t$(UPDATE_FILE) $(srcdir)/{header} {tmpfile}') + rules.append('') + frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): @@ -424,9 +415,36 @@ def regen_makefile(frozenids, frozen): frozenfiles, MAKEFILE, ) + lines = replace_block( + lines, + "# BEGIN: freezing modules", + "# END: freezing modules", + rules, + MAKEFILE, + ) outfile.writelines(lines) +####################################### +# freezing modules + +def freeze_module(modname, pyfile=None, destdir=MODULES_DIR): + """Generate the frozen module .h file for the given module.""" + for modname, pyfile, ispkg in resolve_modules(modname, pyfile): + frozenfile = _resolve_frozen(modname, destdir) + _freeze_module(modname, pyfile, frozenfile) + + +def _freeze_module(frozenid, pyfile, frozenfile): + tmpfile = frozenfile + '.new' + + argv = [TOOL, frozenid, pyfile, tmpfile] + print('#', ' '.join(argv)) + subprocess.run(argv, check=True) + + os.replace(tmpfile, frozenfile) + + ####################################### # the script @@ -434,6 +452,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): import argparse parser = argparse.ArgumentParser(prog=prog) parser.add_argument('--no-regen', dest='regen', action='store_false') + parser.add_argument('--no-freeze', dest='freeze', action='store_false') args = parser.parse_args(argv) ns = vars(args) @@ -441,21 +460,19 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): return ns -def main(*, regen=True): +def main(*, regen=True, freeze=True): # Expand the raw specs, preserving order. specs = list(parse_frozen_specs()) frozen, frozenids = resolve_frozen_files(specs, MODULES_DIR) - # First, freeze the modules. # (We use a consistent order: that of FROZEN above.) - for frozenid in frozenids: - pyfile, frozenfile = frozen[frozenid] - _freeze_module(frozenid, pyfile, frozenfile) - if regen: - # Then regen frozen.c and Makefile.pre.in. regen_frozen(specs, (frozenids, frozen)) regen_makefile(frozenids, frozen) + if freeze: + for frozenid in frozenids: + pyfile, frozenfile = frozen[frozenid] + _freeze_module(frozenid, pyfile, frozenfile) if __name__ == '__main__': From 7a4a876c3e8da5aedbe18f4e4d2544b3b3844ad5 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 25 Aug 2021 15:53:34 -0600 Subject: [PATCH 22/41] Fix a typo. --- Tools/scripts/freeze_modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index d9abf9b6dd2f1e..7b0c4be5696408 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -107,16 +107,16 @@ def parse_frozen_spec(rawspec, knownids=None, section=None): if not modname: modname = frozenid if ispkg: + pkgid = frozenid + pkgname = modname def iter_subs(): - pkgid = frozenid - pkgname = modname for frozenid, pyfile, ispkg in resolved: assert not knownids or frozenid not in knownids, (frozenid, rawspec) if pkgname: modname = frozenid.replace(pkgid, pkgname, 1) else: modname = frozenid - yield frozenid, section, pyfile, modname, ispkg + yield frozenid, pyfile, modname, ispkg, section submodules = iter_subs() spec = (frozenid, pyfile or None, modname, ispkg, section) From 1306a4e1b90549a57e2b294beb960ac09fd2cd66 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 25 Aug 2021 16:03:26 -0600 Subject: [PATCH 23/41] Fix dependencies. --- Tools/scripts/freeze_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 7b0c4be5696408..b3b39ff939803e 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -397,7 +397,7 @@ def regen_makefile(frozenids, frozen): _pyfile = os.path.relpath(pyfile, ROOT_DIR) tmpfile = f'{_pyfile}.new' - rules.append(f'{header}: Programs/_freeze_module {_pyfile}') + rules.append(f'{header}: $(srcdir)/Programs/_freeze_module $(srcdir)/{_pyfile}') rules.append(f'\t$(srcdir)/Programs/_freeze_module {frozenid} \\') rules.append(f'\t\t$(srcdir)/{_pyfile} \\') rules.append(f'\t\t$(srcdir)/{tmpfile}') From f724debf9835e50b3846492409b4725cf3236a93 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 25 Aug 2021 16:32:51 -0600 Subject: [PATCH 24/41] Drop some dead code. --- Tools/scripts/freeze_modules.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index b3b39ff939803e..b529a25e810536 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -169,32 +169,6 @@ def resolve_frozen_files(specs, destdir=MODULES_DIR): return frozen, frozenids -#def expand_frozen(destdir=MODULES_DIR): -# frozen = {} -# frozenids = [] -# headers = [] -# definitions = [] -# for section, specs in FROZEN: -# for spec in specs: -# for frozenid, pyfile, modname, ispkg in parse_frozen_spec(spec, frozen): -# if frozenid not in frozen: -# assert pyfile, spec -# frozenfile = _resolve_frozen(frozenid, destdir) -## assert frozenfile.startswith(destdir), (frozenfile, destdir) -# frozen[frozenid] = (pyfile, frozenfile) -# frozenids.append(frozenid) -# else: -# assert not pyfile, spec -# -# header = os.path.relpath(frozenfile, destdir) -# if header not in headers: -# headers.append((section, header)) -# -# definition = (section, modname, frozenid, ispkg) -# definitions.append(definition) -# return frozen, frozenids, headers, definitions - - ####################################### # generic helpers From 9a4d120324483649ed70d36bb908468b7e421069 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 25 Aug 2021 16:37:08 -0600 Subject: [PATCH 25/41] Fix another typo. --- Makefile.pre.in | 24 ++++++++++++------------ Tools/scripts/freeze_modules.py | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 7a32914c906459..c8851740f1e967 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -748,29 +748,29 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES) # BEGIN: freezing modules -Python/frozen_importlib__bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py +Python/frozen_importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Programs/_freeze_module importlib._bootstrap \ $(srcdir)/Lib/importlib/_bootstrap.py \ - $(srcdir)/Lib/importlib/_bootstrap.py.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_importlib__bootstrap.h Lib/importlib/_bootstrap.py.new + $(srcdir)/Python/frozen_importlib__bootstrap.h.new + $(UPDATE_FILE) $(srcdir)/Python/frozen_importlib__bootstrap.h Python/frozen_importlib__bootstrap.h.new -Python/frozen_importlib__bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py +Python/frozen_importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Programs/_freeze_module importlib._bootstrap_external \ $(srcdir)/Lib/importlib/_bootstrap_external.py \ - $(srcdir)/Lib/importlib/_bootstrap_external.py.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_importlib__bootstrap_external.h Lib/importlib/_bootstrap_external.py.new + $(srcdir)/Python/frozen_importlib__bootstrap_external.h.new + $(UPDATE_FILE) $(srcdir)/Python/frozen_importlib__bootstrap_external.h Python/frozen_importlib__bootstrap_external.h.new -Python/frozen_zipimport.h: Programs/_freeze_module Lib/zipimport.py +Python/frozen_zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py $(srcdir)/Programs/_freeze_module zipimport \ $(srcdir)/Lib/zipimport.py \ - $(srcdir)/Lib/zipimport.py.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_zipimport.h Lib/zipimport.py.new + $(srcdir)/Python/frozen_zipimport.h.new + $(UPDATE_FILE) $(srcdir)/Python/frozen_zipimport.h Python/frozen_zipimport.h.new -Python/frozen_hello.h: Programs/_freeze_module Tools/freeze/flag.py +Python/frozen_hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py $(srcdir)/Programs/_freeze_module hello \ $(srcdir)/Tools/freeze/flag.py \ - $(srcdir)/Tools/freeze/flag.py.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h Tools/freeze/flag.py.new + $(srcdir)/Python/frozen_hello.h.new + $(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h Python/frozen_hello.h.new # END: freezing modules diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index b529a25e810536..0d6462de0df723 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -370,7 +370,7 @@ def regen_makefile(frozenids, frozen): frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') _pyfile = os.path.relpath(pyfile, ROOT_DIR) - tmpfile = f'{_pyfile}.new' + tmpfile = f'{header}.new' rules.append(f'{header}: $(srcdir)/Programs/_freeze_module $(srcdir)/{_pyfile}') rules.append(f'\t$(srcdir)/Programs/_freeze_module {frozenid} \\') rules.append(f'\t\t$(srcdir)/{_pyfile} \\') From 54b01166e027891b2e406d1da5d59eec6cffffde Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 25 Aug 2021 16:45:05 -0600 Subject: [PATCH 26/41] Generate the frozen file in-place rather than with a temp file. --- Makefile.pre.in | 12 ++++-------- Tools/scripts/freeze_modules.py | 5 +++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index c8851740f1e967..caf49eb22bf7ff 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -751,26 +751,22 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES) Python/frozen_importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Programs/_freeze_module importlib._bootstrap \ $(srcdir)/Lib/importlib/_bootstrap.py \ - $(srcdir)/Python/frozen_importlib__bootstrap.h.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_importlib__bootstrap.h Python/frozen_importlib__bootstrap.h.new + $(srcdir)/Python/frozen_importlib__bootstrap.h Python/frozen_importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Programs/_freeze_module importlib._bootstrap_external \ $(srcdir)/Lib/importlib/_bootstrap_external.py \ - $(srcdir)/Python/frozen_importlib__bootstrap_external.h.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_importlib__bootstrap_external.h Python/frozen_importlib__bootstrap_external.h.new + $(srcdir)/Python/frozen_importlib__bootstrap_external.h Python/frozen_zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py $(srcdir)/Programs/_freeze_module zipimport \ $(srcdir)/Lib/zipimport.py \ - $(srcdir)/Python/frozen_zipimport.h.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_zipimport.h Python/frozen_zipimport.h.new + $(srcdir)/Python/frozen_zipimport.h Python/frozen_hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py $(srcdir)/Programs/_freeze_module hello \ $(srcdir)/Tools/freeze/flag.py \ - $(srcdir)/Python/frozen_hello.h.new - $(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h Python/frozen_hello.h.new + $(srcdir)/Python/frozen_hello.h # END: freezing modules diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 0d6462de0df723..b8dc35b03946a1 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -371,11 +371,12 @@ def regen_makefile(frozenids, frozen): _pyfile = os.path.relpath(pyfile, ROOT_DIR) tmpfile = f'{header}.new' + # Note that we freeze the module to the target .h file + # instead of going through an intermediate file like we used to. rules.append(f'{header}: $(srcdir)/Programs/_freeze_module $(srcdir)/{_pyfile}') rules.append(f'\t$(srcdir)/Programs/_freeze_module {frozenid} \\') rules.append(f'\t\t$(srcdir)/{_pyfile} \\') - rules.append(f'\t\t$(srcdir)/{tmpfile}') - rules.append(f'\t$(UPDATE_FILE) $(srcdir)/{header} {tmpfile}') + rules.append(f'\t\t$(srcdir)/{header}') rules.append('') frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") From 16adb8e8554922f7601c223a4dca9f8f24ea640c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 24 Aug 2021 08:50:13 -0600 Subject: [PATCH 27/41] Update Windows build config. --- ...portlib.vcxproj => _freeze_module.vcxproj} | 31 +++++++------ ...filters => _freeze_module.vcxproj.filters} | 13 ++++-- PCbuild/pcbuild.proj | 4 +- PCbuild/pcbuild.sln | 2 +- PCbuild/readme.txt | 7 +-- Tools/scripts/freeze_modules.py | 44 +++++++++++++++++++ 6 files changed, 78 insertions(+), 23 deletions(-) rename PCbuild/{_freeze_importlib.vcxproj => _freeze_module.vcxproj} (80%) rename PCbuild/{_freeze_importlib.vcxproj.filters => _freeze_module.vcxproj.filters} (76%) diff --git a/PCbuild/_freeze_importlib.vcxproj b/PCbuild/_freeze_module.vcxproj similarity index 80% rename from PCbuild/_freeze_importlib.vcxproj rename to PCbuild/_freeze_module.vcxproj index e437412a161ce5..efc67b97bbe642 100644 --- a/PCbuild/_freeze_importlib.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -69,7 +69,7 @@ {19C0C13F-47CA-4432-AFF3-799A296A4DDC} Win32Proj - _freeze_importlib + _freeze_module false @@ -95,7 +95,7 @@ - + @@ -108,31 +108,33 @@ + importlib._bootstrap - $(IntDir)importlib.g.h - $(PySourcePath)Python\importlib.h + $(IntDir)frozen_importlib__bootstrap.g.h + $(PySourcePath)Python\frozen_importlib__bootstrap.h importlib._bootstrap_external - $(IntDir)importlib_external.g.h - $(PySourcePath)Python\importlib_external.h + $(IntDir)frozen_importlib__bootstrap_external.g.h + $(PySourcePath)Python\frozen_importlib__bootstrap_external.h zipimport - $(IntDir)importlib_zipimport.g.h - $(PySourcePath)Python\importlib_zipimport.h + $(IntDir)frozen_zipimport.g.h + $(PySourcePath)Python\frozen_zipimport.h hello $(IntDir)frozen_hello.g.h $(PySourcePath)Python\frozen_hello.h + - + - - + + + DependsOnTargets="_RebuildFrozen"> - + diff --git a/PCbuild/_freeze_importlib.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters similarity index 76% rename from PCbuild/_freeze_importlib.vcxproj.filters rename to PCbuild/_freeze_module.vcxproj.filters index 3ee9eb750d67e8..bed7920fdba638 100644 --- a/PCbuild/_freeze_importlib.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -10,19 +10,24 @@ - + Source Files + - Source Files + Python Files + + + Python Files Python Files - + Python Files + - \ No newline at end of file + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 8e7088d47d2aed..f464ad3b18e44c 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -72,8 +72,8 @@ false - - + + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 3507b972797c96..c774e049717352 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -75,7 +75,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher. {7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446} EndProjectSection EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_module", "_freeze_module.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" EndProject diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 6c25522ea48c00..83e804625aacc2 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -115,9 +115,10 @@ _testembed These are miscellaneous sub-projects that don't really fit the other categories: -_freeze_importlib - _freeze_importlib.exe, used to regenerate Python\importlib.h after - changes have been made to Lib\importlib\_bootstrap.py +_freeze_module + _freeze_module.exe, used to regenerate frozen modules in Python\ + after changes have been made to the corresponding source files + (e.g. Lib\importlib\_bootstrap.py). pyshellext pyshellext.dll, the shell extension deployed with the launcher python3dll diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index b8dc35b03946a1..c7fe0008c1f954 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -23,6 +23,8 @@ FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') +PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj') +PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters') # These are modules that get frozen. FROZEN = [ @@ -400,6 +402,47 @@ def regen_makefile(frozenids, frozen): outfile.writelines(lines) +def regen_pcbuild(frozenids, frozen): + projlines = [] + filterlines = [] + for frozenid in frozenids: + pyfile, frozenfile = frozen[frozenid] + + _pyfile = os.path.relpath(pyfile, ROOT_DIR).replace('/', '\\') + header = os.path.relpath(frozenfile, ROOT_DIR).replace('/', '\\') + intfile = header.split('\\')[-1].strip('.h') + '.g.h' + projlines.append(f' ') + projlines.append(f' {frozenid}') + projlines.append(f' $(IntDir){intfile}') + projlines.append(f' $(PySourcePath){header}') + projlines.append(f' ') + + filterlines.append(f' ') + filterlines.append(' Python Files') + filterlines.append(' ') + + with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): + lines = infile.readlines() + lines = replace_block( + lines, + '', + '', + projlines, + PCBUILD_PROJECT, + ) + outfile.writelines(lines) + with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): + lines = infile.readlines() + lines = replace_block( + lines, + '', + '', + filterlines, + PCBUILD_FILTERS, + ) + outfile.writelines(lines) + + ####################################### # freezing modules @@ -444,6 +487,7 @@ def main(*, regen=True, freeze=True): if regen: regen_frozen(specs, (frozenids, frozen)) regen_makefile(frozenids, frozen) + regen_pcbuild(frozenids, frozen) if freeze: for frozenid in frozenids: pyfile, frozenfile = frozen[frozenid] From 2d51cbadbefed57f7b2b02f432e4db64ba5e0d3f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 26 Aug 2021 12:51:22 -0600 Subject: [PATCH 28/41] Move frozen modules into their own directory. --- Makefile.pre.in | 24 +++++++++---------- PCbuild/_freeze_module.vcxproj | 16 ++++++------- Python/frozen.c | 8 +++---- .../hello.h} | 0 .../importlib__bootstrap.h} | 0 .../importlib__bootstrap_external.h} | 0 .../zipimport.h} | 0 Tools/scripts/freeze_modules.py | 4 ++-- 8 files changed, 26 insertions(+), 26 deletions(-) rename Python/{frozen_hello.h => frozen_modules/hello.h} (100%) rename Python/{frozen_importlib__bootstrap.h => frozen_modules/importlib__bootstrap.h} (100%) rename Python/{frozen_importlib__bootstrap_external.h => frozen_modules/importlib__bootstrap_external.h} (100%) rename Python/{frozen_zipimport.h => frozen_modules/zipimport.h} (100%) diff --git a/Makefile.pre.in b/Makefile.pre.in index caf49eb22bf7ff..cca39f5f024edc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -748,25 +748,25 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES) # BEGIN: freezing modules -Python/frozen_importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py +Python/frozen_modules/importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Programs/_freeze_module importlib._bootstrap \ $(srcdir)/Lib/importlib/_bootstrap.py \ - $(srcdir)/Python/frozen_importlib__bootstrap.h + $(srcdir)/Python/frozen_modules/importlib__bootstrap.h -Python/frozen_importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py +Python/frozen_modules/importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Programs/_freeze_module importlib._bootstrap_external \ $(srcdir)/Lib/importlib/_bootstrap_external.py \ - $(srcdir)/Python/frozen_importlib__bootstrap_external.h + $(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h -Python/frozen_zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py +Python/frozen_modules/zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py $(srcdir)/Programs/_freeze_module zipimport \ $(srcdir)/Lib/zipimport.py \ - $(srcdir)/Python/frozen_zipimport.h + $(srcdir)/Python/frozen_modules/zipimport.h -Python/frozen_hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py +Python/frozen_modules/hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py $(srcdir)/Programs/_freeze_module hello \ $(srcdir)/Tools/freeze/flag.py \ - $(srcdir)/Python/frozen_hello.h + $(srcdir)/Python/frozen_modules/hello.h # END: freezing modules @@ -1001,10 +1001,10 @@ Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \ # FROZEN_FILES is auto-generated by Tools/scripts/freeze_modules.py. FROZEN_FILES = \ - $(srcdir)/Python/frozen_importlib__bootstrap.h \ - $(srcdir)/Python/frozen_importlib__bootstrap_external.h \ - $(srcdir)/Python/frozen_zipimport.h \ - $(srcdir)/Python/frozen_hello.h + $(srcdir)/Python/frozen_modules/importlib__bootstrap.h \ + $(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h \ + $(srcdir)/Python/frozen_modules/zipimport.h \ + $(srcdir)/Python/frozen_modules/hello.h # End FROZEN_FILES Python/frozen.o: $(FROZEN_FILES) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index efc67b97bbe642..a0bedf49e69906 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -111,23 +111,23 @@ importlib._bootstrap - $(IntDir)frozen_importlib__bootstrap.g.h - $(PySourcePath)Python\frozen_importlib__bootstrap.h + $(IntDir)importlib__bootstrap.g.h + $(PySourcePath)Python\frozen_modules\importlib__bootstrap.h importlib._bootstrap_external - $(IntDir)frozen_importlib__bootstrap_external.g.h - $(PySourcePath)Python\frozen_importlib__bootstrap_external.h + $(IntDir)importlib__bootstrap_external.g.h + $(PySourcePath)Python\frozen_modules\importlib__bootstrap_external.h zipimport - $(IntDir)frozen_zipimport.g.h - $(PySourcePath)Python\frozen_zipimport.h + $(IntDir)zipimport.g.h + $(PySourcePath)Python\frozen_modules\zipimport.h hello - $(IntDir)frozen_hello.g.h - $(PySourcePath)Python\frozen_hello.h + $(IntDir)ello.g.h + $(PySourcePath)Python\frozen_modules\hello.h diff --git a/Python/frozen.c b/Python/frozen.c index db258ddef32c3e..860da6a04b1125 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -32,10 +32,10 @@ #include "Python.h" /* Includes for frozen modules: */ -#include "frozen_importlib__bootstrap.h" -#include "frozen_importlib__bootstrap_external.h" -#include "frozen_zipimport.h" -#include "frozen_hello.h" +#include "frozen_modules/importlib__bootstrap.h" +#include "frozen_modules/importlib__bootstrap_external.h" +#include "frozen_modules/zipimport.h" +#include "frozen_modules/hello.h" /* End includes */ /* Note that a negative size indicates a package. */ diff --git a/Python/frozen_hello.h b/Python/frozen_modules/hello.h similarity index 100% rename from Python/frozen_hello.h rename to Python/frozen_modules/hello.h diff --git a/Python/frozen_importlib__bootstrap.h b/Python/frozen_modules/importlib__bootstrap.h similarity index 100% rename from Python/frozen_importlib__bootstrap.h rename to Python/frozen_modules/importlib__bootstrap.h diff --git a/Python/frozen_importlib__bootstrap_external.h b/Python/frozen_modules/importlib__bootstrap_external.h similarity index 100% rename from Python/frozen_importlib__bootstrap_external.h rename to Python/frozen_modules/importlib__bootstrap_external.h diff --git a/Python/frozen_zipimport.h b/Python/frozen_modules/zipimport.h similarity index 100% rename from Python/frozen_zipimport.h rename to Python/frozen_modules/zipimport.h diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index c7fe0008c1f954..2f4a6ce9929228 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -18,7 +18,7 @@ STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') # XXX It may make more sense to put frozen modules in Modules/ or Lib/... -MODULES_DIR = os.path.join(ROOT_DIR, 'Python') +MODULES_DIR = os.path.join(ROOT_DIR, 'Python/frozen_modules') TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') @@ -149,7 +149,7 @@ def resolve_frozen_file(spec, destdir=MODULES_DIR): _, frozenid, _, _, _= spec modname = frozenid # We use a consistent naming convention for all frozen modules. - return os.path.join(destdir, 'frozen_' + modname.replace('.', '_')) + '.h' + return os.path.join(destdir, modname.replace('.', '_')) + '.h' def resolve_frozen_files(specs, destdir=MODULES_DIR): From 5cfd8538e16e847b900abda7d1f9ee22133b7fd3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 26 Aug 2021 13:10:56 -0600 Subject: [PATCH 29/41] Add a NEWS entry. --- .../NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst diff --git a/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst b/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst new file mode 100644 index 00000000000000..d11c6451462bd5 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst @@ -0,0 +1,3 @@ +Generate lines in relevant files for frozen modules. Up until now each of +the files had to be edited manually. This change makes it easier to add to +and modify the frozen modules. From 0905b51a7bc0fbcc81499976c84e46754da253aa Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 26 Aug 2021 13:40:10 -0600 Subject: [PATCH 30/41] Fix a typo. --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index cca39f5f024edc..be0c86ec84374b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -784,7 +784,7 @@ regen-limited-abi: all # Regenerate all generated files regen-all: regen-opcode regen-opcode-targets regen-typeslots \ - regen-token regen-ast regen-keyword regen-frozen-all clinic \ + regen-token regen-ast regen-keyword regen-frozen clinic \ regen-pegen-metaparser regen-pegen regen-test-frozenmain @echo @echo "Note: make regen-stdlib-module-names and autoconf should be run manually" From 55e1ffc443ca18ddfd55f884842e5d1ebde9dec6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 15:12:30 -0600 Subject: [PATCH 31/41] Fix outdated comments. --- PCbuild/readme.txt | 2 +- Programs/_freeze_module.c | 4 ++-- Python/frozen.c | 28 +++++++++++++++++----------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 83e804625aacc2..5ecded06e58893 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -116,7 +116,7 @@ _testembed These are miscellaneous sub-projects that don't really fit the other categories: _freeze_module - _freeze_module.exe, used to regenerate frozen modules in Python\ + _freeze_module.exe, used to regenerate frozen modules in Python after changes have been made to the corresponding source files (e.g. Lib\importlib\_bootstrap.py). pyshellext diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 87004da0dc6bf3..40aba0c651b4d6 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -1,8 +1,8 @@ /* This is built as a stand-alone executable by the Makefile, and helps turn modules into frozen modules (like Lib/importlib/_bootstrap.py - into Python/importlib.h) + into Python/importlib.h). - This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen-*". + This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen". See Python/frozen.c for more info. */ diff --git a/Python/frozen.c b/Python/frozen.c index 860da6a04b1125..a54f61a76616f5 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -2,25 +2,31 @@ /* Frozen modules initializer * * Frozen modules are written to header files by Programs/_freeze_module. - * These files are typically put in Python/. Each holds an array of bytes - * named "_Py_M__", which is used below. + * These files are typically put in Python/frozen_modules/. Each holds + * an array of bytes named "_Py_M__", which is used below. * - * These files must be regenerated any time * the corresponding .pyc - * file would change (e.g. compiler, bytecode format, marshal format). - * This can be done with "make regen-frozen". + * These files must be regenerated any time the corresponding .pyc + * file would change (including with changes to the compiler, bytecode + * format, marshal format). This can be done with "make regen-frozen". + * That make target normally does nothing more than run + * Programs/_freeze_module on each of the modules set to be frozen. * - * That make target simply runs Tools/scripts/freeze_modules.py, which - * does the following: + * Additional modules can be frozen by updating the list at the top of + * Tools/scripts/freeze_modules.py and then run the script + * (or run "make regen-frozen"). + * + * The script does the following (by default): * * 1. run Programs/_freeze_module on the target modules * 2. update the includes and _PyImport_FrozenModules[] in this file * 3. update the FROZEN_FILES variable in Makefile.pre.in + * 4. update the per-module targets in Makefile.pre.in + * 5. update the lists of modules in PCbuild/_freeze_module.vcxproj and + * PCbuild/_freeze_module.vcxproj.filters * - * (Note that most of the date in this file is auto-generated by the script.) + * (Note that most of the data in this file is auto-generated by the script.) * - * Additional modules can be frozen by adding them to freeze_modules.py - * and then re-running "make regen-frozen-all". This can also be done - * manually by following those steps, though this is not recommended. + * Those steps can also be done manually, though this is not recommended. * Expect such manual changes to be removed the next time * freeze_modules.py runs. * */ From b970fa3356f44413e086bd1503ba827c45af6f8f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 15:30:44 -0600 Subject: [PATCH 32/41] Avoid buffer overflows. --- Programs/_freeze_module.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 40aba0c651b4d6..7e9f02aec8a0fa 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -102,9 +102,11 @@ read_text(const char *inpath) static PyObject * compile_and_marshal(const char *name, const char *text) { - char buf[100]; - sprintf(buf, "", name); - PyObject *code = Py_CompileStringExFlags(text, buf, Py_file_input, NULL, 0); + char *filename = (char *) malloc(strlen(name) + 10); + sprintf(filename, "", name); + PyObject *code = Py_CompileStringExFlags(text, filename, + Py_file_input, NULL, 0); + free(filename); if (code == NULL) { return NULL; } @@ -119,20 +121,22 @@ compile_and_marshal(const char *name, const char *text) return marshalled; } -static void -get_varname(const char *name, char *buf) +static char * +get_varname(const char *name, const char *prefix) { - (void)strcpy(buf, "_Py_M__"); - size_t n = 7; + size_t n = strlen(prefix); + char *varname = (char *) malloc(strlen(name) + n + 1); + (void)strcpy(varname, prefix); for (size_t i = 0; name[i] != '\0'; i++) { if (name[i] == '.') { - buf[n++] = '_'; + varname[n++] = '_'; } else { - buf[n++] = name[i]; + varname[n++] = name[i]; } } - buf[n] = '\0'; + varname[n] = '\0'; + return varname; } static void @@ -165,10 +169,10 @@ write_frozen(const char *outpath, const char *inpath, const char *name, return -1; } - char buf[100]; fprintf(outfile, "%s\n", header); - get_varname(name, buf); - write_code(outfile, marshalled, buf); + char *arrayname = get_varname(name, "_Py_M__"); + write_code(outfile, marshalled, arrayname); + free(arrayname); if (ferror(outfile)) { fprintf(stderr, "error when writing to '%s'\n", outpath); From 99daab4b817bfc3306b21090d33495f2c8180427 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 15:59:43 -0600 Subject: [PATCH 33/41] Freeze the modules when running "make regen-frozen". --- Makefile.pre.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index be0c86ec84374b..c7c079b115aca8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -744,7 +744,8 @@ Tools/scripts/freeze_modules.py: Programs/_freeze_module .PHONY: regen-frozen regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES) - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py --no-freeze + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py + @echo "The Makefile was updated, you may need to re-run make." # BEGIN: freezing modules From 10a14debd4ca08087acd40dd1622533da754dd92 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 16:00:38 -0600 Subject: [PATCH 34/41] Be more verbose about what is running in freeze_modules.py. --- Tools/scripts/freeze_modules.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 2f4a6ce9929228..71073313282353 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -339,6 +339,7 @@ def regen_frozen(specs, dest=MODULES_DIR): if line: deflines[i] = indent + line + print(f'# Updating {os.path.relpath(FROZEN_FILE)}') with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): lines = infile.readlines() # TODO: Use more obvious markers, e.g. @@ -383,6 +384,7 @@ def regen_makefile(frozenids, frozen): frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") + print(f'# Updating {os.path.relpath(MAKEFILE)}') with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): lines = infile.readlines() lines = replace_block( @@ -421,6 +423,7 @@ def regen_pcbuild(frozenids, frozen): filterlines.append(' Python Files') filterlines.append(' ') + print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): lines = infile.readlines() lines = replace_block( @@ -431,6 +434,7 @@ def regen_pcbuild(frozenids, frozen): PCBUILD_PROJECT, ) outfile.writelines(lines) + print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() lines = replace_block( @@ -457,8 +461,13 @@ def _freeze_module(frozenid, pyfile, frozenfile): tmpfile = frozenfile + '.new' argv = [TOOL, frozenid, pyfile, tmpfile] - print('#', ' '.join(argv)) - subprocess.run(argv, check=True) + print('#', ' '.join(os.path.relpath(a) for a in argv)) + try: + subprocess.run(argv, check=True) + except subprocess.CalledProcessError: + if not os.path.exists(TOOL): + sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"') + raise # re-raise os.replace(tmpfile, frozenfile) From 26519626199e5258d9228e3f89fdd94fff129ab6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 16:01:15 -0600 Subject: [PATCH 35/41] Update the explanation in frozen.c. --- Python/frozen.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/frozen.c b/Python/frozen.c index a54f61a76616f5..67aff2ed2eba14 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -8,14 +8,14 @@ * These files must be regenerated any time the corresponding .pyc * file would change (including with changes to the compiler, bytecode * format, marshal format). This can be done with "make regen-frozen". - * That make target normally does nothing more than run - * Programs/_freeze_module on each of the modules set to be frozen. + * That make target just runs Tools/scripts/freeze_modules.py. * - * Additional modules can be frozen by updating the list at the top of - * Tools/scripts/freeze_modules.py and then run the script + * The freeze_modules.py script also determines which modules get + * frozen. Update the list at the top of the script to add, remove, + * or modify the target modules. Then run the script * (or run "make regen-frozen"). * - * The script does the following (by default): + * The script does the following: * * 1. run Programs/_freeze_module on the target modules * 2. update the includes and _PyImport_FrozenModules[] in this file From 7406c58befae63ceac4ef59fab623e649d0b3486 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 16:10:39 -0600 Subject: [PATCH 36/41] Mark all frozen modules as "generated". --- .gitattributes | 3 +-- Tools/scripts/freeze_modules.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitattributes b/.gitattributes index 68566e899249f6..b9c08cdd7d65a7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -46,8 +46,7 @@ Modules/clinic/*.h linguist-generated=true Objects/clinic/*.h linguist-generated=true PC/clinic/*.h linguist-generated=true Python/clinic/*.h linguist-generated=true -Python/importlib.h linguist-generated=true -Python/importlib_external.h linguist-generated=true +Python/frozen_modules/*.h linguist-generated=true Include/internal/pycore_ast.h linguist-generated=true Python/Python-ast.c linguist-generated=true Include/opcode.h linguist-generated=true diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 71073313282353..0ac9e677156d49 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -17,7 +17,7 @@ ROOT_DIR = os.path.dirname(TOOLS_DIR) STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') -# XXX It may make more sense to put frozen modules in Modules/ or Lib/... +# If MODULES_DIR is changed then the .gitattributes file needs to be updated. MODULES_DIR = os.path.join(ROOT_DIR, 'Python/frozen_modules') TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module') From f4c136e6d0780b1ac71276a4defb808493be6eb8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 16:12:11 -0600 Subject: [PATCH 37/41] Drop a superfluous comment. --- Tools/scripts/freeze_modules.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 0ac9e677156d49..f90f1c1d76cc49 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -367,8 +367,6 @@ def regen_makefile(frozenids, frozen): for frozenid in frozenids: pyfile, frozenfile = frozen[frozenid] header = os.path.relpath(frozenfile, ROOT_DIR) - # Adding a comment to separate sections here doesn't add much, - # so we don't. relfile = header.replace('\\', '/') frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') From 2730169049f408e859b4349e537e6b4beea846bd Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 16:15:45 -0600 Subject: [PATCH 38/41] Drop the CLI. --- Tools/scripts/freeze_modules.py | 38 ++++++++++++--------------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index f90f1c1d76cc49..4f60e1b9a3a8ba 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -473,34 +473,24 @@ def _freeze_module(frozenid, pyfile, frozenfile): ####################################### # the script -def parse_args(argv=sys.argv[1:], prog=sys.argv[0]): - import argparse - parser = argparse.ArgumentParser(prog=prog) - parser.add_argument('--no-regen', dest='regen', action='store_false') - parser.add_argument('--no-freeze', dest='freeze', action='store_false') - - args = parser.parse_args(argv) - ns = vars(args) - - return ns - - -def main(*, regen=True, freeze=True): +def main(): # Expand the raw specs, preserving order. specs = list(parse_frozen_specs()) frozen, frozenids = resolve_frozen_files(specs, MODULES_DIR) - # (We use a consistent order: that of FROZEN above.) - if regen: - regen_frozen(specs, (frozenids, frozen)) - regen_makefile(frozenids, frozen) - regen_pcbuild(frozenids, frozen) - if freeze: - for frozenid in frozenids: - pyfile, frozenfile = frozen[frozenid] - _freeze_module(frozenid, pyfile, frozenfile) + # Regen build-related files. + regen_frozen(specs, (frozenids, frozen)) + regen_makefile(frozenids, frozen) + regen_pcbuild(frozenids, frozen) + + # Freeze the target modules. + for frozenid in frozenids: + pyfile, frozenfile = frozen[frozenid] + _freeze_module(frozenid, pyfile, frozenfile) if __name__ == '__main__': - kwargs = parse_args() - main(**kwargs) + argv = sys.argv[1:] + if argv: + sys.exit('ERROR: got unexpected args {argv}') + main() From d84fd8551a85b4042f5374bff7c2f2a9f1b0911c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 16:32:06 -0600 Subject: [PATCH 39/41] Use update_file_with_tmpfile() in updating_file_with_tmpfile(). --- Tools/scripts/update_file.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Tools/scripts/update_file.py b/Tools/scripts/update_file.py index ec4cfa354086c2..9d160f2fa071ae 100644 --- a/Tools/scripts/update_file.py +++ b/Tools/scripts/update_file.py @@ -28,14 +28,16 @@ def updating_file_with_tmpfile(filename, tmpfile=None): elif os.path.isdir(tmpfile): tmpfile = os.path.join(tmpfile, filename + '.tmp') + outfile = open(tmpfile, 'w') + infile = open(filename) try: - with open(tmpfile, 'w') as outfile: - with open(filename) as infile: - yield infile, outfile - os.replace(tmpfile, filename) + yield infile, outfile finally: - if os.path.exists(tmpfile): - os.remove(tmpfile) + try: + outfile.close() + finally: + infile.close() + update_file_with_tmpfile(filename, tmpfile) def update_file_with_tmpfile(filename, tmpfile): From cd8fe0aa222914eb3a5aa1dbeff369a57aafa6ef Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 16:34:30 -0600 Subject: [PATCH 40/41] Do not update the file if there was an error. --- Tools/scripts/update_file.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Tools/scripts/update_file.py b/Tools/scripts/update_file.py index 9d160f2fa071ae..cfc4e2b1ab12a3 100644 --- a/Tools/scripts/update_file.py +++ b/Tools/scripts/update_file.py @@ -28,16 +28,10 @@ def updating_file_with_tmpfile(filename, tmpfile=None): elif os.path.isdir(tmpfile): tmpfile = os.path.join(tmpfile, filename + '.tmp') - outfile = open(tmpfile, 'w') - infile = open(filename) - try: - yield infile, outfile - finally: - try: - outfile.close() - finally: - infile.close() - update_file_with_tmpfile(filename, tmpfile) + with open(tmpfile, 'w') as outfile: + with open(filename) as infile: + yield infile, outfile + update_file_with_tmpfile(filename, tmpfile) def update_file_with_tmpfile(filename, tmpfile): From 9a97726a13fd7705479f1df8b203d958dda76041 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 30 Aug 2021 16:46:24 -0600 Subject: [PATCH 41/41] Fix the check_generated_files CI job. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4081b36f926a2a..ff026b9e4f9d87 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,6 +71,7 @@ jobs: make regen-stdlib-module-names - name: Check for changes run: | + git add -u changes=$(git status --porcelain) # Check for changes in regenerated files if ! test -z "$changes"