From b9aeb528ef3bf0a61ac46c28e6e6a81b28d6c78a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 29 Nov 2024 22:57:06 +0000 Subject: [PATCH 01/12] GH-127429: fix sysconfig data generation on cross-builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index ee52700b51fd07..2bcf205c3817b3 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -318,14 +318,22 @@ def get_default_scheme(): def get_makefile_filename(): """Return the path of the Makefile.""" + + # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python. + if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'): + return os.path.join(cross_base, 'Makefile') + if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") + if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: config_dir_name = 'config' + if hasattr(sys.implementation, '_multiarch'): config_dir_name += f'-{sys.implementation._multiarch}' + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') From 6d6c5f3e24f0a7b67e0d1fa76fa3492e8303626b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 29 Nov 2024 23:02:44 +0000 Subject: [PATCH 02/12] Add news MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .../Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst diff --git a/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst new file mode 100644 index 00000000000000..afd3bf3fe991f7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst @@ -0,0 +1,3 @@ +Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being +generated with the host Python's `Makefile`. The data is now generated from +current build's `Makefile`. From 8166e9a38f0f7200b0d5525c1808b7c7416590a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 29 Nov 2024 23:05:28 +0000 Subject: [PATCH 03/12] Fix news' rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .../Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst index afd3bf3fe991f7..708c1a6437d812 100644 --- a/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst +++ b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst @@ -1,3 +1,3 @@ Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being -generated with the host Python's `Makefile`. The data is now generated from -current build's `Makefile`. +generated with the host Python's ``Makefile``. The data is now generated from +current build's ``Makefile``. From d10a36942339234adde87d0a2fdf8a48fc5c607e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 29 Nov 2024 23:06:13 +0000 Subject: [PATCH 04/12] Remove whitespace in .github/workflows/jit.yml to trigger the JIT tests :) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 7dbbe71b2131e7..4ef543d7369734 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -104,7 +104,7 @@ jobs: # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - name: Native macOS if: runner.os == 'macOS' From 53dbc14432ebbd1b2fd4a454d95e4c31ef1bbeb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 30 Nov 2024 00:25:12 +0000 Subject: [PATCH 05/12] Load the prefixes for sysconfig data from the Makefile when cross-compiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 2bcf205c3817b3..5b62584d2b45bc 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -472,18 +472,38 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} + + if os.name == 'nt': + _init_non_posix(_CONFIG_VARS) + _CONFIG_VARS['VPATH'] = sys._vpath + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + + # If we are cross-compiling, load the prefixes from the Makefile instead. + # (only available on POSIX) + if '_PYTHON_PROJECT_BASE' in os.environ and os.name == 'posix': + prefix = _CONFIG_VARS['prefix'] + exec_prefix = _CONFIG_VARS['exec_prefix'] + base_prefix = _CONFIG_VARS['prefix'] + base_exec_prefix = _CONFIG_VARS['exec_prefix'] + else: + prefix = _PREFIX + exec_prefix = _EXEC_PREFIX + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX + _CONFIG_VARS['prefix'] = prefix + _CONFIG_VARS['exec_prefix'] = exec_prefix _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['installed_base'] = base_prefix + _CONFIG_VARS['base'] = prefix + _CONFIG_VARS['installed_platbase'] = base_exec_prefix + _CONFIG_VARS['platbase'] = exec_prefix _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir _CONFIG_VARS['implementation'] = _get_implementation() @@ -498,11 +518,6 @@ def _init_config_vars(): except AttributeError: _CONFIG_VARS['py_version_nodot_plat'] = '' - if os.name == 'nt': - _init_non_posix(_CONFIG_VARS) - _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in From dc3bf73619151b6d9913724d595ed0cfe75a0e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 30 Nov 2024 00:33:07 +0000 Subject: [PATCH 06/12] Keep _init_non_posix after the main initialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 5b62584d2b45bc..7786896433ec27 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -473,9 +473,6 @@ def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} - if os.name == 'nt': - _init_non_posix(_CONFIG_VARS) - _CONFIG_VARS['VPATH'] = sys._vpath if os.name == 'posix': _init_posix(_CONFIG_VARS) @@ -518,6 +515,9 @@ def _init_config_vars(): except AttributeError: _CONFIG_VARS['py_version_nodot_plat'] = '' + if os.name == 'nt': + _init_non_posix(_CONFIG_VARS) + _CONFIG_VARS['VPATH'] = sys._vpath if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in From 02697aaea800a3dbbab63d79f5d2eeda235c58ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 30 Nov 2024 00:34:43 +0000 Subject: [PATCH 07/12] Minor code reformating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 7786896433ec27..caeb0edeeb009d 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -473,21 +473,19 @@ def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} + prefix = _PREFIX + exec_prefix = _EXEC_PREFIX + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + if os.name == 'posix': _init_posix(_CONFIG_VARS) - - # If we are cross-compiling, load the prefixes from the Makefile instead. - # (only available on POSIX) - if '_PYTHON_PROJECT_BASE' in os.environ and os.name == 'posix': - prefix = _CONFIG_VARS['prefix'] - exec_prefix = _CONFIG_VARS['exec_prefix'] - base_prefix = _CONFIG_VARS['prefix'] - base_exec_prefix = _CONFIG_VARS['exec_prefix'] - else: - prefix = _PREFIX - exec_prefix = _EXEC_PREFIX - base_prefix = _BASE_PREFIX - base_exec_prefix = _BASE_EXEC_PREFIX + # If we are cross-compiling, load the prefixes from the Makefile instead. + if '_PYTHON_PROJECT_BASE' in os.environ: + prefix = _CONFIG_VARS['prefix'] + exec_prefix = _CONFIG_VARS['exec_prefix'] + base_prefix = _CONFIG_VARS['prefix'] + base_exec_prefix = _CONFIG_VARS['exec_prefix'] # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the From c18ed8c9c9a15cf3a4c975aedf4f6bdfe873a91a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 30 Nov 2024 02:06:10 +0000 Subject: [PATCH 08/12] Print generated file names on --generate-posix-vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 5660a6c5105b9f..12f6ba8af3e97b 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -229,11 +229,15 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + print(f'Written {destfile}') + # Write a JSON file with the output of sysconfig.get_config_vars jsonfile = os.path.join(pybuilddir, _get_json_data_name()) with open(jsonfile, 'w') as f: json.dump(get_config_vars(), f, indent=2) + print(f'Written {jsonfile}') + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) From ad995af5dd837c89766a3b32d63d7d04bce1d1f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 30 Nov 2024 02:15:18 +0000 Subject: [PATCH 09/12] Fix pybuilddir on debug cross-builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 12f6ba8af3e97b..10728c709e1811 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -7,6 +7,7 @@ _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, + get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -161,7 +162,7 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): + if get_config_var('Py_DEBUG') == '1': pybuilddir += '-pydebug' return pybuilddir From 27a4cd1940060f7269ac2894d21f2a84e763130a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 30 Nov 2024 02:49:25 +0000 Subject: [PATCH 10/12] Disable caching in HOSTRUNNER MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 8fa6cb60900ad1..4cfced10432491 100644 --- a/configure.ac +++ b/configure.ac @@ -1609,8 +1609,8 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform AC_MSG_CHECKING([HOSTRUNNER]) -AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then AS_CASE([$ac_sys_system], From cc4f57a74d02948c417a1dea1267d356abc08b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 30 Nov 2024 02:52:21 +0000 Subject: [PATCH 11/12] Update autotools files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- configure | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/configure b/configure index 84b74ac3584bcd..4e4043260ed2df 100755 --- a/configure +++ b/configure @@ -944,8 +944,8 @@ AR LINK_PYTHON_OBJS LINK_PYTHON_DEPS LIBRARY_DEPS -NODE HOSTRUNNER +NODE STATIC_LIBPYTHON GNULD EXPORTSFROM @@ -1147,7 +1147,6 @@ LDFLAGS LIBS CPPFLAGS CPP -HOSTRUNNER PROFILE_TASK BOLT_INSTRUMENT_FLAGS BOLT_APPLY_FLAGS @@ -1968,7 +1967,6 @@ Some influential environment variables: CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor - HOSTRUNNER Program to run CPython for the host platform PROFILE_TASK Python args for PGO generation task BOLT_INSTRUMENT_FLAGS @@ -7622,9 +7620,9 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 printf %s "checking HOSTRUNNER... " >&6; } - if test -z "$HOSTRUNNER" then case $ac_sys_system in #( From 0184295f4a6bd7f22bc52b0ec21b27d8a3661855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sun, 1 Dec 2024 07:07:13 +0000 Subject: [PATCH 12/12] Fix get_config_var('abiflags') in cross-builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index caeb0edeeb009d..ad86609016e478 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -478,6 +478,11 @@ def _init_config_vars(): base_prefix = _BASE_PREFIX base_exec_prefix = _BASE_EXEC_PREFIX + try: + abiflags = sys.abiflags + except AttributeError: + abiflags = '' + if os.name == 'posix': _init_posix(_CONFIG_VARS) # If we are cross-compiling, load the prefixes from the Makefile instead. @@ -486,6 +491,7 @@ def _init_config_vars(): exec_prefix = _CONFIG_VARS['exec_prefix'] base_prefix = _CONFIG_VARS['prefix'] base_exec_prefix = _CONFIG_VARS['exec_prefix'] + abiflags = _CONFIG_VARS['ABIFLAGS'] # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the @@ -503,11 +509,7 @@ def _init_config_vars(): _CONFIG_VARS['platlibdir'] = sys.platlibdir _CONFIG_VARS['implementation'] = _get_implementation() _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' + _CONFIG_VARS['abiflags'] = abiflags try: _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: