From e6ed0df7ab09ea16d8ec5c1c3b29e7a69e121837 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Jan 2021 11:17:41 +0100 Subject: [PATCH 1/2] bpo-42856: Add --with-wheel-pkg-dir=PATH configure option Add --with-wheel-pkg-dir=PATH option to ./configure. If specified, the :mod:`ensurepip` module also looks for wheel packages in this directory, and picks the most recent versions in ensurepip._bundled and the specified directory. Some Linux distribution packaging policies recommand against bundling dependencies. For example, Fedora installs wheel packages in the /usr/share/python-wheels/ directory and don't install the ensurepip._bundled package. --- Doc/whatsnew/3.10.rst | 12 ++ Lib/ensurepip/__init__.py | 125 ++++++++++++++---- Lib/test/test_ensurepip.py | 85 ++++++++++-- Makefile.pre.in | 2 + .../2021-01-07-12-51-38.bpo-42856.n3cMHV.rst | 9 ++ configure | 28 ++++ configure.ac | 16 +++ 7 files changed, 245 insertions(+), 32 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index e615574fb2fab5..72496d5cc74bac 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -599,6 +599,18 @@ Build Changes don't build nor install test modules. (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) +* Add ``--with-wheel-pkg-dir=PATH`` option to ./configure. If specified, the + :mod:`ensurepip` module also looks for wheel packages in this directory, and + picks the most recent versions in :mod:`ensurepip._bundled` and the specified + directory. + + Some Linux distribution packaging policies recommand against bundling + dependencies. For example, Fedora installs wheel packages in the + ``/usr/share/python-wheels/`` directory and don't install the + ``ensurepip._bundled`` package. + + (Contributed by Victor Stinner in :issue:`42856`.) + C API Changes ============= diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index cb2882e3360fcf..cf0dd9826fb246 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,26 +1,98 @@ -import os +import collections import os.path -import sys +import re import runpy -import tempfile import subprocess +import sys +import sysconfig +import tempfile from importlib import resources -from . import _bundled - __all__ = ["version", "bootstrap"] +_PACKAGE_NAMES = ('setuptools', 'pip') +_Package = collections.namedtuple('Package', ('version', 'filename')) + +# Directory of system wheel packages. Some Linux distribution packaging +# policies recommand against bundling dependencies. For example, Fedora +# installs wheel packages in the /usr/share/python-wheels/ directory and don't +# install the ensurepip._bundled package. +_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') + + +def _get_versions(path): + versions = {} + try: + names = os.listdir(path) + except OSError: + # Ignore: path doesn't exist or permission error + names = () + for name in names: + # name is like 'pip-20.2.3-py2.py3-none-any.whl' + if not name.endswith(".whl"): + continue + for package in _PACKAGE_NAMES: + prefix = package + '-' + if not name.startswith(prefix): + continue + part = name.removeprefix(prefix) + break + else: + continue + + # Extract '20.2.3' from '20.2.3-py2.py3-none-any.whl' + version = [] + part = part.split('-', 1)[0] + for part in part.split('.'): + try: + number = int(part) + except ValueError: + break + version.append(number) + if not version: + # failed to parse the version: ignore the package + continue + + fullname = os.path.join(path, name) + versions[package] = _Package(tuple(version), fullname) + return versions + + +def _get_package_search_paths(): + # last item has the highest priority + paths = [] + if _WHEEL_PKG_DIR: + paths.append(_WHEEL_PKG_DIR) + try: + from . import _bundled + except ImportError: + pass + else: + paths.extend(reversed(_bundled.__path__)) + return paths -_SETUPTOOLS_VERSION = "47.1.0" +def _find_packages(paths): + # package => (version, filename): (None, None) if no package is found + packages = {name: _Package(None, None) for name in _PACKAGE_NAMES} + for path in paths: + versions = _get_versions(path) + for name, item in versions.items(): + if (packages[name].version is None + or packages[name][0] < item.version): + packages[name] = item -_PIP_VERSION = "20.2.3" + return packages -_PROJECTS = [ - ("setuptools", _SETUPTOOLS_VERSION, "py3"), - ("pip", _PIP_VERSION, "py2.py3"), -] + +def _get_packages(): + global _PACKAGES + if _PACKAGES is None: + paths = _get_package_search_paths() + _PACKAGES = _find_packages(paths) + return _PACKAGES +_PACKAGES = None def _run_pip(args, additional_paths=None): @@ -42,7 +114,10 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _PIP_VERSION + version = _get_packages()['pip'].version + if version is None: + return None + return '.'.join(map(str, version)) def _disable_pip_configuration_settings(): # We deliberately ignore all pip environment variables @@ -104,12 +179,14 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] - for project, version, py_tag in _PROJECTS: - wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) - whl = resources.read_binary( - _bundled, - wheel_name, - ) + for name, package in _get_packages().items(): + if package.filename is None: + raise ValueError(f"cannot find {name} wheel package") + + with open(package.filename, "rb") as fp: + whl = fp.read() + + wheel_name = os.path.basename(package.filename) with open(os.path.join(tmpdir, wheel_name), "wb") as fp: fp.write(whl) @@ -126,7 +203,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -140,10 +217,10 @@ def _uninstall_helper(*, verbosity=0): return # If the pip version doesn't match the bundled one, leave it alone - if pip.__version__ != _PIP_VERSION: - msg = ("ensurepip will only uninstall a matching version " - "({!r} installed, {!r} bundled)") - print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + if pip.__version__ != version(): + print(f"ensurepip will only uninstall a matching version " + f"({pip.__version__!r} installed, {version()!r} bundled)", + file=sys.stderr) return _disable_pip_configuration_settings() @@ -153,7 +230,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + return _run_pip([*args, reversed(_PACKAGE_NAMES)]) def _main(argv=None): diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 4786d28f39a3d0..113474c23beaa9 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -5,19 +5,87 @@ import os.path import contextlib import sys +import tempfile import ensurepip import ensurepip._uninstall -class TestEnsurePipVersion(unittest.TestCase): +def clear_caches(): + ensurepip._PACKAGES = None + + +class TestPackages(unittest.TestCase): + def setUp(self): + clear_caches() + + def tearDown(self): + clear_caches() + + def touch(self, directory, filename): + fullname = os.path.join(directory, filename) + open(fullname, "wb").close() + + def test_version(self): + # Test version() + with tempfile.TemporaryDirectory() as tmpdir: + self.touch(tmpdir, "pip-20.2.2-py2.py3-none-any.whl") + with unittest.mock.patch.object(ensurepip, + '_get_package_search_paths', + return_value=[tmpdir]): + self.assertEqual(ensurepip.version(), '20.2.2') + + def test_find_packages_no_dir(self): + # Test _find_packages() with no directories + with tempfile.TemporaryDirectory() as tmpdir: + packages = ensurepip._find_packages([tmpdir]) + self.assertIsNone(packages['pip'].version) + self.assertIsNone(packages['setuptools'].version) + + def test_find_packages_one_dir(self): + # Test _find_packages() with one directory + with tempfile.TemporaryDirectory() as tmpdir: + self.touch(tmpdir, "setuptools-49.1.3-py3-none-any.whl") + self.touch(tmpdir, "pip-20.2.2-py2.py3-none-any.whl") + # not used, make sure that it's ignored + self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") + + packages = ensurepip._find_packages([tmpdir]) + + self.assertEqual(packages['pip'].version, (20, 2, 2)) + self.assertEqual(packages['setuptools'].version, (49, 1, 3)) + self.assertNotIn('wheel', packages) + + with unittest.mock.patch.object(ensurepip, + '_get_package_search_paths', + return_value=[tmpdir]): + self.assertEqual(ensurepip.version(), '20.2.2') + + def test_find_packages_two_dirs(self): + # Test _find_packages() with two directories + with tempfile.TemporaryDirectory() as tmpdir1: + self.touch(tmpdir1, "setuptools-49.1.3-py3-none-any.whl") + self.touch(tmpdir1, "pip-20.2.2-py2.py3-none-any.whl") + with tempfile.TemporaryDirectory() as tmpdir2: + self.touch(tmpdir2, "setuptools-47.1.0-py3-none-any.whl") + self.touch(tmpdir2, "pip-20.2.3-py2.py3-none-any.whl") + + packages = ensurepip._find_packages([tmpdir1, tmpdir2]) + + self.assertEqual(packages['pip'].version, (20, 2, 3)) + self.assertEqual(packages['pip'].filename, + os.path.join(tmpdir2, "pip-20.2.3-py2.py3-none-any.whl")) + self.assertEqual(packages['setuptools'].version, (49, 1, 3)) + self.assertEqual(packages['setuptools'].filename, + os.path.join(tmpdir1, "setuptools-49.1.3-py3-none-any.whl")) + self.assertNotIn('wheel', packages) - def test_returns_version(self): - self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) class EnsurepipMixin: def setUp(self): + clear_caches() + run_pip_patch = unittest.mock.patch("ensurepip._run_pip") self.run_pip = run_pip_patch.start() self.run_pip.return_value = 0 @@ -32,6 +100,9 @@ def setUp(self): patched_os.path = os.path self.os_environ = patched_os.environ = os.environ.copy() + def tearDown(self): + clear_caches() + class TestBootstrap(EnsurepipMixin, unittest.TestCase): @@ -147,7 +218,7 @@ def test_pip_config_file_disabled(self): self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull) @contextlib.contextmanager -def fake_pip(version=ensurepip._PIP_VERSION): +def fake_pip(version=ensurepip.version()): if version is None: pip = None else: @@ -243,8 +314,6 @@ def test_pip_config_file_disabled(self): # Basic testing of the main functions and their argument parsing -EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION - class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): def test_bootstrap_version(self): @@ -252,7 +321,7 @@ def test_bootstrap_version(self): with self.assertRaises(SystemExit): ensurepip._main(["--version"]) result = stdout.getvalue().strip() - self.assertEqual(result, EXPECTED_VERSION_OUTPUT) + self.assertEqual(result, "pip " + ensurepip.version()) self.assertFalse(self.run_pip.called) def test_basic_bootstrapping(self): @@ -283,7 +352,7 @@ def test_uninstall_version(self): with self.assertRaises(SystemExit): ensurepip._uninstall._main(["--version"]) result = stdout.getvalue().strip() - self.assertEqual(result, EXPECTED_VERSION_OUTPUT) + self.assertEqual(result, "pip " + ensurepip.version()) self.assertFalse(self.run_pip.called) def test_basic_uninstall(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index d8b9e8498d51bd..0ced710c0259aa 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -146,6 +146,8 @@ CONFINCLUDEDIR= $(exec_prefix)/include PLATLIBDIR= @PLATLIBDIR@ SCRIPTDIR= $(prefix)/$(PLATLIBDIR) ABIFLAGS= @ABIFLAGS@ +# Variable used by ensurepip +WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ # Detailed destination directories BINLIBDEST= @BINLIBDEST@ diff --git a/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst b/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst new file mode 100644 index 00000000000000..e5f5ebdec02ab1 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst @@ -0,0 +1,9 @@ +Add ``--with-wheel-pkg-dir=PATH`` option to ./configure. If specified, the +:mod:`ensurepip` module also looks for wheel packages in this directory, and +picks the most recent versions in :mod:`ensurepip._bundled` and the +specified directory. + +Some Linux distribution packaging policies recommand against bundling +dependencies. For example, Fedora installs wheel packages in the +``/usr/share/python-wheels/`` directory and don't install the +``ensurepip._bundled`` package. diff --git a/configure b/configure index 5691c27cf66feb..4a942729b3c3e8 100755 --- a/configure +++ b/configure @@ -630,6 +630,7 @@ OPENSSL_INCLUDES ENSUREPIP SRCDIRS THREADHEADERS +WHEEL_PKG_DIR LIBPL PY_ENABLE_SHARED PLATLIBDIR @@ -847,6 +848,7 @@ with_libm with_libc enable_big_digits with_platlibdir +with_wheel_pkg_dir with_computed_gotos with_ensurepip with_openssl @@ -1576,6 +1578,9 @@ Optional Packages: system-dependent) --with-platlibdir=DIRNAME Python library directory name (default is "lib") + --with-wheel-pkg-dir=PATH + Directory of wheel packages used by ensurepip + (default: none) --with-computed-gotos enable computed gotos in evaluation loop (enabled by default on supported compilers) --with-ensurepip[=install|upgrade|no] @@ -15493,6 +15498,29 @@ else fi +# Check for --with-wheel-pkg-dir=PATH + +WHEEL_PKG_DIR="" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-wheel-pkg-dir" >&5 +$as_echo_n "checking for --with-wheel-pkg-dir... " >&6; } + +# Check whether --with-wheel-pkg-dir was given. +if test "${with_wheel_pkg_dir+set}" = set; then : + withval=$with_wheel_pkg_dir; +if test -n "$withval"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + WHEEL_PKG_DIR="$withval" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + # Check whether right shifting a negative integer extends the sign bit # or fills with zeros (like the Cray J90, according to Tim Peters). { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether right shift extends the sign bit" >&5 diff --git a/configure.ac b/configure.ac index 990d6bfdd81b72..b0fb016698e0ed 100644 --- a/configure.ac +++ b/configure.ac @@ -4838,6 +4838,22 @@ else fi AC_SUBST(LIBPL) +# Check for --with-wheel-pkg-dir=PATH +AC_SUBST(WHEEL_PKG_DIR) +WHEEL_PKG_DIR="" +AC_MSG_CHECKING(for --with-wheel-pkg-dir) +AC_ARG_WITH(wheel-pkg-dir, + AS_HELP_STRING([--with-wheel-pkg-dir=PATH], + [Directory of wheel packages used by ensurepip (default: none)]), +[ +if test -n "$withval"; then + AC_MSG_RESULT(yes) + WHEEL_PKG_DIR="$withval" +else + AC_MSG_RESULT(no) +fi], +[AC_MSG_RESULT(no)]) + # Check whether right shifting a negative integer extends the sign bit # or fills with zeros (like the Cray J90, according to Tim Peters). AC_MSG_CHECKING(whether right shift extends the sign bit) From e2d67662b3309f9207330ec0ef24fcc487f3bcf3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Jan 2021 12:55:23 +0100 Subject: [PATCH 2/2] Remove unused imports --- Lib/ensurepip/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index cf0dd9826fb246..18b3f4abd3ce49 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,12 +1,9 @@ import collections import os.path -import re -import runpy import subprocess import sys import sysconfig import tempfile -from importlib import resources