From 180e5cc5e52663aa59b0ddbc2ca7d422a15bada8 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 7 Jun 2022 20:02:49 +0200 Subject: [PATCH 1/2] gh-93584: Make all install+tests targets depends on all All install targets use the "all" target as synchronization point to prevent race conditions with PGO builds. PGO builds use recursive make, which can lead to two parallel `./python setup.py build` processes that step on each others toes. "test" targets now correctly compile PGO build in a clean repo. --- Makefile.pre.in | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a178dd0a65c734..f3c70b99c9e221 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -580,8 +580,8 @@ LIBEXPAT_HEADERS= \ # Default target all: @DEF_MAKE_ALL_RULE@ -build_all: check-clean-src $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks \ - Programs/_testembed python-config +build_all: check-clean-src $(BUILDPYTHON) platform oldsharedmods sharedmods \ + gdbhooks Programs/_testembed python-config build_wasm: check-clean-src $(BUILDPYTHON) platform oldsharedmods python-config # Check that the source is clean when building out of source. @@ -1663,7 +1663,7 @@ cleantest: all # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -test: @DEF_MAKE_RULE@ platform +test: all $(TESTRUNNER) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. @@ -1673,7 +1673,7 @@ test: @DEF_MAKE_RULE@ platform # the bytecode read from a .pyc file had the bug, sometimes the directly # generated bytecode. This is sometimes a very shy bug needing a lot of # sample data. -testall: @DEF_MAKE_RULE@ platform +testall: all -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f @@ -1682,7 +1682,7 @@ testall: @DEF_MAKE_RULE@ platform # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. -testuniversal: @DEF_MAKE_RULE@ platform +testuniversal: all @if [ `arch` != 'i386' ]; then \ echo "This can only be used on OSX/i386" ;\ exit 1 ;\ @@ -1693,7 +1693,7 @@ testuniversal: @DEF_MAKE_RULE@ platform # Like testall, but with only one pass and without multiple processes. # Run an optional script to include information about the build environment. -buildbottest: all platform +buildbottest: all -@if which pybuildbot.identify >/dev/null 2>&1; then \ pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \ fi @@ -1708,7 +1708,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multiprocessing_forkserver \ test_mailbox test_nntplib test_socket test_poll \ test_select test_zipfile test_concurrent_futures -quicktest: @DEF_MAKE_RULE@ platform +quicktest: all $(TESTRUNNER) $(QUICKTESTOPTS) # SSL tests @@ -1719,6 +1719,10 @@ multisslcompile: all multissltest: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/ssl/multissltests.py +# All install targets use the "all" target as synchronization point to +# prevent race conditions with PGO builds. PGO builds use recursive make, +# which can lead to two parallel `./python setup.py build` processes that +# step on each others toes. install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKINSTALLLAST@ if test "x$(ENSUREPIP)" != "xno" ; then \ case $(ENSUREPIP) in \ @@ -1747,7 +1751,7 @@ commoninstall: check-clean-src @FRAMEWORKALTINSTALLFIRST@ \ # Install shared libraries enabled by Setup DESTDIRS= $(exec_prefix) $(LIBDIR) $(BINLIBDEST) $(DESTSHARED) -oldsharedinstall: $(DESTSHARED) $(SHAREDMODS) +oldsharedinstall: $(DESTSHARED) all @for i in X $(SHAREDMODS); do \ if test $$i != X; then \ echo $(INSTALL_SHARED) $$i $(DESTSHARED)/`basename $$i`; \ @@ -2153,7 +2157,7 @@ LIBPL= @LIBPL@ # pkgconfig directory LIBPC= $(LIBDIR)/pkgconfig -libainstall: @DEF_MAKE_RULE@ python-config +libainstall: all python-config @for i in $(LIBDIR) $(LIBPL) $(LIBPC) $(BINDIR); \ do \ if test ! -d $(DESTDIR)$$i; then \ @@ -2207,7 +2211,7 @@ libainstall: @DEF_MAKE_RULE@ python-config # Install the dynamically loadable modules # This goes into $(exec_prefix) -sharedinstall: sharedmods +sharedinstall: all $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \ --prefix=$(prefix) \ --install-scripts=$(BINDIR) \ @@ -2437,7 +2441,7 @@ distclean: clobber -exec rm -f {} ';' # Check that all symbols exported by libpython start with "Py" or "_Py" -smelly: @DEF_MAKE_RULE@ +smelly: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/smelly.py # Find files with funny names @@ -2472,7 +2476,7 @@ funny: -o -print # Perform some verification checks on any modified files. -patchcheck: @DEF_MAKE_RULE@ +patchcheck: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py check-limited-abi: all From 35e5b2aee98e776322d91bad7902fd92a619877b Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 8 Jun 2022 14:28:58 +0200 Subject: [PATCH 2/2] blurb --- .../next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst diff --git a/Misc/NEWS.d/next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst b/Misc/NEWS.d/next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst new file mode 100644 index 00000000000000..07ca5fad592b2d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst @@ -0,0 +1,2 @@ +Address race condition in ``Makefile`` when installing a PGO build. All +``test`` and ``install`` targets now depend on ``all`` target.