diff --git a/utils/build-script b/utils/build-script index 28fdb83235070..8aa752a4264a7 100755 --- a/utils/build-script +++ b/utils/build-script @@ -18,8 +18,11 @@ import shutil import sys import textwrap +# FIXME: Instead of modifying the system path in order to enable imports from +# other directories, all Python modules related to the build script +# should be moved to the `swift_build_support` module. +# For additional information, see: https://bugs.swift.org/browse/SR-237. sys.path.append(os.path.dirname(__file__)) - from SwiftBuildSupport import ( HOME, SWIFT_BUILD_ROOT, @@ -31,6 +34,11 @@ from SwiftBuildSupport import ( quote_shell_command, ) +sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support')) +import swift_build_support.clang +from swift_build_support.migration import migrate_impl_args + + # Main entry point for the preset mode. def main_preset(): parser = argparse.ArgumentParser( @@ -495,6 +503,10 @@ the number of parallel build jobs to use""", dest="build_jobs", default=multiprocessing.cpu_count()) + parser.add_argument("--darwin-xcrun-toolchain", + help="the name of the toolchain to use on Darwin", + default="default") + parser.add_argument("--extra-swift-args", help=textwrap.dedent(""" Pass through extra flags to swift in the form of a cmake list 'module_regexp;flag'. Can be called multiple times to add multiple such module_regexp flag pairs. All semicolons @@ -505,7 +517,8 @@ the number of parallel build jobs to use""", help="", nargs="*") - args = parser.parse_args() + args = parser.parse_args(migrate_impl_args(sys.argv[1:], [ + '--darwin-xcrun-toolchain'])) # Build cmark if any cmark-related options were specified. if (args.cmark_build_variant is not None): @@ -709,9 +722,18 @@ the number of parallel build jobs to use""", if args.clean: shutil.rmtree(build_dir) + host_clang = swift_build_support.clang.host_clang( + xcrun_toolchain=args.darwin_xcrun_toolchain) + if not host_clang: + print_with_argv0( + "Can't find clang. Please install clang-3.5 or a later version.") + return 1 build_script_impl_args = [ os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "build-script-impl"), "--build-dir", build_dir, + "--host-cc", host_clang.cc, + "--host-cxx", host_clang.cxx, + "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain, "--cmark-build-type", args.cmark_build_variant, "--llvm-build-type", args.llvm_build_variant, "--swift-build-type", args.swift_build_variant, diff --git a/utils/build-script-impl b/utils/build-script-impl index 44429d15a8eee..e344662f7485e 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -57,6 +57,8 @@ KNOWN_SETTINGS=( # name default description build-args "" "arguments to the build tool; defaults to -j8 when CMake generator is \"Unix Makefiles\"" build-dir "" "out-of-tree build directory; default is in-tree. **This argument is required**" + host-cc "" "the path to CC, the 'clang' compiler for the host platform. **This argument is requied**" + host-cxx "" "the path to CXX, the 'clang++' compiler for the host platform. **This argument is requied**" darwin-xcrun-toolchain "default" "the name of the toolchain to use on Darwin" build-ninja "" "build the Ninja tool" cmark-build-type "Debug" "the CMake build variant for CommonMark (Debug, RelWithDebInfo, Release, MinSizeRel). Defaults to Debug." @@ -1037,33 +1039,10 @@ if [[ "${EXPORT_COMPILE_COMMANDS}" ]] ; then ) fi -if [ -z "${HOST_CC}" ] ; then - if [ "$(uname -s)" == "Darwin" ] ; then - HOST_CC="$(xcrun_find_tool clang)" - HOST_CXX="$(xcrun_find_tool clang++)" - elif [ "$(uname -s)" == "FreeBSD" ]; then - if [ $(sysctl -n kern.osreldate) -ge 1100000 ]; then - HOST_CC="clang" - HOST_CXX="clang++" - else - for clang_candidate_suffix in "38" "37" "36" "35" ; do - if which "clang${clang_candidate_suffix}" > /dev/null ; then - HOST_CC="clang${clang_candidate_suffix}" - HOST_CXX="clang++${clang_candidate_suffix}" - break - fi - done - fi - else - for clang_candidate_suffix in "" "-3.8" "-3.7" "-3.6" "-3.5" ; do - if which "clang${clang_candidate_suffix}" > /dev/null ; then - HOST_CC="clang${clang_candidate_suffix}" - HOST_CXX="clang++${clang_candidate_suffix}" - break - fi - done - fi -fi +# FIXME: HOST_CC is set and its presence is validated by utils/build-script. +# This check is redundant, but must remain until build-script-impl +# is merged completely with utils/build-script. +# For additional information, see: https://bugs.swift.org/browse/SR-237 if [ -z "${HOST_CC}" ] ; then echo "Can't find clang. Please install clang-3.5 or a later version." exit 1 diff --git a/utils/swift_build_support/README.md b/utils/swift_build_support/README.md new file mode 100644 index 0000000000000..5746cb0174ced --- /dev/null +++ b/utils/swift_build_support/README.md @@ -0,0 +1,10 @@ +# swift_build_support + +`swift_build_support` is a Python module containing functions and data +structures used by the Swift build script. + +You may run unit tests for `swift_build_support` from the command line: + +```sh +apple/swift $ python -m unittest discover -s utils/swift_build_support +``` diff --git a/utils/swift_build_support/swift_build_support/__init__.py b/utils/swift_build_support/swift_build_support/__init__.py new file mode 100644 index 0000000000000..b90cf8b9b2bb9 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/__init__.py @@ -0,0 +1,16 @@ +# swift_build_support/__init__.py - Helpers for building Swift -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- +# +# This file needs to be here in order for Python to treat the +# utils/swift_build_support/ directory as a module. +# +# ---------------------------------------------------------------------------- diff --git a/utils/swift_build_support/swift_build_support/clang.py b/utils/swift_build_support/swift_build_support/clang.py new file mode 100644 index 0000000000000..c2a39acb08352 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/clang.py @@ -0,0 +1,90 @@ +# swift_build_support/clang.py - Detect host machine's Clang -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- +# +# Find the path to a Clang executable on the host machine that is most +# suitable for building Swift. +# +# ---------------------------------------------------------------------------- + +from __future__ import absolute_import + +import collections +import platform +import subprocess + +from . import xcrun +from .which import which + + +# A named tuple consisting of two paths: +# 1. 'cc' is the path to a program used for compiling C. +# 2. 'cxx' is the path to a program used for compiling C++. +CompilerExecutable = collections.namedtuple('CompilerExecutable', 'cc cxx') + + +def _freebsd_release_date(): + """ + Return the release date for FreeBSD operating system on this host. + If the release date cannot be ascertained, return None. + """ + try: + # For details on `sysctl`, see: + # http://www.freebsd.org/cgi/man.cgi?sysctl(8) + return int(subprocess.check_output( + ['sysctl', '-n', 'kern.osreldate']).rstrip()) + except subprocess.CalledProcessError: + return None + + +def _first_clang(suffixes): + """ + Return a CompilerExecutable with the first available versions of clang + and clang++, searching in the order of the given suffixes. + + If no Clang executables are found, return None. + """ + for suffix in suffixes: + cc_path = which('clang{}'.format(suffix)) + cxx_path = which('clang++{}'.format(suffix)) + if cc_path and cxx_path: + return CompilerExecutable(cc=cc_path, cxx=cxx_path) + + return None + + +def host_clang(xcrun_toolchain): + """ + Return a CompilerExecutable for the host platform. + If no appropriate compilers can be found, return None. + """ + if platform.system() == 'Darwin': + cc = xcrun.find(xcrun_toolchain, 'clang') + cxx = xcrun.find(xcrun_toolchain, 'clang++') + if cc and cxx: + return CompilerExecutable(cc=cc, cxx=cxx) + else: + return None + elif platform.system() == 'FreeBSD': + # See: https://github.com/apple/swift/pull/169 + # Building Swift from source requires a recent version of the Clang + # compiler with C++14 support. + freebsd_release_date = _freebsd_release_date() + if freebsd_release_date and freebsd_release_date >= 1100000: + # On newer releases of FreeBSD, the default Clang is sufficient. + return CompilerExecutable(cc='clang', cxx='clang++') + else: + # On older releases, or on releases for which we cannot determine + # the release date, we search for the most modern version + # available. + return _first_clang(['38', '37', '36', '35']) + else: + return _first_clang(['', '-3.8', '-3.7', '-3.6', '-3.5']) diff --git a/utils/swift_build_support/swift_build_support/migration.py b/utils/swift_build_support/swift_build_support/migration.py new file mode 100644 index 0000000000000..72a183e4f9e88 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/migration.py @@ -0,0 +1,56 @@ +# swift_build_support/migration.py - Migrating build-script -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- +# +# utils/build-script takes arguments for its argument parser, as well as +# arguments that are meant to be passed directly to utils/build-script-impl. +# In order to gradually migrate away from build-script-impl, this module +# provides tools to handle parsing of these args. +# +# ---------------------------------------------------------------------------- + + +def migrate_impl_args(argv, migrate_args): + """ + Given a list of arguments of the form: + + --foo --bar=baz -- --flim=flam + + And a list of arguments to migrate, return a list in which the arguments + to migrate come before the '--' separator. For example, were we to migrate + '--flim', we would return: + + --foo --bar=baz --flim=flam -- + + Note that we do not attempt to remove the '--' separator if it is no longer + necessary, nor do we replace '--flim' if it already appears before the + separator. In these cases, argparse "does the right thing": it can handle + a trailing separator, and when options that are specified twice argparse + uses the second value. + """ + try: + split_index = argv.index('--') + except ValueError: + # If there is no separator, then we have nothing to migrate. + return argv + + args = argv[:split_index] + impl_args = argv[split_index:] + impl_args_to_remove = [] + for index, impl_arg in enumerate(impl_args): + if impl_arg.split('=')[0] in migrate_args: + args.append(impl_arg) + impl_args_to_remove.append(impl_arg) + + for impl_arg_to_remove in impl_args_to_remove: + impl_args.remove(impl_arg_to_remove) + + return args + impl_args diff --git a/utils/swift_build_support/swift_build_support/which.py b/utils/swift_build_support/swift_build_support/which.py new file mode 100644 index 0000000000000..7b6a5b212869d --- /dev/null +++ b/utils/swift_build_support/swift_build_support/which.py @@ -0,0 +1,36 @@ +# swift_build_support/which.py - shutil.which() for Python 2.7 -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- +# +# A naive reimplementation of shutil.which() for Python 2.7. This can be +# removed if shutil.which() is backported, or if the Swift build toolchain +# migrates completely to Python 3.3+. +# +# ---------------------------------------------------------------------------- + +import subprocess + + +def which(cmd): + """ + Return the path to an executable which would be run if + the given cmd was called. If no cmd would be called, return None. + + Python 3.3+ provides this behavior via the shutil.which() function; + see: https://docs.python.org/3.3/library/shutil.html#shutil.which + + We provide our own implementation because shutil.which() has not + been backported to Python 2.7, which we support. + """ + try: + return subprocess.check_output(['which', cmd]).rstrip() + except subprocess.CalledProcessError: + return None diff --git a/utils/swift_build_support/swift_build_support/xcrun.py b/utils/swift_build_support/swift_build_support/xcrun.py new file mode 100644 index 0000000000000..0bb387553df81 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/xcrun.py @@ -0,0 +1,34 @@ +# swift_build_support/xcrun.py - Invoke xcrun from Python -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- +# +# Python wrappers for invoking `xcrun` on the command-line. +# +# ---------------------------------------------------------------------------- + +import subprocess + + +def find(toolchain, tool): + """ + Return the path for the given tool, according to `xcrun --find`, using + the given toolchain. If `xcrun --find` cannot find the tool, return None. + """ + try: + # `xcrun --find` prints to stderr when it fails to find the + # given tool. We swallow that output with a pipe. + out = subprocess.check_output(['xcrun', '--sdk', 'macosx', + '--toolchain', toolchain, + '--find', tool], + stderr=subprocess.PIPE) + return out.rstrip() + except subprocess.CalledProcessError: + return None diff --git a/utils/swift_build_support/tests/__init__.py b/utils/swift_build_support/tests/__init__.py new file mode 100644 index 0000000000000..c732b7348f69c --- /dev/null +++ b/utils/swift_build_support/tests/__init__.py @@ -0,0 +1,16 @@ +# swift_build_support/tests/__init__.py - Test module -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- +# +# This file needs to be here in order for Python to treat the +# utils/swift_build_support/tests/ directory as a module. +# +# ---------------------------------------------------------------------------- diff --git a/utils/swift_build_support/tests/test_clang.py b/utils/swift_build_support/tests/test_clang.py new file mode 100644 index 0000000000000..43743c32b3856 --- /dev/null +++ b/utils/swift_build_support/tests/test_clang.py @@ -0,0 +1,31 @@ +# test_clang.py - Unit tests for swift_build_support.clang -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import os +import unittest + +from swift_build_support.clang import host_clang + + +class HostClangTestCase(unittest.TestCase): + def test_clang_available_on_this_platform(self): + # Test that Clang is installed on this platform, as a means of + # testing host_clang(). + clang = host_clang(xcrun_toolchain='default') + + # The CC and CXX from host_clang() should be of the form + # 'path/to/clang', where 'clang' may have a trailing version + # number. + self.assertTrue(os.path.split(clang.cc)[-1].startswith('clang')) + self.assertTrue(os.path.split(clang.cxx)[-1].startswith('clang++')) + + +if __name__ == '__main__': + unittest.main() diff --git a/utils/swift_build_support/tests/test_migration.py b/utils/swift_build_support/tests/test_migration.py new file mode 100644 index 0000000000000..1cadb1a1afed4 --- /dev/null +++ b/utils/swift_build_support/tests/test_migration.py @@ -0,0 +1,31 @@ +# test_migration.py - Tests for swift_build_support.migration -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import unittest + +from swift_build_support.migration import migrate_impl_args + + +class MigrateImplArgsTestCase(unittest.TestCase): + def test_args_moved_before_separator(self): + # Tests that '-RT --foo=bar -- --foo=baz --flim' is parsed as + # '-RT --foo=bar --foo=baz -- --flim' + args = migrate_impl_args( + ['-RT', '--darwin-xcrun-toolchain=foo', '--', + '--darwin-xcrun-toolchain=bar', '--other'], + ['--darwin-xcrun-toolchain']) + + self.assertEqual( + args, + ['-RT', '--darwin-xcrun-toolchain=foo', + '--darwin-xcrun-toolchain=bar', '--', '--other']) + +if __name__ == '__main__': + unittest.main() diff --git a/utils/swift_build_support/tests/test_which.py b/utils/swift_build_support/tests/test_which.py new file mode 100644 index 0000000000000..ed886a1b6bf81 --- /dev/null +++ b/utils/swift_build_support/tests/test_which.py @@ -0,0 +1,26 @@ +# test_which.py - Unit tests for swift_build_support.which -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import os +import unittest + +from swift_build_support.which import which + + +class WhichTestCase(unittest.TestCase): + def test_when_cmd_not_found_returns_none(self): + self.assertIsNone(which('a-tool-that-doesnt-exist')) + + def test_when_cmd_found_returns_path(self): + self.assertEquals(os.path.split(which('ls'))[-1], 'ls') + + +if __name__ == '__main__': + unittest.main() diff --git a/utils/swift_build_support/tests/test_xcrun.py b/utils/swift_build_support/tests/test_xcrun.py new file mode 100644 index 0000000000000..5a54b68473af2 --- /dev/null +++ b/utils/swift_build_support/tests/test_xcrun.py @@ -0,0 +1,32 @@ +# test_xcrun.py - Unit tests for swift_build_support.xcrun -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + +import platform +import unittest + +from swift_build_support import xcrun + + +class FindTestCase(unittest.TestCase): + def setUp(self): + if platform.system() != 'Darwin': + self.skipTest('XCRun tests should only be run on OS X') + + def test_when_tool_not_found_returns_none(self): + self.assertIsNone(xcrun.find( + toolchain='default', tool='a-tool-that-isnt-on-osx')) + + def test_when_tool_found_returns_path(self): + self.assertTrue(xcrun.find( + toolchain='default', tool='clang').endswith('/clang')) + + +if __name__ == '__main__': + unittest.main() diff --git a/validation-test/Python/swift_build_support.swift b/validation-test/Python/swift_build_support.swift new file mode 100644 index 0000000000000..abdee4bef4bef --- /dev/null +++ b/validation-test/Python/swift_build_support.swift @@ -0,0 +1 @@ +// RUN: %{python} -m unittest discover -s %S/../../utils/swift_build_support