[SR-237][build-script] Migrate more of build-script-impl to Python

utils/build-script

@@ -14,6 +14,7 @@ from __future__ import print_function
 import argparse
 import multiprocessing
 import os
+import platform
 import shutil
 import sys
 import textwrap
@@ -32,11 +33,15 @@ from SwiftBuildSupport import (
     get_preset_options,
     print_with_argv0,
     quote_shell_command,
+    WorkingDirectory,
 )
 
 sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support'))
 import swift_build_support.clang
 import swift_build_support.cmake
+import swift_build_support.debug
+import swift_build_support.ninja
+import swift_build_support.tar
 import swift_build_support.targets
 from swift_build_support.migration import migrate_impl_args
 
@@ -333,6 +338,10 @@ details of the setups of other systems or automated environments.""")
     extra_actions_group.add_argument("--export-compile-commands",
         help="generate compilation databases in addition to building",
         action="store_true")
+    extra_actions_group.add_argument("--symbols-package",
+        metavar="PATH",
+        help="if provided, an archive of the symbols directory will be "
+             "generated at this path")
 
     build_variant_group = parser.add_mutually_exclusive_group(required=False)
     build_variant_group.add_argument("-d", "--debug",
@@ -530,6 +539,14 @@ also build for Apple watchos, but disallow tests that require an watchOS device"
 name of the directory under $SWIFT_BUILD_ROOT where the build products will be
 placed""",
         metavar="PATH")
+    parser.add_argument("--install-prefix",
+        help="The installation prefix. This is where built Swift products "
+             "(like bin, lib, and include) will be installed.",
+        metavar="PATH",
+        default=swift_build_support.targets.install_prefix())
+    parser.add_argument("--install-symroot",
+        help="the path to install debug symbols into",
+        metavar="PATH")
 
     parser.add_argument("-j", "--jobs",
         help="""
@@ -544,6 +561,9 @@ the number of parallel build jobs to use""",
     parser.add_argument("--cmake",
                         help="the path to a CMake executable that will be "
                              "used to build Swift")
+    parser.add_argument("--show-sdks",
+                        help="print installed Xcode and SDK versions",
+                        action="store_true")
 
     parser.add_argument("--extra-swift-args", help=textwrap.dedent("""
     Pass through extra flags to swift in the form of a cmake list 'module_regexp;flag'. Can
@@ -559,12 +579,27 @@ the number of parallel build jobs to use""",
         '--darwin-xcrun-toolchain',
         '--cmake',
         '--host-target',
+        '--skip-build',
+        '--show-sdks',
+        '--install-prefix',
+        '--install-symroot',
+        '--symbols-package',
     ]))
 
     if args.host_target is None:
         print_with_argv0("Unknown operating system.")
         return 1
 
+    if args.symbols_package:
+        if not os.path.isabs(args.symbols_package):
+            print('--symbols-package must be an absolute path '
+                  '(was \'{}\')'.format(args.symbols_package))
+            return 1
+        if not args.install_symroot:
+            print_with_argv0("--install-symroot is required when specifying "
+                             "--symbols-package.")
+            return 1
+
     # Build cmark if any cmark-related options were specified.
     if (args.cmark_build_variant is not None):
         args.build_cmark = True
@@ -726,7 +761,32 @@ the number of parallel build jobs to use""",
 
     if args.skip_build:
         build_script_impl_inferred_args += [
-            "--skip-build"
+            "--skip-build-cmark",
+            "--skip-build-llvm",
+            "--skip-build-swift",
+            "--skip-build-osx",
+            "--skip-build-ios",
+            "--skip-build-ios-device",
+            "--skip-build-ios-simulator",
+            "--skip-build-tvos",
+            "--skip-build-tvos_device",
+            "--skip-build-tvos-simulator",
+            "--skip-build-watchos",
+            "--skip-build-watchos-device",
+            "--skip-build-watchos-simulator",
+            "--skip-build-lldb",
+            "--skip-build-llbuild",
+            "--skip-build-swiftpm",
+            "--skip-build-xctest",
+            "--skip-build-foundation",
+            "--skip-build-libdispatch",
+            "--skip-build-benchmarks",
+        ]
+
+    if platform.system() == 'Darwin':
+        build_script_impl_inferred_args += [
+            "--toolchain-prefix",
+            swift_build_support.targets.darwin_toolchain_prefix(args.install_prefix),
         ]
 
     if args.build_subdir is None:
@@ -804,6 +864,7 @@ the number of parallel build jobs to use""",
     build_script_impl_args = [
         os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "build-script-impl"),
         "--build-dir", build_dir,
+        "--install-prefix", os.path.abspath(args.install_prefix),
         "--host-target", args.host_target,
         "--host-cc", host_clang.cc,
         "--host-cxx", host_clang.cxx,
@@ -826,6 +887,9 @@ the number of parallel build jobs to use""",
         build_script_impl_args += [
             "--foundation-build-type", args.foundation_build_variant
         ]
+    if args.cmake_generator == 'Ninja' and \
+       not swift_build_support.ninja.is_ninja_installed():
+        build_script_impl_args += ["--build-ninja"]
     build_script_impl_args += build_script_impl_inferred_args
 
     # If we have extra_swift_args, combine all of them together and then add
@@ -839,8 +903,33 @@ the number of parallel build jobs to use""",
     for v in ['MAKEFLAGS', 'SDKROOT', 'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET']:
         os.environ.pop(v, None)
 
+    if args.show_sdks:
+        swift_build_support.debug.print_xcodebuild_versions([
+            'iphonesimulator',
+            'appletvsimulator',
+            'watchsimulator',
+        ])
+
     check_call(build_script_impl_args)
 
+    if args.symbols_package:
+        print('--- Creating symbols package ---')
+        print('-- Package file: {} --'.format(args.symbols_package))
+
+        if platform.system() == 'Darwin':
+            prefix = swift_build_support.targets.darwin_toolchain_prefix(
+                args.install_prefix)
+        else:
+            prefix = args.install_prefix
+
+        # As a security measure, `tar` normally strips leading '/' from paths
+        # it is archiving. To stay safe, we change working directories, then
+        # run `tar` without the leading '/' (we remove it ourselves to keep
+        # `tar` from emitting a warning).
+        with WorkingDirectory(args.install_symroot):
+            swift_build_support.tar.tar(source=prefix.lstrip('/'),
+                                        destination=args.symbols_package)
+
     return 0
 
 

utils/build-script-impl

@@ -86,22 +86,20 @@ KNOWN_SETTINGS=(
     cmake-generator             "Unix Makefiles" "kind of build system to generate; see output of 'cmake --help' for choices"
     verbose-build               ""               "print the commands executed during the build"
     install-prefix              ""               "installation prefix"
+    toolchain-prefix            ""               "the path to the .xctoolchain directory that houses the install prefix path"
     install-destdir             ""               "the path to use as the filesystem root for the installation"
     install-symroot             ""               "the path to install debug symbols into"
     swift-install-components    ""               "a semicolon-separated list of Swift components to install"
     llvm-install-components    ""                "a semicolon-separated list of LLVM components to install"
     installable-package         ""               "the path to the archive of the installation directory"
     test-installable-package    ""               "whether to run post-packaging tests on the produced package"
-    symbols-package             ""               "the path to the archive of the symbols directory"
-    show-sdks                   ""               "print installed Xcode and SDK versions"
     reconfigure                 ""               "force a CMake configuration run even if CMakeCache.txt already exists"
     swift-sdks                  ""               "build target binaries only for specified SDKs (semicolon-separated list)"
     swift-primary-variant-sdk   ""               "default SDK for target binaries"
     swift-primary-variant-arch  ""               "default arch for target binaries"
     skip-ios                    ""               "set to skip everything iOS-related"
     skip-tvos                   ""               "set to skip everything tvOS-related"
     skip-watchos                ""               "set to skip everything watchOS-related"
-    skip-build                  ""               "set to skip building anything"
     skip-build-cmark            ""               "set to skip building CommonMark"
     skip-build-llvm             ""               "set to skip building LLVM/Clang"
     skip-build-swift            ""               "set to skip building Swift"
@@ -539,29 +537,6 @@ while [[ "$1" ]] ; do
     shift
 done
 
-if [[ "${SKIP_BUILD}" ]]; then
-    SKIP_BUILD_CMARK=1
-    SKIP_BUILD_LLVM=1
-    SKIP_BUILD_SWIFT=1
-    SKIP_BUILD_OSX=1
-    SKIP_BUILD_IOS=1
-    SKIP_BUILD_IOS_DEVICE=1
-    SKIP_BUILD_IOS_SIMULATOR=1
-    SKIP_BUILD_TVOS=1
-    SKIP_BUILD_TVOS_DEVICE=1
-    SKIP_BUILD_TVOS_SIMULATOR=1
-    SKIP_BUILD_WATCHOS=1
-    SKIP_BUILD_WATCHOS_DEVICE=1
-    SKIP_BUILD_WATCHOS_SIMULATOR=1
-    SKIP_BUILD_LLDB=1
-    SKIP_BUILD_LLBUILD=1
-    SKIP_BUILD_SWIFTPM=1
-    SKIP_BUILD_XCTEST=1
-    SKIP_BUILD_FOUNDATION=1
-    SKIP_BUILD_LIBDISPATCH=1
-    SKIP_BUILD_BENCHMARKS=1
-fi
-
 if [[ "${SKIP_IOS}" ]] ; then
     SKIP_BUILD_IOS=1
     SKIP_BUILD_IOS_DEVICE=1
@@ -658,12 +633,6 @@ if [[ "${SKIP_TEST_WATCHOS}" ]] ; then
     SKIP_TEST_WATCHOS_SIMULATOR=1
 fi
 
-if [[ "${CMAKE_GENERATOR}" == "Ninja" && \
-      -z "$(which ninja 2>/dev/null)" &&
-      -z "$(which ninja-build 2>/dev/null)" ]] ; then
-    BUILD_NINJA=1
-fi
-
 # WORKSPACE, BUILD_DIR and INSTALLABLE_PACKAGE must be absolute paths
 case "${WORKSPACE}" in
     /*) ;;
@@ -692,14 +661,6 @@ case "${INSTALLABLE_PACKAGE}" in
         exit 1
         ;;
 esac
-case "${SYMBOLS_PACKAGE}" in
-    /*) ;;
-    "") ;;
-    *)
-        echo "symbols-package must be an absolute path (was '${SYMBOLS_PACKAGE}')"
-        exit 1
-        ;;
-esac
 
 # WORKSPACE must exist
 if [ ! -e "${WORKSPACE}" ] ; then
@@ -745,23 +706,6 @@ function true_false() {
     esac
 }
 
-#
-# Set default values for command-line parameters.
-#
-
-if [[ "${INSTALL_PREFIX}" == "" ]] ; then
-    if [[ "$(uname -s)" == "Darwin" ]] ; then
-        INSTALL_PREFIX="/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr"
-    else
-        INSTALL_PREFIX="/usr"
-    fi
-fi
-
-if [[ "$(uname -s)" == "Darwin" ]] ; then
-    TOOLCHAIN_PREFIX=$(echo ${INSTALL_PREFIX} | sed -E 's/\/usr$//')
-fi
-
-
 # A list of deployment targets to cross-compile the Swift host tools for.
 # We can't run the resulting binaries on the build machine.
 CROSS_TOOLS_DEPLOYMENT_TARGETS=()
@@ -1147,24 +1091,6 @@ if [[ "${CLANG_COMPILER_VERSION}" ]] ; then
     )
 fi
 
-#
-# Record SDK and tools versions for posterity
-#
-if [[ "${SHOW_SDKS}" ]] ; then
-    echo "--- SDK versions ---"
-    xcodebuild -version || :
-    echo
-    if [[ ! "${SKIP_IOS}" ]] ; then
-        xcodebuild -version -sdk iphonesimulator || :
-    fi
-    if [[ ! "${SKIP_TVOS}" ]] ; then
-        xcodebuild -version -sdk appletvsimulator || :
-    fi
-    if [[ ! "${SKIP_WATCHOS}" ]] ; then
-        xcodebuild -version -sdk watchsimulator || :
-    fi
-fi
-
 function build_directory() {
     deployment_target=$1
     product=$2
@@ -2389,15 +2315,3 @@ if [[ "${INSTALLABLE_PACKAGE}" ]] ; then
         { set +x; } 2>/dev/null
     fi
 fi
-
-if [[ "${SYMBOLS_PACKAGE}" ]] ; then
-    echo "--- Creating symbols package ---"
-    echo "-- Package file: ${SYMBOLS_PACKAGE} --"
-    if [[ "$(uname -s)" == "Darwin" ]] ; then
-      (cd "${INSTALL_SYMROOT}" &&
-        tar -c -z -f "${SYMBOLS_PACKAGE}" "${TOOLCHAIN_PREFIX/#\/}")
-    else
-      (cd "${INSTALL_SYMROOT}" &&
-        tar -c -z -f "${SYMBOLS_PACKAGE}" --owner=0 --group=0 "${INSTALL_PREFIX/#\/}")
-    fi
-fi

utils/swift_build_support/swift_build_support/debug.py

@@ -0,0 +1,41 @@
+# swift_build_support/debug.py - Print information on the build -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+#
+# Convenient functions for printing out information on the build process.
+#
+# ----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import subprocess
+import sys
+
+
+def _output(args):
+    try:
+        out = subprocess.check_output(args, stderr=subprocess.PIPE)
+        return out.rstrip()
+    except subprocess.CalledProcessError:
+        return None
+
+
+def print_xcodebuild_versions(sdks, file=sys.stdout):
+	"""
+	Print the host machine's `xcodebuild` version, as well as version
+	information for each of the given SDKs (for a full list of available
+	SDKs, invoke `xcodebuild -showsdks` on the command line).
+	"""
+	print(u'--- SDK versions ---', file=file)
+	print(u'{}\n'.format(_output(['xcodebuild', '-version'])), file=file)
+	for sdk in sdks:
+		print(u'{}\n'.format(_output(['xcodebuild', '-version', '-sdk', sdk])),
+			  file=file)

utils/swift_build_support/swift_build_support/ninja.py

@@ -0,0 +1,21 @@
+# swift_build_support/ninja.py - Detect host machine's Ninja -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+from .which import which
+
+
+def is_ninja_installed():
+    """
+    Return whether `ninja` or `ninja-build` are available on the host machine.
+    """
+    if which('ninja') or which('ninja-build'):
+        return True
+    else:
+        return False