From a313f62522c00e8327084820bd22dc3d2870610e Mon Sep 17 00:00:00 2001 From: Michael Gottesman Date: Mon, 8 Jun 2020 12:05:15 -0700 Subject: [PATCH] [build-script] Add option --infer to infer dependencies. This causes build-script to use the conservative dependency information that I committed. When one uses this option, it is assumed that one wants to also install all built products. Some notes: 1. I included an extra --install-all option so without --infer enabled one can enable this sort of install everything that we want to build behavior. 2. I added %cmake as a lit variable. I did this so I could specify in my build-system unit tests that on Linux they should use the just built cmake (if we built cmake due to an old cmake being on the system). Otherwise, the build system unit tests took way too long. These are meant to be dry-runs, so building this cmake again is just wasteful and doesn't make sense. 3. I unified how we handle cmark, llvm, swift with the rest of the build products by making them conditional on build_* variables, but to preserve current behavior I made it so that they are just enabled by default unlike things like llbuild/swiftpm/foundation/etc. This was necessary since previously we would just pass these flags to build-script-impl and build-script didn't know about them. Now I taught build-script about them so I can manipulate these skip-build-{cmark,llvm,swift} and then just pass them down to build-script-impl if appropriate rather than relying on build-script-impl to control if these are built. Once this lands, I think we are at a good enough place with build-script until we get rid of build-script-impl in terms of high value QoI that will imnprove productivity. Once build-script-impl is destroyed, we can start paring back what build-script itself does. --- test/lit.cfg | 7 +- test/lit.site.cfg.in | 1 + utils/build-script | 77 ++++++++++++++----- .../build_swift/driver_arguments.py | 15 ++++ .../tests/build_swift/test_presets.py | 1 + utils/build_swift/tests/expected_options.py | 11 +++ .../swift_build_support/cmake.py | 2 +- .../infer_implies_install_all.test | 10 +++ validation-test/BuildSystem/install_all.test | 7 ++ .../BuildSystem/install_all_linux.test | 13 ++++ .../BuildSystem/skip_cmark_swift_llvm.test | 33 ++++++++ validation-test/lit.site.cfg.in | 1 + 12 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 validation-test/BuildSystem/infer_implies_install_all.test create mode 100644 validation-test/BuildSystem/install_all.test create mode 100644 validation-test/BuildSystem/install_all_linux.test create mode 100644 validation-test/BuildSystem/skip_cmark_swift_llvm.test diff --git a/test/lit.cfg b/test/lit.cfg index 1bdc5cf705e8f..1e8372b3507cb 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -146,7 +146,7 @@ config.test_format = swift_test.SwiftTest(coverage_mode=config.coverage_mode, # suffixes: A list of file extensions to treat as test files. config.suffixes = ['.swift', '.ll', '.sil', '.gyb', '.m', '.c', - '.swiftinterface', '.test-sh'] + '.swiftinterface', '.test-sh', '.test'] # excludes: A list of directories to exclude from the testsuite. The 'Inputs' # subdirectories contain auxiliary inputs for various tests in their parent @@ -162,6 +162,11 @@ config.test_source_root = os.path.dirname(__file__) # test_exec_root: The root path where tests should be run. swift_obj_root = getattr(config, 'swift_obj_root', None) +# cmake. The path to the cmake executable we used to configure swift. +assert(config.cmake) +config.substitutions.append( ('%cmake', config.cmake) ) +lit_config.note('Using cmake: ' + config.cmake) + # Set llvm_{src,obj}_root for use by others. config.llvm_src_root = getattr(config, 'llvm_src_root', None) config.llvm_obj_root = getattr(config, 'llvm_obj_root', None) diff --git a/test/lit.site.cfg.in b/test/lit.site.cfg.in index a3b4188eb030f..8ae8fb95a8fcd 100644 --- a/test/lit.site.cfg.in +++ b/test/lit.site.cfg.in @@ -14,6 +14,7 @@ import os import platform import sys +config.cmake = "@CMAKE_COMMAND@" config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@" config.llvm_obj_root = "@LLVM_BINARY_DIR@" config.llvm_tools_dir = "@LLVM_TOOLS_DIR@" diff --git a/utils/build-script b/utils/build-script index 494c22893d4de..4d47122f20c03 100755 --- a/utils/build-script +++ b/utils/build-script @@ -35,6 +35,7 @@ from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT import six +from swift_build_support.swift_build_support import build_graph from swift_build_support.swift_build_support import products from swift_build_support.swift_build_support import shell from swift_build_support.swift_build_support import targets @@ -375,6 +376,10 @@ class BuildScriptInvocation(object): self.build_libparser_only = args.build_libparser_only + @property + def install_all(self): + return self.args.install_all or self.args.infer_dependencies + def build_ninja(self): if not os.path.exists(self.workspace.source_dir("ninja")): fatal_error( @@ -548,20 +553,28 @@ class BuildScriptInvocation(object): # Currently we do not build external benchmarks by default. if args.build_external_benchmarks: impl_args += ["--skip-build-external-benchmarks=0"] - if not args.build_foundation: - impl_args += ["--skip-build-foundation"] - if not args.build_xctest: - impl_args += ["--skip-build-xctest"] - if not args.build_lldb: - impl_args += ["--skip-build-lldb"] - if not args.build_llbuild: - impl_args += ["--skip-build-llbuild"] - if not args.build_libcxx: - impl_args += ["--skip-build-libcxx"] - if not args.build_libdispatch: - impl_args += ["--skip-build-libdispatch"] - if not args.build_libicu: - impl_args += ["--skip-build-libicu"] + + # Then add subproject install flags that either skip building them /or/ + # if we are going to build them and install_all is set, we also install + # them. + conditional_subproject_configs = [ + (args.build_cmark, "cmark"), + (args.build_llvm, "llvm"), + (args.build_swift, "swift"), + (args.build_foundation, "foundation"), + (args.build_xctest, "xctest"), + (args.build_lldb, "lldb"), + (args.build_llbuild, "llbuild"), + (args.build_libcxx, "libcxx"), + (args.build_libdispatch, "libdispatch"), + (args.build_libicu, "libicu") + ] + for (should_build, string_name) in conditional_subproject_configs: + if not should_build and not self.args.infer_dependencies: + impl_args += ["--skip-build-{}".format(string_name)] + elif self.install_all: + impl_args += ["--install-{}".format(string_name)] + if args.build_swift_dynamic_stdlib: impl_args += ["--build-swift-dynamic-stdlib"] if args.build_swift_static_stdlib: @@ -816,13 +829,16 @@ class BuildScriptInvocation(object): # FIXME: This is a weird division (returning a list of class objects), # but it matches the existing structure of the `build-script-impl`. impl_product_classes = [] - impl_product_classes.append(products.CMark) - impl_product_classes.append(products.LLVM) + if self.args.build_cmark: + impl_product_classes.append(products.CMark) + if self.args.build_llvm: + impl_product_classes.append(products.LLVM) if self.args.build_libcxx: impl_product_classes.append(products.LibCXX) if self.args.build_libicu: impl_product_classes.append(products.LibICU) - impl_product_classes.append(products.Swift) + if self.args.build_swift: + impl_product_classes.append(products.Swift) if self.args.build_lldb: impl_product_classes.append(products.LLDB) if self.args.build_libdispatch: @@ -868,6 +884,30 @@ class BuildScriptInvocation(object): for prod in product_classes: assert(not prod.is_build_script_impl_product()) + # Now that we have our two lists of product_classes, if we are asked to + # infer dependencies, infer the dependencies now and then re-split the + # list. + if self.args.infer_dependencies: + combined = impl_product_classes + product_classes + + # Now that we have produced the schedule, resplit. We require our + # dependencies to respect our build-script-impl property. This means + # that no build-script-impl products can have dependencies on + # non-build-script-impl products. Otherwise, it would be unsafe to + # re-order build-script-impl products in front of non + # build-script-impl products. + impl_product_classes = [] + product_classes = [] + is_darwin = platform.system() == 'Darwin' + final_schedule = build_graph.produce_scheduled_build(combined)[0] + for p in final_schedule: + if is_darwin and p.is_nondarwin_only_build_product(): + continue + + if p.is_build_script_impl_product(): + impl_product_classes.append(p) + else: + product_classes.append(p) return (impl_product_classes, product_classes) def execute(self): @@ -956,7 +996,8 @@ class BuildScriptInvocation(object): print("--- Running tests for %s ---" % product_name) product.test(host_target) print("--- Finished tests for %s ---" % product_name) - if product.should_install(host_target): + if product.should_install(host_target) or \ + (self.install_all and product.should_build(host_target)): print("--- Installing %s ---" % product_name) product.install(host_target) diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index 868be0ac4944e..04d5eef190834 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -345,6 +345,8 @@ def create_argument_parser(): help='the path to install debug symbols into') option('--install-destdir', store_path, help='the path to use as the filesystem root for the installation') + option('--install-all', toggle_true, + help='Assume all built products should be installed') option(['-j', '--jobs'], store_int('build_jobs'), default=multiprocessing.cpu_count(), @@ -545,6 +547,9 @@ def create_argument_parser(): # ------------------------------------------------------------------------- in_group('Options to select projects') + option('--infer', store_true('infer_dependencies'), + help='Infer any downstream dependencies from enabled projects') + option(['-l', '--lldb'], store_true('build_lldb'), help='build LLDB') @@ -1096,6 +1101,16 @@ def create_argument_parser(): # ------------------------------------------------------------------------- in_group('Build-script-impl arguments (for disambiguation)') + + # We need to represent these options so that we can skip installing them if + # the user is running in install-all mode. + option('--skip-build-cmark', toggle_false('build_cmark'), + help='skip building cmark') + option('--skip-build-llvm', toggle_false('build_llvm'), + help='skip building llvm') + option('--skip-build-swift', toggle_false('build_swift'), + help='skip building swift') + # We need to list --skip-test-swift explicitly because otherwise argparse # will auto-expand arguments like --skip-test-swift to the only known # argument --skip-test-swiftevolve. diff --git a/utils/build_swift/tests/build_swift/test_presets.py b/utils/build_swift/tests/build_swift/test_presets.py index 2436a5bd66153..2668d7a8d790f 100644 --- a/utils/build_swift/tests/build_swift/test_presets.py +++ b/utils/build_swift/tests/build_swift/test_presets.py @@ -39,6 +39,7 @@ 'extra_swift_args': '', 'install_destdir': '/tmp/install', 'install_symroot': '/tmp/install/symroot', + 'install_all': False, 'install_toolchain_dir': '/tmp/install/toolchain', 'install_prefix': '/usr', 'installable_package': '/tmp/install/pkg', diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index 168964feab6c2..160fdcd8e2fe5 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -61,6 +61,9 @@ 'build_cygwin': True, 'build_external_benchmarks': False, 'build_foundation': False, + 'build_cmark': True, + 'build_swift': True, + 'build_llvm': True, 'build_freebsd': True, 'build_ios': True, 'build_ios_device': False, @@ -159,9 +162,11 @@ 'host_test': False, 'only_executable_test': False, 'only_non_executable_test': False, + 'infer_dependencies': False, 'install_prefix': targets.install_prefix(), 'install_symroot': None, 'install_destdir': None, + 'install_all': False, 'ios': False, 'ios_all': False, 'legacy_impl': False, @@ -482,6 +487,7 @@ class BuildScriptImplOption(_BaseOption): SetTrueOption('-p', dest='build_swiftpm'), SetTrueOption('--legacy-impl', dest='legacy_impl'), + SetTrueOption('--infer', dest='infer_dependencies'), EnableOption('--android'), EnableOption('--build-external-benchmarks'), @@ -537,6 +543,10 @@ class BuildScriptImplOption(_BaseOption): EnableOption('--watchos'), EnableOption('--xctest', dest='build_xctest'), + DisableOption('--skip-build-cmark', dest='build_cmark'), + DisableOption('--skip-build-llvm', dest='build_llvm'), + DisableOption('--skip-build-swift', dest='build_swift'), + DisableOption('--skip-build-android', dest='build_android'), DisableOption('--skip-build-benchmarks', dest='build_benchmarks'), DisableOption('--skip-build-cygwin', dest='build_cygwin'), @@ -632,6 +642,7 @@ class BuildScriptImplOption(_BaseOption): PathOption('--install-prefix'), PathOption('--install-symroot'), PathOption('--install-destdir'), + EnableOption('--install-all'), PathOption('--symbols-package'), PathOption('--cmake-c-launcher'), PathOption('--cmake-cxx-launcher'), diff --git a/utils/swift_build_support/swift_build_support/cmake.py b/utils/swift_build_support/swift_build_support/cmake.py index 876caf175392a..85a97dfa747e2 100644 --- a/utils/swift_build_support/swift_build_support/cmake.py +++ b/utils/swift_build_support/swift_build_support/cmake.py @@ -272,7 +272,7 @@ def check_cmake_version(self, source_root, build_root): cmake_binary = 'cmake' installed_ver = self.installed_cmake_version(cmake_binary) - if installed_ver > self.cmake_source_version(cmake_source_dir): + if installed_ver >= self.cmake_source_version(cmake_source_dir): return else: # Build CMake from source and return the path to the executable. diff --git a/validation-test/BuildSystem/infer_implies_install_all.test b/validation-test/BuildSystem/infer_implies_install_all.test new file mode 100644 index 0000000000000..2b36960d1e7a7 --- /dev/null +++ b/validation-test/BuildSystem/infer_implies_install_all.test @@ -0,0 +1,10 @@ +# RUN: %empty-directory(%t) +# RUN: mkdir -p %t +# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --infer --swiftpm --cmake %cmake 2>&1 | %FileCheck %s + +# CHECK: --- Installing cmark --- +# CHECK: --- Installing llvm --- +# CHECK: --- Installing swift --- +# CHECK: --- Installing llbuild --- +# CHECK: --- Building swiftpm --- +# CHECK: --- Installing swiftpm --- diff --git a/validation-test/BuildSystem/install_all.test b/validation-test/BuildSystem/install_all.test new file mode 100644 index 0000000000000..e8888d25beaf9 --- /dev/null +++ b/validation-test/BuildSystem/install_all.test @@ -0,0 +1,7 @@ +# RUN: %empty-directory(%t) +# RUN: mkdir -p %t +# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake 2>&1 | %FileCheck %s + +# CHECK: --- Installing cmark --- +# CHECK: --- Installing llvm --- +# CHECK: --- Installing swift --- diff --git a/validation-test/BuildSystem/install_all_linux.test b/validation-test/BuildSystem/install_all_linux.test new file mode 100644 index 0000000000000..fa2c103503bee --- /dev/null +++ b/validation-test/BuildSystem/install_all_linux.test @@ -0,0 +1,13 @@ +# RUN: %empty-directory(%t) +# RUN: mkdir -p %t +# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --llbuild --swiftpm --foundation --libdispatch --cmake %cmake 2>&1 | %FileCheck %s + +# REQUIRES: OS=linux-gnu + +# CHECK-DAG: --- Installing cmark --- +# CHECK-DAG: --- Installing swift --- +# CHECK-DAG: --- Installing llvm --- +# CHECK-DAG: --- Installing llbuild --- +# CHECK-DAG: --- Installing foundation --- +# CHECK-DAG: --- Installing libdispatch --- +# CHECK-DAG: --- Installing swiftpm --- diff --git a/validation-test/BuildSystem/skip_cmark_swift_llvm.test b/validation-test/BuildSystem/skip_cmark_swift_llvm.test new file mode 100644 index 0000000000000..093aef4d755c8 --- /dev/null +++ b/validation-test/BuildSystem/skip_cmark_swift_llvm.test @@ -0,0 +1,33 @@ +# RUN: %empty-directory(%t) +# RUN: mkdir -p %t +# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake --skip-build-cmark 2>&1 | %FileCheck --check-prefix=SKIP-CMARK-CHECK %s + +# RUN: %empty-directory(%t) +# RUN: mkdir -p %t +# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake --skip-build-llvm 2>&1 | %FileCheck --check-prefix=SKIP-LLVM-CHECK %s + +# RUN: %empty-directory(%t) +# RUN: mkdir -p %t +# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake --skip-build-swift 2>&1 | %FileCheck --check-prefix=SKIP-SWIFT-CHECK %s + +# SKIP-CMARK-CHECK-NOT: cmake --build {{.*}}cmark- +# SKIP-CMARK-CHECK: cmake --build {{.*}}llvm- +# SKIP-CMARK-CHECK: cmake --build {{.*}}swift- +# SKIP-CMARK-CHECK-NOT: --- Installing cmark --- +# SKIP-CMARK-CHECK: --- Installing llvm --- +# SKIP-CMARK-CHECK: --- Installing swift --- + +# SKIP-LLVM-CHECK: cmake --build {{.*}}cmark- +# SKIP-LLVM-CHECK-NOT: cmake --build {{.*}}llvm- +# SKIP-LLVM-CHECK: cmake --build {{.*}}swift- +# SKIP-LLVM-CHECK: --- Installing cmark --- +# SKIP-LLVM-CHECK-NOT: --- Installing llvm --- +# SKIP-LLVM-CHECK: --- Installing swift --- + +# SKIP-SWIFT-CHECK: cmake --build {{.*}}cmark- +# SKIP-SWIFT-CHECK: cmake --build {{.*}}llvm- +# SKIP-SWIFT-CHECK-NOT: cmake --build {{.*}}swift- +# SKIP-SWIFT-CHECK: --- Installing cmark --- +# SKIP-SWIFT-CHECK: --- Installing llvm --- +# SKIP-SWIFT-CHECK-NOT: --- Installing swift --- + diff --git a/validation-test/lit.site.cfg.in b/validation-test/lit.site.cfg.in index 3666329b720d6..343ddb16bb1de 100644 --- a/validation-test/lit.site.cfg.in +++ b/validation-test/lit.site.cfg.in @@ -13,6 +13,7 @@ import sys import platform +config.cmake = "@CMAKE_COMMAND@" config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@" config.llvm_obj_root = "@LLVM_BINARY_DIR@" config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"