From 8109819e214f77d7e1f8a31a1667c8f29436949f Mon Sep 17 00:00:00 2001 From: Michael Gottesman Date: Tue, 17 Aug 2021 19:33:38 -0700 Subject: [PATCH] [build-script-impl] Do not actually try to extract symbols when we perform a dry-run. Otherwise, we can't successfully dry-run a toolchain on darwin. I added an option enable-extract-symbol-dry-run-test that can be set to restore the previous behavior and updating the BuildSystem unit tests that depend on that behavior to pass that flag. --- utils/build-script-impl | 70 ++++++++++--------- .../BuildSystem/dsymutil_jobs.test | 2 +- ...ctsymbols-darwin-symroot-path-filters.test | 6 +- .../extractsymbols-default-behaviour.test | 2 +- 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/utils/build-script-impl b/utils/build-script-impl index 7aea83a787095..70e7375fccccd 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -240,6 +240,7 @@ KNOWN_SETTINGS=( skip-merge-lipo-cross-compile-tools "" "set to skip running merge-lipo after installing cross-compiled host Swift tools" coverage-db "" "If set, coverage database to use when prioritizing testing" skip-local-host-install "" "If we are cross-compiling multiple targets, skip an install pass locally if the hosts match" + enable-extract-symbol-dry-run-test "" "If we are dry-running, still run the extract symbol phase so that we can test it" ) components=( @@ -3267,40 +3268,45 @@ for host in "${ALL_HOSTS[@]}"; do # Copy executables and shared libraries from the `host_install_destdir` to # INSTALL_SYMROOT and run dsymutil on them. - (cd "${CURRENT_INSTALL_DIR}" && - find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | \ - filter_paths "${DARWIN_SYMROOT_PATH_FILTERS}" | \ - cpio --insecure -pdm -v "${host_symroot}") - - dsymutil_path= - if [[ -n "${DARWIN_INSTALL_EXTRACT_SYMBOLS_USE_JUST_BUILT_DSYMUTIL}" ]]; then - dsymutil_path=$(find_just_built_local_host_llvm_tool dsymutil) + if [[ -n "${DRY_RUN}" ]] && [[ -z "${ENABLE_EXTRACT_SYMBOL_DRY_RUN_TEST}" ]]; then + echo "DRY_RUN! Coping executables and shared libraries from the `host_install_destdir` to + INSTALL_SYMROOT and runing dsymutil on them." else - dsymutil_path=$(xcrun_find_tool dsymutil) - fi + (cd "${CURRENT_INSTALL_DIR}" && + find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | \ + filter_paths "${DARWIN_SYMROOT_PATH_FILTERS}" | \ + cpio --insecure -pdm -v "${host_symroot}") + + dsymutil_path= + if [[ -n "${DARWIN_INSTALL_EXTRACT_SYMBOLS_USE_JUST_BUILT_DSYMUTIL}" ]]; then + dsymutil_path=$(find_just_built_local_host_llvm_tool dsymutil) + else + dsymutil_path=$(xcrun_find_tool dsymutil) + fi - # Run dsymutil on executables and shared libraries. - # - # Exclude shell scripts and static archives. - # Tweak carefully the amount of parallelism -- dsymutil can be memory intensive and - # as such too many instance can exhaust the memory and slow down/panic the machine - printJSONStartTimestamp dsymutil - (cd "${host_symroot}" && - find ./"${CURRENT_PREFIX}" -perm -0111 -type f -not -name "*.a" -not -name "*.py" -print | \ - xargs -n 1 -P ${DSYMUTIL_JOBS} ${dsymutil_path}) - printJSONEndTimestamp dsymutil - - # Strip executables, shared libraries and static libraries in - # `host_install_destdir`. - find "${CURRENT_INSTALL_DIR}${CURRENT_PREFIX}/" \ - '(' -perm -0111 -or -name "*.a" ')' -type f -print | \ - xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool strip) -S - - # Codesign dylibs after strip tool - # rdar://45388785 - find "${CURRENT_INSTALL_DIR}${CURRENT_PREFIX}/" \ - '(' -name "*.dylib" ')' -type f -print | \ - xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool codesign) -f -s - + # Run dsymutil on executables and shared libraries. + # + # Exclude shell scripts and static archives. + # Tweak carefully the amount of parallelism -- dsymutil can be memory intensive and + # as such too many instance can exhaust the memory and slow down/panic the machine + printJSONStartTimestamp dsymutil + (cd "${host_symroot}" && + find ./"${CURRENT_PREFIX}" -perm -0111 -type f -not -name "*.a" -not -name "*.py" -print | \ + xargs -n 1 -P ${DSYMUTIL_JOBS} ${dsymutil_path}) + printJSONEndTimestamp dsymutil + + # Strip executables, shared libraries and static libraries in + # `host_install_destdir`. + find "${CURRENT_INSTALL_DIR}${CURRENT_PREFIX}/" \ + '(' -perm -0111 -or -name "*.a" ')' -type f -print | \ + xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool strip) -S + + # Codesign dylibs after strip tool + # rdar://45388785 + find "${CURRENT_INSTALL_DIR}${CURRENT_PREFIX}/" \ + '(' -name "*.dylib" ')' -type f -print | \ + xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool codesign) -f -s - + fi { set +x; } 2>/dev/null fi diff --git a/validation-test/BuildSystem/dsymutil_jobs.test b/validation-test/BuildSystem/dsymutil_jobs.test index 29f29a2fc5042..c0f4d70875ad9 100644 --- a/validation-test/BuildSystem/dsymutil_jobs.test +++ b/validation-test/BuildSystem/dsymutil_jobs.test @@ -4,7 +4,7 @@ # RUN: mkdir -p %t # RUN: mkdir -p %t/destdir # RUN: mkdir -p %t/symroot/macosx-%target-cpu -# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --host-target=macosx-%target-cpu --darwin-install-extract-symbols --dsymutil-jobs 5 --cmake %cmake --install-symroot=%t/symroot --install-destdir=%t/destdir --toolchain-prefix= 2>&1 | %FileCheck %s +# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --enable-extract-symbol-dry-run-test=1 --host-target=macosx-%target-cpu --darwin-install-extract-symbols --dsymutil-jobs 5 --cmake %cmake --install-symroot=%t/symroot --install-destdir=%t/destdir --toolchain-prefix= 2>&1 | %FileCheck %s # CHECK: --- Extracting symbols --- # CHECK: { "command": "dsymutil", "start": " diff --git a/validation-test/BuildSystem/extractsymbols-darwin-symroot-path-filters.test b/validation-test/BuildSystem/extractsymbols-darwin-symroot-path-filters.test index c6a8c66effc2b..997c3ed39fe22 100644 --- a/validation-test/BuildSystem/extractsymbols-darwin-symroot-path-filters.test +++ b/validation-test/BuildSystem/extractsymbols-darwin-symroot-path-filters.test @@ -22,21 +22,21 @@ # RUN: mkdir -p %t/symroot/macosx-%target-cpu # test build-script-impl on its own -# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 --darwin-symroot-path-filters="/lib/ /swift-demangle" 2>&1 | tee %t/build-script-impl-output.txt +# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --enable-extract-symbol-dry-run-test=1 --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 --darwin-symroot-path-filters="/lib/ /swift-demangle" 2>&1 | tee %t/build-script-impl-output.txt # RUN: %FileCheck --input-file=%t/build-script-impl-output.txt %s # RUN: %FileCheck --input-file=%t/build-script-impl-output.txt --check-prefixes CHECK-SKIPPED %s # ensure build-script pass the argument to build-script-impl # RUN: %empty-directory(%t/symroot) # RUN: mkdir -p %t/symroot/macosx-%target-cpu -# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --cmake %cmake --darwin-install-extract-symbols=1 --install-destdir=%t/destdir --toolchain-prefix="" --install-symroot=%t/symroot --darwin-symroot-path-filters="/lib/ /swift-demangle" --jobs=1 --host-target=macosx-%target-cpu 2>&1 | tee %t/build-script-output.txt +# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --enable-extract-symbol-dry-run-test=1 --cmake %cmake --darwin-install-extract-symbols=1 --install-destdir=%t/destdir --toolchain-prefix="" --install-symroot=%t/symroot --darwin-symroot-path-filters="/lib/ /swift-demangle" --jobs=1 --host-target=macosx-%target-cpu 2>&1 | tee %t/build-script-output.txt # RUN: %FileCheck --input-file=%t/build-script-output.txt %s # RUN: %FileCheck --input-file=%t/build-script-output.txt --check-prefixes CHECK-SKIPPED %s # ensure we get all the values if we specify the flag multiple times # RUN: %empty-directory(%t/symroot) # RUN: mkdir -p %t/symroot/macosx-%target-cpu -# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --cmake %cmake --darwin-install-extract-symbols=1 --install-destdir=%t/destdir --toolchain-prefix="" --install-symroot=%t/symroot --darwin-symroot-path-filters="/lib/" --darwin-symroot-path-filters="/swift-demangle" --jobs=1 --host-target=macosx-%target-cpu 2>&1 | tee %t/build-script-output.txt +# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --enable-extract-symbol-dry-run-test=1 --cmake %cmake --darwin-install-extract-symbols=1 --install-destdir=%t/destdir --toolchain-prefix="" --install-symroot=%t/symroot --darwin-symroot-path-filters="/lib/" --darwin-symroot-path-filters="/swift-demangle" --jobs=1 --host-target=macosx-%target-cpu 2>&1 | tee %t/build-script-output.txt # RUN: %FileCheck --input-file=%t/build-script-output.txt %s # RUN: %FileCheck --input-file=%t/build-script-output.txt --check-prefixes CHECK-SKIPPED %s diff --git a/validation-test/BuildSystem/extractsymbols-default-behaviour.test b/validation-test/BuildSystem/extractsymbols-default-behaviour.test index 36640334784e1..1ec173556f4f8 100644 --- a/validation-test/BuildSystem/extractsymbols-default-behaviour.test +++ b/validation-test/BuildSystem/extractsymbols-default-behaviour.test @@ -21,7 +21,7 @@ # RUN: chmod a+x %t/destdir/libswiftCompatibility51.a # RUN: mkdir -p %t/symroot/macosx-%target-cpu -# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 > %t/build-script-impl-output.txt 2>&1 +# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --enable-extract-symbol-dry-run-test=1 --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 > %t/build-script-impl-output.txt 2>&1 # RUN: %FileCheck --input-file=%t/build-script-impl-output.txt %s # RUN: %FileCheck --input-file=%t/build-script-impl-output.txt --check-prefixes CHECK-SKIPPED %s