From 2b0962ca031b70136d9791d4d46cf8deb09fdce3 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 5 Jun 2024 15:06:54 -0500 Subject: [PATCH 01/11] Add swift-testing to update-checkout (cherry picked from commit 19910c6c227ff7e05bd344a85cb66fa1acc4b263) --- utils/update_checkout/update-checkout-config.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index e4e81b215aa3a..bcba447675679 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -34,6 +34,8 @@ "remote": { "id": "apple/swift-system" } }, "swift-stress-tester": { "remote": { "id": "swiftlang/swift-stress-tester" } }, + "swift-testing": { + "remote": { "id": "swiftlang/swift-testing" } }, "swift-corelibs-xctest": { "remote": { "id": "apple/swift-corelibs-xctest" } }, "swift-corelibs-foundation": { @@ -128,6 +130,7 @@ "swift-syntax": "main", "swift-system": "1.3.0", "swift-stress-tester": "main", + "swift-testing": "main", "swift-corelibs-xctest": "main", "swift-corelibs-foundation": "main", "swift-foundation-icu": "main", @@ -277,6 +280,7 @@ "swift-syntax": "main", "swift-system": "1.3.0", "swift-stress-tester": "main", + "swift-testing": "main", "swift-corelibs-xctest": "main", "swift-corelibs-foundation": "main", "swift-corelibs-libdispatch": "main", @@ -654,6 +658,7 @@ "swift-syntax": "main", "swift-system": "1.3.0", "swift-stress-tester": "main", + "swift-testing": "main", "swift-corelibs-xctest": "main", "swift-corelibs-foundation": "main", "swift-corelibs-libdispatch": "main", From 709d909ccad9cc44c9179c8ec4ad728885095631 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 10 Jul 2024 13:42:36 -0500 Subject: [PATCH 02/11] Build script changes to begin building and installing swift-testing into toolchains (on Darwin, so far) (cherry picked from commit 97e85a6bf20b38e2ac754de81d1d87585cea2a95) --- utils/build-presets.ini | 2 + .../build_swift/driver_arguments.py | 5 ++ utils/build_swift/tests/expected_options.py | 4 ++ .../build_script_invocation.py | 2 + .../productpipeline_list_builder.py | 2 +- .../swift_build_support/products/__init__.py | 2 + .../products/swift_testing.py | 67 +++++++++++++++++++ 7 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 utils/swift_build_support/swift_build_support/products/swift_testing.py diff --git a/utils/build-presets.ini b/utils/build-presets.ini index 2e83a50dbbe9a..a77ace78cf400 100644 --- a/utils/build-presets.ini +++ b/utils/build-presets.ini @@ -1309,6 +1309,7 @@ swift-driver # Failing to build in CI: rdar://78408440 # swift-inspect swiftsyntax +swift-testing swiftformat playgroundsupport indexstore-db @@ -1351,6 +1352,7 @@ install-llbuild install-swiftpm install-swift-driver install-swiftsyntax +install-swift-testing install-playgroundsupport install-sourcekit-lsp install-swiftformat diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index 360b507b82e87..6a85a4c490a24 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -811,6 +811,11 @@ def create_argument_parser(): option(['--wasmkit'], toggle_true('build_wasmkit'), help='build WasmKit') + option('--swift-testing', toggle_true('build_swift_testing'), + help='build swift-testing') + option('--install-swift-testing', toggle_true('install_swift_testing'), + help='install swift-testing') + option('--xctest', toggle_true('build_xctest'), help='build xctest') diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index c90e7367d1e17..eeb965f34da54 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -95,6 +95,7 @@ 'build_swiftpm': False, 'build_swift_driver': False, 'build_swift_libexec': True, + 'build_swift_testing': False, 'build_early_swift_driver': True, 'build_early_swiftsyntax': True, 'build_swiftsyntax': False, @@ -111,6 +112,7 @@ 'install_static_linux_config': False, 'install_swiftpm': False, 'install_swiftsyntax': False, + 'install_swift_testing': False, 'install_swift_driver': False, 'install_swiftdocc': False, 'swiftsyntax_verify_generated_files': False, @@ -565,6 +567,7 @@ class BuildScriptImplOption(_BaseOption): SetTrueOption('--swiftpm', dest='build_swiftpm'), SetTrueOption('--swift-driver', dest='build_swift_driver'), SetTrueOption('--swiftsyntax', dest='build_swiftsyntax'), + SetTrueOption('--swift-testing', dest='build_swift_testing'), SetTrueOption('--skstresstester', dest='build_skstresstester'), SetTrueOption('--swiftformat', dest='build_swiftformat'), SetTrueOption('--swiftdocc', dest='build_swiftdocc'), @@ -644,6 +647,7 @@ class BuildScriptImplOption(_BaseOption): EnableOption('--install-llvm', dest='install_llvm'), EnableOption('--install-static-linux-config', dest='install_static_linux_config'), EnableOption('--install-swiftsyntax', dest='install_swiftsyntax'), + EnableOption('--install-swift-testing', dest='install_swift_testing'), EnableOption('--swiftsyntax-verify-generated-files', dest='swiftsyntax_verify_generated_files'), EnableOption('--swiftsyntax-enable-rawsyntax-validation', diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index d14f79359bc24..af32615fce574 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -666,6 +666,8 @@ def compute_product_pipelines(self): is_enabled=self.args.build_swiftpm) builder.add_product(products.SwiftSyntax, is_enabled=self.args.build_swiftsyntax) + builder.add_product(products.SwiftTesting, + is_enabled=self.args.build_swift_testing) builder.add_product(products.SwiftFormat, is_enabled=self.args.build_swiftformat) builder.add_product(products.SKStressTester, diff --git a/utils/swift_build_support/swift_build_support/productpipeline_list_builder.py b/utils/swift_build_support/swift_build_support/productpipeline_list_builder.py index 09c16611090ef..056e8af2f117f 100644 --- a/utils/swift_build_support/swift_build_support/productpipeline_list_builder.py +++ b/utils/swift_build_support/swift_build_support/productpipeline_list_builder.py @@ -101,7 +101,7 @@ def add_product(self, product_cls, is_enabled): self.current_pipeline.append(product_cls, is_enabled) def add_impl_product(self, product_cls, is_enabled): - """Add a non-impl product to the current pipeline begin constructed""" + """Add an impl product to the current pipeline begin constructed""" assert self.current_pipeline is not None assert self.is_current_pipeline_impl assert product_cls.is_build_script_impl_product() diff --git a/utils/swift_build_support/swift_build_support/products/__init__.py b/utils/swift_build_support/swift_build_support/products/__init__.py index 3af36fe224847..087e3eb98b561 100644 --- a/utils/swift_build_support/swift_build_support/products/__init__.py +++ b/utils/swift_build_support/swift_build_support/products/__init__.py @@ -30,6 +30,7 @@ from .sourcekitlsp import SourceKitLSP from .staticswiftlinux import StaticSwiftLinuxConfig from .swift import Swift +from .swift_testing import SwiftTesting from .swiftdocc import SwiftDocC from .swiftdoccrender import SwiftDocCRender from .swiftdriver import SwiftDriver @@ -65,6 +66,7 @@ 'SwiftInspect', 'SwiftPM', 'SwiftDriver', + 'SwiftTesting', 'EarlySwiftDriver', 'XCTest', 'SwiftSyntax', diff --git a/utils/swift_build_support/swift_build_support/products/swift_testing.py b/utils/swift_build_support/swift_build_support/products/swift_testing.py new file mode 100644 index 0000000000000..98213a9f1a2ec --- /dev/null +++ b/utils/swift_build_support/swift_build_support/products/swift_testing.py @@ -0,0 +1,67 @@ +# swift_build_support/products/xctest.py -------------------------*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2024 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- + +import os + +from . import cmake_product +from . import swift +from . import swiftsyntax + +class SwiftTesting(cmake_product.CMakeProduct): + @classmethod + def is_build_script_impl_product(cls): + return False + + @classmethod + def is_before_build_script_impl_product(cls): + return False + + @classmethod + def product_source_name(cls): + return "swift-testing" + + @classmethod + def get_dependencies(cls): + return [swift.Swift, + swiftsyntax.SwiftSyntax] + + def should_build(self, host_target): + return True + + def build(self, host_target): + self.cmake_options.define('BUILD_SHARED_LIBS', 'YES') + + # Use empty CMake install prefix, since the `DESTDIR` env var is set by + # `install_with_cmake` later which already has the same prefix. + self.cmake_options.define('CMAKE_INSTALL_PREFIX', '') + + build_root = os.path.dirname(self.build_dir) + swift_build_dir = os.path.join( + '..', build_root, '%s-%s' % ('swift', host_target)) + swift_cmake_dir = os.path.join(swift_build_dir, 'cmake', 'modules') + self.cmake_options.define('SwiftSyntax_DIR:PATH', swift_cmake_dir) + + self.build_with_cmake([], self.args.build_variant, [], + prefer_native_toolchain=True) + + def should_test(self, host_target): + # TODO + return False + + def should_install(self, host_target): + return self.args.install_swift_testing + + def install(self, host_target): + install_destdir = self.host_install_destdir(host_target) + install_prefix = install_destdir + self.args.install_prefix + + self.install_with_cmake(['install'], install_prefix) From 51d0d2774a8f5661ed41726a0de80302a926c25a Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Thu, 11 Jul 2024 21:40:38 -0500 Subject: [PATCH 03/11] Add swift-testing to Linux presets too (cherry picked from commit ff5ece15bc5e7a0faa6d1a0abbe57f146f349142) --- utils/build-presets.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/build-presets.ini b/utils/build-presets.ini index a77ace78cf400..9a6d47bbfb448 100644 --- a/utils/build-presets.ini +++ b/utils/build-presets.ini @@ -835,6 +835,7 @@ mixin-preset= llbuild swiftpm swift-driver +swift-testing xctest libicu swiftdocc @@ -849,6 +850,7 @@ install-llbuild install-swiftpm install-swift-driver install-swiftsyntax +install-swift-testing install-xctest install-libicu install-prefix=/usr From 136417963e5d14ac4a5b94ad4a40bf6115cd290e Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 16 Jul 2024 08:53:38 -0500 Subject: [PATCH 04/11] Improve description of new build-script flags (cherry picked from commit c8d7d134b01897d16d0b4617a65911c38aec3c5a) --- utils/build_swift/build_swift/driver_arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index 6a85a4c490a24..603ad5693d6a2 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -812,9 +812,9 @@ def create_argument_parser(): help='build WasmKit') option('--swift-testing', toggle_true('build_swift_testing'), - help='build swift-testing') + help='build Swift Testing') option('--install-swift-testing', toggle_true('install_swift_testing'), - help='install swift-testing') + help='install Swift Testing') option('--xctest', toggle_true('build_xctest'), help='build xctest') From d1d2bfa6fb2a6cd396623c0f61c7f3cb788cce75 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 16 Jul 2024 13:38:29 -0500 Subject: [PATCH 05/11] Fix header comment (cherry picked from commit 155e4c857bbee230ea16b228e2f7d3612507d14e) --- .../swift_build_support/products/swift_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/swift_build_support/swift_build_support/products/swift_testing.py b/utils/swift_build_support/swift_build_support/products/swift_testing.py index 98213a9f1a2ec..fb2a91942d5b7 100644 --- a/utils/swift_build_support/swift_build_support/products/swift_testing.py +++ b/utils/swift_build_support/swift_build_support/products/swift_testing.py @@ -1,4 +1,4 @@ -# swift_build_support/products/xctest.py -------------------------*- python -*- +# swift_build_support/products/swift_testing.py -----------------*- python -*- # # This source file is part of the Swift.org open source project # From bd1bbf2f25a38532b70d2dc80d021998c95e63d3 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Wed, 24 Jul 2024 22:34:47 +0000 Subject: [PATCH 06/11] [swift-testing] Build tweak * Remove SwiftSyntax product dependency * Correct CMAKE_BUILD_TYPE (cherry picked from commit e06d38b3ac6b063fc116df9dfcb44b48eeefb5f3) --- .../swift_build_support/products/swift_testing.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/products/swift_testing.py b/utils/swift_build_support/swift_build_support/products/swift_testing.py index fb2a91942d5b7..773a1174f8ff4 100644 --- a/utils/swift_build_support/swift_build_support/products/swift_testing.py +++ b/utils/swift_build_support/swift_build_support/products/swift_testing.py @@ -14,7 +14,7 @@ from . import cmake_product from . import swift -from . import swiftsyntax + class SwiftTesting(cmake_product.CMakeProduct): @classmethod @@ -31,8 +31,7 @@ def product_source_name(cls): @classmethod def get_dependencies(cls): - return [swift.Swift, - swiftsyntax.SwiftSyntax] + return [swift.Swift] def should_build(self, host_target): return True @@ -44,6 +43,8 @@ def build(self, host_target): # `install_with_cmake` later which already has the same prefix. self.cmake_options.define('CMAKE_INSTALL_PREFIX', '') + self.cmake_options.define('CMAKE_BUILD_TYPE', self.args.build_variant) + build_root = os.path.dirname(self.build_dir) swift_build_dir = os.path.join( '..', build_root, '%s-%s' % ('swift', host_target)) From 09f1fe5e5b26a257906d4f0336c2fd1014ec7f28 Mon Sep 17 00:00:00 2001 From: Rintaro Date: Thu, 25 Jul 2024 15:00:37 -0700 Subject: [PATCH 07/11] [Windows] Build and install swift-testing in Windows toolchains * Basically following XCTest scheme. * Build TestingMacro separately from Testing library and install it to the toolchain's `bin` * Testing swift-testing itself is TODO (cherry picked from commit f069aecca504ce98abf85faf80d4919ca59af1cc) --- utils/build.ps1 | 63 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index e4ab86794061b..eaf26909a2aab 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -187,6 +187,7 @@ $ArchX64 = @{ PlatformInstallRoot = "$BinaryCache\x64\Windows.platform"; SDKInstallRoot = "$BinaryCache\x64\Windows.platform\Developer\SDKs\Windows.sdk"; XCTestInstallRoot = "$BinaryCache\x64\Windows.platform\Developer\Library\XCTest-development"; + SwiftTestingInstallRoot = "$BinaryCache\x64\Windows.platform\Developer\Library\Testing-development"; ToolchainInstallRoot = "$BinaryCache\x64\toolchains\$ProductVersion+Asserts"; } @@ -202,6 +203,7 @@ $ArchX86 = @{ PlatformInstallRoot = "$BinaryCache\x86\Windows.platform"; SDKInstallRoot = "$BinaryCache\x86\Windows.platform\Developer\SDKs\Windows.sdk"; XCTestInstallRoot = "$BinaryCache\x86\Windows.platform\Developer\Library\XCTest-development"; + SwiftTestingInstallRoot = "$BinaryCache\x86\Windows.platform\Developer\Library\Testing-development"; } $ArchARM64 = @{ @@ -217,6 +219,7 @@ $ArchARM64 = @{ SDKInstallRoot = "$BinaryCache\arm64\Windows.platform\Developer\SDKs\Windows.sdk"; XCTestInstallRoot = "$BinaryCache\arm64\Windows.platform\Developer\Library\XCTest-development"; ToolchainInstallRoot = "$BinaryCache\arm64\toolchains\$ProductVersion+Asserts"; + SwiftTestingInstallRoot = "$BinaryCache\arm64\Windows.platform\Developer\Library\Testing-development"; } $HostArch = switch ($HostArchName) { @@ -284,6 +287,7 @@ enum TargetComponent { Dispatch Foundation XCTest + SwiftTesting } function Get-TargetProjectBinaryCache($Arch, [TargetComponent]$Project) { @@ -311,6 +315,7 @@ enum HostComponent { LMDB SymbolKit DocC + SwiftTestingMacros } function Get-HostProjectBinaryCache([HostComponent]$Project) { @@ -1542,11 +1547,43 @@ function Build-XCTest([Platform]$Platform, $Arch, [switch]$Test = $false) { dispatch_DIR = "$DispatchBinaryCache\cmake\modules"; Foundation_DIR = "$FoundationBinaryCache\cmake\modules"; } + $TestingDefines) + } +} + +function Build-SwiftTesting([Platform]$Platform, $Arch, [switch]$Test = $false) { + $SwiftTestingBinaryCache = Get-TargetProjectBinaryCache $Arch SwiftTesting + + Isolate-EnvVars { + if ($Test) { + # TODO: Test + return + } else { + $Targets = @("default") + $InstallPath = "$($Arch.SwiftTestingInstallRoot)\usr" + } + Build-CMakeProject ` + -Src $SourceCache\swift-testing ` + -Bin $SwiftTestingBinaryCache ` + -InstallTo $InstallPath ` + -Arch $Arch ` + -Platform $Platform ` + -UseBuiltCompilers C,CXX,Swift ` + -BuildTargets $Targets ` + -Defines (@{ + BUILD_SHARED_LIBS = "YES"; + CMAKE_BUILD_WITH_INSTALL_RPATH = "YES"; + SwiftSyntax_DIR = (Get-HostProjectCMakeModules Compilers); + # FIXME: Build the plugin for the builder and specify the path. + SwiftTesting_MACRO = "NO"; + }) + } +} + +function Write-PlatformInfoPlist([Platform]$Platform, $Arch) { $PList = [IO.Path]::Combine($Arch.BinaryCache, "${Platform}.platform".ToLower(), "Info.plist") - Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" ` + Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFT_TESTING_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" ` -OutFile "$PList" - } } # Copies files installed by CMake from the arch-specific platform root, @@ -1950,6 +1987,21 @@ function Build-SourceKitLSP($Arch) { } } +function Build-SwiftTestingMacros($Arch) { + Build-CMakeProject ` + -Src $SourceCache\swift-testing\Sources\TestingMacros ` + -Bin (Get-HostProjectBinaryCache SwiftTestingMacros) ` + -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` + -Arch $Arch ` + -Platform Windows ` + -UseBuiltCompilers Swift ` + -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` + -BuildTargets default ` + -Defines @{ + SwiftSyntax_DIR = (Get-HostProjectCMakeModules Compilers); + } +} + function Install-HostToolchain() { if ($ToBatch) { return } @@ -2110,6 +2162,8 @@ if (-not $SkipBuild) { Invoke-BuildStep Build-FoundationMacros -Build Windows $BuildArch Invoke-BuildStep Build-Foundation Windows $Arch Invoke-BuildStep Build-XCTest Windows $Arch + Invoke-BuildStep Build-SwiftTesting Windows $Arch + Invoke-BuildStep Write-PlatformInfoPlist Windows $Arch } } @@ -2133,6 +2187,8 @@ if (-not $ToBatch) { } if (-not $SkipBuild) { + # TestingMacros can't be built before the standard library for the host as it is required for the Swift code. + Invoke-BuildStep Build-SwiftTestingMacros $HostArch Invoke-BuildStep Build-SQLite $HostArch Invoke-BuildStep Build-System $HostArch Invoke-BuildStep Build-ToolsSupportCore $HostArch @@ -2187,6 +2243,9 @@ if (-not $IsCrossCompiling) { if ($Test -contains "xctest") { Build-XCTest Windows $HostArch -Test } + if ($Test -contains "testing") { + Build-SwiftTesting Windows $HostArch -Test + } if ($Test -contains "llbuild") { Build-LLBuild $HostArch -Test } if ($Test -contains "swiftpm") { Test-PackageManager $HostArch } } From ddbce082eb49c11e025be6f525b4ae3f9470709f Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Sat, 3 Aug 2024 09:54:27 -0700 Subject: [PATCH 08/11] [build-preset] Add swift-testing to incrmental bots (cherry picked from commit 6a383293c4a916b3e77924cf1c45fecf771c61ef) --- utils/build-presets.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/build-presets.ini b/utils/build-presets.ini index 9a6d47bbfb448..0dc141e12837a 100644 --- a/utils/build-presets.ini +++ b/utils/build-presets.ini @@ -379,6 +379,7 @@ sourcekit-lsp # swift-inspect swiftsyntax swiftformat +swift-testing skstresstester @@ -402,6 +403,7 @@ install-swiftpm install-swift-driver install-swiftsyntax install-swiftformat +install-swift-testing [preset: buildbot_incremental,tools=RA,stdlib=RA,apple_silicon] mixin-preset=buildbot_incremental,tools=RA,stdlib=RA @@ -1171,12 +1173,14 @@ llbuild swiftpm swift-driver xctest +swift-testing foundation libdispatch swiftsyntax swiftformat indexstore-db sourcekit-lsp + install-llvm install-static-linux-config install-swift @@ -1188,6 +1192,7 @@ install-swiftsyntax install-foundation install-libdispatch install-xctest +install-swift-testing install-swiftformat [preset: buildbot_incremental_linux,long_test] @@ -1656,6 +1661,7 @@ swiftpm # swift-inspect swift-driver swiftsyntax +swift-testing swiftformat swiftdocc @@ -1675,6 +1681,7 @@ install-llbuild install-swiftpm install-swift-driver install-swiftsyntax +install-swift-testing install-swiftdocc install-swiftformat From b89bbdbc388abb62e3f745a6a75cd39d4a716995 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Thu, 8 Aug 2024 15:00:27 -0700 Subject: [PATCH 09/11] [build-script] Separate SwiftTestinMacros from SwiftTesting product Also, build them for all hosts including cross compiling host. (cherry picked from commit e98c5eab6111801357176ef10992ec63eef5f071) --- utils/build-presets.ini | 10 ++ .../build_swift/driver_arguments.py | 5 + utils/build_swift/tests/expected_options.py | 5 + .../build_script_invocation.py | 2 + .../swift_build_support/products/__init__.py | 2 + .../swift_build_support/products/product.py | 21 +++- .../products/swift_testing.py | 73 ++++++++++-- .../products/swift_testing_macros.py | 112 ++++++++++++++++++ 8 files changed, 211 insertions(+), 19 deletions(-) create mode 100644 utils/swift_build_support/swift_build_support/products/swift_testing_macros.py diff --git a/utils/build-presets.ini b/utils/build-presets.ini index 0dc141e12837a..b857c8799a39e 100644 --- a/utils/build-presets.ini +++ b/utils/build-presets.ini @@ -380,6 +380,7 @@ sourcekit-lsp swiftsyntax swiftformat swift-testing +swift-testing-macros skstresstester @@ -404,6 +405,7 @@ install-swift-driver install-swiftsyntax install-swiftformat install-swift-testing +install-swift-testing-macros [preset: buildbot_incremental,tools=RA,stdlib=RA,apple_silicon] mixin-preset=buildbot_incremental,tools=RA,stdlib=RA @@ -838,6 +840,7 @@ llbuild swiftpm swift-driver swift-testing +swift-testing-macros xctest libicu swiftdocc @@ -853,6 +856,7 @@ install-swiftpm install-swift-driver install-swiftsyntax install-swift-testing +install-swift-testing-macros install-xctest install-libicu install-prefix=/usr @@ -1174,6 +1178,7 @@ swiftpm swift-driver xctest swift-testing +swift-testing-macros foundation libdispatch swiftsyntax @@ -1193,6 +1198,7 @@ install-foundation install-libdispatch install-xctest install-swift-testing +install-swift-testing-macros install-swiftformat [preset: buildbot_incremental_linux,long_test] @@ -1317,6 +1323,7 @@ swift-driver # swift-inspect swiftsyntax swift-testing +swift-testing-macros swiftformat playgroundsupport indexstore-db @@ -1360,6 +1367,7 @@ install-swiftpm install-swift-driver install-swiftsyntax install-swift-testing +install-swift-testing-macros install-playgroundsupport install-sourcekit-lsp install-swiftformat @@ -1662,6 +1670,7 @@ swiftpm swift-driver swiftsyntax swift-testing +swift-testing-macros swiftformat swiftdocc @@ -1682,6 +1691,7 @@ install-swiftpm install-swift-driver install-swiftsyntax install-swift-testing +install-swift-testing-macros install-swiftdocc install-swiftformat diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index 603ad5693d6a2..cd9480cc7b3a3 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -815,6 +815,11 @@ def create_argument_parser(): help='build Swift Testing') option('--install-swift-testing', toggle_true('install_swift_testing'), help='install Swift Testing') + option('--swift-testing-macros', toggle_true('build_swift_testing_macros'), + help='build Swift Testing macro plugin') + option('--install-swift-testing-macros', + toggle_true('install_swift_testing_macros'), + help='install Swift Testing macro plugin') option('--xctest', toggle_true('build_xctest'), help='build xctest') diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index eeb965f34da54..5d1a6a6898aac 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -96,6 +96,7 @@ 'build_swift_driver': False, 'build_swift_libexec': True, 'build_swift_testing': False, + 'build_swift_testing_macros': False, 'build_early_swift_driver': True, 'build_early_swiftsyntax': True, 'build_swiftsyntax': False, @@ -113,6 +114,7 @@ 'install_swiftpm': False, 'install_swiftsyntax': False, 'install_swift_testing': False, + 'install_swift_testing_macros': False, 'install_swift_driver': False, 'install_swiftdocc': False, 'swiftsyntax_verify_generated_files': False, @@ -568,6 +570,7 @@ class BuildScriptImplOption(_BaseOption): SetTrueOption('--swift-driver', dest='build_swift_driver'), SetTrueOption('--swiftsyntax', dest='build_swiftsyntax'), SetTrueOption('--swift-testing', dest='build_swift_testing'), + SetTrueOption('--swift-testing-macros', dest='build_swift_testing_macros'), SetTrueOption('--skstresstester', dest='build_skstresstester'), SetTrueOption('--swiftformat', dest='build_swiftformat'), SetTrueOption('--swiftdocc', dest='build_swiftdocc'), @@ -648,6 +651,8 @@ class BuildScriptImplOption(_BaseOption): EnableOption('--install-static-linux-config', dest='install_static_linux_config'), EnableOption('--install-swiftsyntax', dest='install_swiftsyntax'), EnableOption('--install-swift-testing', dest='install_swift_testing'), + EnableOption('--install-swift-testing-macros', + dest='install_swift_testing_macros'), EnableOption('--swiftsyntax-verify-generated-files', dest='swiftsyntax_verify_generated_files'), EnableOption('--swiftsyntax-enable-rawsyntax-validation', diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index af32615fce574..56cbcb44ce771 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -666,6 +666,8 @@ def compute_product_pipelines(self): is_enabled=self.args.build_swiftpm) builder.add_product(products.SwiftSyntax, is_enabled=self.args.build_swiftsyntax) + builder.add_product(products.SwiftTestingMacros, + is_enabled=self.args.build_swift_testing_macros) builder.add_product(products.SwiftTesting, is_enabled=self.args.build_swift_testing) builder.add_product(products.SwiftFormat, diff --git a/utils/swift_build_support/swift_build_support/products/__init__.py b/utils/swift_build_support/swift_build_support/products/__init__.py index 087e3eb98b561..e2d657702b169 100644 --- a/utils/swift_build_support/swift_build_support/products/__init__.py +++ b/utils/swift_build_support/swift_build_support/products/__init__.py @@ -31,6 +31,7 @@ from .staticswiftlinux import StaticSwiftLinuxConfig from .swift import Swift from .swift_testing import SwiftTesting +from .swift_testing_macros import SwiftTestingMacros from .swiftdocc import SwiftDocC from .swiftdoccrender import SwiftDocCRender from .swiftdriver import SwiftDriver @@ -67,6 +68,7 @@ 'SwiftPM', 'SwiftDriver', 'SwiftTesting', + 'SwiftTestingMacros', 'EarlySwiftDriver', 'XCTest', 'SwiftSyntax', diff --git a/utils/swift_build_support/swift_build_support/products/product.py b/utils/swift_build_support/swift_build_support/products/product.py index ababc351f95ee..a3f2bc91c2148 100644 --- a/utils/swift_build_support/swift_build_support/products/product.py +++ b/utils/swift_build_support/swift_build_support/products/product.py @@ -293,7 +293,8 @@ def target_for_platform(self, platform, arch, include_version=True): arch, self.args.darwin_deployment_version_xros) return target - def generate_darwin_toolchain_file(self, platform, arch): + def generate_darwin_toolchain_file(self, platform, arch, + macos_deployment_version=None): """ Generates a new CMake tolchain file that specifies Darwin as a target plaftorm. @@ -306,9 +307,14 @@ def generate_darwin_toolchain_file(self, platform, arch): cmake_osx_sysroot = xcrun.sdk_path(platform) - target = self.target_for_platform(platform, arch) - if not target: - raise RuntimeError('Unhandled platform {}?!'.format(platform)) + if platform == 'macosx': + if macos_deployment_version is None: + macos_deployment_version = self.args.darwin_deployment_version_osx + target = '{}-apple-macosx{}'.format(arch, macos_deployment_version) + else: + target = self.target_for_platform(platform, arch) + if not target: + raise RuntimeError('Unhandled platform {}?!'.format(platform)) toolchain_args = {} @@ -429,7 +435,8 @@ def generate_linux_toolchain_file(self, platform, arch): return toolchain_file - def generate_toolchain_file_for_darwin_or_linux(self, host_target): + def generate_toolchain_file_for_darwin_or_linux( + self, host_target, override_macos_deployment_version=None): """ Checks `host_target` platform and generates a new CMake tolchain file appropriate for that target plaftorm (either Darwin or Linux). Defines @@ -448,7 +455,9 @@ def generate_toolchain_file_for_darwin_or_linux(self, host_target): toolchain_file = None if self.is_darwin_host(host_target): - toolchain_file = self.generate_darwin_toolchain_file(platform, arch) + toolchain_file = self.generate_darwin_toolchain_file( + platform, arch, + macos_deployment_version=override_macos_deployment_version) self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file) elif platform == "linux": toolchain_file = self.generate_linux_toolchain_file(platform, arch) diff --git a/utils/swift_build_support/swift_build_support/products/swift_testing.py b/utils/swift_build_support/swift_build_support/products/swift_testing.py index 773a1174f8ff4..0e96f71f14c06 100644 --- a/utils/swift_build_support/swift_build_support/products/swift_testing.py +++ b/utils/swift_build_support/swift_build_support/products/swift_testing.py @@ -12,11 +12,14 @@ import os +from build_swift.build_swift.versions import Version + from . import cmake_product +from . import product from . import swift -class SwiftTesting(cmake_product.CMakeProduct): +class SwiftTesting(product.Product): @classmethod def is_build_script_impl_product(cls): return False @@ -36,7 +39,59 @@ def get_dependencies(cls): def should_build(self, host_target): return True + def should_test(self, host_target): + # TODO: Implement. + return False + + def should_install(self, host_target): + return self.args.install_swift_testing_macros + + def _cmake_product(self, host_target): + build_root = os.path.dirname(self.build_dir) + build_dir = os.path.join( + build_root, '%s-%s' % (self.product_name(), host_target)) + + return SwiftTestingCMakeShim( + args=self.args, + toolchain=self.toolchain, + source_dir=self.source_dir, + build_dir=build_dir) + + def _build_with_cmake(self, host_target): + self._cmake_product(host_target).build(host_target) + + def build(self, host_target): + self._build_with_cmake(host_target) + + # For Darwin host, 'build' is only called for the builder. + # Manually iterate the cross compile hosts. + if self.has_cross_compile_hosts() and self.is_darwin_host(host_target): + for target in self.args.cross_compile_hosts: + self._build_with_cmake(target) + + # FIXME: build testing library for 'stdlib_deployment_targets'? + pass + + def _install_with_cmake(self, host_target): + self._cmake_product(host_target).install(host_target) + + def install(self, host_target): + self._install_with_cmake(host_target) + + # For Darwin host, 'install' is only called for the builder. + # Manually iterate the cross compile hosts. + if self.has_cross_compile_hosts() and self.is_darwin_host(host_target): + for target in self.args.cross_compile_hosts: + self._install_with_cmake(target) + + +class SwiftTestingCMakeShim(cmake_product.CMakeProduct): def build(self, host_target): + override_deployment_version = None + if host_target.startswith('macosx'): + if Version(self.args.darwin_deployment_version_osx) < Version('10.15'): + override_deployment_version = '10.15' + self.cmake_options.define('BUILD_SHARED_LIBS', 'YES') # Use empty CMake install prefix, since the `DESTDIR` env var is set by @@ -45,22 +100,14 @@ def build(self, host_target): self.cmake_options.define('CMAKE_BUILD_TYPE', self.args.build_variant) - build_root = os.path.dirname(self.build_dir) - swift_build_dir = os.path.join( - '..', build_root, '%s-%s' % ('swift', host_target)) - swift_cmake_dir = os.path.join(swift_build_dir, 'cmake', 'modules') - self.cmake_options.define('SwiftSyntax_DIR:PATH', swift_cmake_dir) + # FIXME: If we build macros for the builder, specify the path. + self.cmake_options.define('SwiftTesting_MACRO', 'NO') + self.generate_toolchain_file_for_darwin_or_linux( + host_target, override_macos_deployment_version=override_deployment_version) self.build_with_cmake([], self.args.build_variant, [], prefer_native_toolchain=True) - def should_test(self, host_target): - # TODO - return False - - def should_install(self, host_target): - return self.args.install_swift_testing - def install(self, host_target): install_destdir = self.host_install_destdir(host_target) install_prefix = install_destdir + self.args.install_prefix diff --git a/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py b/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py new file mode 100644 index 0000000000000..38e2d8ab3a8fa --- /dev/null +++ b/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py @@ -0,0 +1,112 @@ +# swift_build_support/products/swift_testing_macros.py ----------*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2024 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- + +import os + +from build_swift.build_swift.versions import Version + +from . import cmake_product +from . import product +from . import swift + + +class SwiftTestingMacros(product.Product): + @classmethod + def is_build_script_impl_product(cls): + return False + + @classmethod + def is_before_build_script_impl_product(cls): + return False + + @classmethod + def product_source_name(cls): + return "swift-testing/Sources/TestingMacros" + + @classmethod + def get_dependencies(cls): + return [swift.Swift] + + def should_build(self, host_target): + return True + + def should_test(self, host_target): + return False + + def should_install(self, host_target): + return self.args.install_swift_testing_macros + + def _cmake_product(self, host_target): + build_root = os.path.dirname(self.build_dir) + build_dir = os.path.join( + build_root, '%s-%s' % (self.product_name(), host_target)) + + return SwiftTestingMacrosCMakeShim( + args=self.args, + toolchain=self.toolchain, + source_dir=self.source_dir, + build_dir=build_dir) + + def _build_with_cmake(self, host_target): + self._cmake_product(host_target).build(host_target) + + def build(self, host_target): + self._build_with_cmake(host_target) + + # For Darwin host, 'build' is only called for the builder. + # Manually iterate the cross compile hosts. + if self.has_cross_compile_hosts() and self.is_darwin_host(host_target): + for target in self.args.cross_compile_hosts: + self._build_with_cmake(target) + + def _install_with_cmake(self, host_target): + self._cmake_product(host_target).install(host_target) + + def install(self, host_target): + self._install_with_cmake(host_target) + + # For Darwin host, 'install' is only called for the builder. + # Manually iterate the cross compile hosts. + if self.has_cross_compile_hosts() and self.is_darwin_host(host_target): + for target in self.args.cross_compile_hosts: + self._install_with_cmake(target) + + +class SwiftTestingMacrosCMakeShim(cmake_product.CMakeProduct): + def build(self, host_target): + override_deployment_version = None + if host_target.startswith('macosx'): + if Version(self.args.darwin_deployment_version_osx) < Version('10.15'): + override_deployment_version = '10.15' + + # Use empty CMake install prefix, since the `DESTDIR` env var is set by + # `install_with_cmake` later which already has the same prefix. + self.cmake_options.define('CMAKE_INSTALL_PREFIX', '') + + self.cmake_options.define('CMAKE_BUILD_TYPE', self.args.build_variant) + + build_root = os.path.dirname(self.build_dir) + swift_build_dir = os.path.join( + '..', build_root, '%s-%s' % ('swift', host_target)) + swift_cmake_dir = os.path.join(swift_build_dir, 'cmake', 'modules') + self.cmake_options.define('SwiftSyntax_DIR:PATH', swift_cmake_dir) + + self.generate_toolchain_file_for_darwin_or_linux( + host_target, override_macos_deployment_version=override_deployment_version) + self.build_with_cmake([], self.args.build_variant, [], + prefer_native_toolchain=True) + + def install(self, host_target): + install_destdir = self.host_install_destdir(host_target) + install_prefix = install_destdir + self.args.install_prefix + + self.install_with_cmake(['install'], install_prefix) From c48717fb72d7a4a1d837385327c1bf3964eb1d48 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Mon, 12 Aug 2024 09:08:34 -0700 Subject: [PATCH 10/11] [6.0] Add swift-testing to update-checkout-config.json To release/6.0 scheme. --- utils/update_checkout/update-checkout-config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index bcba447675679..c88d88422f2c8 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -182,6 +182,7 @@ "swift-syntax": "release/6.0", "swift-system": "1.3.0", "swift-stress-tester": "release/6.0", + "swift-testing": "release/6.0", "swift-corelibs-xctest": "release/6.0", "swift-corelibs-foundation": "release/6.0", "swift-foundation": "release/6.0", From a324b5c4deb625eb88f209ba7da8f8e63b95f6ea Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Thu, 15 Aug 2024 11:26:45 -0700 Subject: [PATCH 11/11] [build-script] Fix product dependencies SwiftPM now depends on `SwiftTesting`, `SwiftTesting` depends on `SwiftTestingMacros` rdar://133946466 (cherry picked from commit 852aa7798ff25c7ce50bdf7477f724fbc624662b) --- .../swift_build_support/build_script_invocation.py | 8 ++++---- .../swift_build_support/products/swift_testing.py | 4 +++- .../swift_build_support/products/swiftpm.py | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index 56cbcb44ce771..772c6e0f15988 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -662,14 +662,14 @@ def compute_product_pipelines(self): # Begin the post build-script-impl build phase. builder.begin_pipeline() - builder.add_product(products.SwiftPM, - is_enabled=self.args.build_swiftpm) - builder.add_product(products.SwiftSyntax, - is_enabled=self.args.build_swiftsyntax) builder.add_product(products.SwiftTestingMacros, is_enabled=self.args.build_swift_testing_macros) builder.add_product(products.SwiftTesting, is_enabled=self.args.build_swift_testing) + builder.add_product(products.SwiftPM, + is_enabled=self.args.build_swiftpm) + builder.add_product(products.SwiftSyntax, + is_enabled=self.args.build_swiftsyntax) builder.add_product(products.SwiftFormat, is_enabled=self.args.build_swiftformat) builder.add_product(products.SKStressTester, diff --git a/utils/swift_build_support/swift_build_support/products/swift_testing.py b/utils/swift_build_support/swift_build_support/products/swift_testing.py index 0e96f71f14c06..474724baddb0e 100644 --- a/utils/swift_build_support/swift_build_support/products/swift_testing.py +++ b/utils/swift_build_support/swift_build_support/products/swift_testing.py @@ -17,6 +17,7 @@ from . import cmake_product from . import product from . import swift +from . import swift_testing_macros class SwiftTesting(product.Product): @@ -34,7 +35,8 @@ def product_source_name(cls): @classmethod def get_dependencies(cls): - return [swift.Swift] + return [swift.Swift, + swift_testing_macros.SwiftTestingMacros] def should_build(self, host_target): return True diff --git a/utils/swift_build_support/swift_build_support/products/swiftpm.py b/utils/swift_build_support/swift_build_support/products/swiftpm.py index c9024e64bdaab..ea6e71d193ef2 100644 --- a/utils/swift_build_support/swift_build_support/products/swiftpm.py +++ b/utils/swift_build_support/swift_build_support/products/swiftpm.py @@ -21,6 +21,7 @@ from . import llvm from . import product from . import swift +from . import swift_testing from . import xctest from .. import shell from ..targets import StdlibDeploymentTarget @@ -143,4 +144,5 @@ def get_dependencies(cls): libdispatch.LibDispatch, foundation.Foundation, xctest.XCTest, - llbuild.LLBuild] + llbuild.LLBuild, + swift_testing.SwiftTesting]