From 031e2fe0ef748f812ddd2dd1658afbe75f999332 Mon Sep 17 00:00:00 2001 From: Kristopher Cieplak Date: Thu, 27 Feb 2025 15:15:18 -0500 Subject: [PATCH 1/4] Enable Windows linker discovery * Make Platform Registry initialization async * Change the function signature of additionalPlatformExecutableSearchPaths to allow passing of a filesystem for discovery. Plugins will need to be updated to match new signatures * Add the visual studio install directory to platform search paths for executables. * Enable Windows linker discovery to get link.exe type and version * Add Windows platform plugin extension to get install directory * Add test for discovery of windows linker * Paths in the clang ouput have double slashes so this needs to be handled. --- Sources/SWBCore/Core.swift | 46 ++++---- .../Extensions/PlatformInfoExtension.swift | 4 +- Sources/SWBCore/PlatformRegistry.swift | 28 ++--- .../Tools/LinkerTools.swift | 16 ++- Sources/SWBWindowsPlatform/Plugin.swift | 43 +++++--- Tests/SWBBuildSystemTests/LinkerTests.swift | 103 +++++++++++------- ...mmandLineToolSpecDiscoveredInfoTests.swift | 16 +++ .../SWBCoreTests/PlatformRegistryTests.swift | 2 +- 8 files changed, 162 insertions(+), 96 deletions(-) diff --git a/Sources/SWBCore/Core.swift b/Sources/SWBCore/Core.swift index 6bef93f1..24ea9a0c 100644 --- a/Sources/SWBCore/Core.swift +++ b/Sources/SWBCore/Core.swift @@ -98,6 +98,8 @@ public final class Core: Sendable { await core.initializeSpecRegistry() + await core.initializePlatformRegistry() + await core.initializeToolchainRegistry() // Force loading SDKs. @@ -315,27 +317,10 @@ public final class Core: Sendable { @_spi(Testing) public var toolchainPaths: [(Path, strict: Bool)] /// The platform registry. - public lazy var platformRegistry: PlatformRegistry = { - // FIXME: We should support building the platforms (with symlinks) locally (for `inferiorProductsPath`). - - // Search the default location first (unless directed not to), then search any extra locations we've been passed. - var searchPaths: [Path] - let fs = localFS - if let onlySearchAdditionalPlatformPaths = getEnvironmentVariable("XCODE_ONLY_EXTRA_PLATFORM_FOLDERS"), onlySearchAdditionalPlatformPaths.boolValue { - searchPaths = [] - } - else { - let platformsDir = self.developerPath.join("Platforms") - searchPaths = [platformsDir] - } - if let additionalPlatformSearchPaths = getEnvironmentVariable("XCODE_EXTRA_PLATFORM_FOLDERS") { - for searchPath in additionalPlatformSearchPaths.split(separator: ":") { - searchPaths.append(Path(searchPath)) - } - } - searchPaths += UserDefaults.additionalPlatformSearchPaths - return PlatformRegistry(delegate: self.registryDelegate, searchPaths: searchPaths, hostOperatingSystem: hostOperatingSystem, fs: fs) - }() + let _platformRegistry: UnsafeDelayedInitializationSendableWrapper = .init() + public var platformRegistry: PlatformRegistry { + _platformRegistry.value + } @PluginExtensionSystemActor public var loadedPluginPaths: [Path] { pluginManager.pluginsByIdentifier.values.map(\.path) @@ -397,6 +382,25 @@ public final class Core: Sendable { private var _specRegistry: SpecRegistry? + private func initializePlatformRegistry() async { + var searchPaths: [Path] + let fs = localFS + if let onlySearchAdditionalPlatformPaths = getEnvironmentVariable("XCODE_ONLY_EXTRA_PLATFORM_FOLDERS"), onlySearchAdditionalPlatformPaths.boolValue { + searchPaths = [] + } else { + let platformsDir = self.developerPath.join("Platforms") + searchPaths = [platformsDir] + } + if let additionalPlatformSearchPaths = getEnvironmentVariable("XCODE_EXTRA_PLATFORM_FOLDERS") { + for searchPath in additionalPlatformSearchPaths.split(separator: Path.pathEnvironmentSeparator) { + searchPaths.append(Path(searchPath)) + } + } + searchPaths += UserDefaults.additionalPlatformSearchPaths + _platformRegistry.initialize(to: await PlatformRegistry(delegate: self.registryDelegate, searchPaths: searchPaths, hostOperatingSystem: hostOperatingSystem, fs: fs)) + } + + private func initializeToolchainRegistry() async { self.toolchainRegistry = await ToolchainRegistry(delegate: self.registryDelegate, searchPaths: self.toolchainPaths, fs: localFS, hostOperatingSystem: hostOperatingSystem) } diff --git a/Sources/SWBCore/Extensions/PlatformInfoExtension.swift b/Sources/SWBCore/Extensions/PlatformInfoExtension.swift index a5b21d34..584d4cbc 100644 --- a/Sources/SWBCore/Extensions/PlatformInfoExtension.swift +++ b/Sources/SWBCore/Extensions/PlatformInfoExtension.swift @@ -30,7 +30,7 @@ public protocol PlatformInfoExtension: Sendable { func additionalKnownTestLibraryPathSuffixes() -> [Path] - func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path) -> [Path] + func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path, fs: any FSProxy) async -> [Path] func additionalToolchainExecutableSearchPaths(toolchainIdentifier: String, toolchainPath: Path) -> [Path] @@ -56,7 +56,7 @@ extension PlatformInfoExtension { [] } - public func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path) -> [Path] { + public func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path, fs: any FSProxy) async -> [Path] { [] } diff --git a/Sources/SWBCore/PlatformRegistry.swift b/Sources/SWBCore/PlatformRegistry.swift index 8ad27ef2..59b970c2 100644 --- a/Sources/SWBCore/PlatformRegistry.swift +++ b/Sources/SWBCore/PlatformRegistry.swift @@ -324,23 +324,23 @@ public final class PlatformRegistry { }) } - @_spi(Testing) public init(delegate: any PlatformRegistryDelegate, searchPaths: [Path], hostOperatingSystem: OperatingSystem, fs: any FSProxy) { + @_spi(Testing) public init(delegate: any PlatformRegistryDelegate, searchPaths: [Path], hostOperatingSystem: OperatingSystem, fs: any FSProxy) async { self.delegate = delegate for path in searchPaths { - registerPlatformsInDirectory(path, fs) + await registerPlatformsInDirectory(path, fs) } do { if hostOperatingSystem.createFallbackSystemToolchain { - try registerFallbackSystemPlatform(operatingSystem: hostOperatingSystem, fs: fs) + try await registerFallbackSystemPlatform(operatingSystem: hostOperatingSystem, fs: fs) } } catch { delegate.error(error) } - @preconcurrency @PluginExtensionSystemActor func platformInfoExtensions() -> [any PlatformInfoExtensionPoint.ExtensionProtocol] { - delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self) + @preconcurrency @PluginExtensionSystemActor func platformInfoExtensions() async -> [any PlatformInfoExtensionPoint.ExtensionProtocol] { + return await delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self) } struct Context: PlatformInfoExtensionAdditionalPlatformsContext { @@ -352,16 +352,17 @@ public final class PlatformRegistry { for platformExtension in platformInfoExtensions() { do { for (path, data) in try platformExtension.additionalPlatforms(context: Context(hostOperatingSystem: hostOperatingSystem, developerPath: delegate.developerPath, fs: fs)) { - registerPlatform(path, .plDict(data), fs) + await registerPlatform(path, .plDict(data), fs) } } catch { delegate.error(error) + } } } - private func registerFallbackSystemPlatform(operatingSystem: OperatingSystem, fs: any FSProxy) throws { - try registerPlatform(Path("/"), .plDict(fallbackSystemPlatformSettings(operatingSystem: operatingSystem)), fs) + private func registerFallbackSystemPlatform(operatingSystem: OperatingSystem, fs: any FSProxy) async throws { + try await registerPlatform(Path("/"), .plDict(fallbackSystemPlatformSettings(operatingSystem: operatingSystem)), fs) } private func fallbackSystemPlatformSettings(operatingSystem: OperatingSystem) throws -> [String: PropertyListItem] { @@ -428,7 +429,7 @@ public final class PlatformRegistry { } /// Register all platforms in the given directory. - private func registerPlatformsInDirectory(_ path: Path, _ fs: any FSProxy) { + private func registerPlatformsInDirectory(_ path: Path, _ fs: any FSProxy) async { for item in (try? localFS.listdir(path))?.sorted(by: <) ?? [] { let itemPath = path.join(item) @@ -446,14 +447,14 @@ public final class PlatformRegistry { // Silently skip loading the platform if it does not have an Info.plist at all. (We will still error below if it has an Info.plist which is malformed.) continue } - registerPlatform(itemPath, infoPlist, fs) + await registerPlatform(itemPath, infoPlist, fs) } catch let err { delegate.error(itemPath, "unable to load platform: 'Info.plist' was malformed: \(err)") } } } - private func registerPlatform(_ path: Path, _ data: PropertyListItem, _ fs: any FSProxy) { + private func registerPlatform(_ path: Path, _ data: PropertyListItem, _ fs: any FSProxy) async { // The data should always be a dictionary. guard case .plDict(let items) = data else { delegate.error(path, "unexpected platform data") @@ -617,7 +618,7 @@ public final class PlatformRegistry { delegate.pluginManager.extensions(of: PlatformInfoExtensionPoint.self) } - for platformExtension in platformInfoExtensions() { + for platformExtension in await platformInfoExtensions() { if let value = platformExtension.preferredArchValue(for: name) { preferredArchValue = value } @@ -632,9 +633,10 @@ public final class PlatformRegistry { ] for platformExtension in platformInfoExtensions() { - executableSearchPaths.append(contentsOf: platformExtension.additionalPlatformExecutableSearchPaths(platformName: name, platformPath: path)) + await executableSearchPaths.append(contentsOf: platformExtension.additionalPlatformExecutableSearchPaths(platformName: name, platformPath: path)) platformExtension.adjustPlatformSDKSearchPaths(platformName: name, platformPath: path, sdkSearchPaths: &sdkSearchPaths) + } executableSearchPaths.append(contentsOf: [ diff --git a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift index 1f38b0d2..895a1401 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift @@ -1309,15 +1309,14 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // // Note: On Linux you cannot invoke the llvm linker by the direct name for determining the version, // you need to use ld. - var linkerPath = Path("ld") + var linkerPath = producer.hostOperatingSystem == .windows ? Path("link") : Path("ld") if alternateLinker != "" && alternateLinker != "ld" { linkerPath = Path(producer.hostOperatingSystem.imageFormat.executableName(basename: "ld.\(alternateLinker)")) } - // Create the cache key. This is just the path to the linker we would invoke if we were invoking the linker directly. - guard let toolPath = producer.executableSearchPaths.lookup(linkerPath) else { + guard let toolPath = producer.executableSearchPaths.findExecutable(operatingSystem: producer.hostOperatingSystem, basename: linkerPath.str) else { return nil } - + // Create the cache key. This is just the path to the linker we would invoke if we were invoking the linker directly. return await discoveredLinkerToolsInfo(producer, delegate, at: toolPath) } } @@ -1638,10 +1637,19 @@ public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegat #/GNU gold \(GNU Binutils.*\) (?[\d.]+)/#, // Ubuntu "GNU gold (GNU Binutils for Ubuntu 2.38) 1.16", Debian "GNU gold (GNU Binutils for Debian 2.40) 1.16" #/GNU gold \(version .*\) (?[\d.]+)/#, // Fedora "GNU gold (version 2.40-14.fc39) 1.16", RHEL "GNU gold (version 2.35.2-54.el9) 1.16", Amazon "GNU gold (version 2.29.1-31.amzn2.0.1) 1.14" ] + if let match = try goLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first { return DiscoveredLdLinkerToolSpecInfo(linker: .gold, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set()) } + // link.exe has no option to simply dump the version, running, the program will no arguments or an invalid one will dump a header that contains the version. + let linkExe = [ + #/Microsoft \(R\) Incremental Linker Version (?[\d.]+)/# + ] + if let match = try linkExe.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first { + return DiscoveredLdLinkerToolSpecInfo(linker: .linkExe, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set()) + } + struct LDVersionDetails: Decodable { let version: Version let architectures: Set diff --git a/Sources/SWBWindowsPlatform/Plugin.swift b/Sources/SWBWindowsPlatform/Plugin.swift index 933f7d2c..5c93fc9d 100644 --- a/Sources/SWBWindowsPlatform/Plugin.swift +++ b/Sources/SWBWindowsPlatform/Plugin.swift @@ -44,23 +44,29 @@ struct WindowsPlatformSpecsExtension: SpecificationsExtension { @_spi(Testing) public func additionalEnvironmentVariables(context: any EnvironmentExtensionAdditionalEnvironmentVariablesContext) async throws -> [String: String] { if context.hostOperatingSystem == .windows { - // Add the environment variable for the MSVC toolset for Swift and Clang to find it let vcToolsInstallDir = "VCToolsInstallDir" - let installations = try await plugin.cachedVSInstallations() - .sorted(by: { $0.installationVersion > $1.installationVersion }) - if let latest = installations.first { - let msvcDir = latest.installationPath.join("VC").join("Tools").join("MSVC") - if context.fs.exists(msvcDir) { - let versions = try context.fs.listdir(msvcDir).map { try Version($0) }.sorted { $0 > $1 } - if let latestVersion = versions.first { - let dir = msvcDir.join(latestVersion.description).str - return [vcToolsInstallDir: dir] - } - } + guard let dir = try? await findLatestInstallDirectory(fs: context.fs) else { + return [:] } + return [vcToolsInstallDir: dir.str] } - return [:] } + + @_spi(Testing)public func findLatestInstallDirectory(fs: any FSProxy) async throws -> Path? { + let plugin: WindowsPlugin + let installations = try await plugin.cachedVSInstallations() + .sorted(by: { $0.installationVersion > $1.installationVersion }) + if let latest = installations.first { + let msvcDir = latest.installationPath.join("VC").join("Tools").join("MSVC") + if fs.exists(msvcDir) { + let versions = try fs.listdir(msvcDir).map { try Version($0) }.sorted { $0 > $1 } + if let latestVersion = versions.first { + return msvcDir.join(latestVersion.description) + } + } + } + return nil +} } struct WindowsPlatformExtension: PlatformInfoExtension { @@ -107,6 +113,17 @@ struct WindowsPlatformExtension: PlatformInfoExtension { sdkSearchPaths = [] } } + + public func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path, fs: any FSProxy) async -> [Path] { + guard let dir = try? await findLatestInstallDirectory(fs: fs) else { + return [] + } + if Architecture.hostStringValue == "aarch64" { + return [dir.join("bin/Hostarm64/arm64")] + } else { + return [dir.join("bin/Hostx64/x64")] + } + } } struct WindowsSDKRegistryExtension: SDKRegistryExtension { diff --git a/Tests/SWBBuildSystemTests/LinkerTests.swift b/Tests/SWBBuildSystemTests/LinkerTests.swift index 489ce717..230e06be 100644 --- a/Tests/SWBBuildSystemTests/LinkerTests.swift +++ b/Tests/SWBBuildSystemTests/LinkerTests.swift @@ -29,23 +29,25 @@ fileprivate struct LinkerTests: CoreBasedTests { groupTree: TestGroup( "SomeFiles", children: [ - TestFile("source.swift"), + TestFile("source.swift") ]), targets: [ TestStandardTarget( "testTarget", type: .framework, buildConfigurations: [ - TestBuildConfiguration("Debug", buildSettings: [ - "GENERATE_INFOPLIST_FILE": "YES", - "PRODUCT_NAME": "$(TARGET_NAME)", - "SWIFT_VERSION": swiftVersion, - "OTHER_LDFLAGS": "-not-a-real-flag" - ]), + TestBuildConfiguration( + "Debug", + buildSettings: [ + "GENERATE_INFOPLIST_FILE": "YES", + "PRODUCT_NAME": "$(TARGET_NAME)", + "SWIFT_VERSION": swiftVersion, + "OTHER_LDFLAGS": "-not-a-real-flag", + ]) ], buildPhases: [ - TestSourcesBuildPhase(["source.swift"]), + TestSourcesBuildPhase(["source.swift"]) ] - ), + ) ]) let tester = try await BuildOperationTester(getCore(), testProject, simulated: false) @@ -74,37 +76,41 @@ fileprivate struct LinkerTests: CoreBasedTests { "SomeFiles", children: [ TestFile("source.swift"), - TestFile("source.mm") + TestFile("source.mm"), ]), targets: [ TestStandardTarget( "testTarget", type: .application, buildConfigurations: [ - TestBuildConfiguration("Debug", buildSettings: [ - "GENERATE_INFOPLIST_FILE": "YES", - "PRODUCT_NAME": "$(TARGET_NAME)", - "SWIFT_VERSION": swiftVersion, - ]), + TestBuildConfiguration( + "Debug", + buildSettings: [ + "GENERATE_INFOPLIST_FILE": "YES", + "PRODUCT_NAME": "$(TARGET_NAME)", + "SWIFT_VERSION": swiftVersion, + ]) ], buildPhases: [ - TestSourcesBuildPhase(["source.mm"]), + TestSourcesBuildPhase(["source.mm"]) ], dependencies: [TestTargetDependency("testFramework")] ), TestStandardTarget( "testFramework", type: .framework, buildConfigurations: [ - TestBuildConfiguration("Debug", buildSettings: [ - "GENERATE_INFOPLIST_FILE": "YES", - "PRODUCT_NAME": "$(TARGET_NAME)", - "SWIFT_VERSION": swiftVersion, - "SWIFT_OBJC_INTEROP_MODE": enableInterop ? "objcxx" : "objc", - ]), + TestBuildConfiguration( + "Debug", + buildSettings: [ + "GENERATE_INFOPLIST_FILE": "YES", + "PRODUCT_NAME": "$(TARGET_NAME)", + "SWIFT_VERSION": swiftVersion, + "SWIFT_OBJC_INTEROP_MODE": enableInterop ? "objcxx" : "objc", + ]) ], buildPhases: [ TestSourcesBuildPhase(["source.swift"]) ] - ) + ), ]) } @@ -120,7 +126,7 @@ fileprivate struct LinkerTests: CoreBasedTests { } try await tester.checkBuild(runDestination: .macOS) { results in try results.checkTasks(.matchRuleType("Ld")) { tasks in - let task = try #require(tasks.first(where: { $0.outputPaths[0].ends(with: "testTarget") })) + let task = try #require(tasks.first(where: { $0.outputPaths[0].ends(with: "testTarget") })) task.checkCommandLineMatches([StringPattern.and(StringPattern.prefix("-L"), StringPattern.suffix("usr/lib/swift/macosx"))]) task.checkCommandLineContains(["-L/usr/lib/swift", "-lswiftCore"]) task.checkCommandLineMatches([StringPattern.suffix("testTarget.app/Contents/MacOS/testTarget")]) @@ -147,7 +153,7 @@ fileprivate struct LinkerTests: CoreBasedTests { } try await tester.checkBuild(runDestination: .macOS) { results in results.checkTasks(.matchRuleType("Ld")) { tasks in - let task = tasks.first(where: { $0.outputPaths[0].ends(with: "testTarget") })! + let task = tasks.first(where: { $0.outputPaths[0].ends(with: "testTarget") })! task.checkCommandLineNoMatch([StringPattern.and(StringPattern.prefix("-L"), StringPattern.suffix("usr/lib/swift/macosx"))]) task.checkCommandLineDoesNotContain("-L/usr/lib/swift") task.checkCommandLineDoesNotContain("-lswiftCore") @@ -165,15 +171,19 @@ fileprivate struct LinkerTests: CoreBasedTests { /// There is no reliable way to determine from a final linked binary which /// linker was used, so the test enables some verbosity to see which linker /// clang invokes. - /// Note: There is an output parser in the LinkerTool spec that does - /// error parsing and creates a new build error diagnostic with - /// a capaitalized error snippet, so this needs to be handled. + /// Notes: + /// * There is an output parser in the LinkerTool spec that does + /// error parsing and creates a new build error diagnostic with + /// a capaitalized error snippet, so this needs to be handled. + /// * The clang output on Windows has paths that have double slashes, not + /// quite valid command quoted. i.e. "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC" + /// This needs to be taken into account. @Test(.requireSDKs(.host)) func alternateLinkerSelection() async throws { let runDestination: RunDestinationInfo = .host let swiftVersion = try await self.swiftVersion try await withTemporaryDirectory { tmpDir in - let testProject = try await TestProject( + let testProject = TestProject( "TestProject", sourceRoot: tmpDir, groupTree: TestGroup( @@ -233,11 +243,7 @@ fileprivate struct LinkerTests: CoreBasedTests { let ldLinkerPath = try await self.ldPath let lldLinkerPath = try await self.lldPath let goldLinkerPath = try await self.goldPath - var linkLinkerPath = try await self.linkPath - if runDestination == .windows { - // Issue: Finding link.exe will fail until https://github.com/swiftlang/swift-build/pull/163 is merged. Clang will find it via PATH. - linkLinkerPath = Path("link.exe") - } + let linkLinkerPath = try await self.linkPath let installedLinkerPaths = [lldLinkerPath, ldLinkerPath, goldLinkerPath, linkLinkerPath].compactMap { $0 } // Default Linker @@ -247,7 +253,14 @@ fileprivate struct LinkerTests: CoreBasedTests { results.checkTaskOutput(task) { taskOutput in results.checkTaskOutput(task) { output in // Expect that one of the installed linkers is used, we are not sure which one. - #expect(installedLinkerPaths.map { $0.str }.contains(where: output.asString.contains)) + if runDestination == .windows && Architecture.hostStringValue == "aarch64" { + withKnownIssue("'clang' picks the wrong binary for link.exe using x86 version") { + // On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\arm64\link.exe" + #expect(installedLinkerPaths.map { $0.str }.contains(where: output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains)) + } + } else { + #expect(installedLinkerPaths.map { $0.str }.contains(where: output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains)) + } } } } @@ -282,13 +295,12 @@ fileprivate struct LinkerTests: CoreBasedTests { results.checkTask(.matchRuleType("Ld")) { task in task.checkCommandLineContains(["-fuse-ld=lld"]) results.checkTaskOutput(task) { output in - // Expect that the default linker is called by clang + // Expect that the llvm linker is called by clang if runDestination == .windows { // clang will choose to run lld-link rather than ld.lld.exe. - // clang output will have escaped slashes in stdout. #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(lldLinkerPath.dirname.join("lld-link").str)) } else { - #expect(output.asString.contains(lldLinkerPath.str)) + #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(lldLinkerPath.str)) } } } @@ -303,8 +315,8 @@ fileprivate struct LinkerTests: CoreBasedTests { results.checkTask(.matchRuleType("Ld")) { task in task.checkCommandLineContains(["-fuse-ld=gold"]) results.checkTaskOutput(task) { output in - // Expect that the default linker is called by clang - #expect(output.asString.contains(goldLinkerPath.str)) + // Expect that the gold linker is called by clang + #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(goldLinkerPath.str)) } } results.checkNoDiagnostics() @@ -318,8 +330,15 @@ fileprivate struct LinkerTests: CoreBasedTests { results.checkTask(.matchRuleType("Ld")) { task in task.checkCommandLineContains(["-fuse-ld=link"]) results.checkTaskOutput(task) { output in - // Expect that the default linker is called by clang - #expect(output.asString.contains(linkLinkerPath.str)) + // Expect that the 'link' linker is called by clang + if runDestination == .windows && Architecture.hostStringValue == "aarch64" { + withKnownIssue("'clang' picks the wrong binary for link.exe using Hostx86 version") { + // On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\arm64\link.exe" + #expect(output.asString.replacingOccurrences(of: "Hostx86", with: "Hostarm64").contains(linkLinkerPath.str)) + } + } else { + #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(linkLinkerPath.str)) + } } } results.checkNoDiagnostics() diff --git a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift index 4500b083..30d57a8d 100644 --- a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift +++ b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift @@ -176,6 +176,22 @@ import Testing } } + @Test(.requireHostOS(.windows)) + func discoveredLdLinkerSpecInfoWindows() async throws { + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows") { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + } + try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("Microsoft (R) Incremental Linker Version 14.41.34120.0\n".utf8), stderr: Data()), platform: "windows") { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion == Version(14, 41, 34120)) + #expect(info.architectures == Set()) + } + } + @Test(.skipHostOS(.windows), .requireSystemPackages(apt: "libtool", yum: "libtool")) func discoveredLibtoolSpecInfo() async throws { try await withSpec(LibtoolLinkerSpec.self, .deferred) { (info: DiscoveredLibtoolLinkerToolSpecInfo) in diff --git a/Tests/SWBCoreTests/PlatformRegistryTests.swift b/Tests/SWBCoreTests/PlatformRegistryTests.swift index e3b4fb6a..fd74bfc6 100644 --- a/Tests/SWBCoreTests/PlatformRegistryTests.swift +++ b/Tests/SWBCoreTests/PlatformRegistryTests.swift @@ -70,7 +70,7 @@ import SWBMacro } let delegate = await TestDataDelegate(pluginManager: PluginManager(skipLoadingPluginIdentifiers: [])) - let registry = PlatformRegistry(delegate: delegate, searchPaths: [tmpDirPath], hostOperatingSystem: try ProcessInfo.processInfo.hostOperatingSystem(), fs: localFS) + let registry = await PlatformRegistry(delegate: delegate, searchPaths: [tmpDirPath], hostOperatingSystem: try ProcessInfo.processInfo.hostOperatingSystem(), fs: localFS) try await perform(registry, delegate) } } From ca13f219ebe5b4cec12378116bf0c5d0a515108f Mon Sep 17 00:00:00 2001 From: Kristopher Cieplak Date: Fri, 7 Mar 2025 09:21:46 -0500 Subject: [PATCH 2/4] Make linkers that do not support multiple targets discoverable. The link.exe cannot link for multiple architectures there is a distinct link.exe for arm86 and x86_64. The path to the specific linkers should not be added to the global search path, as when building a target we do not know the architecture until task creation time. * Only add up until the Host[x86|arm]/bin directory to the global search path. * Let the LD spec determine the correct prefix directory to find the proper link.exe * Introduce two build settings LD_MULTIARCH - A boolean indicator for multiple architecture support LD_MULTIARCH_PREFIX_MAP - A prefix directory map from architecture:prefix * Add in specfic tests for discoveredLdLinkerSpecInfo for each linkers * Add x86_64 as a supported architecture for windows. A seperate PR is inflight to add the true supported set. --- Sources/SWBCore/SDKRegistry.swift | 10 +- Sources/SWBCore/Settings/BuiltinMacros.swift | 4 + .../Tools/LinkerTools.swift | 39 ++++++- Sources/SWBCore/Specs/CoreBuildSystem.xcspec | 5 + .../Specs/en.lproj/CoreBuildSystem.strings | 7 ++ Sources/SWBTestSupport/CoreBasedTests.swift | 44 ++++---- .../SWBWindowsPlatform/Specs/WindowsLd.xcspec | 12 ++ Tests/SWBBuildSystemTests/LinkerTests.swift | 62 +++++----- ...mmandLineToolSpecDiscoveredInfoTests.swift | 106 ++++++++++++++++-- 9 files changed, 230 insertions(+), 59 deletions(-) diff --git a/Sources/SWBCore/SDKRegistry.swift b/Sources/SWBCore/SDKRegistry.swift index ef477eb4..078f710f 100644 --- a/Sources/SWBCore/SDKRegistry.swift +++ b/Sources/SWBCore/SDKRegistry.swift @@ -646,6 +646,14 @@ public final class SDKRegistry: SDKRegistryLookup, CustomStringConvertible, Send tripleEnvironment = "" } + let archs: PropertyListItem = + switch operatingSystem { + case .windows: + .plArray([.plString("x86_64"), .plString("aarch64")]) + default: + .plArray([.plString(Architecture.hostStringValue ?? "unknown")]) + } + return try [ "Type": .plString("SDK"), "Version": .plString(Version(ProcessInfo.processInfo.operatingSystemVersion).zeroTrimmed.description), @@ -656,7 +664,7 @@ public final class SDKRegistry: SDKRegistryLookup, CustomStringConvertible, Send ].merging(defaultProperties, uniquingKeysWith: { _, new in new })), "SupportedTargets": .plDict([ operatingSystem.xcodePlatformName: .plDict([ - "Archs": .plArray([.plString(Architecture.hostStringValue ?? "unknown")]), + "Archs": archs, "LLVMTargetTripleEnvironment": .plString(tripleEnvironment), "LLVMTargetTripleSys": .plString(operatingSystem.xcodePlatformName), "LLVMTargetTripleVendor": .plString("unknown"), diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index 58104578..e4ccaa62 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -803,6 +803,8 @@ public final class BuiltinMacros { public static let LD_RUNPATH_SEARCH_PATHS = BuiltinMacros.declareStringListMacro("LD_RUNPATH_SEARCH_PATHS") public static let LD_SDK_IMPORTS_FILE = BuiltinMacros.declarePathMacro("LD_SDK_IMPORTS_FILE") public static let LD_WARN_UNUSED_DYLIBS = BuiltinMacros.declareBooleanMacro("LD_WARN_UNUSED_DYLIBS") + public static let LD_MULTIARCH = BuiltinMacros.declareBooleanMacro("LD_MULTIARCH") + public static let LD_MULTIARCH_PREFIX_MAP = BuiltinMacros.declareStringListMacro("LD_MULTIARCH_PREFIX_MAP") public static let LEX = BuiltinMacros.declarePathMacro("LEX") public static let LEXFLAGS = BuiltinMacros.declareStringListMacro("LEXFLAGS") public static let LIBRARIAN = BuiltinMacros.declareStringMacro("LIBRARIAN") @@ -1858,6 +1860,8 @@ public final class BuiltinMacros { LD_RUNPATH_SEARCH_PATHS, LD_SDK_IMPORTS_FILE, LD_WARN_UNUSED_DYLIBS, + LD_MULTIARCH, + LD_MULTIARCH_PREFIX_MAP, LEGACY_DEVELOPER_DIR, LEX, LEXFLAGS, diff --git a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift index 895a1401..64681c3a 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift @@ -1273,7 +1273,6 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec } override public func discoveredCommandLineToolSpecInfo(_ producer: any CommandProducer, _ scope: MacroEvaluationScope, _ delegate: any CoreClientTargetDiagnosticProducingDelegate) async -> (any DiscoveredCommandLineToolSpecInfo)? { - let alternateLinker = scope.evaluate(BuiltinMacros.ALTERNATE_LINKER) // The ALTERNATE_LINKER is the 'name' of the linker not the executable name, clang will find the linker binary based on name passed via -fuse-ld, but we need to discover // its properties by executing the actual binary. There is a common filename when the linker is not "ld" across all platforms using "ld.(.exe)" // macOS (Xcode SDK) @@ -1309,9 +1308,43 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // // Note: On Linux you cannot invoke the llvm linker by the direct name for determining the version, // you need to use ld. - var linkerPath = producer.hostOperatingSystem == .windows ? Path("link") : Path("ld") - if alternateLinker != "" && alternateLinker != "ld" { + let alternateLinker = scope.evaluate(BuiltinMacros.ALTERNATE_LINKER) + let isLinkerMultiarch = scope.evaluate(BuiltinMacros.LD_MULTIARCH) + + var linkerPath = producer.hostOperatingSystem == .windows ? Path("ld.lld") : Path("ld") + if alternateLinker != "" && alternateLinker != "ld" && alternateLinker != "link" { linkerPath = Path(producer.hostOperatingSystem.imageFormat.executableName(basename: "ld.\(alternateLinker)")) + } else if alternateLinker != "" { + linkerPath = Path(alternateLinker) + } + // If the linker does not support multiple architectures update the path to include a subfolder based on the prefix map + // to find the architecture specific executable. + if !isLinkerMultiarch { + let archMap = scope.evaluate(BuiltinMacros.LD_MULTIARCH_PREFIX_MAP) + let archMappings = archMap.reduce(into: [String: String]()) { mappings, map in + let split = map.components(separatedBy: ":") + if !split.isEmpty { + return mappings[split[0]] = split[1] + } + } + if archMappings.isEmpty { + delegate.error("LD_MULTIARCH is 'false', but no prefix mappings are present in LD_MULTIARCH_PREFIX_MAP") + return nil + } + // Linkers that don't support multiple architectures cannot support universal binaries, so ARCHS will + // contain the target architecture and can only be a single value. + let arch = scope.evaluate(BuiltinMacros.ARCHS) + if arch.count > 1 { + delegate.error("LD_MULTIARCH is 'false', but multiple ARCHS have been given, this is invalid") + return nil + } + if let prefix = archMappings[arch[0]] { + // Add in the target architecture prefix directory to path for search. + linkerPath = Path(prefix).join(linkerPath) + } else { + delegate.error("Could not find prefix mapping for \(arch[0]) in LD_MULTIARCH_PREFIX_MAP") + return nil + } } guard let toolPath = producer.executableSearchPaths.findExecutable(operatingSystem: producer.hostOperatingSystem, basename: linkerPath.str) else { return nil diff --git a/Sources/SWBCore/Specs/CoreBuildSystem.xcspec b/Sources/SWBCore/Specs/CoreBuildSystem.xcspec index 85a10454..932b4eb7 100644 --- a/Sources/SWBCore/Specs/CoreBuildSystem.xcspec +++ b/Sources/SWBCore/Specs/CoreBuildSystem.xcspec @@ -1148,6 +1148,11 @@ When `GENERATE_INFOPLIST_FILE` is enabled, sets the value of the [CFBundleIdenti DefaultValue = NO; Category = "Linking - Warnings"; }, + { + Name = "LD_MULTIARCH"; + Type = Boolean; + DefaultValue = YES; + }, { Name = "LIBRARY_SEARCH_PATHS"; Type = PathList; diff --git a/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings b/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings index a3ba08df..5627ef9a 100644 --- a/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings +++ b/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings @@ -423,6 +423,13 @@ You cannot create a PIE from `.o` files compiled with `-mdynamic-no-pic`. Using "[LD_WARN_DUPLICATE_LIBRARIES]-name" = "Duplicate Libraries"; "[LD_WARN_DUPLICATE_LIBRARIES]-description" = "Warn for linking the same library multiple times."; +"[LD_MULTIARCH]-name" = "Multiple Architecture Supporte Linker"; +"[LD_MULTIARCH]-description" = "Linker supports linking multiple target architectures."; + +"[LD_MULTIARCH_PREFIX_MAP]-name" = "Linker subfolder architecture map"; +"[LD_MULTIARCH_PREFIX_MAP]-description" = "Mapping of architecture to subfolder. :"; + + // Localization Settings "[Localization]-category" = "Localization"; diff --git a/Sources/SWBTestSupport/CoreBasedTests.swift b/Sources/SWBTestSupport/CoreBasedTests.swift index 7aa6d82a..ee073ff1 100644 --- a/Sources/SWBTestSupport/CoreBasedTests.swift +++ b/Sources/SWBTestSupport/CoreBasedTests.swift @@ -272,53 +272,55 @@ extension CoreBasedTests { return nil } } - package var linkPath: Path? { + package var lldPath: Path? { get async throws { let (core, defaultToolchain) = try await coreAndToolchain() - if core.hostOperatingSystem != .windows { - // Most unixes have a link executable, but that is not a linker - return nil - } - if let executable = defaultToolchain.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "link") { + if let executable = defaultToolchain.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld.lld") { return executable } for platform in core.platformRegistry.platforms { - if let executable = platform.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "link") { + if let executable = platform.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld.lld") { return executable } } return nil } } - - package var lldPath: Path? { + package var goldPath: Path? { get async throws { let (core, defaultToolchain) = try await coreAndToolchain() - if let executable = defaultToolchain.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld.ld") { + if let executable = defaultToolchain.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld.gold") { return executable } for platform in core.platformRegistry.platforms { - if let executable = platform.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld.ld") { + if let executable = platform.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld.gold") { return executable } } return nil } } + package func linkPath(_ targetArchitecture: String) async throws -> Path? { + let (core, defaultToolchain) = try await self.coreAndToolchain() + let prefixMapping = [ "x86_64": "x64", "aarch64": "arm64", "arm64": "arm64" ] - package var goldPath: Path? { - get async throws { - let (core, defaultToolchain) = try await coreAndToolchain() - if let executable = defaultToolchain.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld.gold") { + guard let prefix = prefixMapping[targetArchitecture] else { + return nil + } + let linkerPath = Path(prefix).join("link").str + if core.hostOperatingSystem != .windows { + // Most unixes have a link executable, but that is not a linker + return nil + } + if let executable = defaultToolchain.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: linkerPath) { + return executable + } + for platform in core.platformRegistry.platforms { + if let executable = platform.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: linkerPath) { return executable } - for platform in core.platformRegistry.platforms { - if let executable = platform.executableSearchPaths.findExecutable(operatingSystem: core.hostOperatingSystem, basename: "ld.gold") { - return executable - } - } - return nil } + return nil } } diff --git a/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec b/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec index 4678c8a6..30d51e47 100644 --- a/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec +++ b/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec @@ -52,6 +52,18 @@ CommandLineFlag = "--sysroot"; IsInputDependency = Yes; }, + { + Name = LD_MULTIARCH; + Type = Boolean; + DefaultValue = NO; + Condition = "$(ALTERNATE_LINKER) == link"; + }, + { + Name = LD_MULTIARCH_PREFIX_MAP; + Type = StringList; + DefaultValue = "x86_64:x64 aarch64:arm64 arm64:arm64"; + Condition = "$(ALTERNATE_LINKER) == link"; + }, { // No such concept Name = LD_DETERMINISTIC_MODE; diff --git a/Tests/SWBBuildSystemTests/LinkerTests.swift b/Tests/SWBBuildSystemTests/LinkerTests.swift index 230e06be..eb458624 100644 --- a/Tests/SWBBuildSystemTests/LinkerTests.swift +++ b/Tests/SWBBuildSystemTests/LinkerTests.swift @@ -42,6 +42,7 @@ fileprivate struct LinkerTests: CoreBasedTests { "PRODUCT_NAME": "$(TARGET_NAME)", "SWIFT_VERSION": swiftVersion, "OTHER_LDFLAGS": "-not-a-real-flag", + "ARCHS" : "x86_64 aarch64" ]) ], buildPhases: [ @@ -243,25 +244,22 @@ fileprivate struct LinkerTests: CoreBasedTests { let ldLinkerPath = try await self.ldPath let lldLinkerPath = try await self.lldPath let goldLinkerPath = try await self.goldPath - let linkLinkerPath = try await self.linkPath - let installedLinkerPaths = [lldLinkerPath, ldLinkerPath, goldLinkerPath, linkLinkerPath].compactMap { $0 } + let linkLinkerPathX86 = try await self.linkPath("x86_64") + let linkLinkerPathAarch64 = try await self.linkPath("aarch64") + var installedLinkerPaths = [ldLinkerPath, lldLinkerPath, goldLinkerPath, linkLinkerPathX86, linkLinkerPathAarch64].compactMap { $0 } // Default Linker - var parameters = BuildParameters(configuration: "Debug", overrides: ["ALTERNATE_LINKER": ""]) + var parameters = BuildParameters(configuration: "Debug") try await tester.checkBuild(parameters: parameters, runDestination: .host) { results in results.checkTask(.matchRuleType("Ld")) { task in - results.checkTaskOutput(task) { taskOutput in - results.checkTaskOutput(task) { output in - // Expect that one of the installed linkers is used, we are not sure which one. - if runDestination == .windows && Architecture.hostStringValue == "aarch64" { - withKnownIssue("'clang' picks the wrong binary for link.exe using x86 version") { - // On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\arm64\link.exe" - #expect(installedLinkerPaths.map { $0.str }.contains(where: output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains)) - } - } else { - #expect(installedLinkerPaths.map { $0.str }.contains(where: output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains)) + results.checkTaskOutput(task) { output in + if runDestination == .windows { + // clang will choose to run lld-link rather than ld.lld.exe. + if let lldLinkerPath { + installedLinkerPaths.append(lldLinkerPath.dirname.join("lld-link")) } } + #expect(installedLinkerPaths.map { $0.str }.contains(where: output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains)) } } results.checkNoDiagnostics() @@ -286,12 +284,6 @@ fileprivate struct LinkerTests: CoreBasedTests { if let lldLinkerPath { parameters = BuildParameters(configuration: "Debug", overrides: ["ALTERNATE_LINKER": "lld"]) try await tester.checkBuild(parameters: parameters, runDestination: .host) { results in - if runDestination == .windows { - // Issue: Linker cannot find dependent library - results.checkError(.contains("Linker command failed with exit code 1")) - results.checkError(.contains("lld-link: error: could not open 'Library.lib'")) - } - results.checkTask(.matchRuleType("Ld")) { task in task.checkCommandLineContains(["-fuse-ld=lld"]) results.checkTaskOutput(task) { output in @@ -324,26 +316,46 @@ fileprivate struct LinkerTests: CoreBasedTests { } // link.exe - if let linkLinkerPath { - parameters = BuildParameters(configuration: "Debug", overrides: ["ALTERNATE_LINKER": "link"]) + if let linkLinkerPathX86 { + parameters = BuildParameters(configuration: "Debug", overrides: ["ARCHS": "x86_64", "ALTERNATE_LINKER": "link"]) try await tester.checkBuild(parameters: parameters, runDestination: .host) { results in results.checkTask(.matchRuleType("Ld")) { task in task.checkCommandLineContains(["-fuse-ld=link"]) results.checkTaskOutput(task) { output in // Expect that the 'link' linker is called by clang if runDestination == .windows && Architecture.hostStringValue == "aarch64" { - withKnownIssue("'clang' picks the wrong binary for link.exe using Hostx86 version") { - // On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\arm64\link.exe" - #expect(output.asString.replacingOccurrences(of: "Hostx86", with: "Hostarm64").contains(linkLinkerPath.str)) + // On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\x64\link.exe" + withKnownIssue("'clang' picks the wrong binary for link.exe using the Hostx86 version") { + #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(linkLinkerPathX86.str)) } } else { - #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(linkLinkerPath.str)) + #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(linkLinkerPathX86.str)) } } } results.checkNoDiagnostics() } } + if let linkLinkerPathAarch64 { + parameters = BuildParameters(configuration: "Debug", overrides: ["ARCHS": "aarch64", "ALTERNATE_LINKER": "link"]) + try await tester.checkBuild(parameters: parameters, runDestination: .host) { results in + results.checkTask(.matchRuleType("Ld")) { task in + task.checkCommandLineContains(["-fuse-ld=link"]) + results.checkTaskOutput(task) { output in + // Expect that the 'link' linker is called by clang + if runDestination == .windows && Architecture.hostStringValue == "aarch64" { + // On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\x64\link.exe" + withKnownIssue("'clang' picks the wrong binary for link.exe using the Hostx86 version") { + #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(linkLinkerPathAarch64.str)) + } + } else { + #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(linkLinkerPathAarch64.str)) + } + } + } + results.checkNoDiagnostics() + } + } } } } diff --git a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift index 30d57a8d..d13e5f70 100644 --- a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift +++ b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift @@ -16,6 +16,7 @@ import SWBProtocol import SWBTestSupport import SWBUtil import Testing +import SWBMacro @Suite fileprivate struct CommandLineToolSpecDiscoveredInfoTests: CoreBasedTests { @Test(.skipHostOS(.windows, "Failed to obtain command line tool spec info but no errors were emitted")) @@ -149,47 +150,134 @@ import Testing } } - @Test(.skipHostOS(.windows, "Failed to obtain command line tool spec info but no errors were emitted")) + // Linker tool discovery is a bit more complex as it afffected by the ALTERNATE_LINKER build setting. + func ldMacroTable() async throws -> MacroValueAssignmentTable { + let core = try await getCore() + return MacroValueAssignmentTable(namespace: core.specRegistry.internalMacroNamespace) + } + + @Test func discoveredLdLinkerSpecInfo() async throws { - try await withSpec(LdLinkerSpec.self, .deferred) { (info: DiscoveredLdLinkerToolSpecInfo) in + var table = try await ldMacroTable() + table.push(BuiltinMacros.LD_MULTIARCH, literal: true) + // Default Linker, just check we have one. + try await withSpec(LdLinkerSpec.self, .deferred, additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + } + } + + @Test(.requireSDKs(.macOS)) + func discoveredLdLinkerSpecInfo_macOS() async throws { + var table = try await ldMacroTable() + table.push(BuiltinMacros.LD_MULTIARCH, literal: true) + // Default Linker + try await withSpec(LdLinkerSpec.self, .deferred, additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) #expect(info.toolVersion != nil) if let toolVersion = info.toolVersion { #expect(toolVersion > Version(0, 0, 0)) } + #expect(info.linker == .ld64) // rdar://112109825 (ld_prime only reports arm64 and arm64e architectures in ld -version_details) // let expectedArchs = Set(["armv7", "armv7k", "armv7s", "arm64", "arm64e", "i386", "x86_64"]) // XCTAssertEqual(info.architectures.intersection(expectedArchs), expectedArchs) // XCTAssertFalse(info.architectures.contains("(tvOS)")) } - - try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("GNU ld (GNU Binutils for Debian) 2.40\n".utf8), stderr: Data())) { (info: DiscoveredLdLinkerToolSpecInfo) in + } + @Test(.requireSDKs(.linux)) + func discoveredLdLinkerSpecInfo_Linux() async throws { + var table = try await ldMacroTable() + table.push(BuiltinMacros.LD_MULTIARCH, literal: true) + // Default Linker + try await withSpec(LdLinkerSpec.self, .deferred, additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .gnuld) + } + try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("GNU ld (GNU Binutils for Debian) 2.40\n".utf8), stderr: Data()), additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) #expect(info.toolVersion == Version(2, 40)) #expect(info.architectures == Set()) } - try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("GNU ld version 2.29.1-31.amzn2.0.1\n".utf8), stderr: Data())) { (info: DiscoveredLdLinkerToolSpecInfo) in + try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("GNU ld version 2.29.1-31.amzn2.0.1\n".utf8), stderr: Data()), additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) #expect(info.toolVersion == Version(2, 29, 1)) #expect(info.architectures == Set()) } + // llvm-ld + table.push(BuiltinMacros.ALTERNATE_LINKER, literal: "lld") + try await withSpec(LdLinkerSpec.self, .deferred, additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .lld) + } + // gold + table.push(BuiltinMacros.ALTERNATE_LINKER, literal: "gold") + try await withSpec(LdLinkerSpec.self, .deferred, additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .gold) + } } - @Test(.requireHostOS(.windows)) - func discoveredLdLinkerSpecInfoWindows() async throws { - try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows") { (info: DiscoveredLdLinkerToolSpecInfo) in + @Test(.requireSDKs(.windows)) + func discoveredLdLinkerSpecInfo_Windows() async throws { + var table = try await ldMacroTable() + table.push(BuiltinMacros.LD_MULTIARCH, literal: true) + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) #expect(info.toolVersion != nil) if let toolVersion = info.toolVersion { #expect(toolVersion > Version(0, 0, 0)) } + #expect(info.linker == .lld) } - try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("Microsoft (R) Incremental Linker Version 14.41.34120.0\n".utf8), stderr: Data()), platform: "windows") { (info: DiscoveredLdLinkerToolSpecInfo) in + try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("Microsoft (R) Incremental Linker Version 14.41.34120.0\n".utf8), stderr: Data()), platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) #expect(info.toolVersion == Version(14, 41, 34120)) #expect(info.architectures == Set()) } + + // link.exe cannot be used for multipler architectures and requires a distinct link.exe for each target architecture + table.push(BuiltinMacros.ALTERNATE_LINKER, literal: "link") + table.push(BuiltinMacros.LD_MULTIARCH, literal: false) + table.push(BuiltinMacros.LD_MULTIARCH_PREFIX_MAP, literal: ["x86_64:x64", "aarch64:arm64", "arm64:arm64"]) + + // x86_64 + table.push(BuiltinMacros.ARCHS, literal: ["x86_64"]) + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .linkExe) + } + + // // link aarch64 + // table.push(BuiltinMacros.ARCHS, literal: ["aarch64"]) + // try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + // #expect(!info.toolPath.isEmpty) + // #expect(info.toolVersion != nil) + // if let toolVersion = info.toolVersion { + // #expect(toolVersion > Version(0, 0, 0)) + // } + // #expect(info.linker == .linkExe) + // } } @Test(.skipHostOS(.windows), .requireSystemPackages(apt: "libtool", yum: "libtool")) From c872bc6b7fae33a4e3812aa0c28dbfe79adc427e Mon Sep 17 00:00:00 2001 From: Kristopher Cieplak Date: Tue, 11 Mar 2025 08:07:19 -0400 Subject: [PATCH 3/4] Changes based on review feedback. * Rename LD_MULTIARCH to _LD_MULTIARCH. This build setting should not be used by external clients, so hide it using the underscore notation. * Add in mappings for future supported architectures i386 and thumbv7. * Drop arm64->arm64 mapping, handled by aarch64. * Fix logic parsing for architecture prefix mappings, could have crashed accessing out of bound element. * Rebase changes ontop of SDK platform registration refactor. --- Sources/SWBCore/PlatformRegistry.swift | 6 +-- Sources/SWBCore/Settings/BuiltinMacros.swift | 8 ++-- .../Tools/LinkerTools.swift | 21 +++++----- Sources/SWBCore/Specs/CoreBuildSystem.xcspec | 2 +- .../Specs/en.lproj/CoreBuildSystem.strings | 7 ---- Sources/SWBTestSupport/CoreBasedTests.swift | 2 +- Sources/SWBWindowsPlatform/Plugin.swift | 42 +++++++++++++++---- .../SWBWindowsPlatform/Specs/WindowsLd.xcspec | 6 +-- ...mmandLineToolSpecDiscoveredInfoTests.swift | 13 +++--- 9 files changed, 63 insertions(+), 44 deletions(-) diff --git a/Sources/SWBCore/PlatformRegistry.swift b/Sources/SWBCore/PlatformRegistry.swift index 59b970c2..4ada3d85 100644 --- a/Sources/SWBCore/PlatformRegistry.swift +++ b/Sources/SWBCore/PlatformRegistry.swift @@ -349,7 +349,7 @@ public final class PlatformRegistry { var fs: any FSProxy } - for platformExtension in platformInfoExtensions() { + for platformExtension in await platformInfoExtensions() { do { for (path, data) in try platformExtension.additionalPlatforms(context: Context(hostOperatingSystem: hostOperatingSystem, developerPath: delegate.developerPath, fs: fs)) { await registerPlatform(path, .plDict(data), fs) @@ -632,8 +632,8 @@ public final class PlatformRegistry { path.join("Developer").join("SDKs") ] - for platformExtension in platformInfoExtensions() { - await executableSearchPaths.append(contentsOf: platformExtension.additionalPlatformExecutableSearchPaths(platformName: name, platformPath: path)) + for platformExtension in await platformInfoExtensions() { + await executableSearchPaths.append(contentsOf: platformExtension.additionalPlatformExecutableSearchPaths(platformName: name, platformPath: path, fs: localFS)) platformExtension.adjustPlatformSDKSearchPaths(platformName: name, platformPath: path, sdkSearchPaths: &sdkSearchPaths) diff --git a/Sources/SWBCore/Settings/BuiltinMacros.swift b/Sources/SWBCore/Settings/BuiltinMacros.swift index e4ccaa62..f0cba65e 100644 --- a/Sources/SWBCore/Settings/BuiltinMacros.swift +++ b/Sources/SWBCore/Settings/BuiltinMacros.swift @@ -803,8 +803,8 @@ public final class BuiltinMacros { public static let LD_RUNPATH_SEARCH_PATHS = BuiltinMacros.declareStringListMacro("LD_RUNPATH_SEARCH_PATHS") public static let LD_SDK_IMPORTS_FILE = BuiltinMacros.declarePathMacro("LD_SDK_IMPORTS_FILE") public static let LD_WARN_UNUSED_DYLIBS = BuiltinMacros.declareBooleanMacro("LD_WARN_UNUSED_DYLIBS") - public static let LD_MULTIARCH = BuiltinMacros.declareBooleanMacro("LD_MULTIARCH") - public static let LD_MULTIARCH_PREFIX_MAP = BuiltinMacros.declareStringListMacro("LD_MULTIARCH_PREFIX_MAP") + public static let _LD_MULTIARCH = BuiltinMacros.declareBooleanMacro("_LD_MULTIARCH") + public static let _LD_MULTIARCH_PREFIX_MAP = BuiltinMacros.declareStringListMacro("_LD_MULTIARCH_PREFIX_MAP") public static let LEX = BuiltinMacros.declarePathMacro("LEX") public static let LEXFLAGS = BuiltinMacros.declareStringListMacro("LEXFLAGS") public static let LIBRARIAN = BuiltinMacros.declareStringMacro("LIBRARIAN") @@ -1860,8 +1860,8 @@ public final class BuiltinMacros { LD_RUNPATH_SEARCH_PATHS, LD_SDK_IMPORTS_FILE, LD_WARN_UNUSED_DYLIBS, - LD_MULTIARCH, - LD_MULTIARCH_PREFIX_MAP, + _LD_MULTIARCH, + _LD_MULTIARCH_PREFIX_MAP, LEGACY_DEVELOPER_DIR, LEX, LEXFLAGS, diff --git a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift index 64681c3a..1b062522 100644 --- a/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift +++ b/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift @@ -1309,7 +1309,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // Note: On Linux you cannot invoke the llvm linker by the direct name for determining the version, // you need to use ld. let alternateLinker = scope.evaluate(BuiltinMacros.ALTERNATE_LINKER) - let isLinkerMultiarch = scope.evaluate(BuiltinMacros.LD_MULTIARCH) + let isLinkerMultiarch = scope.evaluate(BuiltinMacros._LD_MULTIARCH) var linkerPath = producer.hostOperatingSystem == .windows ? Path("ld.lld") : Path("ld") if alternateLinker != "" && alternateLinker != "ld" && alternateLinker != "link" { @@ -1320,29 +1320,28 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec // If the linker does not support multiple architectures update the path to include a subfolder based on the prefix map // to find the architecture specific executable. if !isLinkerMultiarch { - let archMap = scope.evaluate(BuiltinMacros.LD_MULTIARCH_PREFIX_MAP) + let archMap = scope.evaluate(BuiltinMacros._LD_MULTIARCH_PREFIX_MAP) let archMappings = archMap.reduce(into: [String: String]()) { mappings, map in - let split = map.components(separatedBy: ":") - if !split.isEmpty { - return mappings[split[0]] = split[1] + let (arch, prefixDir) = map.split(":") + if !arch.isEmpty && !prefixDir.isEmpty { + return mappings[arch] = prefixDir } } if archMappings.isEmpty { - delegate.error("LD_MULTIARCH is 'false', but no prefix mappings are present in LD_MULTIARCH_PREFIX_MAP") + delegate.error("_LD_MULTIARCH is 'false', but no prefix mappings are present in _LD_MULTIARCH_PREFIX_MAP") return nil } // Linkers that don't support multiple architectures cannot support universal binaries, so ARCHS will // contain the target architecture and can only be a single value. - let arch = scope.evaluate(BuiltinMacros.ARCHS) - if arch.count > 1 { - delegate.error("LD_MULTIARCH is 'false', but multiple ARCHS have been given, this is invalid") + guard let arch = scope.evaluate(BuiltinMacros.ARCHS).only else { + delegate.error("_LD_MULTIARCH is 'false', but multiple ARCHS have been given, this is invalid") return nil } - if let prefix = archMappings[arch[0]] { + if let prefix = archMappings[arch] { // Add in the target architecture prefix directory to path for search. linkerPath = Path(prefix).join(linkerPath) } else { - delegate.error("Could not find prefix mapping for \(arch[0]) in LD_MULTIARCH_PREFIX_MAP") + delegate.error("Could not find prefix mapping for \(arch) in _LD_MULTIARCH_PREFIX_MAP") return nil } } diff --git a/Sources/SWBCore/Specs/CoreBuildSystem.xcspec b/Sources/SWBCore/Specs/CoreBuildSystem.xcspec index 932b4eb7..d481110e 100644 --- a/Sources/SWBCore/Specs/CoreBuildSystem.xcspec +++ b/Sources/SWBCore/Specs/CoreBuildSystem.xcspec @@ -1149,7 +1149,7 @@ When `GENERATE_INFOPLIST_FILE` is enabled, sets the value of the [CFBundleIdenti Category = "Linking - Warnings"; }, { - Name = "LD_MULTIARCH"; + Name = "_LD_MULTIARCH"; Type = Boolean; DefaultValue = YES; }, diff --git a/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings b/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings index 5627ef9a..a3ba08df 100644 --- a/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings +++ b/Sources/SWBCore/Specs/en.lproj/CoreBuildSystem.strings @@ -423,13 +423,6 @@ You cannot create a PIE from `.o` files compiled with `-mdynamic-no-pic`. Using "[LD_WARN_DUPLICATE_LIBRARIES]-name" = "Duplicate Libraries"; "[LD_WARN_DUPLICATE_LIBRARIES]-description" = "Warn for linking the same library multiple times."; -"[LD_MULTIARCH]-name" = "Multiple Architecture Supporte Linker"; -"[LD_MULTIARCH]-description" = "Linker supports linking multiple target architectures."; - -"[LD_MULTIARCH_PREFIX_MAP]-name" = "Linker subfolder architecture map"; -"[LD_MULTIARCH_PREFIX_MAP]-description" = "Mapping of architecture to subfolder. :"; - - // Localization Settings "[Localization]-category" = "Localization"; diff --git a/Sources/SWBTestSupport/CoreBasedTests.swift b/Sources/SWBTestSupport/CoreBasedTests.swift index ee073ff1..9b61573e 100644 --- a/Sources/SWBTestSupport/CoreBasedTests.swift +++ b/Sources/SWBTestSupport/CoreBasedTests.swift @@ -302,7 +302,7 @@ extension CoreBasedTests { } package func linkPath(_ targetArchitecture: String) async throws -> Path? { let (core, defaultToolchain) = try await self.coreAndToolchain() - let prefixMapping = [ "x86_64": "x64", "aarch64": "arm64", "arm64": "arm64" ] + let prefixMapping = [ "x86_64": "x64", "aarch64": "arm64", "i386": "x86", "thumbv7" : "arm" ] guard let prefix = prefixMapping[targetArchitecture] else { return nil diff --git a/Sources/SWBWindowsPlatform/Plugin.swift b/Sources/SWBWindowsPlatform/Plugin.swift index 5c93fc9d..f8d359fd 100644 --- a/Sources/SWBWindowsPlatform/Plugin.swift +++ b/Sources/SWBWindowsPlatform/Plugin.swift @@ -18,12 +18,13 @@ import Foundation let plugin = WindowsPlugin() manager.register(WindowsPlatformSpecsExtension(), type: SpecificationsExtensionPoint.self) manager.register(WindowsEnvironmentExtension(plugin: plugin), type: EnvironmentExtensionPoint.self) - manager.register(WindowsPlatformExtension(), type: PlatformInfoExtensionPoint.self) + manager.register(WindowsPlatformExtension(plugin: plugin), type: PlatformInfoExtensionPoint.self) manager.register(WindowsSDKRegistryExtension(), type: SDKRegistryExtensionPoint.self) } public final class WindowsPlugin: Sendable { private let vsInstallations = AsyncSingleValueCache<[VSInstallation], any Error>() + private let latestVsInstalltionDirectory = AsyncSingleValueCache() public func cachedVSInstallations() async throws -> [VSInstallation] { try await vsInstallations.value { @@ -31,6 +32,23 @@ public final class WindowsPlugin: Sendable { try await VSInstallation.findInstallations(fs: localFS) } } + + func cachedLatestVSInstallDirectory(fs: any FSProxy) async throws -> Path? { + try await latestVsInstalltionDirectory.value { + let installations = try await cachedVSInstallations() + .sorted(by: { $0.installationVersion > $1.installationVersion }) + if let latest = installations.first { + let msvcDir = latest.installationPath.join("VC").join("Tools").join("MSVC") + if fs.exists(msvcDir) { + let versions = try fs.listdir(msvcDir).map { try Version($0) }.sorted { $0 > $1 } + if let latestVersion = versions.first { + return msvcDir.join(latestVersion.description) + } + } + } + return nil + } + } } struct WindowsPlatformSpecsExtension: SpecificationsExtension { @@ -49,10 +67,12 @@ struct WindowsPlatformSpecsExtension: SpecificationsExtension { return [:] } return [vcToolsInstallDir: dir.str] + } else { + return [:] } } - @_spi(Testing)public func findLatestInstallDirectory(fs: any FSProxy) async throws -> Path? { + @_spi(Testing)public func findLatestInstallDirectory(fs: any FSProxy) async throws -> Path? { let plugin: WindowsPlugin let installations = try await plugin.cachedVSInstallations() .sorted(by: { $0.installationVersion > $1.installationVersion }) @@ -67,9 +87,9 @@ struct WindowsPlatformSpecsExtension: SpecificationsExtension { } return nil } -} struct WindowsPlatformExtension: PlatformInfoExtension { + let plugin: WindowsPlugin func additionalPlatforms(context: any PlatformInfoExtensionAdditionalPlatformsContext) throws -> [(path: Path, data: [String: PropertyListItem])] { let operatingSystem = context.hostOperatingSystem guard operatingSystem == .windows else { @@ -115,13 +135,19 @@ struct WindowsPlatformExtension: PlatformInfoExtension { } public func additionalPlatformExecutableSearchPaths(platformName: String, platformPath: Path, fs: any FSProxy) async -> [Path] { - guard let dir = try? await findLatestInstallDirectory(fs: fs) else { + guard let dir = try? await plugin.cachedLatestVSInstallDirectory(fs: fs) else { return [] } - if Architecture.hostStringValue == "aarch64" { - return [dir.join("bin/Hostarm64/arm64")] - } else { - return [dir.join("bin/Hostx64/x64")] + + // Note: Do not add in the target directories under the host as these will end up in the global search paths, i.e. PATH + // Let the commandlinetool discovery add in the target subdirectory based on the targeted architecture. + switch Architecture.hostStringValue { + case "aarch64": + return [dir.join("bin/Hostarm64")] + case "x86_64": + return [dir.join("bin/Hostx64")] + default: + return [] } } } diff --git a/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec b/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec index 30d51e47..206eaafc 100644 --- a/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec +++ b/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec @@ -53,15 +53,15 @@ IsInputDependency = Yes; }, { - Name = LD_MULTIARCH; + Name = _LD_MULTIARCH; Type = Boolean; DefaultValue = NO; Condition = "$(ALTERNATE_LINKER) == link"; }, { - Name = LD_MULTIARCH_PREFIX_MAP; + Name = _LD_MULTIARCH_PREFIX_MAP; Type = StringList; - DefaultValue = "x86_64:x64 aarch64:arm64 arm64:arm64"; + DefaultValue = "x86_64:x64 aarch64:arm64 i386:x86 thumbv7:arm"; Condition = "$(ALTERNATE_LINKER) == link"; }, { diff --git a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift index d13e5f70..7f0a5bfe 100644 --- a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift +++ b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift @@ -159,7 +159,7 @@ import SWBMacro @Test func discoveredLdLinkerSpecInfo() async throws { var table = try await ldMacroTable() - table.push(BuiltinMacros.LD_MULTIARCH, literal: true) + table.push(BuiltinMacros._LD_MULTIARCH, literal: true) // Default Linker, just check we have one. try await withSpec(LdLinkerSpec.self, .deferred, additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) @@ -173,7 +173,7 @@ import SWBMacro @Test(.requireSDKs(.macOS)) func discoveredLdLinkerSpecInfo_macOS() async throws { var table = try await ldMacroTable() - table.push(BuiltinMacros.LD_MULTIARCH, literal: true) + table.push(BuiltinMacros._LD_MULTIARCH, literal: true) // Default Linker try await withSpec(LdLinkerSpec.self, .deferred, additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) @@ -191,7 +191,7 @@ import SWBMacro @Test(.requireSDKs(.linux)) func discoveredLdLinkerSpecInfo_Linux() async throws { var table = try await ldMacroTable() - table.push(BuiltinMacros.LD_MULTIARCH, literal: true) + table.push(BuiltinMacros._LD_MULTIARCH, literal: true) // Default Linker try await withSpec(LdLinkerSpec.self, .deferred, additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) @@ -237,7 +237,7 @@ import SWBMacro @Test(.requireSDKs(.windows)) func discoveredLdLinkerSpecInfo_Windows() async throws { var table = try await ldMacroTable() - table.push(BuiltinMacros.LD_MULTIARCH, literal: true) + table.push(BuiltinMacros._LD_MULTIARCH, literal: true) try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in #expect(!info.toolPath.isEmpty) #expect(info.toolVersion != nil) @@ -254,8 +254,8 @@ import SWBMacro // link.exe cannot be used for multipler architectures and requires a distinct link.exe for each target architecture table.push(BuiltinMacros.ALTERNATE_LINKER, literal: "link") - table.push(BuiltinMacros.LD_MULTIARCH, literal: false) - table.push(BuiltinMacros.LD_MULTIARCH_PREFIX_MAP, literal: ["x86_64:x64", "aarch64:arm64", "arm64:arm64"]) + table.push(BuiltinMacros._LD_MULTIARCH, literal: false) + table.push(BuiltinMacros._LD_MULTIARCH_PREFIX_MAP, literal: ["x86_64:x64", "aarch64:arm64", "i386:x86", "thumbv7:arm"]) // x86_64 table.push(BuiltinMacros.ARCHS, literal: ["x86_64"]) @@ -268,6 +268,7 @@ import SWBMacro #expect(info.linker == .linkExe) } + // FIXME: Fails in swift-ci missing arm64 toolchain on x86_64 host. // // link aarch64 // table.push(BuiltinMacros.ARCHS, literal: ["aarch64"]) // try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in From 31ecfbda47c932622d1caca250af9349fda80dd6 Mon Sep 17 00:00:00 2001 From: Kristopher Cieplak Date: Mon, 17 Mar 2025 09:09:38 -0400 Subject: [PATCH 4/4] Move discoveredLdLinkerSpecInfo_Windows to platform tests. * Move discoveredLdLinkerSpecInfo_Windows to the windows platform tests. * Use the helper functions to make linker tests conditional on the presence of specific host SDK components. * Update target architectures to match windows SDK. --- Sources/SWBCore/SDKRegistry.swift | 10 +- Sources/SWBTestSupport/CoreBasedTests.swift | 2 +- Sources/SWBWindowsPlatform/Plugin.swift | 21 +--- .../SWBWindowsPlatform/Specs/WindowsLd.xcspec | 2 +- Tests/SWBBuildSystemTests/LinkerTests.swift | 4 +- ...mmandLineToolSpecDiscoveredInfoTests.swift | 47 -------- .../SWBWindowsPlatformTests.swift | 100 ++++++++++++++++++ 7 files changed, 108 insertions(+), 78 deletions(-) diff --git a/Sources/SWBCore/SDKRegistry.swift b/Sources/SWBCore/SDKRegistry.swift index 078f710f..ef477eb4 100644 --- a/Sources/SWBCore/SDKRegistry.swift +++ b/Sources/SWBCore/SDKRegistry.swift @@ -646,14 +646,6 @@ public final class SDKRegistry: SDKRegistryLookup, CustomStringConvertible, Send tripleEnvironment = "" } - let archs: PropertyListItem = - switch operatingSystem { - case .windows: - .plArray([.plString("x86_64"), .plString("aarch64")]) - default: - .plArray([.plString(Architecture.hostStringValue ?? "unknown")]) - } - return try [ "Type": .plString("SDK"), "Version": .plString(Version(ProcessInfo.processInfo.operatingSystemVersion).zeroTrimmed.description), @@ -664,7 +656,7 @@ public final class SDKRegistry: SDKRegistryLookup, CustomStringConvertible, Send ].merging(defaultProperties, uniquingKeysWith: { _, new in new })), "SupportedTargets": .plDict([ operatingSystem.xcodePlatformName: .plDict([ - "Archs": archs, + "Archs": .plArray([.plString(Architecture.hostStringValue ?? "unknown")]), "LLVMTargetTripleEnvironment": .plString(tripleEnvironment), "LLVMTargetTripleSys": .plString(operatingSystem.xcodePlatformName), "LLVMTargetTripleVendor": .plString("unknown"), diff --git a/Sources/SWBTestSupport/CoreBasedTests.swift b/Sources/SWBTestSupport/CoreBasedTests.swift index 9b61573e..93d8b80f 100644 --- a/Sources/SWBTestSupport/CoreBasedTests.swift +++ b/Sources/SWBTestSupport/CoreBasedTests.swift @@ -302,7 +302,7 @@ extension CoreBasedTests { } package func linkPath(_ targetArchitecture: String) async throws -> Path? { let (core, defaultToolchain) = try await self.coreAndToolchain() - let prefixMapping = [ "x86_64": "x64", "aarch64": "arm64", "i386": "x86", "thumbv7" : "arm" ] + let prefixMapping = ["aarch64" : "arm64", "arm64ec" : "arm64", "armv7" : "arm", "x86_64": "x64", "i686": "x86"] guard let prefix = prefixMapping[targetArchitecture] else { return nil diff --git a/Sources/SWBWindowsPlatform/Plugin.swift b/Sources/SWBWindowsPlatform/Plugin.swift index f8d359fd..9ddfdf17 100644 --- a/Sources/SWBWindowsPlatform/Plugin.swift +++ b/Sources/SWBWindowsPlatform/Plugin.swift @@ -24,7 +24,7 @@ import Foundation public final class WindowsPlugin: Sendable { private let vsInstallations = AsyncSingleValueCache<[VSInstallation], any Error>() - private let latestVsInstalltionDirectory = AsyncSingleValueCache() + private let latestVsInstallationDirectory = AsyncSingleValueCache() public func cachedVSInstallations() async throws -> [VSInstallation] { try await vsInstallations.value { @@ -34,7 +34,7 @@ public final class WindowsPlugin: Sendable { } func cachedLatestVSInstallDirectory(fs: any FSProxy) async throws -> Path? { - try await latestVsInstalltionDirectory.value { + try await latestVsInstallationDirectory.value { let installations = try await cachedVSInstallations() .sorted(by: { $0.installationVersion > $1.installationVersion }) if let latest = installations.first { @@ -63,7 +63,7 @@ struct WindowsPlatformSpecsExtension: SpecificationsExtension { @_spi(Testing) public func additionalEnvironmentVariables(context: any EnvironmentExtensionAdditionalEnvironmentVariablesContext) async throws -> [String: String] { if context.hostOperatingSystem == .windows { let vcToolsInstallDir = "VCToolsInstallDir" - guard let dir = try? await findLatestInstallDirectory(fs: context.fs) else { + guard let dir = try? await plugin.cachedLatestVSInstallDirectory(fs: context.fs) else { return [:] } return [vcToolsInstallDir: dir.str] @@ -71,21 +71,6 @@ struct WindowsPlatformSpecsExtension: SpecificationsExtension { return [:] } } - - @_spi(Testing)public func findLatestInstallDirectory(fs: any FSProxy) async throws -> Path? { - let plugin: WindowsPlugin - let installations = try await plugin.cachedVSInstallations() - .sorted(by: { $0.installationVersion > $1.installationVersion }) - if let latest = installations.first { - let msvcDir = latest.installationPath.join("VC").join("Tools").join("MSVC") - if fs.exists(msvcDir) { - let versions = try fs.listdir(msvcDir).map { try Version($0) }.sorted { $0 > $1 } - if let latestVersion = versions.first { - return msvcDir.join(latestVersion.description) - } - } - } - return nil } struct WindowsPlatformExtension: PlatformInfoExtension { diff --git a/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec b/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec index 206eaafc..13c8f67b 100644 --- a/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec +++ b/Sources/SWBWindowsPlatform/Specs/WindowsLd.xcspec @@ -61,7 +61,7 @@ { Name = _LD_MULTIARCH_PREFIX_MAP; Type = StringList; - DefaultValue = "x86_64:x64 aarch64:arm64 i386:x86 thumbv7:arm"; + DefaultValue = "aarch64:arm64 armv7:arm arm64ec:arm64 x86_64:x64 i686:x86"; Condition = "$(ALTERNATE_LINKER) == link"; }, { diff --git a/Tests/SWBBuildSystemTests/LinkerTests.swift b/Tests/SWBBuildSystemTests/LinkerTests.swift index eb458624..8a2f3d25 100644 --- a/Tests/SWBBuildSystemTests/LinkerTests.swift +++ b/Tests/SWBBuildSystemTests/LinkerTests.swift @@ -324,7 +324,7 @@ fileprivate struct LinkerTests: CoreBasedTests { results.checkTaskOutput(task) { output in // Expect that the 'link' linker is called by clang if runDestination == .windows && Architecture.hostStringValue == "aarch64" { - // On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\x64\link.exe" + // rdar://145868953 - On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\x64\link.exe" withKnownIssue("'clang' picks the wrong binary for link.exe using the Hostx86 version") { #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(linkLinkerPathX86.str)) } @@ -344,7 +344,7 @@ fileprivate struct LinkerTests: CoreBasedTests { results.checkTaskOutput(task) { output in // Expect that the 'link' linker is called by clang if runDestination == .windows && Architecture.hostStringValue == "aarch64" { - // On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\x64\link.exe" + // rdar://145868953 - On windows aarch64 'clang' picks the wrong host architecture for link.exe, choosing "MSVC\14.41.34120\bin\Hostx86\x64\link.exe" withKnownIssue("'clang' picks the wrong binary for link.exe using the Hostx86 version") { #expect(output.asString.replacingOccurrences(of: "\\\\", with: "\\").contains(linkLinkerPathAarch64.str)) } diff --git a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift index 7f0a5bfe..a72051d7 100644 --- a/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift +++ b/Tests/SWBCoreTests/CommandLineToolSpecDiscoveredInfoTests.swift @@ -234,53 +234,6 @@ import SWBMacro } } - @Test(.requireSDKs(.windows)) - func discoveredLdLinkerSpecInfo_Windows() async throws { - var table = try await ldMacroTable() - table.push(BuiltinMacros._LD_MULTIARCH, literal: true) - try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in - #expect(!info.toolPath.isEmpty) - #expect(info.toolVersion != nil) - if let toolVersion = info.toolVersion { - #expect(toolVersion > Version(0, 0, 0)) - } - #expect(info.linker == .lld) - } - try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("Microsoft (R) Incremental Linker Version 14.41.34120.0\n".utf8), stderr: Data()), platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in - #expect(!info.toolPath.isEmpty) - #expect(info.toolVersion == Version(14, 41, 34120)) - #expect(info.architectures == Set()) - } - - // link.exe cannot be used for multipler architectures and requires a distinct link.exe for each target architecture - table.push(BuiltinMacros.ALTERNATE_LINKER, literal: "link") - table.push(BuiltinMacros._LD_MULTIARCH, literal: false) - table.push(BuiltinMacros._LD_MULTIARCH_PREFIX_MAP, literal: ["x86_64:x64", "aarch64:arm64", "i386:x86", "thumbv7:arm"]) - - // x86_64 - table.push(BuiltinMacros.ARCHS, literal: ["x86_64"]) - try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in - #expect(!info.toolPath.isEmpty) - #expect(info.toolVersion != nil) - if let toolVersion = info.toolVersion { - #expect(toolVersion > Version(0, 0, 0)) - } - #expect(info.linker == .linkExe) - } - - // FIXME: Fails in swift-ci missing arm64 toolchain on x86_64 host. - // // link aarch64 - // table.push(BuiltinMacros.ARCHS, literal: ["aarch64"]) - // try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in - // #expect(!info.toolPath.isEmpty) - // #expect(info.toolVersion != nil) - // if let toolVersion = info.toolVersion { - // #expect(toolVersion > Version(0, 0, 0)) - // } - // #expect(info.linker == .linkExe) - // } - } - @Test(.skipHostOS(.windows), .requireSystemPackages(apt: "libtool", yum: "libtool")) func discoveredLibtoolSpecInfo() async throws { try await withSpec(LibtoolLinkerSpec.self, .deferred) { (info: DiscoveredLibtoolLinkerToolSpecInfo) in diff --git a/Tests/SWBWindowsPlatformTests/SWBWindowsPlatformTests.swift b/Tests/SWBWindowsPlatformTests/SWBWindowsPlatformTests.swift index 380508dc..25771aa5 100644 --- a/Tests/SWBWindowsPlatformTests/SWBWindowsPlatformTests.swift +++ b/Tests/SWBWindowsPlatformTests/SWBWindowsPlatformTests.swift @@ -9,4 +9,104 @@ // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // //===----------------------------------------------------------------------===// +import Foundation +@_spi(Testing) import SWBCore +import SWBProtocol +import SWBTestSupport +import SWBUtil +import Testing +import SWBMacro +@Suite fileprivate struct CommandLineToolSpecDiscoveredInfoTests: CoreBasedTests { + // Linker tool discovery is a bit more complex as it afffected by the ALTERNATE_LINKER build setting. + func ldMacroTable() async throws -> MacroValueAssignmentTable { + let core = try await getCore() + return MacroValueAssignmentTable(namespace: core.specRegistry.internalMacroNamespace) + } + + @Test(.requireSDKs(.windows)) + func discoveredLdLinkerSpecInfo_Windows() async throws { + var table = try await ldMacroTable() + let core = try await getCore() + table.push(BuiltinMacros._LD_MULTIARCH, literal: true) + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .lld) + } + try await withSpec(LdLinkerSpec.self, .result(status: .exit(0), stdout: Data("Microsoft (R) Incremental Linker Version 14.41.34120.0\n".utf8), stderr: Data()), platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion == Version(14, 41, 34120)) + #expect(info.architectures == Set()) + } + + // link.exe cannot be used for multipler architectures and requires a distinct link.exe for each target architecture + table.push(BuiltinMacros.ALTERNATE_LINKER, literal: "link") + table.push(BuiltinMacros._LD_MULTIARCH, literal: false) + table.push(BuiltinMacros._LD_MULTIARCH_PREFIX_MAP, literal: ["aarch64:arm64", "arm64ec:arm64", "armv7:arm", "x86_64:x64", "i686:x86"]) + + // link x86_64 + if try await core.hasVisualStudioComponent(.visualCppBuildTools_x86_x64) { + table.push(BuiltinMacros.ARCHS, literal: ["x86_64"]) + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .linkExe) + } + } + // link i686 + if try await core.hasVisualStudioComponent(.visualCppBuildTools_x86_x64) { + table.push(BuiltinMacros.ARCHS, literal: ["i686"]) + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .linkExe) + } + } + // link aarch64 + if try await core.hasVisualStudioComponent(.visualCppBuildTools_arm64) { + table.push(BuiltinMacros.ARCHS, literal: ["aarch64"]) + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .linkExe) + } + } + // link armv7 + if try await core.hasVisualStudioComponent(.visualCppBuildTools_arm) { + table.push(BuiltinMacros.ARCHS, literal: ["armv7"]) + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .linkExe) + } + } + // link arm64ec + if try await core.hasVisualStudioComponent(.visualCppBuildTools_arm64ec) { + table.push(BuiltinMacros.ARCHS, literal: ["arm64ec"]) + try await withSpec(LdLinkerSpec.self, .deferred, platform: "windows", additionalTable: table) { (info: DiscoveredLdLinkerToolSpecInfo) in + #expect(!info.toolPath.isEmpty) + #expect(info.toolVersion != nil) + if let toolVersion = info.toolVersion { + #expect(toolVersion > Version(0, 0, 0)) + } + #expect(info.linker == .linkExe) + } + } + } +} \ No newline at end of file