From d91abb30c3898d2a940773d745b2a63474b24e02 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 23 Apr 2024 12:34:35 +0100 Subject: [PATCH 01/36] =?UTF-8?q?Cover=20`=E2=80=93-toolchain`=20argument?= =?UTF-8?q?=20in=20`SwiftCommandStateTests`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should prevent possible regressions in how this argument is handled. --- .../FileSystem/FileSystem+Extensions.swift | 3 +- .../FileSystem/InMemoryFileSystem.swift | 496 ++++++++++++++++++ Sources/CoreCommands/SwiftCommandState.swift | 18 +- Sources/PackageModel/UserToolchain.swift | 166 ++++-- Sources/SPMTestSupport/Toolchain.swift | 2 +- Sources/Workspace/Workspace.swift | 4 +- .../SwiftCommandStateTests.swift | 64 ++- .../PackageModelTests/PackageModelTests.swift | 29 +- 8 files changed, 703 insertions(+), 79 deletions(-) create mode 100644 Sources/Basics/FileSystem/InMemoryFileSystem.swift diff --git a/Sources/Basics/FileSystem/FileSystem+Extensions.swift b/Sources/Basics/FileSystem/FileSystem+Extensions.swift index d16637d5872..4a61f56f006 100644 --- a/Sources/Basics/FileSystem/FileSystem+Extensions.swift +++ b/Sources/Basics/FileSystem/FileSystem+Extensions.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2020-2021 Apple Inc. and the Swift project authors +// Copyright (c) 2020-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -21,7 +21,6 @@ import class TSCBasic.FileLock import enum TSCBasic.FileMode import protocol TSCBasic.FileSystem import enum TSCBasic.FileSystemAttribute -import class TSCBasic.InMemoryFileSystem import var TSCBasic.localFileSystem import protocol TSCBasic.WritableByteStream diff --git a/Sources/Basics/FileSystem/InMemoryFileSystem.swift b/Sources/Basics/FileSystem/InMemoryFileSystem.swift new file mode 100644 index 00000000000..04420e98210 --- /dev/null +++ b/Sources/Basics/FileSystem/InMemoryFileSystem.swift @@ -0,0 +1,496 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors + */ + +import class Foundation.NSLock +import class Dispatch.DispatchQueue +import struct TSCBasic.AbsolutePath +import struct TSCBasic.ByteString +import class TSCBasic.FileLock +import enum TSCBasic.FileMode +import struct TSCBasic.FileSystemError + +/// Concrete FileSystem implementation which simulates an empty disk. +public final class InMemoryFileSystem: FileSystem { + /// Private internal representation of a file system node. + /// Not thread-safe. + private class Node { + /// The actual node data. + let contents: NodeContents + + /// Whether the node has executable bit enabled. + var isExecutable: Bool + + init(_ contents: NodeContents, isExecutable: Bool = false) { + self.contents = contents + self.isExecutable = isExecutable + } + + /// Creates deep copy of the object. + func copy() -> Node { + return Node(contents.copy()) + } + } + + /// Private internal representation the contents of a file system node. + /// Not thread-safe. + private enum NodeContents { + case file(ByteString) + case directory(DirectoryContents) + case symlink(String) + + /// Creates deep copy of the object. + func copy() -> NodeContents { + switch self { + case .file(let bytes): + return .file(bytes) + case .directory(let contents): + return .directory(contents.copy()) + case .symlink(let path): + return .symlink(path) + } + } + } + + /// Private internal representation the contents of a directory. + /// Not thread-safe. + private final class DirectoryContents { + var entries: [String: Node] + + init(entries: [String: Node] = [:]) { + self.entries = entries + } + + /// Creates deep copy of the object. + func copy() -> DirectoryContents { + let contents = DirectoryContents() + for (key, node) in entries { + contents.entries[key] = node.copy() + } + return contents + } + } + + /// The root node of the filesystem. + private var root: Node + + /// Protects `root` and everything underneath it. + /// FIXME: Using a single lock for this is a performance problem, but in + /// reality, the only practical use for InMemoryFileSystem is for unit + /// tests. + private let lock = NSLock() + /// A map that keeps weak references to all locked files. + private var lockFiles = Dictionary>() + /// Used to access lockFiles in a thread safe manner. + private let lockFilesLock = NSLock() + + /// Exclusive file system lock vended to clients through `withLock()`. + /// Used to ensure that DispatchQueues are released when they are no longer in use. + private struct WeakReference { + weak var reference: Value? + + init(_ value: Value?) { + self.reference = value + } + } + + public init() { + root = Node(.directory(DirectoryContents())) + } + + /// Creates deep copy of the object. + public func copy() -> InMemoryFileSystem { + return lock.withLock { + let fs = InMemoryFileSystem() + fs.root = root.copy() + return fs + } + } + + /// Private function to look up the node corresponding to a path. + /// Not thread-safe. + private func getNode(_ path: TSCBasic.AbsolutePath, followSymlink: Bool = true) throws -> Node? { + func getNodeInternal(_ path: TSCBasic.AbsolutePath) throws -> Node? { + // If this is the root node, return it. + if path.isRoot { + return root + } + + // Otherwise, get the parent node. + guard let parent = try getNodeInternal(path.parentDirectory) else { + return nil + } + + // If we didn't find a directory, this is an error. + guard case .directory(let contents) = parent.contents else { + throw FileSystemError(.notDirectory, path.parentDirectory) + } + + // Return the directory entry. + let node = contents.entries[path.basename] + + switch node?.contents { + case .directory, .file: + return node + case .symlink(let destination): + let destination = try TSCBasic.AbsolutePath(validating: destination, relativeTo: path.parentDirectory) + return followSymlink ? try getNodeInternal(destination) : node + case .none: + return nil + } + } + + // Get the node that corresponds to the path. + return try getNodeInternal(path) + } + + // MARK: FileSystem Implementation + + public func exists(_ path: TSCBasic.AbsolutePath, followSymlink: Bool) -> Bool { + return lock.withLock { + do { + switch try getNode(path, followSymlink: followSymlink)?.contents { + case .file, .directory, .symlink: return true + case .none: return false + } + } catch { + return false + } + } + } + + public func isDirectory(_ path: TSCBasic.AbsolutePath) -> Bool { + return lock.withLock { + do { + if case .directory? = try getNode(path)?.contents { + return true + } + return false + } catch { + return false + } + } + } + + public func isFile(_ path: TSCBasic.AbsolutePath) -> Bool { + return lock.withLock { + do { + if case .file? = try getNode(path)?.contents { + return true + } + return false + } catch { + return false + } + } + } + + public func isSymlink(_ path: TSCBasic.AbsolutePath) -> Bool { + return lock.withLock { + do { + if case .symlink? = try getNode(path, followSymlink: false)?.contents { + return true + } + return false + } catch { + return false + } + } + } + + public func isReadable(_ path: TSCBasic.AbsolutePath) -> Bool { + self.exists(path) + } + + public func isWritable(_ path: TSCBasic.AbsolutePath) -> Bool { + self.exists(path) + } + + public func isExecutableFile(_ path: TSCBasic.AbsolutePath) -> Bool { + (try? self.getNode(path)?.isExecutable) ?? false + } + + public func updatePermissions(_ path: AbsolutePath, isExecutable: Bool) throws { + try lock.withLock { + guard let node = try self.getNode(path.underlying, followSymlink: true) else { + throw FileSystemError(.noEntry, path) + } + node.isExecutable = isExecutable + } + } + + /// Virtualized current working directory. + public var currentWorkingDirectory: TSCBasic.AbsolutePath? { + return try? .init(validating: "/") + } + + public func changeCurrentWorkingDirectory(to path: TSCBasic.AbsolutePath) throws { + throw FileSystemError(.unsupported, path) + } + + public var homeDirectory: TSCBasic.AbsolutePath { + get throws { + // FIXME: Maybe we should allow setting this when creating the fs. + return try .init(validating: "/home/user") + } + } + + public var cachesDirectory: TSCBasic.AbsolutePath? { + return try? self.homeDirectory.appending(component: "caches") + } + + public var tempDirectory: TSCBasic.AbsolutePath { + get throws { + return try .init(validating: "/tmp") + } + } + + public func getDirectoryContents(_ path: TSCBasic.AbsolutePath) throws -> [String] { + return try lock.withLock { + guard let node = try getNode(path) else { + throw FileSystemError(.noEntry, path) + } + guard case .directory(let contents) = node.contents else { + throw FileSystemError(.notDirectory, path) + } + + // FIXME: Perhaps we should change the protocol to allow lazy behavior. + return [String](contents.entries.keys) + } + } + + /// Not thread-safe. + private func _createDirectory(_ path: TSCBasic.AbsolutePath, recursive: Bool) throws { + // Ignore if client passes root. + guard !path.isRoot else { + return + } + // Get the parent directory node. + let parentPath = path.parentDirectory + guard let parent = try getNode(parentPath) else { + // If the parent doesn't exist, and we are recursive, then attempt + // to create the parent and retry. + if recursive && path != parentPath { + // Attempt to create the parent. + try _createDirectory(parentPath, recursive: true) + + // Re-attempt creation, non-recursively. + return try _createDirectory(path, recursive: false) + } else { + // Otherwise, we failed. + throw FileSystemError(.noEntry, parentPath) + } + } + + // Check that the parent is a directory. + guard case .directory(let contents) = parent.contents else { + // The parent isn't a directory, this is an error. + throw FileSystemError(.notDirectory, parentPath) + } + + // Check if the node already exists. + if let node = contents.entries[path.basename] { + // Verify it is a directory. + guard case .directory = node.contents else { + // The path itself isn't a directory, this is an error. + throw FileSystemError(.notDirectory, path) + } + + // We are done. + return + } + + // Otherwise, the node does not exist, create it. + contents.entries[path.basename] = Node(.directory(DirectoryContents())) + } + + public func createDirectory(_ path: TSCBasic.AbsolutePath, recursive: Bool) throws { + return try lock.withLock { + try _createDirectory(path, recursive: recursive) + } + } + + public func createSymbolicLink( + _ path: TSCBasic.AbsolutePath, + pointingAt destination: TSCBasic.AbsolutePath, + relative: Bool + ) throws { + return try lock.withLock { + // Create directory to destination parent. + guard let destinationParent = try getNode(path.parentDirectory) else { + throw FileSystemError(.noEntry, path.parentDirectory) + } + + // Check that the parent is a directory. + guard case .directory(let contents) = destinationParent.contents else { + throw FileSystemError(.notDirectory, path.parentDirectory) + } + + guard contents.entries[path.basename] == nil else { + throw FileSystemError(.alreadyExistsAtDestination, path) + } + + let destination = relative ? destination.relative(to: path.parentDirectory).pathString : destination.pathString + + contents.entries[path.basename] = Node(.symlink(destination)) + } + } + + public func readFileContents(_ path: TSCBasic.AbsolutePath) throws -> ByteString { + return try lock.withLock { + // Get the node. + guard let node = try getNode(path) else { + throw FileSystemError(.noEntry, path) + } + + // Check that the node is a file. + guard case .file(let contents) = node.contents else { + // The path is a directory, this is an error. + throw FileSystemError(.isDirectory, path) + } + + // Return the file contents. + return contents + } + } + + public func writeFileContents(_ path: TSCBasic.AbsolutePath, bytes: ByteString) throws { + return try lock.withLock { + // It is an error if this is the root node. + let parentPath = path.parentDirectory + guard path != parentPath else { + throw FileSystemError(.isDirectory, path) + } + + // Get the parent node. + guard let parent = try getNode(parentPath) else { + throw FileSystemError(.noEntry, parentPath) + } + + // Check that the parent is a directory. + guard case .directory(let contents) = parent.contents else { + // The parent isn't a directory, this is an error. + throw FileSystemError(.notDirectory, parentPath) + } + + // Check if the node exists. + if let node = contents.entries[path.basename] { + // Verify it is a file. + guard case .file = node.contents else { + // The path is a directory, this is an error. + throw FileSystemError(.isDirectory, path) + } + } + + // Write the file. + contents.entries[path.basename] = Node(.file(bytes)) + } + } + + public func writeFileContents(_ path: TSCBasic.AbsolutePath, bytes: ByteString, atomically: Bool) throws { + // In memory file system's writeFileContents is already atomic, so ignore the parameter here + // and just call the base implementation. + try writeFileContents(path, bytes: bytes) + } + + public func removeFileTree(_ path: TSCBasic.AbsolutePath) throws { + return lock.withLock { + // Ignore root and get the parent node's content if its a directory. + guard !path.isRoot, + let parent = try? getNode(path.parentDirectory), + case .directory(let contents) = parent.contents else { + return + } + // Set it to nil to release the contents. + contents.entries[path.basename] = nil + } + } + + public func chmod(_ mode: FileMode, path: TSCBasic.AbsolutePath, options: Set) throws { + // FIXME: We don't have these semantics in InMemoryFileSystem. + } + + /// Private implementation of core copying function. + /// Not thread-safe. + private func _copy(from sourcePath: TSCBasic.AbsolutePath, to destinationPath: TSCBasic.AbsolutePath) throws { + // Get the source node. + guard let source = try getNode(sourcePath) else { + throw FileSystemError(.noEntry, sourcePath) + } + + // Create directory to destination parent. + guard let destinationParent = try getNode(destinationPath.parentDirectory) else { + throw FileSystemError(.noEntry, destinationPath.parentDirectory) + } + + // Check that the parent is a directory. + guard case .directory(let contents) = destinationParent.contents else { + throw FileSystemError(.notDirectory, destinationPath.parentDirectory) + } + + guard contents.entries[destinationPath.basename] == nil else { + throw FileSystemError(.alreadyExistsAtDestination, destinationPath) + } + + contents.entries[destinationPath.basename] = source + } + + public func copy(from sourcePath: TSCBasic.AbsolutePath, to destinationPath: TSCBasic.AbsolutePath) throws { + return try lock.withLock { + try _copy(from: sourcePath, to: destinationPath) + } + } + + public func move(from sourcePath: TSCBasic.AbsolutePath, to destinationPath: TSCBasic.AbsolutePath) throws { + return try lock.withLock { + // Get the source parent node. + guard let sourceParent = try getNode(sourcePath.parentDirectory) else { + throw FileSystemError(.noEntry, sourcePath.parentDirectory) + } + + // Check that the parent is a directory. + guard case .directory(let contents) = sourceParent.contents else { + throw FileSystemError(.notDirectory, sourcePath.parentDirectory) + } + + try _copy(from: sourcePath, to: destinationPath) + + contents.entries[sourcePath.basename] = nil + } + } + + public func withLock( + on path: TSCBasic.AbsolutePath, + type: FileLock.LockType = .exclusive, + _ body: () throws -> T + ) throws -> T { + let resolvedPath: TSCBasic.AbsolutePath = try lock.withLock { + if case let .symlink(destination) = try getNode(path)?.contents { + return try .init(validating: destination, relativeTo: path.parentDirectory) + } else { + return path + } + } + + let fileQueue: DispatchQueue = lockFilesLock.withLock { + if let queueReference = lockFiles[resolvedPath], let queue = queueReference.reference { + return queue + } else { + let queue = DispatchQueue(label: "org.swift.swiftpm.in-memory-file-system.file-queue", attributes: .concurrent) + lockFiles[resolvedPath] = WeakReference(queue) + return queue + } + } + + return try fileQueue.sync(flags: type == .exclusive ? .barrier : .init() , execute: body) + } +} + +// Internal state of `InMemoryFileSystem` is protected with a lock in all of its `public` methods. +extension InMemoryFileSystem: @unchecked Sendable {} diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 754a5401ea4..d1547d03ca8 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -294,9 +294,10 @@ package final class SwiftCommandState { options: GlobalOptions, toolWorkspaceConfiguration: ToolWorkspaceConfiguration, workspaceDelegateProvider: @escaping WorkspaceDelegateProvider, - workspaceLoaderProvider: @escaping WorkspaceLoaderProvider + workspaceLoaderProvider: @escaping WorkspaceLoaderProvider, + fileSystem: any FileSystem = localFileSystem ) throws { - self.fileSystem = localFileSystem + self.fileSystem = fileSystem // first, bootstrap the observability system self.logLevel = options.logging.logLevel self.observabilityHandler = SwiftCommandObservabilityHandler(outputStream: outputStream, logLevel: self.logLevel) @@ -842,16 +843,19 @@ package final class SwiftCommandState { return self._hostToolchain } - return Result(catching: { try UserToolchain(swiftSDK: swiftSDK) }) + return Result(catching: { try UserToolchain(swiftSDK: swiftSDK, fileSystem: self.fileSystem) }) }() /// Lazily compute the host toolchain used to compile the package description. private lazy var _hostToolchain: Result = { return Result(catching: { - try UserToolchain(swiftSDK: SwiftSDK.hostSwiftSDK( - originalWorkingDirectory: self.originalWorkingDirectory, - observabilityScope: self.observabilityScope - )) + try UserToolchain( + swiftSDK: SwiftSDK.hostSwiftSDK( + originalWorkingDirectory: self.originalWorkingDirectory, + observabilityScope: self.observabilityScope + ), + fileSystem: self.fileSystem + ) }) }() diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 9b7042332bb..ba6ff8eba3b 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -58,6 +58,8 @@ public final class UserToolchain: Toolchain { self.swiftCompilerPath.parentDirectory.appending("swift" + hostExecutableSuffix) } + private let fileSystem: any FileSystem + /// The compilation destination object. @available(*, deprecated, renamed: "swiftSDK") public var destination: SwiftSDK { swiftSDK } @@ -103,7 +105,7 @@ public final class UserToolchain: Toolchain { ) // Ensure that the runtime is present. - guard localFileSystem.exists(runtime) else { + guard fileSystem.exists(runtime) else { throw InvalidToolchainDiagnostic("Missing runtime for \(sanitizer) sanitizer") } @@ -120,13 +122,17 @@ public final class UserToolchain: Toolchain { lookupExecutablePath(filename: environment[variable], searchPaths: searchPaths) } - private static func getTool(_ name: String, binDirectories: [AbsolutePath]) throws -> AbsolutePath { + private static func getTool( + _ name: String, + binDirectories: [AbsolutePath], + fileSystem: any FileSystem + ) throws -> AbsolutePath { let executableName = "\(name)\(hostExecutableSuffix)" var toolPath: AbsolutePath? for dir in binDirectories { let path = dir.appending(component: executableName) - guard localFileSystem.isExecutableFile(path) else { + guard fileSystem.isExecutableFile(path) else { continue } toolPath = path @@ -142,7 +148,8 @@ public final class UserToolchain: Toolchain { private static func findTool( _ name: String, envSearchPaths: [AbsolutePath], - useXcrun: Bool + useXcrun: Bool, + fileSystem: any FileSystem ) throws -> AbsolutePath { if useXcrun { #if os(macOS) @@ -152,7 +159,7 @@ public final class UserToolchain: Toolchain { #endif } - return try getTool(name, binDirectories: envSearchPaths) + return try getTool(name, binDirectories: envSearchPaths, fileSystem: fileSystem) } // MARK: - public API @@ -163,10 +170,9 @@ public final class UserToolchain: Toolchain { useXcrun: Bool, environment: EnvironmentVariables, searchPaths: [AbsolutePath], - extraSwiftFlags: [String] - ) throws - -> AbsolutePath - { + extraSwiftFlags: [String], + fileSystem: any FileSystem + ) throws -> AbsolutePath { let variable: String = triple.isApple() ? "LIBTOOL" : "AR" let tool: String = { if triple.isApple() { return "libtool" } @@ -196,25 +202,25 @@ public final class UserToolchain: Toolchain { searchPaths: searchPaths, environment: environment ) { - if localFileSystem.isExecutableFile(librarian) { + if fileSystem.isExecutableFile(librarian) { return librarian } } - if let librarian = try? UserToolchain.getTool(tool, binDirectories: binDirectories) { + if let librarian = try? UserToolchain.getTool(tool, binDirectories: binDirectories, fileSystem: fileSystem) { return librarian } if triple.isApple() || triple.isWindows() { - return try UserToolchain.findTool(tool, envSearchPaths: searchPaths, useXcrun: useXcrun) + return try UserToolchain.findTool(tool, envSearchPaths: searchPaths, useXcrun: useXcrun, fileSystem: fileSystem) } else { - if let librarian = try? UserToolchain.findTool(tool, envSearchPaths: searchPaths, useXcrun: false) { + if let librarian = try? UserToolchain.findTool(tool, envSearchPaths: searchPaths, useXcrun: false, fileSystem: fileSystem) { return librarian } // Fall back to looking for binutils `ar` if `llvm-ar` can't be found. - if let librarian = try? UserToolchain.getTool("ar", binDirectories: binDirectories) { + if let librarian = try? UserToolchain.getTool("ar", binDirectories: binDirectories, fileSystem: fileSystem) { return librarian } - return try UserToolchain.findTool("ar", envSearchPaths: searchPaths, useXcrun: false) + return try UserToolchain.findTool("ar", envSearchPaths: searchPaths, useXcrun: false, fileSystem: fileSystem) } } @@ -223,11 +229,12 @@ public final class UserToolchain: Toolchain { binDirectories: [AbsolutePath], useXcrun: Bool, environment: EnvironmentVariables, - searchPaths: [AbsolutePath] + searchPaths: [AbsolutePath], + fileSystem: any FileSystem ) throws -> SwiftCompilers { func validateCompiler(at path: AbsolutePath?) throws { guard let path else { return } - guard localFileSystem.isExecutableFile(path) else { + guard fileSystem.isExecutableFile(path) else { throw InvalidToolchainDiagnostic( "could not find the `swiftc\(hostExecutableSuffix)` at expected path \(path)" ) @@ -248,7 +255,7 @@ public final class UserToolchain: Toolchain { let resolvedBinDirCompiler: AbsolutePath if let SWIFT_EXEC { resolvedBinDirCompiler = SWIFT_EXEC - } else if let binDirCompiler = try? UserToolchain.getTool("swiftc", binDirectories: binDirectories) { + } else if let binDirCompiler = try? UserToolchain.getTool("swiftc", binDirectories: binDirectories, fileSystem: fileSystem) { resolvedBinDirCompiler = binDirCompiler } else { // Try to lookup swift compiler on the system which is possible when @@ -256,13 +263,14 @@ public final class UserToolchain: Toolchain { resolvedBinDirCompiler = try UserToolchain.findTool( "swiftc", envSearchPaths: searchPaths, - useXcrun: useXcrun + useXcrun: useXcrun, + fileSystem: fileSystem ) } // The compiler for compilation tasks is SWIFT_EXEC or the bin dir compiler. // The compiler for manifest is either SWIFT_EXEC_MANIFEST or the bin dir compiler. - return (SWIFT_EXEC ?? resolvedBinDirCompiler, SWIFT_EXEC_MANIFEST ?? resolvedBinDirCompiler) + return (compile: SWIFT_EXEC ?? resolvedBinDirCompiler, manifest: SWIFT_EXEC_MANIFEST ?? resolvedBinDirCompiler) } /// Returns the path to clang compiler tool. @@ -283,15 +291,22 @@ public final class UserToolchain: Toolchain { } // Then, check the toolchain. - do { - if let toolPath = try? UserToolchain.getTool("clang", binDirectories: self.swiftSDK.toolset.rootPaths) { - self._clangCompiler = toolPath - return toolPath - } + if let toolPath = try? UserToolchain.getTool( + "clang", + binDirectories: self.swiftSDK.toolset.rootPaths, + fileSystem: self.fileSystem + ) { + self._clangCompiler = toolPath + return toolPath } // Otherwise, lookup it up on the system. - let toolPath = try UserToolchain.findTool("clang", envSearchPaths: self.envSearchPaths, useXcrun: useXcrun) + let toolPath = try UserToolchain.findTool( + "clang", + envSearchPaths: self.envSearchPaths, + useXcrun: useXcrun, + fileSystem: self.fileSystem + ) self._clangCompiler = toolPath return toolPath } @@ -309,21 +324,38 @@ public final class UserToolchain: Toolchain { /// Returns the path to lldb. public func getLLDB() throws -> AbsolutePath { // Look for LLDB next to the compiler first. - if let lldbPath = try? UserToolchain.getTool("lldb", binDirectories: [self.swiftCompilerPath.parentDirectory]) { + if let lldbPath = try? UserToolchain.getTool( + "lldb", + binDirectories: [self.swiftCompilerPath.parentDirectory], + fileSystem: self.fileSystem + ) { return lldbPath } // If that fails, fall back to xcrun, PATH, etc. - return try UserToolchain.findTool("lldb", envSearchPaths: self.envSearchPaths, useXcrun: useXcrun) + return try UserToolchain.findTool( + "lldb", + envSearchPaths: self.envSearchPaths, + useXcrun: useXcrun, + fileSystem: self.fileSystem + ) } /// Returns the path to llvm-cov tool. public func getLLVMCov() throws -> AbsolutePath { - try UserToolchain.getTool("llvm-cov", binDirectories: [self.swiftCompilerPath.parentDirectory]) + try UserToolchain.getTool( + "llvm-cov", + binDirectories: [self.swiftCompilerPath.parentDirectory], + fileSystem: self.fileSystem + ) } /// Returns the path to llvm-prof tool. public func getLLVMProf() throws -> AbsolutePath { - try UserToolchain.getTool("llvm-profdata", binDirectories: [self.swiftCompilerPath.parentDirectory]) + try UserToolchain.getTool( + "llvm-profdata", + binDirectories: [self.swiftCompilerPath.parentDirectory], + fileSystem: self.fileSystem + ) } public func getSwiftAPIDigester() throws -> AbsolutePath { @@ -334,7 +366,12 @@ public final class UserToolchain: Toolchain { ) { return envValue } - return try UserToolchain.getTool("swift-api-digester", binDirectories: [self.swiftCompilerPath.parentDirectory]) + return try UserToolchain.getTool( + "swift-api-digester", + binDirectories: [self.swiftCompilerPath.parentDirectory], + fileSystem: self.fileSystem + + ) } public func getSymbolGraphExtract() throws -> AbsolutePath { @@ -345,13 +382,18 @@ public final class UserToolchain: Toolchain { ) { return envValue } - return try UserToolchain.getTool("swift-symbolgraph-extract", binDirectories: [self.swiftCompilerPath.parentDirectory]) + return try UserToolchain.getTool( + "swift-symbolgraph-extract", + binDirectories: [self.swiftCompilerPath.parentDirectory], + fileSystem: self.fileSystem + ) } internal static func deriveSwiftCFlags( triple: Triple, swiftSDK: SwiftSDK, - environment: EnvironmentVariables + environment: EnvironmentVariables, + fileSystem: any FileSystem ) throws -> [String] { var swiftCompilerFlags = swiftSDK.toolset.knownTools[.swiftCompiler]?.extraCLIOptions ?? [] @@ -372,7 +414,7 @@ public final class UserToolchain: Toolchain { if let settings = WindowsSDKSettings( reading: sdkroot.appending("SDKSettings.plist"), observabilityScope: nil, - filesystem: localFileSystem + filesystem: fileSystem ) { switch settings.defaults.runtime { case .multithreadedDebugDLL: @@ -397,7 +439,7 @@ public final class UserToolchain: Toolchain { if let info = WindowsPlatformInfo( reading: platform.appending("Info.plist"), observabilityScope: nil, - filesystem: localFileSystem + filesystem: fileSystem ) { let installation: AbsolutePath = platform.appending("Developer") @@ -438,7 +480,7 @@ public final class UserToolchain: Toolchain { validating: "usr/lib/swift/windows/XCTest.lib", relativeTo: installation ) - if localFileSystem.exists(implib) { + if fileSystem.exists(implib) { xctest.append(contentsOf: ["-L", implib.parentDirectory.pathString]) } @@ -477,7 +519,8 @@ public final class UserToolchain: Toolchain { swiftSDK: destination, environment: environment, searchStrategy: searchStrategy, - customLibrariesLocation: customLibrariesLocation + customLibrariesLocation: customLibrariesLocation, + fileSystem: localFileSystem ) } @@ -487,7 +530,8 @@ public final class UserToolchain: Toolchain { searchStrategy: SearchStrategy = .default, customLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation? = nil, customInstalledSwiftPMConfiguration: InstalledSwiftPMConfiguration? = nil, - customProvidedLibraries: [LibraryMetadata]? = nil + customProvidedLibraries: [LibraryMetadata]? = nil, + fileSystem: any FileSystem = localFileSystem ) throws { self.swiftSDK = swiftSDK self.environment = environment @@ -497,7 +541,7 @@ public final class UserToolchain: Toolchain { // Get the search paths from PATH. self.envSearchPaths = getEnvSearchPaths( pathString: environment.path, - currentWorkingDirectory: localFileSystem.currentWorkingDirectory + currentWorkingDirectory: fileSystem.currentWorkingDirectory ) self.useXcrun = true case .custom(let searchPaths, let useXcrun): @@ -509,7 +553,8 @@ public final class UserToolchain: Toolchain { binDirectories: swiftSDK.toolset.rootPaths, useXcrun: useXcrun, environment: environment, - searchPaths: envSearchPaths + searchPaths: envSearchPaths, + fileSystem: fileSystem ) self.swiftCompilerPath = swiftCompilers.compile self.architectures = swiftSDK.architectures @@ -517,7 +562,7 @@ public final class UserToolchain: Toolchain { #if canImport(Darwin) let toolchainPlistPath = self.swiftCompilerPath.parentDirectory.parentDirectory.parentDirectory .appending(component: "Info.plist") - if localFileSystem.exists(toolchainPlistPath), let toolchainPlist = try? NSDictionary( + if fileSystem.exists(toolchainPlistPath), let toolchainPlist = try? NSDictionary( contentsOf: URL(fileURLWithPath: toolchainPlistPath.pathString), error: () ), let overrideBuildSettings = toolchainPlist["OverrideBuildSettings"] as? NSDictionary, @@ -572,7 +617,9 @@ public final class UserToolchain: Toolchain { swiftCompilerFlags: try Self.deriveSwiftCFlags( triple: triple, swiftSDK: swiftSDK, - environment: environment), + environment: environment, + fileSystem: fileSystem + ), linkerFlags: swiftSDK.toolset.knownTools[.linker]?.extraCLIOptions ?? [], xcbuildFlags: swiftSDK.toolset.knownTools[.xcbuild]?.extraCLIOptions ?? []) @@ -585,7 +632,8 @@ public final class UserToolchain: Toolchain { useXcrun: useXcrun, environment: environment, searchPaths: envSearchPaths, - extraSwiftFlags: self.extraFlags.swiftCompilerFlags + extraSwiftFlags: self.extraFlags.swiftCompilerFlags, + fileSystem: fileSystem ) if let sdkDir = swiftSDK.pathsConfiguration.sdkRootPath { @@ -598,7 +646,7 @@ public final class UserToolchain: Toolchain { if let settings = WindowsSDKSettings( reading: root.appending("SDKSettings.plist"), observabilityScope: nil, - filesystem: localFileSystem + filesystem: fileSystem ) { switch settings.defaults.runtime { case .multithreadedDebugDLL: @@ -634,7 +682,8 @@ public final class UserToolchain: Toolchain { let swiftPMLibrariesLocation = try customLibrariesLocation ?? Self.deriveSwiftPMLibrariesLocation( swiftCompilerPath: swiftCompilerPath, swiftSDK: swiftSDK, - environment: environment + environment: environment, + fileSystem: fileSystem ) let xctestPath: AbsolutePath? @@ -644,7 +693,8 @@ public final class UserToolchain: Toolchain { xctestPath = try Self.deriveXCTestPath( swiftSDK: self.swiftSDK, triple: triple, - environment: environment + environment: environment, + fileSystem: fileSystem ) } @@ -657,12 +707,15 @@ public final class UserToolchain: Toolchain { sdkRootPath: self.swiftSDK.pathsConfiguration.sdkRootPath, xctestPath: xctestPath ) + + self.fileSystem = fileSystem } private static func deriveSwiftPMLibrariesLocation( swiftCompilerPath: AbsolutePath, swiftSDK: SwiftSDK, - environment: EnvironmentVariables + environment: EnvironmentVariables, + fileSystem: any FileSystem ) throws -> ToolchainConfiguration.SwiftPMLibrariesLocation? { // Look for an override in the env. if let pathEnvVariable = environment["SWIFTPM_CUSTOM_LIBS_DIR"] ?? environment["SWIFTPM_PD_LIBS"] { @@ -678,7 +731,7 @@ public final class UserToolchain: Toolchain { #endif let paths = pathEnvVariable.split(separator: separator).map(String.init) for pathString in paths { - if let path = try? AbsolutePath(validating: pathString), localFileSystem.exists(path) { + if let path = try? AbsolutePath(validating: pathString), fileSystem.exists(path) { // we found the custom one return .init(root: path) } @@ -697,7 +750,7 @@ public final class UserToolchain: Toolchain { for applicationPath in swiftSDK.toolset.rootPaths { // this is the normal case when using the toolchain let librariesPath = applicationPath.parentDirectory.appending(components: "lib", "swift", "pm") - if localFileSystem.exists(librariesPath) { + if fileSystem.exists(librariesPath) { return .init(root: librariesPath) } @@ -707,7 +760,7 @@ public final class UserToolchain: Toolchain { "PackageDescription.framework" ) let pluginFrameworksPath = applicationPath.appending(components: "PackageFrameworks", "PackagePlugin.framework") - if localFileSystem.exists(manifestFrameworksPath), localFileSystem.exists(pluginFrameworksPath) { + if fileSystem.exists(manifestFrameworksPath), fileSystem.exists(pluginFrameworksPath) { return .init( manifestLibraryPath: manifestFrameworksPath, pluginLibraryPath: pluginFrameworksPath @@ -752,7 +805,8 @@ public final class UserToolchain: Toolchain { private static func deriveXCTestPath( swiftSDK: SwiftSDK, triple: Triple, - environment: EnvironmentVariables + environment: EnvironmentVariables, + fileSystem: any FileSystem ) throws -> AbsolutePath? { if triple.isDarwin() { // XCTest is optional on macOS, for example when Xcode is not installed @@ -784,7 +838,7 @@ public final class UserToolchain: Toolchain { if let info = WindowsPlatformInfo( reading: platform.appending("Info.plist"), observabilityScope: nil, - filesystem: localFileSystem + filesystem: fileSystem ) { let xctest: AbsolutePath = platform.appending("Developer") @@ -804,7 +858,7 @@ public final class UserToolchain: Toolchain { let path: AbsolutePath = xctest.appending("usr") .appending("bin64") - if localFileSystem.exists(path) { + if fileSystem.exists(path) { return path } @@ -812,7 +866,7 @@ public final class UserToolchain: Toolchain { let path: AbsolutePath = xctest.appending("usr") .appending("bin32") - if localFileSystem.exists(path) { + if fileSystem.exists(path) { return path } @@ -820,7 +874,7 @@ public final class UserToolchain: Toolchain { let path: AbsolutePath = xctest.appending("usr") .appending("bin32a") - if localFileSystem.exists(path) { + if fileSystem.exists(path) { return path } @@ -828,7 +882,7 @@ public final class UserToolchain: Toolchain { let path: AbsolutePath = xctest.appending("usr") .appending("bin64a") - if localFileSystem.exists(path) { + if fileSystem.exists(path) { return path } diff --git a/Sources/SPMTestSupport/Toolchain.swift b/Sources/SPMTestSupport/Toolchain.swift index 35a1a2c3c11..ae2914432ad 100644 --- a/Sources/SPMTestSupport/Toolchain.swift +++ b/Sources/SPMTestSupport/Toolchain.swift @@ -49,7 +49,7 @@ extension SwiftSDK { extension UserToolchain { package static var `default`: Self { get throws { - return try .init(swiftSDK: SwiftSDK.default) + return try .init(swiftSDK: SwiftSDK.default, fileSystem: localFileSystem) } } } diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index e6016e3d554..2b12c4fe89d 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -426,7 +426,7 @@ public class Workspace { ) let currentToolsVersion = customToolsVersion ?? ToolsVersion.current - let hostToolchain = try customHostToolchain ?? UserToolchain(swiftSDK: .hostSwiftSDK()) + let hostToolchain = try customHostToolchain ?? UserToolchain(swiftSDK: .hostSwiftSDK(), fileSystem: fileSystem) var manifestLoader = customManifestLoader ?? ManifestLoader( toolchain: hostToolchain, cacheDir: location.sharedManifestsCacheDirectory, diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index bafa0b5f801..a87c031034a 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -26,7 +26,6 @@ import SPMTestSupport import XCTest import class TSCBasic.BufferedOutputByteStream -import class TSCBasic.InMemoryFileSystem import protocol TSCBasic.OutputByteStream import var TSCBasic.stderrStream @@ -322,12 +321,69 @@ final class SwiftCommandStateTests: CommandsTestCase { try XCTAssertMatch(plan.buildProducts.compactMap { $0 as? Build.ProductBuildDescription }.first?.linkArguments() ?? [], [.anySequence, "-gnone", .anySequence]) } + + func testToolchainArgument() throws { + let fs = InMemoryFileSystem(emptyFiles: [ + "/Pkg/Sources/exe/main.swift", + "/path/to/toolchain/usr/bin/swiftc", + "/path/to/toolchain/usr/bin/llvm-ar", + ]) + + let customTargetToolchain = AbsolutePath("/path/to/toolchain") + try fs.createDirectory(customTargetToolchain, recursive: true) + + let swiftcPath = customTargetToolchain.appending(components: ["usr", "bin" , "swiftc"]) + let arPath = customTargetToolchain.appending(components: ["usr", "bin", "llvm-ar"]) + try fs.updatePermissions(swiftcPath, isExecutable: true) + try fs.updatePermissions(arPath, isExecutable: true) + + let observer = ObservabilitySystem.makeForTesting() + let graph = try loadModulesGraph( + fileSystem: fs, + manifests: [ + Manifest.createRootManifest( + displayName: "Pkg", + path: "/Pkg", + targets: [TargetDescription(name: "exe")] + ) + ], + observabilityScope: observer.topScope + ) + + let options = try GlobalOptions.parse( + [ + "--toolchain", customTargetToolchain.pathString, + "--triple", "x86_64-unknown-linux-gnu", + ] + ) + let swiftCommandState = try SwiftCommandState.makeMockState(options: options, fileSystem: fs) + XCTAssertEqual( + try swiftCommandState.getTargetToolchain().swiftCompilerPath, + swiftcPath + ) + XCTAssertEqual( + try swiftCommandState.getTargetToolchain().swiftSDK.toolset.knownTools[.swiftCompiler]?.path, + nil + ) + let plan = try BuildPlan( + destinationBuildParameters: swiftCommandState.productsBuildParameters, + toolsBuildParameters: swiftCommandState.toolsBuildParameters, + graph: graph, + fileSystem: fs, + observabilityScope: observer.topScope + ) + + let arguments = try plan.buildProducts.compactMap { $0 as? Build.ProductBuildDescription }.first?.linkArguments() ?? [] + + XCTAssertMatch(arguments, [.contains("/path/to/toolchain")]) + } } extension SwiftCommandState { static func makeMockState( outputStream: OutputByteStream = stderrStream, - options: GlobalOptions + options: GlobalOptions, + fileSystem: any FileSystem = localFileSystem ) throws -> SwiftCommandState { return try SwiftCommandState( outputStream: outputStream, @@ -346,6 +402,8 @@ extension SwiftCommandState { fileSystem: $0, observabilityScope: $1 ) - }) + }, + fileSystem: fileSystem + ) } } diff --git a/Tests/PackageModelTests/PackageModelTests.swift b/Tests/PackageModelTests/PackageModelTests.swift index 54c32fa276e..1a0dc648a94 100644 --- a/Tests/PackageModelTests/PackageModelTests.swift +++ b/Tests/PackageModelTests/PackageModelTests.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2014-2021 Apple Inc. and the Swift project authors +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -17,7 +17,7 @@ import XCTest import struct TSCBasic.ByteString -class PackageModelTests: XCTestCase { +final class PackageModelTests: XCTestCase { func testProductTypeCodable() throws { struct Foo: Codable, Equatable { var type: ProductType @@ -58,8 +58,11 @@ class PackageModelTests: XCTestCase { func testAndroidCompilerFlags() throws { let triple = try Triple("x86_64-unknown-linux-android") + let fileSystem = InMemoryFileSystem() let sdkDir = AbsolutePath("/some/path/to/an/SDK.sdk") + try fileSystem.createDirectory(sdkDir) let toolchainPath = AbsolutePath("/some/path/to/a/toolchain.xctoolchain") + try fileSystem.createDirectory(toolchainPath) let swiftSDK = SwiftSDK( targetTriple: triple, @@ -68,10 +71,16 @@ class PackageModelTests: XCTestCase { ) XCTAssertEqual( - try UserToolchain.deriveSwiftCFlags(triple: triple, swiftSDK: swiftSDK, environment: .process()), + try UserToolchain.deriveSwiftCFlags( + triple: triple, + swiftSDK: swiftSDK, + environment: .process(), + fileSystem: fileSystem + ), [ // Needed when cross‐compiling for Android. 2020‐03‐01 - "-sdk", sdkDir.pathString, + "-sdk", + sdkDir.pathString, ] ) } @@ -120,7 +129,8 @@ class PackageModelTests: XCTestCase { try XCTAssertEqual( UserToolchain.determineLibrarian( triple: triple, binDirectories: [bin], useXcrun: false, environment: [:], searchPaths: [], - extraSwiftFlags: ["-Xswiftc", "-use-ld=lld"] + extraSwiftFlags: ["-Xswiftc", "-use-ld=lld"], + fileSystem: fs ), lld ) @@ -128,7 +138,8 @@ class PackageModelTests: XCTestCase { try XCTAssertEqual( UserToolchain.determineLibrarian( triple: triple, binDirectories: [bin], useXcrun: false, environment: [:], searchPaths: [], - extraSwiftFlags: ["-Xswiftc", "-use-ld=not-link"] + extraSwiftFlags: ["-Xswiftc", "-use-ld=not-link"], + fileSystem: fs ), not ) @@ -136,7 +147,8 @@ class PackageModelTests: XCTestCase { try XCTAssertThrowsError( UserToolchain.determineLibrarian( triple: triple, binDirectories: [bin], useXcrun: false, environment: [:], searchPaths: [], - extraSwiftFlags: [] + extraSwiftFlags: [], + fileSystem: fs ) ) } @@ -170,7 +182,8 @@ class PackageModelTests: XCTestCase { binDirectories: [toolchainBinDir], useXcrun: false, environment: [:], - searchPaths: binDirs + searchPaths: binDirs, + fileSystem: fs ) // The first swiftc in the search paths should be chosen. From 82c0b88f81de674615d097299d5eea6fd709b66f Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 23 Apr 2024 12:39:02 +0100 Subject: [PATCH 02/36] Fix `InMemoryFileSystem.swift` not present in `CMakeLists.txt` --- Sources/Basics/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/Basics/CMakeLists.txt b/Sources/Basics/CMakeLists.txt index bb3948ef432..30aaaee42e0 100644 --- a/Sources/Basics/CMakeLists.txt +++ b/Sources/Basics/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(Basics Errors.swift FileSystem/AbsolutePath.swift FileSystem/FileSystem+Extensions.swift + FileSystem/InMemoryFileSystem.swift FileSystem/NativePathExtensions.swift FileSystem/RelativePath.swift FileSystem/TemporaryFile.swift From e5530ce2cd5b3683f80ad8d8ee23a31735c14b7c Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 23 Apr 2024 13:03:29 +0100 Subject: [PATCH 03/36] Fix `SourceKitLSPAPITests` build error --- Tests/SourceKitLSPAPITests/SourceKitLSPAPITests.swift | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Tests/SourceKitLSPAPITests/SourceKitLSPAPITests.swift b/Tests/SourceKitLSPAPITests/SourceKitLSPAPITests.swift index 0ef54fbc308..a17a47b5244 100644 --- a/Tests/SourceKitLSPAPITests/SourceKitLSPAPITests.swift +++ b/Tests/SourceKitLSPAPITests/SourceKitLSPAPITests.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -19,10 +19,9 @@ import PackageGraph import PackageModel import SourceKitLSPAPI import SPMTestSupport -import TSCBasic import XCTest -class SourceKitLSPAPITests: XCTestCase { +final class SourceKitLSPAPITests: XCTestCase { func testBasicSwiftPackage() throws { let fs = InMemoryFileSystem(emptyFiles: "/Pkg/Sources/exe/main.swift", From e546eb796100030e6a468208ac969eef1615febf Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 23 Apr 2024 13:22:49 +0100 Subject: [PATCH 04/36] Fix `testAndroidCompilerFlags` --- Tests/PackageModelTests/PackageModelTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/PackageModelTests/PackageModelTests.swift b/Tests/PackageModelTests/PackageModelTests.swift index 1a0dc648a94..c166780663e 100644 --- a/Tests/PackageModelTests/PackageModelTests.swift +++ b/Tests/PackageModelTests/PackageModelTests.swift @@ -60,9 +60,9 @@ final class PackageModelTests: XCTestCase { let triple = try Triple("x86_64-unknown-linux-android") let fileSystem = InMemoryFileSystem() let sdkDir = AbsolutePath("/some/path/to/an/SDK.sdk") - try fileSystem.createDirectory(sdkDir) + try fileSystem.createDirectory(sdkDir, recursive: true) let toolchainPath = AbsolutePath("/some/path/to/a/toolchain.xctoolchain") - try fileSystem.createDirectory(toolchainPath) + try fileSystem.createDirectory(toolchainPath, recursive: true) let swiftSDK = SwiftSDK( targetTriple: triple, From dc40bcb012f0dde33d9061e34d210a7c791dbd89 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 23 Apr 2024 15:01:42 +0100 Subject: [PATCH 05/36] Formatting cleanup --- Sources/PackageModel/UserToolchain.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index ba6ff8eba3b..75ba74a68f6 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -551,9 +551,9 @@ public final class UserToolchain: Toolchain { let swiftCompilers = try UserToolchain.determineSwiftCompilers( binDirectories: swiftSDK.toolset.rootPaths, - useXcrun: useXcrun, + useXcrun: self.useXcrun, environment: environment, - searchPaths: envSearchPaths, + searchPaths: self.envSearchPaths, fileSystem: fileSystem ) self.swiftCompilerPath = swiftCompilers.compile From a16b5868fc73b066284799e0935e5cff06f66455 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Tue, 23 Apr 2024 17:53:21 +0100 Subject: [PATCH 06/36] Reproduce the issue with `SWIFTPM_CUSTOM_BIN_DIR` --- Sources/CoreCommands/SwiftCommandState.swift | 19 +++++-------- Sources/PackageModel/SwiftSDKs/SwiftSDK.swift | 20 ++++--------- Sources/swift-bootstrap/main.swift | 6 +--- .../SwiftCommandStateTests.swift | 28 ++++++++++++------- 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index d1547d03ca8..31ec9279043 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -302,7 +302,8 @@ package final class SwiftCommandState { self.logLevel = options.logging.logLevel self.observabilityHandler = SwiftCommandObservabilityHandler(outputStream: outputStream, logLevel: self.logLevel) let observabilitySystem = ObservabilitySystem(self.observabilityHandler) - self.observabilityScope = observabilitySystem.topScope + let observabilityScope = observabilitySystem.topScope + self.observabilityScope = observabilityScope self.shouldDisableSandbox = options.security.shouldDisableSandbox self.toolWorkspaceConfiguration = toolWorkspaceConfiguration self.workspaceDelegateProvider = workspaceDelegateProvider @@ -355,6 +356,10 @@ package final class SwiftCommandState { explicitDirectory: options.locations.swiftSDKsDirectory ) + self._hostToolchain = Result(catching: { + try UserToolchain(swiftSDK: SwiftSDK.hostSwiftSDK(fileSystem: fileSystem)) + }) + // set global process logging handler Process.loggingHandler = { self.observabilityScope.emit(debug: $0) } } @@ -847,17 +852,7 @@ package final class SwiftCommandState { }() /// Lazily compute the host toolchain used to compile the package description. - private lazy var _hostToolchain: Result = { - return Result(catching: { - try UserToolchain( - swiftSDK: SwiftSDK.hostSwiftSDK( - originalWorkingDirectory: self.originalWorkingDirectory, - observabilityScope: self.observabilityScope - ), - fileSystem: self.fileSystem - ) - }) - }() + private let _hostToolchain: Result private lazy var _manifestLoader: Result = { return Result(catching: { diff --git a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift index 665c4fe1351..30513ec70a0 100644 --- a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift +++ b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift @@ -480,14 +480,10 @@ public struct SwiftSDK: Equatable { } /// Returns the bin directory for the host. - /// - /// - Parameter originalWorkingDirectory: The working directory when the program was launched. private static func hostBinDir( - fileSystem: FileSystem, - originalWorkingDirectory: AbsolutePath? = nil + fileSystem: FileSystem ) throws -> AbsolutePath { - let originalWorkingDirectory = originalWorkingDirectory ?? fileSystem.currentWorkingDirectory - guard let cwd = originalWorkingDirectory else { + guard let cwd = fileSystem.currentWorkingDirectory else { return try AbsolutePath(validating: CommandLine.arguments[0]).parentDirectory } return try AbsolutePath(validating: CommandLine.arguments[0], relativeTo: cwd).parentDirectory @@ -500,27 +496,23 @@ public struct SwiftSDK: Equatable { originalWorkingDirectory: AbsolutePath? = nil, environment: [String: String] = ProcessEnv.vars ) throws -> SwiftSDK { - try self.hostSwiftSDK(binDir, originalWorkingDirectory: originalWorkingDirectory, environment: environment) + try self.hostSwiftSDK(binDir, environment: environment) } /// The Swift SDK for the host platform. public static func hostSwiftSDK( _ binDir: AbsolutePath? = nil, - originalWorkingDirectory: AbsolutePath? = nil, environment: [String: String] = ProcessEnv.vars, - observabilityScope: ObservabilityScope? = nil + observabilityScope: ObservabilityScope? = nil, + fileSystem: any FileSystem = localFileSystem ) throws -> SwiftSDK { - let originalWorkingDirectory = originalWorkingDirectory ?? localFileSystem.currentWorkingDirectory // Select the correct binDir. if ProcessEnv.block["SWIFTPM_CUSTOM_BINDIR"] != nil { print("SWIFTPM_CUSTOM_BINDIR was deprecated in favor of SWIFTPM_CUSTOM_BIN_DIR") } let customBinDir = (ProcessEnv.block["SWIFTPM_CUSTOM_BIN_DIR"] ?? ProcessEnv.block["SWIFTPM_CUSTOM_BINDIR"]) .flatMap { try? AbsolutePath(validating: $0) } - let binDir = try customBinDir ?? binDir ?? SwiftSDK.hostBinDir( - fileSystem: localFileSystem, - originalWorkingDirectory: originalWorkingDirectory - ) + let binDir = try customBinDir ?? binDir ?? SwiftSDK.hostBinDir(fileSystem: fileSystem) let sdkPath: AbsolutePath? #if os(macOS) diff --git a/Sources/swift-bootstrap/main.swift b/Sources/swift-bootstrap/main.swift index 6cf38354370..4ff228415c3 100644 --- a/Sources/swift-bootstrap/main.swift +++ b/Sources/swift-bootstrap/main.swift @@ -222,13 +222,9 @@ struct SwiftBootstrapBuildTool: ParsableCommand { ] init(fileSystem: FileSystem, observabilityScope: ObservabilityScope, logLevel: Basics.Diagnostic.Severity) throws { - guard let cwd: AbsolutePath = fileSystem.currentWorkingDirectory else { - throw ExitCode.failure - } - self.identityResolver = DefaultIdentityResolver() self.dependencyMapper = DefaultDependencyMapper(identityResolver: self.identityResolver) - self.hostToolchain = try UserToolchain(swiftSDK: SwiftSDK.hostSwiftSDK(originalWorkingDirectory: cwd)) + self.hostToolchain = try UserToolchain(swiftSDK: SwiftSDK.hostSwiftSDK(fileSystem: fileSystem)) self.targetToolchain = hostToolchain // TODO: support cross-compilation? self.fileSystem = fileSystem self.observabilityScope = observabilityScope diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index a87c031034a..f24e7ac5d60 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -25,6 +25,7 @@ import func PackageGraph.loadModulesGraph import SPMTestSupport import XCTest +import enum TSCBasic.ProcessEnv import class TSCBasic.BufferedOutputByteStream import protocol TSCBasic.OutputByteStream import var TSCBasic.stderrStream @@ -323,19 +324,25 @@ final class SwiftCommandStateTests: CommandsTestCase { } func testToolchainArgument() throws { + let hostBinDir = AbsolutePath("/swiftpm/bin") + let hostSwiftcPath = hostBinDir.appending(components: ["swiftc"]) + + let customTargetToolchain = AbsolutePath("/path/to/toolchain") + let targetSwiftcPath = customTargetToolchain.appending(components: ["usr", "bin" , "swiftc"]) + let targetArPath = customTargetToolchain.appending(components: ["usr", "bin", "llvm-ar"]) + let fs = InMemoryFileSystem(emptyFiles: [ "/Pkg/Sources/exe/main.swift", - "/path/to/toolchain/usr/bin/swiftc", - "/path/to/toolchain/usr/bin/llvm-ar", + hostSwiftcPath.pathString, + targetSwiftcPath.pathString, + targetArPath.pathString ]) - let customTargetToolchain = AbsolutePath("/path/to/toolchain") - try fs.createDirectory(customTargetToolchain, recursive: true) - - let swiftcPath = customTargetToolchain.appending(components: ["usr", "bin" , "swiftc"]) - let arPath = customTargetToolchain.appending(components: ["usr", "bin", "llvm-ar"]) - try fs.updatePermissions(swiftcPath, isExecutable: true) - try fs.updatePermissions(arPath, isExecutable: true) + try ProcessEnv.setVar("SWIFTPM_CUSTOM_BIN_DIR", value: hostBinDir.pathString) + + try fs.updatePermissions(hostSwiftcPath, isExecutable: true) + try fs.updatePermissions(targetSwiftcPath, isExecutable: true) + try fs.updatePermissions(targetArPath, isExecutable: true) let observer = ObservabilitySystem.makeForTesting() let graph = try loadModulesGraph( @@ -357,9 +364,10 @@ final class SwiftCommandStateTests: CommandsTestCase { ] ) let swiftCommandState = try SwiftCommandState.makeMockState(options: options, fileSystem: fs) + XCTAssertEqual(swiftCommandState.originalWorkingDirectory, fs.currentWorkingDirectory) XCTAssertEqual( try swiftCommandState.getTargetToolchain().swiftCompilerPath, - swiftcPath + targetSwiftcPath ) XCTAssertEqual( try swiftCommandState.getTargetToolchain().swiftSDK.toolset.knownTools[.swiftCompiler]?.path, From ff1ed064c49de64c768eb1cc53dd9e71c2559480 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 2 May 2024 17:04:07 +0100 Subject: [PATCH 07/36] Prepend toolset paths in `SwiftSDK` to respect `--toolchain` option --- Sources/PackageModel/SwiftSDKs/SwiftSDK.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift index 862bd1262f5..bfcbae67f4d 100644 --- a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift +++ b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift @@ -729,7 +729,7 @@ public struct SwiftSDK: Equatable { /// Appends a path to the array of toolset root paths. /// - Parameter toolsetRootPath: new path to add to Swift SDK's toolset. public mutating func add(toolsetRootPath: AbsolutePath) { - self.toolset.rootPaths.append(toolsetRootPath) + self.toolset.rootPaths.insert(toolsetRootPath, at: 0) } } From f903c7ffa1724063abec4868c48faa7bbde8cc9e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 2 May 2024 18:57:12 +0100 Subject: [PATCH 08/36] Restore environment after updating it --- Tests/CommandsTests/SwiftCommandStateTests.swift | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index f24e7ac5d60..1088c21e0dd 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -338,7 +338,16 @@ final class SwiftCommandStateTests: CommandsTestCase { targetArPath.pathString ]) - try ProcessEnv.setVar("SWIFTPM_CUSTOM_BIN_DIR", value: hostBinDir.pathString) + let envKey = "SWIFTPM_CUSTOM_BIN_DIR" + let realCustomBinDir = ProcessEnv.vars[envKey] + try ProcessEnv.setVar(envKey, value: hostBinDir.pathString) + defer { + if let realCustomBinDir { + try! ProcessEnv.setVar(envKey, value: realCustomBinDir) + } else { + try! ProcessEnv.unsetVar(envKey) + } + } try fs.updatePermissions(hostSwiftcPath, isExecutable: true) try fs.updatePermissions(targetSwiftcPath, isExecutable: true) From 5b4763664bb5235fcd45af6f1fa195f53b11df9c Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 2 May 2024 20:17:52 +0100 Subject: [PATCH 09/36] Inject environment consistent with injected file system --- Sources/CoreCommands/SwiftCommandState.swift | 15 +++++++++--- Sources/PackageModel/SwiftSDKs/SwiftSDK.swift | 10 ++++---- Sources/PackageModel/UserToolchain.swift | 2 +- Sources/SPMTestSupport/MockWorkspace.swift | 1 + Sources/SPMTestSupport/Toolchain.swift | 4 ++-- .../PluginTargetBuildDescription.swift | 14 ++++++++++- .../SwiftSDKCommand/SwiftSDKSubcommand.swift | 8 ++++++- Sources/Workspace/Workspace.swift | 14 ++++++++++- Sources/swift-bootstrap/main.swift | 9 ++++++- Tests/BuildTests/BuildPlanTests.swift | 6 ++--- Tests/BuildTests/PluginsBuildPlanTests.swift | 2 +- Tests/CommandsTests/PackageCommandTests.swift | 8 +++++-- .../SwiftCommandStateTests.swift | 24 +++++++++---------- .../SwiftSDKBundleTests.swift | 6 ++--- .../PluginInvocationTests.swift | 13 +++++++--- .../RegistryPackageContainerTests.swift | 8 ++++--- .../SourceControlPackageContainerTests.swift | 11 +++++++-- Tests/WorkspaceTests/WorkspaceTests.swift | 14 ++++++----- 18 files changed, 118 insertions(+), 51 deletions(-) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index adca11ab050..61a035a8e79 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -265,6 +265,8 @@ package final class SwiftCommandState { fileprivate var buildSystemProvider: BuildSystemProvider? + private let environment: EnvironmentVariables + /// Create an instance of this tool. /// /// - parameter options: The command line options to be passed to this tool. @@ -295,9 +297,11 @@ package final class SwiftCommandState { toolWorkspaceConfiguration: ToolWorkspaceConfiguration, workspaceDelegateProvider: @escaping WorkspaceDelegateProvider, workspaceLoaderProvider: @escaping WorkspaceLoaderProvider, - fileSystem: any FileSystem = localFileSystem + fileSystem: any FileSystem = localFileSystem, + environment: EnvironmentVariables = ProcessEnv.vars ) throws { self.fileSystem = fileSystem + self.environment = environment // first, bootstrap the observability system self.logLevel = options.logging.logLevel self.observabilityHandler = SwiftCommandObservabilityHandler(outputStream: outputStream, logLevel: self.logLevel) @@ -362,7 +366,10 @@ package final class SwiftCommandState { ) self._hostToolchain = Result(catching: { - try UserToolchain(swiftSDK: SwiftSDK.hostSwiftSDK(fileSystem: fileSystem)) + try UserToolchain( + swiftSDK: SwiftSDK.hostSwiftSDK(environment: environment, fileSystem: fileSystem), + environment: environment + ) }) // set global process logging handler @@ -858,7 +865,9 @@ package final class SwiftCommandState { return self._hostToolchain } - return Result(catching: { try UserToolchain(swiftSDK: swiftSDK, fileSystem: self.fileSystem) }) + return Result(catching: { + try UserToolchain(swiftSDK: swiftSDK, environment: self.environment, fileSystem: self.fileSystem) + }) }() /// Lazily compute the host toolchain used to compile the package description. diff --git a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift index bfcbae67f4d..3798a52f539 100644 --- a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift +++ b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift @@ -496,7 +496,7 @@ public struct SwiftSDK: Equatable { public static func hostDestination( _ binDir: AbsolutePath? = nil, originalWorkingDirectory: AbsolutePath? = nil, - environment: [String: String] = ProcessEnv.vars + environment: [String: String] ) throws -> SwiftSDK { try self.hostSwiftSDK(binDir, environment: environment) } @@ -504,22 +504,22 @@ public struct SwiftSDK: Equatable { /// The Swift SDK for the host platform. public static func hostSwiftSDK( _ binDir: AbsolutePath? = nil, - environment: [String: String] = ProcessEnv.vars, + environment: [String: String], observabilityScope: ObservabilityScope? = nil, fileSystem: any FileSystem = localFileSystem ) throws -> SwiftSDK { // Select the correct binDir. - if ProcessEnv.block["SWIFTPM_CUSTOM_BINDIR"] != nil { + if environment["SWIFTPM_CUSTOM_BINDIR"] != nil { print("SWIFTPM_CUSTOM_BINDIR was deprecated in favor of SWIFTPM_CUSTOM_BIN_DIR") } - let customBinDir = (ProcessEnv.block["SWIFTPM_CUSTOM_BIN_DIR"] ?? ProcessEnv.block["SWIFTPM_CUSTOM_BINDIR"]) + let customBinDir = (environment["SWIFTPM_CUSTOM_BIN_DIR"] ?? environment["SWIFTPM_CUSTOM_BINDIR"]) .flatMap { try? AbsolutePath(validating: $0) } let binDir = try customBinDir ?? binDir ?? SwiftSDK.hostBinDir(fileSystem: fileSystem) let sdkPath: AbsolutePath? #if os(macOS) // Get the SDK. - if let value = ProcessEnv.block["SDKROOT"] { + if let value = environment["SDKROOT"] { sdkPath = try AbsolutePath(validating: value) } else { // No value in env, so search for it. diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 3112f642fc6..0a0c0232e22 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -526,7 +526,7 @@ public final class UserToolchain: Toolchain { public init( swiftSDK: SwiftSDK, - environment: EnvironmentVariables = .process(), + environment: EnvironmentVariables, searchStrategy: SearchStrategy = .default, customLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation? = nil, customInstalledSwiftPMConfiguration: InstalledSwiftPMConfiguration? = nil, diff --git a/Sources/SPMTestSupport/MockWorkspace.swift b/Sources/SPMTestSupport/MockWorkspace.swift index f2f24792322..c9871cef62c 100644 --- a/Sources/SPMTestSupport/MockWorkspace.swift +++ b/Sources/SPMTestSupport/MockWorkspace.swift @@ -274,6 +274,7 @@ package final class MockWorkspace { let workspace = try Workspace._init( fileSystem: self.fileSystem, + environment: .empty(), location: .init( scratchDirectory: self.sandbox.appending(".build"), editsDirectory: self.sandbox.appending("edits"), diff --git a/Sources/SPMTestSupport/Toolchain.swift b/Sources/SPMTestSupport/Toolchain.swift index ae2914432ad..7b09b5e59f9 100644 --- a/Sources/SPMTestSupport/Toolchain.swift +++ b/Sources/SPMTestSupport/Toolchain.swift @@ -41,7 +41,7 @@ extension SwiftSDK { package static var `default`: Self { get throws { let binDir = try resolveBinDir() - return try! SwiftSDK.hostSwiftSDK(binDir) + return try! SwiftSDK.hostSwiftSDK(binDir, environment: .process()) } } } @@ -49,7 +49,7 @@ extension SwiftSDK { extension UserToolchain { package static var `default`: Self { get throws { - return try .init(swiftSDK: SwiftSDK.default, fileSystem: localFileSystem) + return try .init(swiftSDK: SwiftSDK.default, environment: .process(), fileSystem: localFileSystem) } } } diff --git a/Sources/SourceKitLSPAPI/PluginTargetBuildDescription.swift b/Sources/SourceKitLSPAPI/PluginTargetBuildDescription.swift index 8e6a147cc3e..6af3a19660a 100644 --- a/Sources/SourceKitLSPAPI/PluginTargetBuildDescription.swift +++ b/Sources/SourceKitLSPAPI/PluginTargetBuildDescription.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +private import Basics + import struct Foundation.URL import struct PackageGraph.ResolvedModule @@ -18,6 +20,8 @@ private import class PackageLoading.ManifestLoader internal import struct PackageModel.ToolsVersion private import class PackageModel.UserToolchain +private import enum TSCBasic.ProcessEnv + struct PluginTargetBuildDescription: BuildTarget { private let target: ResolvedModule private let toolsVersion: ToolsVersion @@ -36,7 +40,15 @@ struct PluginTargetBuildDescription: BuildTarget { func compileArguments(for fileURL: URL) throws -> [String] { // FIXME: This is very odd and we should clean this up by merging `ManifestLoader` and `DefaultPluginScriptRunner` again. - let loader = ManifestLoader(toolchain: try UserToolchain(swiftSDK: .hostSwiftSDK())) + let environment = EnvironmentVariables.process() + let loader = ManifestLoader( + toolchain: try UserToolchain( + swiftSDK: .hostSwiftSDK( + environment: environment + ), + environment: environment + ) + ) var args = loader.interpreterFlags(for: self.toolsVersion) // Note: we ignore the `fileURL` here as the expectation is that we get a commandline for the entire target in case of Swift. Plugins are always assumed to only consist of Swift files. args += sources.map { $0.path } diff --git a/Sources/SwiftSDKCommand/SwiftSDKSubcommand.swift b/Sources/SwiftSDKCommand/SwiftSDKSubcommand.swift index 0f93f6b2274..5e37f8e78c4 100644 --- a/Sources/SwiftSDKCommand/SwiftSDKSubcommand.swift +++ b/Sources/SwiftSDKCommand/SwiftSDKSubcommand.swift @@ -60,7 +60,13 @@ extension SwiftSDKSubcommand { let observabilityScope = observabilitySystem.topScope let swiftSDKsDirectory = try self.getOrCreateSwiftSDKsDirectory() - let hostToolchain = try UserToolchain(swiftSDK: SwiftSDK.hostSwiftSDK()) + let environment = EnvironmentVariables.process() + let hostToolchain = try UserToolchain( + swiftSDK: SwiftSDK.hostSwiftSDK( + environment: environment + ), + environment: environment + ) let triple = try Triple.getHostTriple(usingSwiftCompiler: hostToolchain.swiftCompilerPath) var commandError: Error? = nil diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index a0688d60d95..17346928a27 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -173,6 +173,7 @@ public class Workspace { /// - delegate: Delegate for workspace events public convenience init( fileSystem: any FileSystem, + environment: EnvironmentVariables = .process(), location: Location, authorizationProvider: (any AuthorizationProvider)? = .none, registryAuthorizationProvider: (any AuthorizationProvider)? = .none, @@ -189,6 +190,7 @@ public class Workspace { ) throws { try self.init( fileSystem: fileSystem, + environment: environment, location: location, authorizationProvider: authorizationProvider, registryAuthorizationProvider: registryAuthorizationProvider, @@ -236,6 +238,7 @@ public class Workspace { /// - delegate: Delegate for workspace events public convenience init( fileSystem: FileSystem? = .none, + environment: EnvironmentVariables = .process(), forRootPackage packagePath: AbsolutePath, authorizationProvider: AuthorizationProvider? = .none, registryAuthorizationProvider: AuthorizationProvider? = .none, @@ -331,6 +334,7 @@ public class Workspace { public static func _init( // core fileSystem: FileSystem, + environment: EnvironmentVariables, location: Location, authorizationProvider: AuthorizationProvider? = .none, registryAuthorizationProvider: AuthorizationProvider? = .none, @@ -359,6 +363,7 @@ public class Workspace { ) throws -> Workspace { try .init( fileSystem: fileSystem, + environment: environment, location: location, authorizationProvider: authorizationProvider, registryAuthorizationProvider: registryAuthorizationProvider, @@ -388,6 +393,7 @@ public class Workspace { private init( // core fileSystem: FileSystem, + environment: EnvironmentVariables, location: Location, authorizationProvider: AuthorizationProvider?, registryAuthorizationProvider: AuthorizationProvider?, @@ -426,7 +432,13 @@ public class Workspace { ) let currentToolsVersion = customToolsVersion ?? ToolsVersion.current - let hostToolchain = try customHostToolchain ?? UserToolchain(swiftSDK: .hostSwiftSDK(), fileSystem: fileSystem) + let hostToolchain = try customHostToolchain ?? UserToolchain( + swiftSDK: .hostSwiftSDK( + environment: environment + ), + environment: environment, + fileSystem: fileSystem + ) var manifestLoader = customManifestLoader ?? ManifestLoader( toolchain: hostToolchain, cacheDir: location.sharedManifestsCacheDirectory, diff --git a/Sources/swift-bootstrap/main.swift b/Sources/swift-bootstrap/main.swift index b0bf66c6423..aa2df2e3d83 100644 --- a/Sources/swift-bootstrap/main.swift +++ b/Sources/swift-bootstrap/main.swift @@ -224,7 +224,14 @@ struct SwiftBootstrapBuildTool: ParsableCommand { init(fileSystem: FileSystem, observabilityScope: ObservabilityScope, logLevel: Basics.Diagnostic.Severity) throws { self.identityResolver = DefaultIdentityResolver() self.dependencyMapper = DefaultDependencyMapper(identityResolver: self.identityResolver) - self.hostToolchain = try UserToolchain(swiftSDK: SwiftSDK.hostSwiftSDK(fileSystem: fileSystem)) + let environment = EnvironmentVariables.process() + self.hostToolchain = try UserToolchain( + swiftSDK: SwiftSDK.hostSwiftSDK( + environment: environment, + fileSystem: fileSystem + ), + environment: environment + ) self.targetToolchain = hostToolchain // TODO: support cross-compilation? self.fileSystem = fileSystem self.observabilityScope = observabilityScope diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 24a4570325d..3c4c93e4550 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4369,7 +4369,7 @@ final class BuildPlanTests: XCTestCase { swiftStaticResourcesPath: "/fake/lib/swift_static" ) ) - let mockToolchain = try UserToolchain(swiftSDK: userSwiftSDK) + let mockToolchain = try UserToolchain(swiftSDK: userSwiftSDK, environment: .empty()) let extraBuildParameters = mockBuildParameters( toolchain: mockToolchain, flags: BuildFlags( @@ -4511,7 +4511,7 @@ final class BuildPlanTests: XCTestCase { ), toolset: toolset ) - let toolchain = try UserToolchain(swiftSDK: swiftSDK) + let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .empty()) let buildParameters = mockBuildParameters( toolchain: toolchain, flags: BuildFlags( @@ -4667,7 +4667,7 @@ final class BuildPlanTests: XCTestCase { .swiftCompiler: .init(extraCLIOptions: ["-use-ld=lld"]), ]) ) - let toolchain = try UserToolchain(swiftSDK: swiftSDK) + let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .empty()) let buildParameters = mockBuildParameters(toolchain: toolchain) let result = try BuildPlanResult(plan: BuildPlan( buildParameters: buildParameters, diff --git a/Tests/BuildTests/PluginsBuildPlanTests.swift b/Tests/BuildTests/PluginsBuildPlanTests.swift index 7b2a1a78378..62ea679cf06 100644 --- a/Tests/BuildTests/PluginsBuildPlanTests.swift +++ b/Tests/BuildTests/PluginsBuildPlanTests.swift @@ -36,7 +36,7 @@ final class PluginsBuildPlanTests: XCTestCase { try XCTSkipIf(true, "test is only supported on macOS") #endif - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK()) + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) let hostTriple = try! hostToolchain.targetTriple.withoutVersion().tripleString let x86Triple = "x86_64-apple-macosx" diff --git a/Tests/CommandsTests/PackageCommandTests.swift b/Tests/CommandsTests/PackageCommandTests.swift index f67e148dac1..91f6a2d89ef 100644 --- a/Tests/CommandsTests/PackageCommandTests.swift +++ b/Tests/CommandsTests/PackageCommandTests.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2014-2022 Apple Inc. and the Swift project authors +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -1809,7 +1809,11 @@ final class PackageCommandTests: CommandsTestCase { """ ) - let hostTriple = try UserToolchain(swiftSDK: .hostSwiftSDK()).targetTriple + let environment = EnvironmentVariables.process() + let hostTriple = try UserToolchain( + swiftSDK: .hostSwiftSDK(environment: environment), + environment: environment + ).targetTriple let hostTripleString = if hostTriple.isDarwin() { hostTriple.tripleString(forPlatformVersion: "") } else { diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index 1088c21e0dd..c9b73a9f738 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -338,16 +338,6 @@ final class SwiftCommandStateTests: CommandsTestCase { targetArPath.pathString ]) - let envKey = "SWIFTPM_CUSTOM_BIN_DIR" - let realCustomBinDir = ProcessEnv.vars[envKey] - try ProcessEnv.setVar(envKey, value: hostBinDir.pathString) - defer { - if let realCustomBinDir { - try! ProcessEnv.setVar(envKey, value: realCustomBinDir) - } else { - try! ProcessEnv.unsetVar(envKey) - } - } try fs.updatePermissions(hostSwiftcPath, isExecutable: true) try fs.updatePermissions(targetSwiftcPath, isExecutable: true) @@ -372,7 +362,13 @@ final class SwiftCommandStateTests: CommandsTestCase { "--triple", "x86_64-unknown-linux-gnu", ] ) - let swiftCommandState = try SwiftCommandState.makeMockState(options: options, fileSystem: fs) + let swiftCommandState = try SwiftCommandState.makeMockState( + options: options, + fileSystem: fs, + environment: [ + "SWIFTPM_CUSTOM_BIN_DIR": hostBinDir.pathString + ] + ) XCTAssertEqual(swiftCommandState.originalWorkingDirectory, fs.currentWorkingDirectory) XCTAssertEqual( try swiftCommandState.getTargetToolchain().swiftCompilerPath, @@ -400,7 +396,8 @@ extension SwiftCommandState { static func makeMockState( outputStream: OutputByteStream = stderrStream, options: GlobalOptions, - fileSystem: any FileSystem = localFileSystem + fileSystem: any FileSystem = localFileSystem, + environment: EnvironmentVariables = .process() ) throws -> SwiftCommandState { return try SwiftCommandState( outputStream: outputStream, @@ -420,7 +417,8 @@ extension SwiftCommandState { observabilityScope: $1 ) }, - fileSystem: fileSystem + fileSystem: fileSystem, + environment: environment ) } } diff --git a/Tests/PackageModelTests/SwiftSDKBundleTests.swift b/Tests/PackageModelTests/SwiftSDKBundleTests.swift index 0491ba1e344..c45e5b9d078 100644 --- a/Tests/PackageModelTests/SwiftSDKBundleTests.swift +++ b/Tests/PackageModelTests/SwiftSDKBundleTests.swift @@ -365,7 +365,7 @@ final class SwiftSDKBundleTests: XCTestCase { ]) } - func testTargetSDKDeriviation() async throws { + func testTargetSDKDerivation() async throws { let toolsetRootPath = AbsolutePath("/path/to/toolpath") let (fileSystem, bundles, swiftSDKsDirectory) = try generateTestFileSystem( bundleArtifacts: [ @@ -374,7 +374,7 @@ final class SwiftSDKBundleTests: XCTestCase { ] ) let system = ObservabilitySystem.makeForTesting() - let hostSwiftSDK = try SwiftSDK.hostSwiftSDK() + let hostSwiftSDK = try SwiftSDK.hostSwiftSDK(environment: .empty()) let hostTriple = try! Triple("arm64-apple-macosx14.0") let archiver = MockArchiver() let store = SwiftSDKBundleStore( @@ -424,7 +424,7 @@ final class SwiftSDKBundleTests: XCTestCase { fileSystem: fileSystem ) // With toolset in the target SDK, it should contain the host toolset roots at the end. - XCTAssertEqual(targetSwiftSDK.toolset.rootPaths, [toolsetRootPath] + hostSwiftSDK.toolset.rootPaths) + XCTAssertEqual(targetSwiftSDK.toolset.rootPaths, hostSwiftSDK.toolset.rootPaths + [toolsetRootPath]) } do { diff --git a/Tests/SPMBuildCoreTests/PluginInvocationTests.swift b/Tests/SPMBuildCoreTests/PluginInvocationTests.swift index eb9be4cd970..fd66d654d93 100644 --- a/Tests/SPMBuildCoreTests/PluginInvocationTests.swift +++ b/Tests/SPMBuildCoreTests/PluginInvocationTests.swift @@ -30,7 +30,6 @@ import class TSCBasic.InMemoryFileSystem import struct TSCUtility.SerializedDiagnostics final class PluginInvocationTests: XCTestCase { - func testBasics() throws { // Construct a canned file system and package graph with a single package and a library that uses a build tool plugin that invokes a tool. let fileSystem = InMemoryFileSystem(emptyFiles: @@ -1039,10 +1038,17 @@ final class PluginInvocationTests: XCTestCase { ///////// // Load a workspace from the package. let observability = ObservabilitySystem.makeForTesting() + let environment = EnvironmentVariables.process() let workspace = try Workspace( fileSystem: localFileSystem, location: try Workspace.Location(forRootPackage: packageDir, fileSystem: localFileSystem), - customHostToolchain: UserToolchain(swiftSDK: .hostSwiftSDK(), customLibrariesLocation: .init(manifestLibraryPath: fakeExtraModulesDir, pluginLibraryPath: fakeExtraModulesDir)), + customHostToolchain: UserToolchain( + swiftSDK: .hostSwiftSDK( + environment: environment + ), + environment: environment, + customLibrariesLocation: .init(manifestLibraryPath: fakeExtraModulesDir, pluginLibraryPath: fakeExtraModulesDir) + ), customManifestLoader: ManifestLoader(toolchain: UserToolchain.default), delegate: MockWorkspaceDelegate() ) @@ -1224,7 +1230,8 @@ final class PluginInvocationTests: XCTestCase { targetTriple: hostTriple, toolset: swiftSDK.toolset, pathsConfiguration: swiftSDK.pathsConfiguration - ) + ), + environment: .process() ) // Create a plugin script runner for the duration of the test. diff --git a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift index ea669b6cff5..53f49a628c3 100644 --- a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift +++ b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -24,8 +24,7 @@ import class TSCBasic.InMemoryFileSystem import struct TSCUtility.Version -class RegistryPackageContainerTests: XCTestCase { - +final class RegistryPackageContainerTests: XCTestCase { func testToolsVersionCompatibleVersions() async throws { let fs = InMemoryFileSystem() @@ -88,6 +87,7 @@ class RegistryPackageContainerTests: XCTestCase { return try Workspace._init( fileSystem: fs, + environment: .empty(), location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, customManifestLoader: MockManifestLoader(manifests: [:]), @@ -153,6 +153,7 @@ class RegistryPackageContainerTests: XCTestCase { return try Workspace._init( fileSystem: fs, + environment: .empty(), location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, customManifestLoader: MockManifestLoader(manifests: [:]), @@ -246,6 +247,7 @@ class RegistryPackageContainerTests: XCTestCase { return try Workspace._init( fileSystem: fs, + environment: .empty(), location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, customManifestLoader: MockManifestLoader(), diff --git a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift index 199b1041bb9..0d844f75e44 100644 --- a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift +++ b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift open source project // -// Copyright (c) 2014-2020 Apple Inc. and the Swift project authors +// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See http://swift.org/LICENSE.txt for license information @@ -192,7 +192,7 @@ private let v1: Version = "1.0.0" private let v2: Version = "2.0.0" private let v1Range: VersionSetSpecifier = .range("1.0.0" ..< "2.0.0") -class SourceControlPackageContainerTests: XCTestCase { +final class SourceControlPackageContainerTests: XCTestCase { func testVprefixVersions() async throws { let fs = InMemoryFileSystem() @@ -224,6 +224,7 @@ class SourceControlPackageContainerTests: XCTestCase { let provider = try Workspace._init( fileSystem: fs, + environment: .empty(), location: .init(forRootPackage: repoPath, fileSystem: fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager @@ -277,6 +278,7 @@ class SourceControlPackageContainerTests: XCTestCase { func createProvider(_ currentToolsVersion: ToolsVersion) throws -> PackageContainerProvider { return try Workspace._init( fileSystem: fs, + environment: .empty(), location: .init(forRootPackage: repoPath, fileSystem: fs), customToolsVersion: currentToolsVersion, customManifestLoader: MockManifestLoader(manifests: [:]), @@ -361,6 +363,7 @@ class SourceControlPackageContainerTests: XCTestCase { let provider = try Workspace._init( fileSystem: fs, + environment: .empty(), location: .init(forRootPackage: repoPath, fileSystem: fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager @@ -410,6 +413,7 @@ class SourceControlPackageContainerTests: XCTestCase { let provider = try Workspace._init( fileSystem: fs, + environment: .empty(), location: .init(forRootPackage: repoPath, fileSystem: fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager @@ -594,6 +598,7 @@ class SourceControlPackageContainerTests: XCTestCase { ) let containerProvider = try Workspace._init( fileSystem: localFileSystem, + environment: .empty(), location: .init(forRootPackage: packageDir, fileSystem: localFileSystem), customManifestLoader: MockManifestLoader(manifests: [.init(url: packageDir.pathString, version: nil): manifest]), customRepositoryManager: repositoryManager @@ -646,6 +651,7 @@ class SourceControlPackageContainerTests: XCTestCase { let containerProvider = try Workspace._init( fileSystem: localFileSystem, + environment: .empty(), location: .init(forRootPackage: packageDirectory, fileSystem: localFileSystem), customManifestLoader: MockManifestLoader( manifests: [ @@ -757,6 +763,7 @@ class SourceControlPackageContainerTests: XCTestCase { ) let containerProvider = try Workspace._init( fileSystem: localFileSystem, + environment: .empty(), location: .init(forRootPackage: packageDirectory, fileSystem: localFileSystem), customManifestLoader: MockManifestLoader( manifests: [.init(url: packageDirectory.pathString, version: Version(1, 0, 0)): manifest] diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index 783329c01f1..bf3484f5e16 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -7838,7 +7838,7 @@ final class WorkspaceTests: XCTestCase { let binaryArtifactsManager = try Workspace.BinaryArtifactsManager( fileSystem: fs, authorizationProvider: .none, - hostToolchain: UserToolchain(swiftSDK: .hostSwiftSDK()), + hostToolchain: UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()), checksumAlgorithm: checksumAlgorithm, cachePath: .none, customHTTPClient: .none, @@ -9078,7 +9078,7 @@ final class WorkspaceTests: XCTestCase { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() let downloads = ThreadSafeKeyValueStore() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK()) + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) let ariFiles = [ """ @@ -9367,7 +9367,7 @@ final class WorkspaceTests: XCTestCase { func testDownloadArchiveIndexFileBadChecksum() throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK()) + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) let ari = """ { @@ -9486,7 +9486,7 @@ final class WorkspaceTests: XCTestCase { func testDownloadArchiveIndexFileBadArchivesChecksum() throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK()) + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) let ari = """ { @@ -9596,7 +9596,7 @@ final class WorkspaceTests: XCTestCase { func testDownloadArchiveIndexFileArchiveNotFound() throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK()) + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) let ari = """ { @@ -9672,7 +9672,7 @@ final class WorkspaceTests: XCTestCase { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK()) + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) let androidTriple = try Triple("x86_64-unknown-linux-android") let macTriple = try Triple("arm64-apple-macosx") let notHostTriple = hostToolchain.targetTriple == androidTriple ? macTriple : androidTriple @@ -12299,6 +12299,7 @@ final class WorkspaceTests: XCTestCase { let delegate = MockWorkspaceDelegate() let workspace = try Workspace( fileSystem: fs, + environment: .empty(), forRootPackage: .root, customManifestLoader: TestLoader(error: .none), delegate: delegate @@ -12315,6 +12316,7 @@ final class WorkspaceTests: XCTestCase { let delegate = MockWorkspaceDelegate() let workspace = try Workspace( fileSystem: fs, + environment: .empty(), forRootPackage: .root, customManifestLoader: TestLoader(error: StringError("boom")), delegate: delegate From f8f78225181d3131c9673bce403df88161c307db Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 2 May 2024 20:23:36 +0100 Subject: [PATCH 10/36] Fix `Workspace` environment injection --- Sources/Workspace/Workspace.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 17346928a27..6f1eedaa190 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -256,6 +256,7 @@ public class Workspace { let location = try Location(forRootPackage: packagePath, fileSystem: fileSystem) try self.init( fileSystem: fileSystem, + environment: environment, location: location, authorizationProvider: authorizationProvider, registryAuthorizationProvider: registryAuthorizationProvider, From 666cf96dfd72cbd724d327ad508df5021eefddb4 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 3 May 2024 16:54:38 +0100 Subject: [PATCH 11/36] Use process environment for determining the host toolchain --- Sources/CoreCommands/SwiftCommandState.swift | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 61a035a8e79..7d0a2c6b853 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -365,13 +365,6 @@ package final class SwiftCommandState { explicitDirectory: options.locations.swiftSDKsDirectory ?? options.locations.deprecatedSwiftSDKsDirectory ) - self._hostToolchain = Result(catching: { - try UserToolchain( - swiftSDK: SwiftSDK.hostSwiftSDK(environment: environment, fileSystem: fileSystem), - environment: environment - ) - }) - // set global process logging handler Process.loggingHandler = { self.observabilityScope.emit(debug: $0) } } @@ -871,7 +864,18 @@ package final class SwiftCommandState { }() /// Lazily compute the host toolchain used to compile the package description. - private let _hostToolchain: Result + private lazy var _hostToolchain: Result = { + return Result(catching: { + try UserToolchain( + swiftSDK: SwiftSDK.hostSwiftSDK( + environment: .process(), + observabilityScope: self.observabilityScope + ), + environment: .process(), + fileSystem: self.fileSystem + ) + }) + }() private lazy var _manifestLoader: Result = { return Result(catching: { From b733dd6d7f5bab53588091745b242233b96a89af Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 3 May 2024 18:43:14 +0100 Subject: [PATCH 12/36] Use local file system for the host toolchain --- Sources/CoreCommands/SwiftCommandState.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 7d0a2c6b853..7c7baa58cb6 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -872,7 +872,7 @@ package final class SwiftCommandState { observabilityScope: self.observabilityScope ), environment: .process(), - fileSystem: self.fileSystem + fileSystem: localFileSystem ) }) }() From 5650a823a7e682e07e6291ad3ee8fd30a247d5f9 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Thu, 9 May 2024 20:32:05 +0100 Subject: [PATCH 13/36] Fix build issues after merge --- Sources/PackageModel/SwiftSDKs/SwiftSDK.swift | 2 +- Sources/PackageModel/UserToolchain.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift index 52a85fb21c3..d1942246255 100644 --- a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift +++ b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift @@ -505,7 +505,7 @@ public struct SwiftSDK: Equatable { /// The Swift SDK for the host platform. public static func hostSwiftSDK( _ binDir: AbsolutePath? = nil, - environment: [String: String], + environment: EnvironmentVariables = .process(), observabilityScope: ObservabilityScope? = nil, fileSystem: any FileSystem = localFileSystem ) throws -> SwiftSDK { diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 3aadc372f55..64b663b6a6d 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -524,7 +524,7 @@ public final class UserToolchain: Toolchain { public init( swiftSDK: SwiftSDK, - environment: EnvironmentVariables, + environment: EnvironmentVariables = .process(), searchStrategy: SearchStrategy = .default, customLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation? = nil, customInstalledSwiftPMConfiguration: InstalledSwiftPMConfiguration? = nil, From c36881c3929683c595491dcebe78b155df3e3377 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 15:43:09 +0100 Subject: [PATCH 14/36] Create fake `swiftc` on `InMemoryFileSystem` in `MockWorkspace` --- Sources/Basics/FileSystem/InMemoryFileSystem.swift | 4 ++++ Sources/SPMTestSupport/InMemoryGitRepository.swift | 1 - Sources/SPMTestSupport/MockWorkspace.swift | 4 ++-- Tests/SourceControlTests/InMemoryGitRepositoryTests.swift | 2 -- Tests/WorkspaceTests/SourceControlPackageContainerTests.swift | 2 -- Tests/WorkspaceTests/WorkspaceTests.swift | 1 - 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Sources/Basics/FileSystem/InMemoryFileSystem.swift b/Sources/Basics/FileSystem/InMemoryFileSystem.swift index 04420e98210..56e6450aa55 100644 --- a/Sources/Basics/FileSystem/InMemoryFileSystem.swift +++ b/Sources/Basics/FileSystem/InMemoryFileSystem.swift @@ -490,6 +490,10 @@ public final class InMemoryFileSystem: FileSystem { return try fileQueue.sync(flags: type == .exclusive ? .barrier : .init() , execute: body) } + + public func withLock(on path: TSCBasic.AbsolutePath, type: FileLock.LockType, blocking: Bool, _ body: () throws -> T) throws -> T { + try self.withLock(on: path, type: type, body) + } } // Internal state of `InMemoryFileSystem` is protected with a lock in all of its `public` methods. diff --git a/Sources/SPMTestSupport/InMemoryGitRepository.swift b/Sources/SPMTestSupport/InMemoryGitRepository.swift index 550dd3ed298..bebaf648b22 100644 --- a/Sources/SPMTestSupport/InMemoryGitRepository.swift +++ b/Sources/SPMTestSupport/InMemoryGitRepository.swift @@ -18,7 +18,6 @@ import SourceControl import struct TSCBasic.ByteString import enum TSCBasic.FileMode import struct TSCBasic.FileSystemError -import class TSCBasic.InMemoryFileSystem /// The error encountered during in memory git repository operations. package enum InMemoryGitRepositoryError: Swift.Error { diff --git a/Sources/SPMTestSupport/MockWorkspace.swift b/Sources/SPMTestSupport/MockWorkspace.swift index c9871cef62c..bfd61e2912e 100644 --- a/Sources/SPMTestSupport/MockWorkspace.swift +++ b/Sources/SPMTestSupport/MockWorkspace.swift @@ -19,8 +19,6 @@ import SourceControl import Workspace import XCTest -import class TSCBasic.InMemoryFileSystem - import struct TSCUtility.Version package final class MockWorkspace { @@ -62,6 +60,8 @@ package final class MockWorkspace { sourceControlToRegistryDependencyTransformation: WorkspaceConfiguration.SourceControlToRegistryDependencyTransformation = .disabled, defaultRegistry: Registry? = .none ) throws { + fileSystem.createEmptyFiles(at: AbsolutePath.root, files: "/usr/bin/swiftc") + try fileSystem.updatePermissions("/usr/bin/swiftc", isExecutable: true) self.sandbox = sandbox self.fileSystem = fileSystem self.roots = roots diff --git a/Tests/SourceControlTests/InMemoryGitRepositoryTests.swift b/Tests/SourceControlTests/InMemoryGitRepositoryTests.swift index 8b643ec8a4f..cef996045e2 100644 --- a/Tests/SourceControlTests/InMemoryGitRepositoryTests.swift +++ b/Tests/SourceControlTests/InMemoryGitRepositoryTests.swift @@ -15,8 +15,6 @@ import SourceControl import SPMTestSupport import XCTest -import class TSCBasic.InMemoryFileSystem - class InMemoryGitRepositoryTests: XCTestCase { func testBasics() throws { let fs = InMemoryFileSystem() diff --git a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift index 0d844f75e44..fd8ed7aa031 100644 --- a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift +++ b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift @@ -20,8 +20,6 @@ import SPMTestSupport @testable import Workspace import XCTest -import class TSCBasic.InMemoryFileSystem - import enum TSCUtility.Git import struct TSCUtility.Version diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index 12a9f67550f..113a3a333b2 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -24,7 +24,6 @@ import SPMTestSupport import XCTest import struct TSCBasic.ByteString -import class TSCBasic.InMemoryFileSystem import struct TSCUtility.Version From 53fa2917cd73f5a722b6f325ec179202bd95892c Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 15:51:27 +0100 Subject: [PATCH 15/36] Create all required tools as executables in `MockWorkspace` --- Sources/SPMTestSupport/MockWorkspace.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/SPMTestSupport/MockWorkspace.swift b/Sources/SPMTestSupport/MockWorkspace.swift index bfd61e2912e..e50500048b8 100644 --- a/Sources/SPMTestSupport/MockWorkspace.swift +++ b/Sources/SPMTestSupport/MockWorkspace.swift @@ -60,8 +60,12 @@ package final class MockWorkspace { sourceControlToRegistryDependencyTransformation: WorkspaceConfiguration.SourceControlToRegistryDependencyTransformation = .disabled, defaultRegistry: Registry? = .none ) throws { - fileSystem.createEmptyFiles(at: AbsolutePath.root, files: "/usr/bin/swiftc") - try fileSystem.updatePermissions("/usr/bin/swiftc", isExecutable: true) + let files = ["/usr/bin/swiftc", "/usr/bin/ar"] + fileSystem.createEmptyFiles(at: AbsolutePath.root, files: files) + for toolPath in files { + try fileSystem.updatePermissions(.init(toolPath), isExecutable: true) + } + self.sandbox = sandbox self.fileSystem = fileSystem self.roots = roots From bb2fdcb4e9d7f530f873ac6affbcd212a6147708 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 16:53:04 +0100 Subject: [PATCH 16/36] Make sure toolchain is correctly mocked in failing tests --- Sources/SPMTestSupport/MockWorkspace.swift | 22 ++++++++++++++----- Tests/BuildTests/BuildPlanTests.swift | 7 +++--- .../RegistryPackageContainerTests.swift | 11 +++++----- .../SourceControlPackageContainerTests.swift | 14 +++++++----- Tests/WorkspaceTests/WorkspaceTests.swift | 20 +++++++++++------ 5 files changed, 48 insertions(+), 26 deletions(-) diff --git a/Sources/SPMTestSupport/MockWorkspace.swift b/Sources/SPMTestSupport/MockWorkspace.swift index e50500048b8..4a93d650b39 100644 --- a/Sources/SPMTestSupport/MockWorkspace.swift +++ b/Sources/SPMTestSupport/MockWorkspace.swift @@ -21,6 +21,20 @@ import XCTest import struct TSCUtility.Version +extension EnvironmentVariables { + package static var mockEnvironment: Self { ["PATH": "/usr/bin"] } +} + +extension InMemoryFileSystem { + package func createMockToolchain() throws { + let files = ["/usr/bin/swiftc", "/usr/bin/ar"] + self.createEmptyFiles(at: AbsolutePath.root, files: files) + for toolPath in files { + try self.updatePermissions(.init(toolPath), isExecutable: true) + } + } +} + package final class MockWorkspace { let sandbox: AbsolutePath let fileSystem: InMemoryFileSystem @@ -60,11 +74,7 @@ package final class MockWorkspace { sourceControlToRegistryDependencyTransformation: WorkspaceConfiguration.SourceControlToRegistryDependencyTransformation = .disabled, defaultRegistry: Registry? = .none ) throws { - let files = ["/usr/bin/swiftc", "/usr/bin/ar"] - fileSystem.createEmptyFiles(at: AbsolutePath.root, files: files) - for toolPath in files { - try fileSystem.updatePermissions(.init(toolPath), isExecutable: true) - } + try fileSystem.createMockToolchain() self.sandbox = sandbox self.fileSystem = fileSystem @@ -278,7 +288,7 @@ package final class MockWorkspace { let workspace = try Workspace._init( fileSystem: self.fileSystem, - environment: .empty(), + environment: .mockEnvironment, location: .init( scratchDirectory: self.sandbox.appending(".build"), editsDirectory: self.sandbox.appending("edits"), diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 3c4c93e4550..67d3145c341 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -31,7 +31,6 @@ import Workspace import XCTest import struct TSCBasic.ByteString -import class TSCBasic.InMemoryFileSystem import enum TSCUtility.Diagnostics @@ -4334,6 +4333,7 @@ final class BuildPlanTests: XCTestCase { "/Pkg/Sources/lib/lib.c", "/Pkg/Sources/lib/include/lib.h" ) + try fs.createMockToolchain() let observability = ObservabilitySystem.makeForTesting() let graph = try loadModulesGraph( @@ -4369,7 +4369,7 @@ final class BuildPlanTests: XCTestCase { swiftStaticResourcesPath: "/fake/lib/swift_static" ) ) - let mockToolchain = try UserToolchain(swiftSDK: userSwiftSDK, environment: .empty()) + let mockToolchain = try UserToolchain(swiftSDK: userSwiftSDK, environment: .mockEnvironment, fileSystem: fs) let extraBuildParameters = mockBuildParameters( toolchain: mockToolchain, flags: BuildFlags( @@ -4635,6 +4635,7 @@ final class BuildPlanTests: XCTestCase { "/Pkg/Sources/cLib/cLib.c", "/Pkg/Sources/cLib/include/cLib.h" ) + try fileSystem.createMockToolchain() let observability = ObservabilitySystem.makeForTesting() let graph = try loadModulesGraph( @@ -4667,7 +4668,7 @@ final class BuildPlanTests: XCTestCase { .swiftCompiler: .init(extraCLIOptions: ["-use-ld=lld"]), ]) ) - let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .empty()) + let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment, fileSystem: fileSystem) let buildParameters = mockBuildParameters(toolchain: toolchain) let result = try BuildPlanResult(plan: BuildPlan( buildParameters: buildParameters, diff --git a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift index 53f49a628c3..6aa9f34000c 100644 --- a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift +++ b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift @@ -20,13 +20,12 @@ import SPMTestSupport @testable import Workspace import XCTest -import class TSCBasic.InMemoryFileSystem - import struct TSCUtility.Version final class RegistryPackageContainerTests: XCTestCase { func testToolsVersionCompatibleVersions() async throws { let fs = InMemoryFileSystem() + try fs.createMockToolchain() let packageIdentity = PackageIdentity.plain("org.foo") let packageVersion = Version("1.0.0") @@ -87,7 +86,7 @@ final class RegistryPackageContainerTests: XCTestCase { return try Workspace._init( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, customManifestLoader: MockManifestLoader(manifests: [:]), @@ -122,6 +121,7 @@ final class RegistryPackageContainerTests: XCTestCase { func testAlternateManifests() async throws { let fs = InMemoryFileSystem() + try fs.createMockToolchain() let packageIdentity = PackageIdentity.plain("org.foo") let packageVersion = Version("1.0.0") @@ -153,7 +153,7 @@ final class RegistryPackageContainerTests: XCTestCase { return try Workspace._init( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, customManifestLoader: MockManifestLoader(manifests: [:]), @@ -209,6 +209,7 @@ final class RegistryPackageContainerTests: XCTestCase { func testLoadManifest() async throws { let fs = InMemoryFileSystem() + try fs.createMockToolchain() let packageIdentity = PackageIdentity.plain("org.foo") let packageVersion = Version("1.0.0") @@ -247,7 +248,7 @@ final class RegistryPackageContainerTests: XCTestCase { return try Workspace._init( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, customManifestLoader: MockManifestLoader(), diff --git a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift index fd8ed7aa031..efce8a4b160 100644 --- a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift +++ b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift @@ -193,6 +193,7 @@ private let v1Range: VersionSetSpecifier = .range("1.0.0" ..< "2.0.0") final class SourceControlPackageContainerTests: XCTestCase { func testVprefixVersions() async throws { let fs = InMemoryFileSystem() + try fs.createMockToolchain() let repoPath = AbsolutePath.root let filePath = repoPath.appending("Package.swift") @@ -222,7 +223,7 @@ final class SourceControlPackageContainerTests: XCTestCase { let provider = try Workspace._init( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, location: .init(forRootPackage: repoPath, fileSystem: fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager @@ -236,6 +237,7 @@ final class SourceControlPackageContainerTests: XCTestCase { func testVersions() async throws { let fs = InMemoryFileSystem() + try fs.createMockToolchain() let repoPath = AbsolutePath.root let filePath = repoPath.appending("Package.swift") @@ -276,7 +278,7 @@ final class SourceControlPackageContainerTests: XCTestCase { func createProvider(_ currentToolsVersion: ToolsVersion) throws -> PackageContainerProvider { return try Workspace._init( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, location: .init(forRootPackage: repoPath, fileSystem: fs), customToolsVersion: currentToolsVersion, customManifestLoader: MockManifestLoader(manifests: [:]), @@ -330,6 +332,7 @@ final class SourceControlPackageContainerTests: XCTestCase { func testPreReleaseVersions() async throws { let fs = InMemoryFileSystem() + try fs.createMockToolchain() let repoPath = AbsolutePath.root let filePath = repoPath.appending("Package.swift") @@ -361,7 +364,7 @@ final class SourceControlPackageContainerTests: XCTestCase { let provider = try Workspace._init( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, location: .init(forRootPackage: repoPath, fileSystem: fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager @@ -375,6 +378,7 @@ final class SourceControlPackageContainerTests: XCTestCase { func testSimultaneousVersions() async throws { let fs = InMemoryFileSystem() + try fs.createMockToolchain() let repoPath = AbsolutePath.root let filePath = repoPath.appending("Package.swift") @@ -411,7 +415,7 @@ final class SourceControlPackageContainerTests: XCTestCase { let provider = try Workspace._init( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, location: .init(forRootPackage: repoPath, fileSystem: fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager @@ -596,7 +600,7 @@ final class SourceControlPackageContainerTests: XCTestCase { ) let containerProvider = try Workspace._init( fileSystem: localFileSystem, - environment: .empty(), + environment: .mockEnvironment, location: .init(forRootPackage: packageDir, fileSystem: localFileSystem), customManifestLoader: MockManifestLoader(manifests: [.init(url: packageDir.pathString, version: nil): manifest]), customRepositoryManager: repositoryManager diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index 113a3a333b2..db9665ab811 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -7830,6 +7830,7 @@ final class WorkspaceTests: XCTestCase { func testArtifactChecksum() throws { let fs = InMemoryFileSystem() + try fs.createMockToolchain() let sandbox = AbsolutePath("/tmp/ws/") try fs.createDirectory(sandbox, recursive: true) @@ -7837,7 +7838,7 @@ final class WorkspaceTests: XCTestCase { let binaryArtifactsManager = try Workspace.BinaryArtifactsManager( fileSystem: fs, authorizationProvider: .none, - hostToolchain: UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()), + hostToolchain: UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment), checksumAlgorithm: checksumAlgorithm, cachePath: .none, customHTTPClient: .none, @@ -9076,8 +9077,9 @@ final class WorkspaceTests: XCTestCase { func testDownloadArchiveIndexFilesHappyPath() throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() + try fs.createMockToolchain() let downloads = ThreadSafeKeyValueStore() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) let ariFiles = [ """ @@ -9366,7 +9368,8 @@ final class WorkspaceTests: XCTestCase { func testDownloadArchiveIndexFileBadChecksum() throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) + try fs.createMockToolchain() + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) let ari = """ { @@ -9485,7 +9488,8 @@ final class WorkspaceTests: XCTestCase { func testDownloadArchiveIndexFileBadArchivesChecksum() throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) + try fs.createMockToolchain() + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) let ari = """ { @@ -9595,7 +9599,8 @@ final class WorkspaceTests: XCTestCase { func testDownloadArchiveIndexFileArchiveNotFound() throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) + try fs.createMockToolchain() + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) let ari = """ { @@ -12279,6 +12284,7 @@ final class WorkspaceTests: XCTestCase { } let fs = InMemoryFileSystem() + try fs.createMockToolchain() let observability = ObservabilitySystem.makeForTesting() // write a manifest @@ -12294,7 +12300,7 @@ final class WorkspaceTests: XCTestCase { let delegate = MockWorkspaceDelegate() let workspace = try Workspace( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, forRootPackage: .root, customManifestLoader: TestLoader(error: .none), delegate: delegate @@ -12311,7 +12317,7 @@ final class WorkspaceTests: XCTestCase { let delegate = MockWorkspaceDelegate() let workspace = try Workspace( fileSystem: fs, - environment: .empty(), + environment: .mockEnvironment, forRootPackage: .root, customManifestLoader: TestLoader(error: StringError("boom")), delegate: delegate From 637e69442e00e4aa60592874e1ce26b34a0c19a0 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 17:00:46 +0100 Subject: [PATCH 17/36] Fix wrong environment used in `testRepositoryPackageContainerCache` --- Tests/WorkspaceTests/SourceControlPackageContainerTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift index efce8a4b160..d593bab8475 100644 --- a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift +++ b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift @@ -765,7 +765,7 @@ final class SourceControlPackageContainerTests: XCTestCase { ) let containerProvider = try Workspace._init( fileSystem: localFileSystem, - environment: .empty(), + environment: .process(), location: .init(forRootPackage: packageDirectory, fileSystem: localFileSystem), customManifestLoader: MockManifestLoader( manifests: [.init(url: packageDirectory.pathString, version: Version(1, 0, 0)): manifest] From d114ad4de47d8f46cd61816b27ac643b7ec47113 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 17:21:40 +0100 Subject: [PATCH 18/36] Fix remaining test failures --- Tests/BuildTests/BuildPlanTests.swift | 3 ++- Tests/WorkspaceTests/SourceControlPackageContainerTests.swift | 2 +- Tests/WorkspaceTests/WorkspaceTests.swift | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 67d3145c341..592739f0669 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4468,6 +4468,7 @@ final class BuildPlanTests: XCTestCase { "/Pkg/Sources/cxxLib/cxxLib.c", "/Pkg/Sources/cxxLib/include/cxxLib.h" ) + try fileSystem.createMockToolchain() let observability = ObservabilitySystem.makeForTesting() let graph = try loadModulesGraph( @@ -4511,7 +4512,7 @@ final class BuildPlanTests: XCTestCase { ), toolset: toolset ) - let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .empty()) + let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment) let buildParameters = mockBuildParameters( toolchain: toolchain, flags: BuildFlags( diff --git a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift index d593bab8475..f1a8bac1e60 100644 --- a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift +++ b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift @@ -653,7 +653,7 @@ final class SourceControlPackageContainerTests: XCTestCase { let containerProvider = try Workspace._init( fileSystem: localFileSystem, - environment: .empty(), + environment: .process(), location: .init(forRootPackage: packageDirectory, fileSystem: localFileSystem), customManifestLoader: MockManifestLoader( manifests: [ diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index db9665ab811..a661d6a8b9b 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -9675,8 +9675,9 @@ final class WorkspaceTests: XCTestCase { func testDownloadArchiveIndexTripleNotFound() throws { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() + try fs.createMockToolchain() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .empty()), environment: .empty()) + let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) let androidTriple = try Triple("x86_64-unknown-linux-android") let macTriple = try Triple("arm64-apple-macosx") let notHostTriple = hostToolchain.targetTriple == androidTriple ? macTriple : androidTriple From 7c61ed51e46a535774185db629073965ab47f693 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 18:20:02 +0100 Subject: [PATCH 19/36] Prepopulate host triple in `MockWorkspace` --- Sources/SPMTestSupport/MockWorkspace.swift | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Sources/SPMTestSupport/MockWorkspace.swift b/Sources/SPMTestSupport/MockWorkspace.swift index 4a93d650b39..4553df842c9 100644 --- a/Sources/SPMTestSupport/MockWorkspace.swift +++ b/Sources/SPMTestSupport/MockWorkspace.swift @@ -22,12 +22,12 @@ import XCTest import struct TSCUtility.Version extension EnvironmentVariables { - package static var mockEnvironment: Self { ["PATH": "/usr/bin"] } + package static var mockEnvironment: Self { ["PATH": "/fake/path/to"] } } extension InMemoryFileSystem { package func createMockToolchain() throws { - let files = ["/usr/bin/swiftc", "/usr/bin/ar"] + let files = ["/fake/path/to/swiftc", "/fake/path/to/ar"] self.createEmptyFiles(at: AbsolutePath.root, files: files) for toolPath in files { try self.updatePermissions(.init(toolPath), isExecutable: true) @@ -41,6 +41,7 @@ package final class MockWorkspace { let roots: [MockPackage] let packages: [MockPackage] let customToolsVersion: ToolsVersion? + private let customHostToolchain: UserToolchain let fingerprints: MockPackageFingerprintStorage let signingEntities: MockPackageSigningEntityStorage let mirrors: DependencyMirrors @@ -107,6 +108,13 @@ package final class MockWorkspace { httpClient: LegacyHTTPClient.mock(fileSystem: fileSystem), archiver: MockArchiver() ) + var hostSwiftSDK = try SwiftSDK.hostSwiftSDK(environment: .mockEnvironment, fileSystem: fileSystem) + hostSwiftSDK.targetTriple = hostTriple + self.customHostToolchain = try UserToolchain( + swiftSDK: hostSwiftSDK, + environment: .mockEnvironment, + fileSystem: fileSystem + ) try self.create() } @@ -315,6 +323,7 @@ package final class MockWorkspace { customFingerprints: self.fingerprints, customMirrors: self.mirrors, customToolsVersion: self.customToolsVersion, + customHostToolchain: self.customHostToolchain, customManifestLoader: self.manifestLoader, customPackageContainerProvider: self.customPackageContainerProvider, customRepositoryProvider: self.repositoryProvider, From 64b42c40bdc8b3cecfbc92fc50b4e6a1003dbe11 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 21:26:12 +0100 Subject: [PATCH 20/36] Generalize `customHostToolchain` of `MockWorkspace` --- Sources/SPMTestSupport/MockWorkspace.swift | 20 +++++++++++++------- Sources/Workspace/Workspace.swift | 2 ++ Tests/WorkspaceTests/WorkspaceTests.swift | 4 ++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Sources/SPMTestSupport/MockWorkspace.swift b/Sources/SPMTestSupport/MockWorkspace.swift index 4553df842c9..cf6269ca879 100644 --- a/Sources/SPMTestSupport/MockWorkspace.swift +++ b/Sources/SPMTestSupport/MockWorkspace.swift @@ -21,6 +21,18 @@ import XCTest import struct TSCUtility.Version +extension UserToolchain { + package static func mockHostToolchain(_ fileSystem: InMemoryFileSystem) throws -> UserToolchain { + var hostSwiftSDK = try SwiftSDK.hostSwiftSDK(environment: .mockEnvironment, fileSystem: fileSystem) + hostSwiftSDK.targetTriple = hostTriple + return try UserToolchain( + swiftSDK: hostSwiftSDK, + environment: .mockEnvironment, + fileSystem: fileSystem + ) + } +} + extension EnvironmentVariables { package static var mockEnvironment: Self { ["PATH": "/fake/path/to"] } } @@ -108,13 +120,7 @@ package final class MockWorkspace { httpClient: LegacyHTTPClient.mock(fileSystem: fileSystem), archiver: MockArchiver() ) - var hostSwiftSDK = try SwiftSDK.hostSwiftSDK(environment: .mockEnvironment, fileSystem: fileSystem) - hostSwiftSDK.targetTriple = hostTriple - self.customHostToolchain = try UserToolchain( - swiftSDK: hostSwiftSDK, - environment: .mockEnvironment, - fileSystem: fileSystem - ) + self.customHostToolchain = try UserToolchain.mockHostToolchain(fileSystem) try self.create() } diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index f35ee5798ef..135fc1f05c9 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -246,6 +246,7 @@ public class Workspace { cancellator: Cancellator? = .none, initializationWarningHandler: ((String) -> Void)? = .none, // optional customization used for advanced integration situations + customHostToolchain: UserToolchain? = .none, customManifestLoader: ManifestLoaderProtocol? = .none, customPackageContainerProvider: PackageContainerProvider? = .none, customRepositoryProvider: RepositoryProvider? = .none, @@ -263,6 +264,7 @@ public class Workspace { configuration: configuration, cancellator: cancellator, initializationWarningHandler: initializationWarningHandler, + customHostToolchain: customHostToolchain, customManifestLoader: customManifestLoader, customPackageContainerProvider: customPackageContainerProvider, customRepositoryProvider: customRepositoryProvider, diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index a661d6a8b9b..2cc8ea37b2c 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -12296,6 +12296,8 @@ final class WorkspaceTests: XCTestCase { fileSystem: fs ) + let customHostToolchain = try UserToolchain.mockHostToolchain(fs) + do { // no error let delegate = MockWorkspaceDelegate() @@ -12303,6 +12305,7 @@ final class WorkspaceTests: XCTestCase { fileSystem: fs, environment: .mockEnvironment, forRootPackage: .root, + customHostToolchain: customHostToolchain, customManifestLoader: TestLoader(error: .none), delegate: delegate ) @@ -12320,6 +12323,7 @@ final class WorkspaceTests: XCTestCase { fileSystem: fs, environment: .mockEnvironment, forRootPackage: .root, + customHostToolchain: customHostToolchain, customManifestLoader: TestLoader(error: StringError("boom")), delegate: delegate ) From dffdb7e870d53c4407e0c45c6e7a23e548c9525b Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 21:30:04 +0100 Subject: [PATCH 21/36] Fix `SourceControlPackageContainerTests` --- .../WorkspaceTests/SourceControlPackageContainerTests.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift index f1a8bac1e60..de87b5c1a9e 100644 --- a/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift +++ b/Tests/WorkspaceTests/SourceControlPackageContainerTests.swift @@ -225,6 +225,7 @@ final class SourceControlPackageContainerTests: XCTestCase { fileSystem: fs, environment: .mockEnvironment, location: .init(forRootPackage: repoPath, fileSystem: fs), + customHostToolchain: .mockHostToolchain(fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager ) @@ -281,6 +282,7 @@ final class SourceControlPackageContainerTests: XCTestCase { environment: .mockEnvironment, location: .init(forRootPackage: repoPath, fileSystem: fs), customToolsVersion: currentToolsVersion, + customHostToolchain: .mockHostToolchain(fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager ) @@ -366,6 +368,7 @@ final class SourceControlPackageContainerTests: XCTestCase { fileSystem: fs, environment: .mockEnvironment, location: .init(forRootPackage: repoPath, fileSystem: fs), + customHostToolchain: .mockHostToolchain(fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager ) @@ -417,6 +420,7 @@ final class SourceControlPackageContainerTests: XCTestCase { fileSystem: fs, environment: .mockEnvironment, location: .init(forRootPackage: repoPath, fileSystem: fs), + customHostToolchain: .mockHostToolchain(fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRepositoryManager: repositoryManager ) @@ -600,7 +604,7 @@ final class SourceControlPackageContainerTests: XCTestCase { ) let containerProvider = try Workspace._init( fileSystem: localFileSystem, - environment: .mockEnvironment, + environment: .process(), location: .init(forRootPackage: packageDir, fileSystem: localFileSystem), customManifestLoader: MockManifestLoader(manifests: [.init(url: packageDir.pathString, version: nil): manifest]), customRepositoryManager: repositoryManager From d27e0a7b33bb3ab8e3c9a7764e5c3281715387fb Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 21:31:27 +0100 Subject: [PATCH 22/36] Fix `RegistryPackageContainerTests` --- Tests/WorkspaceTests/RegistryPackageContainerTests.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift index 6aa9f34000c..4ff02d352b3 100644 --- a/Tests/WorkspaceTests/RegistryPackageContainerTests.swift +++ b/Tests/WorkspaceTests/RegistryPackageContainerTests.swift @@ -89,6 +89,7 @@ final class RegistryPackageContainerTests: XCTestCase { environment: .mockEnvironment, location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, + customHostToolchain: .mockHostToolchain(fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRegistryClient: registryClient ) @@ -156,6 +157,7 @@ final class RegistryPackageContainerTests: XCTestCase { environment: .mockEnvironment, location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, + customHostToolchain: .mockHostToolchain(fs), customManifestLoader: MockManifestLoader(manifests: [:]), customRegistryClient: registryClient ) @@ -251,6 +253,7 @@ final class RegistryPackageContainerTests: XCTestCase { environment: .mockEnvironment, location: .init(forRootPackage: packagePath, fileSystem: fs), customToolsVersion: toolsVersion, + customHostToolchain: .mockHostToolchain(fs), customManifestLoader: MockManifestLoader(), customRegistryClient: registryClient ) From f53c9c95f325d03c1432f0f158a0d75ca4f79ba3 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 21:33:58 +0100 Subject: [PATCH 23/36] Fix remaining `WorkspaceTests` --- Tests/WorkspaceTests/WorkspaceTests.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index 2cc8ea37b2c..6dca5522b85 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -7838,7 +7838,7 @@ final class WorkspaceTests: XCTestCase { let binaryArtifactsManager = try Workspace.BinaryArtifactsManager( fileSystem: fs, authorizationProvider: .none, - hostToolchain: UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment), + hostToolchain: UserToolchain.mockHostToolchain(fs), checksumAlgorithm: checksumAlgorithm, cachePath: .none, customHTTPClient: .none, @@ -9079,7 +9079,7 @@ final class WorkspaceTests: XCTestCase { let fs = InMemoryFileSystem() try fs.createMockToolchain() let downloads = ThreadSafeKeyValueStore() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) + let hostToolchain = try UserToolchain.mockHostToolchain(fs) let ariFiles = [ """ @@ -9369,7 +9369,7 @@ final class WorkspaceTests: XCTestCase { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() try fs.createMockToolchain() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) + let hostToolchain = try UserToolchain.mockHostToolchain(fs) let ari = """ { @@ -9489,7 +9489,7 @@ final class WorkspaceTests: XCTestCase { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() try fs.createMockToolchain() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) + let hostToolchain = try UserToolchain.mockHostToolchain(fs) let ari = """ { @@ -9600,7 +9600,7 @@ final class WorkspaceTests: XCTestCase { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() try fs.createMockToolchain() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) + let hostToolchain = try UserToolchain.mockHostToolchain(fs) let ari = """ { @@ -9677,7 +9677,7 @@ final class WorkspaceTests: XCTestCase { let fs = InMemoryFileSystem() try fs.createMockToolchain() - let hostToolchain = try UserToolchain(swiftSDK: .hostSwiftSDK(environment: .mockEnvironment), environment: .mockEnvironment) + let hostToolchain = try UserToolchain.mockHostToolchain(fs) let androidTriple = try Triple("x86_64-unknown-linux-android") let macTriple = try Triple("arm64-apple-macosx") let notHostTriple = hostToolchain.targetTriple == androidTriple ? macTriple : androidTriple From 1f2a1ca3b85f43a1c2d5ce7a01fcdd498e7737af Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 21:41:33 +0100 Subject: [PATCH 24/36] Specify host triple for mock Swift SDK in `testUserToolchainCompileFlags` --- Tests/BuildTests/BuildPlanTests.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 592739f0669..144af22f62b 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4356,6 +4356,7 @@ final class BuildPlanTests: XCTestCase { XCTAssertNoDiagnostics(observability.diagnostics) let userSwiftSDK = try SwiftSDK( + hostTriple: hostTriple, toolset: .init( knownTools: [ .cCompiler: .init(extraCLIOptions: ["-I/fake/sdk/sysroot", "-clang-flag-from-json"]), From f1f86fd2ce5e49fccc4eaa451b1c850b5dc4a18e Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 21:44:07 +0100 Subject: [PATCH 25/36] Fixes in `BuildPlanTests` --- Tests/BuildTests/BuildPlanTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 144af22f62b..9268fe666d0 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4362,7 +4362,7 @@ final class BuildPlanTests: XCTestCase { .cCompiler: .init(extraCLIOptions: ["-I/fake/sdk/sysroot", "-clang-flag-from-json"]), .swiftCompiler: .init(extraCLIOptions: ["-use-ld=lld", "-swift-flag-from-json"]), ], - rootPaths: UserToolchain.default.swiftSDK.toolset.rootPaths + rootPaths: UserToolchain.mockHostToolchain(fs).swiftSDK.toolset.rootPaths ), pathsConfiguration: .init( sdkRootPath: "/fake/sdk", @@ -4502,7 +4502,7 @@ final class BuildPlanTests: XCTestCase { .librarian: .init(path: "/fake/toolchain/usr/bin/librarian"), .linker: .init(path: "/fake/toolchain/usr/bin/linker", extraCLIOptions: [jsonFlag(tool: .linker)]), ], - rootPaths: UserToolchain.default.swiftSDK.toolset.rootPaths + rootPaths: UserToolchain.mockHostToolchain(fileSystem).swiftSDK.toolset.rootPaths ) let targetTriple = try Triple("armv7em-unknown-none-macho") let swiftSDK = try SwiftSDK( From 91a7809e1decaad7d7f6e5520812c4dbb07f7004 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 21:49:32 +0100 Subject: [PATCH 26/36] Fix target triple in `testUserToolchainCompileFlags` --- Tests/BuildTests/BuildPlanTests.swift | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 9268fe666d0..e3ee3583d9b 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4356,7 +4356,8 @@ final class BuildPlanTests: XCTestCase { XCTAssertNoDiagnostics(observability.diagnostics) let userSwiftSDK = try SwiftSDK( - hostTriple: hostTriple, + hostTriple: .arm64Linux, + targetTriple: .wasi, toolset: .init( knownTools: [ .cCompiler: .init(extraCLIOptions: ["-I/fake/sdk/sysroot", "-clang-flag-from-json"]), @@ -4391,11 +4392,7 @@ final class BuildPlanTests: XCTestCase { let lib = try result.target(for: "lib").clangTarget() var args: [StringPattern] = [.anySequence] - #if os(macOS) - args += ["-isysroot"] - #else args += ["--sysroot"] - #endif args += [ "\(userSwiftSDK.pathsConfiguration.sdkRootPath!)", "-I/fake/sdk/sysroot", From ce7058225954acb4a1bb65dd3938209c43b55667 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 21:53:55 +0100 Subject: [PATCH 27/36] Precompute host triple in `testUserToolchainWithToolsetCompileFlags` --- Tests/BuildTests/BuildPlanTests.swift | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index e3ee3583d9b..0b635272780 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4502,13 +4502,14 @@ final class BuildPlanTests: XCTestCase { rootPaths: UserToolchain.mockHostToolchain(fileSystem).swiftSDK.toolset.rootPaths ) let targetTriple = try Triple("armv7em-unknown-none-macho") - let swiftSDK = try SwiftSDK( + let swiftSDK = SwiftSDK( + hostTriple: .arm64Linux, targetTriple: targetTriple, - properties: .init( + toolset: toolset, + pathsConfiguration: .init( sdkRootPath: "/fake/sdk", swiftStaticResourcesPath: "/usr/lib/swift_static/none" - ), - toolset: toolset + ) ) let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment) let buildParameters = mockBuildParameters( From 74909d7fea372bd0b581a9e6efe60417f2589846 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Fri, 10 May 2024 22:02:07 +0100 Subject: [PATCH 28/36] Fix missing `fileSystem` argument in `testUserToolchainWithToolsetCompileFlags` --- Tests/BuildTests/BuildPlanTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 0b635272780..560501ee20f 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4511,7 +4511,7 @@ final class BuildPlanTests: XCTestCase { swiftStaticResourcesPath: "/usr/lib/swift_static/none" ) ) - let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment) + let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment, fileSystem: fileSystem) let buildParameters = mockBuildParameters( toolchain: toolchain, flags: BuildFlags( From 28d1484df8ec0a6bd4a52e8278e0993afa1de39f Mon Sep 17 00:00:00 2001 From: Pavel Yaskevich Date: Thu, 16 May 2024 10:47:46 -0700 Subject: [PATCH 29/36] Revert "Prepend toolset paths in `SwiftSDK` to respect `--toolchain` option" This reverts commit ff1ed064c49de64c768eb1cc53dd9e71c2559480. --- Sources/PackageModel/SwiftSDKs/SwiftSDK.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift index d1942246255..8be13f6980f 100644 --- a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift +++ b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift @@ -730,7 +730,7 @@ public struct SwiftSDK: Equatable { /// Appends a path to the array of toolset root paths. /// - Parameter toolsetRootPath: new path to add to Swift SDK's toolset. public mutating func add(toolsetRootPath: AbsolutePath) { - self.toolset.rootPaths.insert(toolsetRootPath, at: 0) + self.toolset.rootPaths.append(toolsetRootPath) } } From d7f50e029815c2495db08526634d6faa7434e89e Mon Sep 17 00:00:00 2001 From: Pavel Yaskevich Date: Thu, 16 May 2024 13:14:29 -0700 Subject: [PATCH 30/36] NFC: Rename `SwiftSDK.add` to `SwiftSDK.append` to better reflect the semantics of the operation --- Sources/PackageModel/SwiftSDKs/SwiftSDK.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift index 8be13f6980f..14f1e5f9cdb 100644 --- a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift +++ b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift @@ -698,7 +698,7 @@ public struct SwiftSDK: Equatable { ) } - swiftSDK.add(toolsetRootPath: binDir.appending(components: "usr", "bin")) + swiftSDK.append(toolsetRootPath: binDir.appending(components: "usr", "bin")) } if let sdk = customCompileSDK { swiftSDK.pathsConfiguration.sdkRootPath = sdk @@ -709,7 +709,7 @@ public struct SwiftSDK: Equatable { // Append the host toolchain's toolset paths at the end for the case the target Swift SDK // doesn't have some of the tools (e.g. swift-frontend might be shared between the host and // target Swift SDKs). - hostSwiftSDK.toolset.rootPaths.forEach { swiftSDK.add(toolsetRootPath: $0) } + hostSwiftSDK.toolset.rootPaths.forEach { swiftSDK.append(toolsetRootPath: $0) } } return swiftSDK @@ -729,7 +729,7 @@ public struct SwiftSDK: Equatable { /// Appends a path to the array of toolset root paths. /// - Parameter toolsetRootPath: new path to add to Swift SDK's toolset. - public mutating func add(toolsetRootPath: AbsolutePath) { + public mutating func append(toolsetRootPath: AbsolutePath) { self.toolset.rootPaths.append(toolsetRootPath) } } From 04904e292665be79b60a7a595d0bc342a65ab34f Mon Sep 17 00:00:00 2001 From: Pavel Yaskevich Date: Thu, 16 May 2024 13:31:35 -0700 Subject: [PATCH 31/36] [SwiftSDK] Prepend custom toolchain into root paths to give it priority --- Sources/PackageModel/SwiftSDKs/SwiftSDK.swift | 16 +++++++++++++++- .../PackageModelTests/SwiftSDKBundleTests.swift | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift index 14f1e5f9cdb..65d4678499f 100644 --- a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift +++ b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift @@ -698,7 +698,8 @@ public struct SwiftSDK: Equatable { ) } - swiftSDK.append(toolsetRootPath: binDir.appending(components: "usr", "bin")) + // `--tooolchain` should override existing anything in the SDK and search paths. + swiftSDK.prepend(toolsetRootPath: binDir.appending(components: "usr", "bin")) } if let sdk = customCompileSDK { swiftSDK.pathsConfiguration.sdkRootPath = sdk @@ -727,7 +728,20 @@ public struct SwiftSDK: Equatable { self.toolset.knownTools[.swiftCompiler] = properties } + /// Prepends a path to the array of toolset root paths. + /// + /// Note: Use this operation if you want new root path to take priority over existing paths. + /// + /// - Parameter toolsetRootPath: new path to add to Swift SDK's toolset. + public mutating func prepend(toolsetRootPath path: AbsolutePath) { + self.toolset.rootPaths.insert(path, at: 0) + } + /// Appends a path to the array of toolset root paths. + /// + /// Note: The paths are evaluated in insertion order which means that newly added path would + /// have a lower priority vs. existing paths. + /// /// - Parameter toolsetRootPath: new path to add to Swift SDK's toolset. public mutating func append(toolsetRootPath: AbsolutePath) { self.toolset.rootPaths.append(toolsetRootPath) diff --git a/Tests/PackageModelTests/SwiftSDKBundleTests.swift b/Tests/PackageModelTests/SwiftSDKBundleTests.swift index c45e5b9d078..b1cedd92dc1 100644 --- a/Tests/PackageModelTests/SwiftSDKBundleTests.swift +++ b/Tests/PackageModelTests/SwiftSDKBundleTests.swift @@ -424,7 +424,7 @@ final class SwiftSDKBundleTests: XCTestCase { fileSystem: fileSystem ) // With toolset in the target SDK, it should contain the host toolset roots at the end. - XCTAssertEqual(targetSwiftSDK.toolset.rootPaths, hostSwiftSDK.toolset.rootPaths + [toolsetRootPath]) + XCTAssertEqual(targetSwiftSDK.toolset.rootPaths, [toolsetRootPath] + hostSwiftSDK.toolset.rootPaths) } do { From 133fbeef91b75e13895e9d9acde764afc4d8a82c Mon Sep 17 00:00:00 2001 From: Pavel Yaskevich Date: Thu, 16 May 2024 13:58:28 -0700 Subject: [PATCH 32/36] [CoreCommands] Pipe environment through to the host toolchain --- Sources/CoreCommands/SwiftCommandState.swift | 4 ++-- Tests/CommandsTests/SwiftCommandStateTests.swift | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 7c7baa58cb6..b71e8736606 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -868,10 +868,10 @@ package final class SwiftCommandState { return Result(catching: { try UserToolchain( swiftSDK: SwiftSDK.hostSwiftSDK( - environment: .process(), + environment: environment, observabilityScope: self.observabilityScope ), - environment: .process(), + environment: environment, fileSystem: localFileSystem ) }) diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index c9b73a9f738..8a60de4ec03 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -362,13 +362,16 @@ final class SwiftCommandStateTests: CommandsTestCase { "--triple", "x86_64-unknown-linux-gnu", ] ) + + var env: EnvironmentVariables = .process() + env.appendPath("SWIFTPM_CUSTOM_BIN_DIR", value: hostBinDir.pathString) + let swiftCommandState = try SwiftCommandState.makeMockState( options: options, fileSystem: fs, - environment: [ - "SWIFTPM_CUSTOM_BIN_DIR": hostBinDir.pathString - ] + environment: env ) + XCTAssertEqual(swiftCommandState.originalWorkingDirectory, fs.currentWorkingDirectory) XCTAssertEqual( try swiftCommandState.getTargetToolchain().swiftCompilerPath, From dfa56d377ca7cbf4ef638895f4a3d81a06ab4f8e Mon Sep 17 00:00:00 2001 From: Pavel Yaskevich Date: Fri, 17 May 2024 10:13:09 -0700 Subject: [PATCH 33/36] Revert "[CoreCommands] Pipe environment through to the host toolchain" This reverts commit 133fbeef91b75e13895e9d9acde764afc4d8a82c. --- Sources/CoreCommands/SwiftCommandState.swift | 4 ++-- Tests/CommandsTests/SwiftCommandStateTests.swift | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 2acd48fbe0d..f29f0a4bfab 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -868,10 +868,10 @@ public final class SwiftCommandState { return Result(catching: { try UserToolchain( swiftSDK: SwiftSDK.hostSwiftSDK( - environment: environment, + environment: .process(), observabilityScope: self.observabilityScope ), - environment: environment, + environment: .process(), fileSystem: localFileSystem ) }) diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index 4c319ad6f68..bb325aa8bb3 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -359,16 +359,13 @@ final class SwiftCommandStateTests: CommandsTestCase { "--triple", "x86_64-unknown-linux-gnu", ] ) - - var env: EnvironmentVariables = .process() - env.appendPath("SWIFTPM_CUSTOM_BIN_DIR", value: hostBinDir.pathString) - let swiftCommandState = try SwiftCommandState.makeMockState( options: options, fileSystem: fs, - environment: env + environment: [ + "SWIFTPM_CUSTOM_BIN_DIR": hostBinDir.pathString + ] ) - XCTAssertEqual(swiftCommandState.originalWorkingDirectory, fs.currentWorkingDirectory) XCTAssertEqual( try swiftCommandState.getTargetToolchain().swiftCompilerPath, From a7f8121ae0dab3b85e12720c1c8c3fefa39adc08 Mon Sep 17 00:00:00 2001 From: Pavel Yaskevich Date: Fri, 17 May 2024 10:41:14 -0700 Subject: [PATCH 34/36] [Tests] NFC: Improve SwiftCommandStateTests.testToolchaArgument Drop `SWIFTPM_CUSTOM_BIN_DIR` and related files because they are not passed through to the `hostSwiftSDK` anyway and even if they were we cannot use non-existant binaries because `targetTriple` of host toolchain gets computed by invoking `swiftc -print-target-info`. --- Tests/CommandsTests/SwiftCommandStateTests.swift | 11 +---------- Tests/PackageModelTests/SwiftSDKBundleTests.swift | 4 ++++ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index bb325aa8bb3..c952af5572e 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -321,22 +321,16 @@ final class SwiftCommandStateTests: CommandsTestCase { } func testToolchainArgument() throws { - let hostBinDir = AbsolutePath("/swiftpm/bin") - let hostSwiftcPath = hostBinDir.appending(components: ["swiftc"]) - let customTargetToolchain = AbsolutePath("/path/to/toolchain") let targetSwiftcPath = customTargetToolchain.appending(components: ["usr", "bin" , "swiftc"]) let targetArPath = customTargetToolchain.appending(components: ["usr", "bin", "llvm-ar"]) let fs = InMemoryFileSystem(emptyFiles: [ "/Pkg/Sources/exe/main.swift", - hostSwiftcPath.pathString, targetSwiftcPath.pathString, targetArPath.pathString ]) - - try fs.updatePermissions(hostSwiftcPath, isExecutable: true) try fs.updatePermissions(targetSwiftcPath, isExecutable: true) try fs.updatePermissions(targetArPath, isExecutable: true) @@ -361,10 +355,7 @@ final class SwiftCommandStateTests: CommandsTestCase { ) let swiftCommandState = try SwiftCommandState.makeMockState( options: options, - fileSystem: fs, - environment: [ - "SWIFTPM_CUSTOM_BIN_DIR": hostBinDir.pathString - ] + fileSystem: fs ) XCTAssertEqual(swiftCommandState.originalWorkingDirectory, fs.currentWorkingDirectory) XCTAssertEqual( diff --git a/Tests/PackageModelTests/SwiftSDKBundleTests.swift b/Tests/PackageModelTests/SwiftSDKBundleTests.swift index c49283bfcc5..58b9e96e462 100644 --- a/Tests/PackageModelTests/SwiftSDKBundleTests.swift +++ b/Tests/PackageModelTests/SwiftSDKBundleTests.swift @@ -447,6 +447,10 @@ final class SwiftSDKBundleTests: XCTestCase { ) XCTAssertEqual(targetSwiftSDK.architectures, archs) XCTAssertEqual(targetSwiftSDK.pathsConfiguration.sdkRootPath, customCompileSDK) + XCTAssertEqual( + targetSwiftSDK.toolset.rootPaths, + [customCompileToolchain.appending(components: ["usr", "bin"])] + hostSwiftSDK.toolset.rootPaths + ) } } } From 3ed79e8f9d9c0d69f31ace90a7b417df74d13155 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 5 Jun 2024 17:47:05 +0100 Subject: [PATCH 35/36] Fix `testToolchainArgument` on all platforms --- Sources/CoreCommands/SwiftCommandState.swift | 6 +++--- Tests/CommandsTests/SwiftCommandStateTests.swift | 14 ++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 280a95026ad..812b4bb15e4 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -874,11 +874,11 @@ public final class SwiftCommandState { return Result(catching: { try UserToolchain( swiftSDK: SwiftSDK.hostSwiftSDK( - environment: .process(), + environment: self.environment, observabilityScope: self.observabilityScope ), - environment: .process(), - fileSystem: localFileSystem + environment: self.environment, + fileSystem: self.fileSystem ) }) }() diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index c952af5572e..39973ec3857 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -246,7 +246,7 @@ final class SwiftCommandStateTests: CommandsTestCase { ]) let observer = ObservabilitySystem.makeForTesting() - let graph = try loadPackageGraph(fileSystem: fs, manifests: [ + let graph = try loadModulesGraph(fileSystem: fs, manifests: [ Manifest.createRootManifest(displayName: "Pkg", path: "/Pkg", targets: [TargetDescription(name: "exe")]) @@ -322,17 +322,22 @@ final class SwiftCommandStateTests: CommandsTestCase { func testToolchainArgument() throws { let customTargetToolchain = AbsolutePath("/path/to/toolchain") + let hostSwiftcPath = AbsolutePath("/usr/bin/swiftc") + let hostArPath = AbsolutePath("/usr/bin/ar") let targetSwiftcPath = customTargetToolchain.appending(components: ["usr", "bin" , "swiftc"]) let targetArPath = customTargetToolchain.appending(components: ["usr", "bin", "llvm-ar"]) let fs = InMemoryFileSystem(emptyFiles: [ "/Pkg/Sources/exe/main.swift", + hostSwiftcPath.pathString, + hostArPath.pathString, targetSwiftcPath.pathString, targetArPath.pathString ]) - try fs.updatePermissions(targetSwiftcPath, isExecutable: true) - try fs.updatePermissions(targetArPath, isExecutable: true) + for path in [hostSwiftcPath, hostArPath, targetSwiftcPath, targetArPath,] { + try fs.updatePermissions(path, isExecutable: true) + } let observer = ObservabilitySystem.makeForTesting() let graph = try loadModulesGraph( @@ -355,7 +360,8 @@ final class SwiftCommandStateTests: CommandsTestCase { ) let swiftCommandState = try SwiftCommandState.makeMockState( options: options, - fileSystem: fs + fileSystem: fs, + environment: ["PATH": "/usr/bin"] ) XCTAssertEqual(swiftCommandState.originalWorkingDirectory, fs.currentWorkingDirectory) XCTAssertEqual( From e4968d04c33c4b468e40077a50d4bc173080894c Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 5 Jun 2024 18:18:28 +0100 Subject: [PATCH 36/36] Override host triple in `SwiftCommandStateTests` --- Sources/CoreCommands/SwiftCommandState.swift | 17 ++++++++++++----- .../CommandsTests/SwiftCommandStateTests.swift | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 812b4bb15e4..7d8e406a575 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -267,6 +267,8 @@ public final class SwiftCommandState { private let environment: EnvironmentVariables + private let hostTriple: Basics.Triple? + /// Create an instance of this tool. /// /// - parameter options: The command line options to be passed to this tool. @@ -297,9 +299,11 @@ public final class SwiftCommandState { toolWorkspaceConfiguration: ToolWorkspaceConfiguration, workspaceDelegateProvider: @escaping WorkspaceDelegateProvider, workspaceLoaderProvider: @escaping WorkspaceLoaderProvider, + hostTriple: Basics.Triple? = nil, fileSystem: any FileSystem = localFileSystem, environment: EnvironmentVariables = ProcessEnv.vars ) throws { + self.hostTriple = hostTriple self.fileSystem = fileSystem self.environment = environment // first, bootstrap the observability system @@ -872,11 +876,14 @@ public final class SwiftCommandState { /// Lazily compute the host toolchain used to compile the package description. private lazy var _hostToolchain: Result = { return Result(catching: { - try UserToolchain( - swiftSDK: SwiftSDK.hostSwiftSDK( - environment: self.environment, - observabilityScope: self.observabilityScope - ), + var hostSwiftSDK = try SwiftSDK.hostSwiftSDK( + environment: self.environment, + observabilityScope: self.observabilityScope + ) + hostSwiftSDK.targetTriple = self.hostTriple + + return try UserToolchain( + swiftSDK: hostSwiftSDK, environment: self.environment, fileSystem: self.fileSystem ) diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index 39973ec3857..05cbfe26bc5 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -411,6 +411,7 @@ extension SwiftCommandState { observabilityScope: $1 ) }, + hostTriple: .arm64Linux, fileSystem: fileSystem, environment: environment )