From cc35dcafb8a35525d9bdf5a192b36a954173470a Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Mon, 12 Apr 2021 10:59:17 -0700 Subject: [PATCH] Add support for spawning a compile job that reads input from stdin. This change builds on https://github.com/apple/swift-tools-support-core/pull/208 to allow forwarding of the driver's Standard Input to a compile job. Resolves rdar://76454784 --- Package.resolved | 12 +-- .../Execution/ProcessProtocol.swift | 31 ++++++- .../MultiJobExecutor.swift | 34 ++++++-- .../SwiftDriverExecutor.swift | 10 ++- Tests/SwiftDriverTests/JobExecutorTests.swift | 83 +++++++++++++++++++ 5 files changed, 153 insertions(+), 17 deletions(-) diff --git a/Package.resolved b/Package.resolved index 76c65aefa..be02840c3 100644 --- a/Package.resolved +++ b/Package.resolved @@ -6,8 +6,8 @@ "repositoryURL": "https://github.com/apple/swift-argument-parser.git", "state": { "branch": null, - "revision": "92646c0cdbaca076c8d3d0207891785b3379cbff", - "version": "0.3.1" + "revision": "9564d61b08a5335ae0a36f789a7d71493eacadfc", + "version": "0.3.2" } }, { @@ -15,7 +15,7 @@ "repositoryURL": "https://github.com/apple/swift-llbuild.git", "state": { "branch": "main", - "revision": "53e245a2cf429e0d97633c633ca9968f5eb21b15", + "revision": "b7b4c5eaa798708d6233ca07908a7cab75620040", "version": null } }, @@ -24,7 +24,7 @@ "repositoryURL": "https://github.com/apple/swift-tools-support-core.git", "state": { "branch": "main", - "revision": "207f9c6bc5e33eb8383266c0cd3129c1276f534b", + "revision": "435a2708a6e486d69ea7d7aaa3f4ad243bc3b408", "version": null } }, @@ -33,8 +33,8 @@ "repositoryURL": "https://github.com/jpsim/Yams.git", "state": { "branch": null, - "revision": "1bce5b89a11912fb8a8c48bd404bd24979472f27", - "version": "4.0.3" + "revision": "9ff1cc9327586db4e0c8f46f064b6a82ec1566fa", + "version": "4.0.6" } } ] diff --git a/Sources/SwiftDriver/Execution/ProcessProtocol.swift b/Sources/SwiftDriver/Execution/ProcessProtocol.swift index 3c9bef09b..2e7deae9c 100644 --- a/Sources/SwiftDriver/Execution/ProcessProtocol.swift +++ b/Sources/SwiftDriver/Execution/ProcessProtocol.swift @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// import TSCBasic +import Foundation /// Abstraction for functionality that allows working with subprocesses. public protocol ProcessProtocol { @@ -19,7 +20,7 @@ public protocol ProcessProtocol { /// a negative number to represent a "quasi-pid". /// /// - SeeAlso: https://github.com/apple/swift/blob/main/docs/DriverParseableOutput.rst#quasi-pids - var processID: Process.ProcessID { get } + var processID: TSCBasic.Process.ProcessID { get } /// Wait for the process to finish execution. @discardableResult @@ -29,15 +30,39 @@ public protocol ProcessProtocol { arguments: [String], env: [String: String] ) throws -> Self + + static func launchProcessAndWriteInput( + arguments: [String], + env: [String: String], + inputFileHandle: FileHandle + ) throws -> Self } -extension Process: ProcessProtocol { +extension TSCBasic.Process: ProcessProtocol { public static func launchProcess( arguments: [String], env: [String: String] - ) throws -> Process { + ) throws -> TSCBasic.Process { let process = Process(arguments: arguments, environment: env) try process.launch() return process } + + public static func launchProcessAndWriteInput( + arguments: [String], + env: [String: String], + inputFileHandle: FileHandle + ) throws -> TSCBasic.Process { + let process = Process(arguments: arguments, environment: env) + let processInputStream = try process.launch() + var input: Data + // Write out the contents of the input handle and close the input stream + repeat { + input = inputFileHandle.availableData + processInputStream.write(input) + } while (input.count > 0) + processInputStream.flush() + try processInputStream.close() + return process + } } diff --git a/Sources/SwiftDriverExecution/MultiJobExecutor.swift b/Sources/SwiftDriverExecution/MultiJobExecutor.swift index fcc2f566f..248656e62 100644 --- a/Sources/SwiftDriverExecution/MultiJobExecutor.swift +++ b/Sources/SwiftDriverExecution/MultiJobExecutor.swift @@ -82,6 +82,9 @@ public final class MultiJobExecutor { /// The type to use when launching new processes. This mostly serves as an override for testing. let processType: ProcessProtocol.Type + /// The standard input `FileHandle` override for testing. + let testInputHandle: FileHandle? + /// If a job fails, the driver needs to stop running jobs. private(set) var isBuildCancelled = false @@ -100,7 +103,8 @@ public final class MultiJobExecutor { forceResponseFiles: Bool, recordedInputModificationDates: [TypedVirtualPath: Date], diagnosticsEngine: DiagnosticsEngine, - processType: ProcessProtocol.Type = Process.self + processType: ProcessProtocol.Type = Process.self, + inputHandleOverride: FileHandle? = nil ) { ( jobs: self.jobs, @@ -121,6 +125,7 @@ public final class MultiJobExecutor { self.recordedInputModificationDates = recordedInputModificationDates self.diagnosticsEngine = diagnosticsEngine self.processType = processType + self.testInputHandle = inputHandleOverride } private static func fillInJobsAndProducers(_ workload: DriverExecutorWorkload @@ -248,6 +253,9 @@ public final class MultiJobExecutor { /// The type to use when launching new processes. This mostly serves as an override for testing. private let processType: ProcessProtocol.Type + /// The standard input `FileHandle` override for testing. + let testInputHandle: FileHandle? + public init( workload: DriverExecutorWorkload, resolver: ArgsResolver, @@ -257,7 +265,8 @@ public final class MultiJobExecutor { processSet: ProcessSet? = nil, forceResponseFiles: Bool = false, recordedInputModificationDates: [TypedVirtualPath: Date] = [:], - processType: ProcessProtocol.Type = Process.self + processType: ProcessProtocol.Type = Process.self, + inputHandleOverride: FileHandle? = nil ) { self.workload = workload self.argsResolver = resolver @@ -268,6 +277,7 @@ public final class MultiJobExecutor { self.forceResponseFiles = forceResponseFiles self.recordedInputModificationDates = recordedInputModificationDates self.processType = processType + self.testInputHandle = inputHandleOverride } /// Execute all jobs. @@ -314,7 +324,8 @@ public final class MultiJobExecutor { forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputModificationDates, diagnosticsEngine: diagnosticsEngine, - processType: processType + processType: processType, + inputHandleOverride: testInputHandle ) } } @@ -566,9 +577,20 @@ class ExecuteJobRule: LLBuildRule { let arguments: [String] = try resolver.resolveArgumentList(for: job, forceResponseFiles: context.forceResponseFiles) - let process = try context.processType.launchProcess( - arguments: arguments, env: env - ) + + let process : ProcessProtocol + // If the input comes from standard input, forward the driver's input to the compile job. + if job.inputs.contains(TypedVirtualPath(file: .standardInput, type: .swift)) { + let inputFileHandle = context.testInputHandle ?? FileHandle.standardInput + process = try context.processType.launchProcessAndWriteInput( + arguments: arguments, env: env, inputFileHandle: inputFileHandle + ) + } else { + process = try context.processType.launchProcess( + arguments: arguments, env: env + ) + } + pid = Int(process.processID) // Add it to the process set if it's a real process. diff --git a/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift index f285896df..dbe9494b5 100644 --- a/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift +++ b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift @@ -51,8 +51,14 @@ public final class SwiftDriverExecutor: DriverExecutor { } else { var childEnv = env childEnv.merge(job.extraEnvironment, uniquingKeysWith: { (_, new) in new }) - - let process = try Process.launchProcess(arguments: arguments, env: childEnv) + let process : ProcessProtocol + if job.inputs.contains(TypedVirtualPath(file: .standardInput, type: .swift)) { + process = try Process.launchProcessAndWriteInput( + arguments: arguments, env: childEnv, inputFileHandle: FileHandle.standardInput + ) + } else { + process = try Process.launchProcess(arguments: arguments, env: childEnv) + } return try process.waitUntilExit() } } diff --git a/Tests/SwiftDriverTests/JobExecutorTests.swift b/Tests/SwiftDriverTests/JobExecutorTests.swift index f5e0d5e20..83d39185a 100644 --- a/Tests/SwiftDriverTests/JobExecutorTests.swift +++ b/Tests/SwiftDriverTests/JobExecutorTests.swift @@ -29,6 +29,11 @@ class JobCollectingDelegate: JobExecutionDelegate { return .init() } + static func launchProcessAndWriteInput(arguments: [String], env: [String : String], + inputFileHandle: FileHandle) throws -> StubProcess { + return .init() + } + var processID: TSCBasic.Process.ProcessID { .init(-1) } func waitUntilExit() throws -> ProcessResult { @@ -207,6 +212,84 @@ final class JobExecutorTests: XCTestCase { #endif } + /// Ensure the executor is capable of forwarding its standard input to the compile job that requires it. + func testInputForwarding() throws { +#if os(macOS) + let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(), + processSet: ProcessSet(), + fileSystem: localFileSystem, + env: ProcessEnv.vars) + let toolchain = DarwinToolchain(env: ProcessEnv.vars, executor: executor) + try withTemporaryDirectory { path in + let exec = path.appending(component: "main") + let compile = Job( + moduleName: "main", + kind: .compile, + tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), + commandLine: [ + "-frontend", + "-c", + "-primary-file", + // This compile job must read the input from STDIN + "-", + "-target", "x86_64-apple-darwin18.7.0", + "-enable-objc-interop", + "-sdk", + .path(.absolute(try toolchain.sdk.get())), + "-module-name", "main", + "-o", .path(.temporary(RelativePath("main.o"))), + ], + inputs: [TypedVirtualPath(file: .standardInput, type: .swift )], + primaryInputs: [TypedVirtualPath(file: .standardInput, type: .swift )], + outputs: [.init(file: VirtualPath.temporary(RelativePath("main.o")).intern(), + type: .object)] + ) + let link = Job( + moduleName: "main", + kind: .link, + tool: .absolute(try toolchain.getToolPath(.dynamicLinker)), + commandLine: [ + .path(.temporary(RelativePath("main.o"))), + .path(.absolute(try toolchain.clangRT.get())), + "-syslibroot", .path(.absolute(try toolchain.sdk.get())), + "-lobjc", "-lSystem", "-arch", "x86_64", + "-force_load", .path(.absolute(try toolchain.compatibility50.get())), + "-force_load", .path(.absolute(try toolchain.compatibilityDynamicReplacements.get())), + "-L", .path(.absolute(try toolchain.resourcesDirectory.get())), + "-L", .path(.absolute(try toolchain.sdkStdlib(sdk: toolchain.sdk.get()))), + "-rpath", "/usr/lib/swift", "-macosx_version_min", "10.14.0", "-no_objc_category_merging", + "-o", .path(.absolute(exec)), + ], + inputs: [ + .init(file: VirtualPath.temporary(RelativePath("main.o")).intern(), type: .object), + ], + primaryInputs: [], + outputs: [.init(file: VirtualPath.relative(RelativePath("main")).intern(), type: .image)] + ) + + // Create a file with inpuit + let inputFile = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(inputFile) { + $0 <<< "print(\"Hello, World\")" + } + // We are going to override he executors standard input FileHandle to the above + // input file, to simulate it being piped over standard input to this compilation. + let testFile: FileHandle = FileHandle(forReadingAtPath: inputFile.description)! + let delegate = JobCollectingDelegate() + let resolver = try ArgsResolver(fileSystem: localFileSystem) + let executor = MultiJobExecutor(workload: .all([compile, link]), + resolver: resolver, executorDelegate: delegate, + diagnosticsEngine: DiagnosticsEngine(), + inputHandleOverride: testFile) + try executor.execute(env: toolchain.env, fileSystem: localFileSystem) + + // Execute the resulting program + let output = try TSCBasic.Process.checkNonZeroExit(args: exec.pathString) + XCTAssertEqual(output, "Hello, World\n") + } +#endif + } + func testStubProcessProtocol() throws { // This test fails intermittently on Linux // rdar://70067844