diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 6f2aa1051..9a85d6b5c 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -231,7 +231,8 @@ extension Driver { outputType: FileType?, addJobOutputs: ([TypedVirtualPath]) -> Void, pchCompileJob: Job?, - emitModuleTrace: Bool) + emitModuleTrace: Bool, + produceCacheKey: Bool) throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [] diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 357ec699d..ed801aba3 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -90,16 +90,17 @@ extension Driver { incrementalCompilationState = nil } - return try ( - // For compatibility with swiftpm, the driver produces batched jobs - // for every job, even when run in incremental mode, so that all jobs - // can be returned from `planBuild`. - // But in that case, don't emit lifecycle messages. - formBatchedJobs(jobsInPhases.allJobs, - showJobLifecycle: showJobLifecycle && incrementalCompilationState == nil, - jobCreatingPch: jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH})), - incrementalCompilationState - ) + let batchedJobs: [Job] + // If the jobs are batched during the incremental build, reuse the computation rather than computing the batches again. + if let incrementalState = incrementalCompilationState { + batchedJobs = incrementalState.mandatoryJobsInOrder + incrementalState.jobsAfterCompiles + } else { + batchedJobs = try formBatchedJobs(jobsInPhases.allJobs, + showJobLifecycle: showJobLifecycle, + jobCreatingPch: jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH})) + } + + return (batchedJobs, incrementalCompilationState) } /// If performing an explicit module build, compute an inter-module dependency graph. @@ -355,7 +356,8 @@ extension Driver { outputType: compilerOutputType, addJobOutputs: addJobOutputs, pchCompileJob: pchCompileJob, - emitModuleTrace: emitModuleTrace) + emitModuleTrace: emitModuleTrace, + produceCacheKey: true) addJob(compile) return compile } @@ -446,11 +448,14 @@ extension Driver { // We can skip the compile jobs if all we want is a module when it's // built separately. if parsedOptions.hasArgument(.driverExplicitModuleBuild), canSkipIfOnlyModule { return } + // If we are in the batch mode, the constructed jobs here will be batched + // later. There is no need to produce cache key for the job. let compile = try compileJob(primaryInputs: [primaryInput], outputType: compilerOutputType, addJobOutputs: addJobOutputs, pchCompileJob: pchCompileJob, - emitModuleTrace: emitModuleTrace) + emitModuleTrace: emitModuleTrace, + produceCacheKey: !compilerMode.isBatchCompile) addCompileJob(compile) } @@ -872,7 +877,8 @@ extension Driver { outputType: compilerOutputType, addJobOutputs: {_ in }, pchCompileJob: jobCreatingPch, - emitModuleTrace: constituentsEmittedModuleTrace) + emitModuleTrace: constituentsEmittedModuleTrace, + produceCacheKey: true) } return batchedCompileJobs + noncompileJobs } diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 6e6818a30..ed878f944 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -674,20 +674,45 @@ private extension swiftscan_functions_t { } } -// TODO: Move to TSC? -/// Perform an `action` passing it a `const char **` constructed out of `[String]` -@_spi(Testing) public func withArrayOfCStrings(_ strings: [String], - _ action: (UnsafeMutablePointer?>?) -> T) -> T -{ -#if os(Windows) - let cstrings = strings.map { _strdup($0) } + [nil] -#else - let cstrings = strings.map { strdup($0) } + [nil] -#endif - let unsafeCStrings = cstrings.map { UnsafePointer($0) } - let result = unsafeCStrings.withUnsafeBufferPointer { - action(UnsafeMutablePointer(mutating: $0.baseAddress)) +// TODO: Move the following functions to TSC? +/// Helper function to scan a sequence type to help generate pointers for C String Arrays. +func scan< + S: Sequence, U +>(_ seq: S, _ initial: U, _ combine: (U, S.Element) -> U) -> [U] { + var result: [U] = [] + result.reserveCapacity(seq.underestimatedCount) + var runningResult = initial + for element in seq { + runningResult = combine(runningResult, element) + result.append(runningResult) } - for ptr in cstrings { if let ptr = ptr { free(ptr) } } return result } + +/// Perform an `action` passing it a `const char **` constructed out of `[String]` +@_spi(Testing) public func withArrayOfCStrings( + _ args: [String], + _ body: (UnsafeMutablePointer?>?) -> T +) -> T { + let argsCounts = Array(args.map { $0.utf8.count + 1 }) + let argsOffsets = [0] + scan(argsCounts, 0, +) + let argsBufferSize = argsOffsets.last! + var argsBuffer: [UInt8] = [] + argsBuffer.reserveCapacity(argsBufferSize) + for arg in args { + argsBuffer.append(contentsOf: arg.utf8) + argsBuffer.append(0) + } + return argsBuffer.withUnsafeMutableBufferPointer { + (argsBuffer) in + let ptr = UnsafeRawPointer(argsBuffer.baseAddress!).bindMemory( + to: Int8.self, capacity: argsBuffer.count) + var cStrings: [UnsafePointer?] = argsOffsets.map { ptr + $0 } + cStrings[cStrings.count - 1] = nil + return cStrings.withUnsafeMutableBufferPointer { + let unsafeString = UnsafeMutableRawPointer($0.baseAddress!).bindMemory( + to: UnsafePointer?.self, capacity: $0.count) + return body(unsafeString) + } + } +} diff --git a/Tests/SwiftDriverTests/CachingBuildTests.swift b/Tests/SwiftDriverTests/CachingBuildTests.swift index d883e985f..192b8deba 100644 --- a/Tests/SwiftDriverTests/CachingBuildTests.swift +++ b/Tests/SwiftDriverTests/CachingBuildTests.swift @@ -986,17 +986,60 @@ final class CachingBuildTests: XCTestCase { XCTFail("Cached compilation doesn't have a CAS") } try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem) + } + } - // try replan the job and make sure some key command-line options are generated. - let rebuildJobs = try driver.planBuild() - for job in rebuildJobs { - if job.kind == .compile || job.kind == .emitModule { - XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules")))) - XCTAssertTrue(job.commandLine.contains(.flag(String("-cache-compile-job")))) - XCTAssertTrue(job.commandLine.contains(.flag(String("-cas-path")))) - XCTAssertTrue(job.commandLine.contains(.flag(String("-bridging-header-pch-key")))) - } + func testCacheBatchBuildPlan() throws { + try withTemporaryDirectory { path in + try localFileSystem.changeCurrentWorkingDirectory(to: path) + let moduleCachePath = path.appending(component: "ModuleCache") + let casPath = path.appending(component: "cas") + try localFileSystem.createDirectory(moduleCachePath) + let main = path.appending(component: "testCachingBuild.swift") + let mainFileContent = "import C;import E;import G;" + try localFileSystem.writeFileContents(main) { + $0.send(mainFileContent) } + let ofm = path.appending(component: "ofm.json") + let inputPathsAndContents: [(AbsolutePath, String)] = [(main, mainFileContent)] + OutputFileMapCreator.write( + module: "Test", inputPaths: inputPathsAndContents.map {$0.0}, + derivedData: path, to: ofm, excludeMainEntry: false) + + let cHeadersPath: AbsolutePath = + try testInputsPath.appending(component: "ExplicitModuleBuilds") + .appending(component: "CHeaders") + let swiftModuleInterfacesPath: AbsolutePath = + try testInputsPath.appending(component: "ExplicitModuleBuilds") + .appending(component: "Swift") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + let bridgingHeaderpath: AbsolutePath = + cHeadersPath.appending(component: "Bridging.h") + var driver = try Driver(args: ["swiftc", + "-I", cHeadersPath.nativePathString(escaped: true), + "-I", swiftModuleInterfacesPath.nativePathString(escaped: true), + "-explicit-module-build", "-Rcache-compile-job", "-incremental", + "-module-cache-path", moduleCachePath.nativePathString(escaped: true), + "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true), + "-import-objc-header", bridgingHeaderpath.nativePathString(escaped: true), + "-output-file-map", ofm.nativePathString(escaped: true), + "-working-directory", path.nativePathString(escaped: true), + main.nativePathString(escaped: true)] + sdkArgumentsForTesting, + interModuleDependencyOracle: dependencyOracle) + let jobs = try driver.planBuild() + try driver.run(jobs: jobs) + XCTAssertFalse(driver.diagnosticEngine.hasErrors) + + let scanLibPath = try XCTUnwrap(driver.getSwiftScanLibPath()) + try dependencyOracle.verifyOrCreateScannerInstance(swiftScanLibPath: scanLibPath) + + let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: []) + if let driverCAS = driver.cas { + XCTAssertEqual(cas, driverCAS, "CAS should only be created once") + } else { + XCTFail("Cached compilation doesn't have a CAS") + } + try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem) } }