From 9064163bc90e92c5aa2a04e203fe9562f2a423ec Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 26 Feb 2025 15:46:52 -0800 Subject: [PATCH 1/3] [SwiftScan] Improve the bridging between `[String]` and `const char **` Borrow the implementation from stdlib test for a faster way to bridging swift `[String]` to `const char **`. Instead of strdup every string which can be costly if the argument list is very long, use one single big allocation to hold all the strings. --- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 53 ++++++++++++++----- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 6e6818a30..ed878f944 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -674,20 +674,45 @@ private extension swiftscan_functions_t { } } -// TODO: Move to TSC? -/// Perform an `action` passing it a `const char **` constructed out of `[String]` -@_spi(Testing) public func withArrayOfCStrings(_ strings: [String], - _ action: (UnsafeMutablePointer?>?) -> T) -> T -{ -#if os(Windows) - let cstrings = strings.map { _strdup($0) } + [nil] -#else - let cstrings = strings.map { strdup($0) } + [nil] -#endif - let unsafeCStrings = cstrings.map { UnsafePointer($0) } - let result = unsafeCStrings.withUnsafeBufferPointer { - action(UnsafeMutablePointer(mutating: $0.baseAddress)) +// TODO: Move the following functions to TSC? +/// Helper function to scan a sequence type to help generate pointers for C String Arrays. +func scan< + S: Sequence, U +>(_ seq: S, _ initial: U, _ combine: (U, S.Element) -> U) -> [U] { + var result: [U] = [] + result.reserveCapacity(seq.underestimatedCount) + var runningResult = initial + for element in seq { + runningResult = combine(runningResult, element) + result.append(runningResult) } - for ptr in cstrings { if let ptr = ptr { free(ptr) } } return result } + +/// Perform an `action` passing it a `const char **` constructed out of `[String]` +@_spi(Testing) public func withArrayOfCStrings( + _ args: [String], + _ body: (UnsafeMutablePointer?>?) -> T +) -> T { + let argsCounts = Array(args.map { $0.utf8.count + 1 }) + let argsOffsets = [0] + scan(argsCounts, 0, +) + let argsBufferSize = argsOffsets.last! + var argsBuffer: [UInt8] = [] + argsBuffer.reserveCapacity(argsBufferSize) + for arg in args { + argsBuffer.append(contentsOf: arg.utf8) + argsBuffer.append(0) + } + return argsBuffer.withUnsafeMutableBufferPointer { + (argsBuffer) in + let ptr = UnsafeRawPointer(argsBuffer.baseAddress!).bindMemory( + to: Int8.self, capacity: argsBuffer.count) + var cStrings: [UnsafePointer?] = argsOffsets.map { ptr + $0 } + cStrings[cStrings.count - 1] = nil + return cStrings.withUnsafeMutableBufferPointer { + let unsafeString = UnsafeMutableRawPointer($0.baseAddress!).bindMemory( + to: UnsafePointer?.self, capacity: $0.count) + return body(unsafeString) + } + } +} From 7ba9d485ab44cca5c61cf974679d6fba91feb1ca Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 27 Feb 2025 13:18:57 -0800 Subject: [PATCH 2/3] [Planning] Avoid batching compile job twice When using incremental build, the FirstWaveComputer has already batched all the compile jobs. `planStandardCompile()` can just reuse the batch job information from incremental state to construct the jobs. --- Sources/SwiftDriver/Jobs/Planning.swift | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 357ec699d..4851cb62b 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -90,16 +90,17 @@ extension Driver { incrementalCompilationState = nil } - return try ( - // For compatibility with swiftpm, the driver produces batched jobs - // for every job, even when run in incremental mode, so that all jobs - // can be returned from `planBuild`. - // But in that case, don't emit lifecycle messages. - formBatchedJobs(jobsInPhases.allJobs, - showJobLifecycle: showJobLifecycle && incrementalCompilationState == nil, - jobCreatingPch: jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH})), - incrementalCompilationState - ) + let batchedJobs: [Job] + // If the jobs are batched during the incremental build, reuse the computation rather than computing the batches again. + if let incrementalState = incrementalCompilationState { + batchedJobs = incrementalState.mandatoryJobsInOrder + incrementalState.jobsAfterCompiles + } else { + batchedJobs = try formBatchedJobs(jobsInPhases.allJobs, + showJobLifecycle: showJobLifecycle, + jobCreatingPch: jobsInPhases.allJobs.first(where: {$0.kind == .generatePCH})) + } + + return (batchedJobs, incrementalCompilationState) } /// If performing an explicit module build, compute an inter-module dependency graph. From 5ea08fecbc342446c763f89e283e9f6892802471 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Thu, 27 Feb 2025 13:24:11 -0800 Subject: [PATCH 3/3] [Caching] Don't produce cache key for initial compile job in batch mode During batch mode, the initially computed compilation tasks will be batched later. There is no need to compute cache keys for those Jobs since they will not be executed. --- Sources/SwiftDriver/Jobs/CompileJob.swift | 3 +- Sources/SwiftDriver/Jobs/Planning.swift | 11 +++- .../SwiftDriverTests/CachingBuildTests.swift | 61 ++++++++++++++++--- 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 6f2aa1051..9a85d6b5c 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -231,7 +231,8 @@ extension Driver { outputType: FileType?, addJobOutputs: ([TypedVirtualPath]) -> Void, pchCompileJob: Job?, - emitModuleTrace: Bool) + emitModuleTrace: Bool, + produceCacheKey: Bool) throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [] diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index 4851cb62b..ed801aba3 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -356,7 +356,8 @@ extension Driver { outputType: compilerOutputType, addJobOutputs: addJobOutputs, pchCompileJob: pchCompileJob, - emitModuleTrace: emitModuleTrace) + emitModuleTrace: emitModuleTrace, + produceCacheKey: true) addJob(compile) return compile } @@ -447,11 +448,14 @@ extension Driver { // We can skip the compile jobs if all we want is a module when it's // built separately. if parsedOptions.hasArgument(.driverExplicitModuleBuild), canSkipIfOnlyModule { return } + // If we are in the batch mode, the constructed jobs here will be batched + // later. There is no need to produce cache key for the job. let compile = try compileJob(primaryInputs: [primaryInput], outputType: compilerOutputType, addJobOutputs: addJobOutputs, pchCompileJob: pchCompileJob, - emitModuleTrace: emitModuleTrace) + emitModuleTrace: emitModuleTrace, + produceCacheKey: !compilerMode.isBatchCompile) addCompileJob(compile) } @@ -873,7 +877,8 @@ extension Driver { outputType: compilerOutputType, addJobOutputs: {_ in }, pchCompileJob: jobCreatingPch, - emitModuleTrace: constituentsEmittedModuleTrace) + emitModuleTrace: constituentsEmittedModuleTrace, + produceCacheKey: true) } return batchedCompileJobs + noncompileJobs } diff --git a/Tests/SwiftDriverTests/CachingBuildTests.swift b/Tests/SwiftDriverTests/CachingBuildTests.swift index d883e985f..192b8deba 100644 --- a/Tests/SwiftDriverTests/CachingBuildTests.swift +++ b/Tests/SwiftDriverTests/CachingBuildTests.swift @@ -986,17 +986,60 @@ final class CachingBuildTests: XCTestCase { XCTFail("Cached compilation doesn't have a CAS") } try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem) + } + } - // try replan the job and make sure some key command-line options are generated. - let rebuildJobs = try driver.planBuild() - for job in rebuildJobs { - if job.kind == .compile || job.kind == .emitModule { - XCTAssertTrue(job.commandLine.contains(.flag(String("-disable-implicit-swift-modules")))) - XCTAssertTrue(job.commandLine.contains(.flag(String("-cache-compile-job")))) - XCTAssertTrue(job.commandLine.contains(.flag(String("-cas-path")))) - XCTAssertTrue(job.commandLine.contains(.flag(String("-bridging-header-pch-key")))) - } + func testCacheBatchBuildPlan() throws { + try withTemporaryDirectory { path in + try localFileSystem.changeCurrentWorkingDirectory(to: path) + let moduleCachePath = path.appending(component: "ModuleCache") + let casPath = path.appending(component: "cas") + try localFileSystem.createDirectory(moduleCachePath) + let main = path.appending(component: "testCachingBuild.swift") + let mainFileContent = "import C;import E;import G;" + try localFileSystem.writeFileContents(main) { + $0.send(mainFileContent) } + let ofm = path.appending(component: "ofm.json") + let inputPathsAndContents: [(AbsolutePath, String)] = [(main, mainFileContent)] + OutputFileMapCreator.write( + module: "Test", inputPaths: inputPathsAndContents.map {$0.0}, + derivedData: path, to: ofm, excludeMainEntry: false) + + let cHeadersPath: AbsolutePath = + try testInputsPath.appending(component: "ExplicitModuleBuilds") + .appending(component: "CHeaders") + let swiftModuleInterfacesPath: AbsolutePath = + try testInputsPath.appending(component: "ExplicitModuleBuilds") + .appending(component: "Swift") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + let bridgingHeaderpath: AbsolutePath = + cHeadersPath.appending(component: "Bridging.h") + var driver = try Driver(args: ["swiftc", + "-I", cHeadersPath.nativePathString(escaped: true), + "-I", swiftModuleInterfacesPath.nativePathString(escaped: true), + "-explicit-module-build", "-Rcache-compile-job", "-incremental", + "-module-cache-path", moduleCachePath.nativePathString(escaped: true), + "-cache-compile-job", "-cas-path", casPath.nativePathString(escaped: true), + "-import-objc-header", bridgingHeaderpath.nativePathString(escaped: true), + "-output-file-map", ofm.nativePathString(escaped: true), + "-working-directory", path.nativePathString(escaped: true), + main.nativePathString(escaped: true)] + sdkArgumentsForTesting, + interModuleDependencyOracle: dependencyOracle) + let jobs = try driver.planBuild() + try driver.run(jobs: jobs) + XCTAssertFalse(driver.diagnosticEngine.hasErrors) + + let scanLibPath = try XCTUnwrap(driver.getSwiftScanLibPath()) + try dependencyOracle.verifyOrCreateScannerInstance(swiftScanLibPath: scanLibPath) + + let cas = try dependencyOracle.getOrCreateCAS(pluginPath: nil, onDiskPath: casPath, pluginOptions: []) + if let driverCAS = driver.cas { + XCTAssertEqual(cas, driverCAS, "CAS should only be created once") + } else { + XCTFail("Cached compilation doesn't have a CAS") + } + try checkCASForResults(jobs: jobs, cas: cas, fs: driver.fileSystem) } }