external dependency hashing for incremental

benb · benb · commit 20fd81e15344 · 2025-06-24T17:42:20.000+01:00
diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt
@@ -114,6 +114,7 @@ add_library(SwiftDriver
   Utilities/DateAdditions.swift
   Utilities/Diagnostics.swift
   Utilities/FileList.swift
+  Utilities/FileMetadata.swift
   Utilities/FileType.swift
   Utilities/PredictableRandomNumberGenerator.swift
   Utilities/RelativePathAdditions.swift
diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift
@@ -45,15 +45,6 @@ extension Driver.ErrorDiagnostics: CustomStringConvertible {
   }
 }
 
-public struct FileMetadata {
-    public let mTime: TimePoint
-    public let hash: String
-    init(mTime: TimePoint, hash: String = "") {
-        self.mTime = mTime
-        self.hash = hash
-    }
-}
-
 /// The Swift driver.
 public struct Driver {
   public enum Error: Swift.Error, Equatable, DiagnosticData {
@@ -994,8 +985,8 @@ public struct Driver {
     self.recordedInputMetadata = .init(uniqueKeysWithValues:
       Set(inputFiles).compactMap { inputFile -> (TypedVirtualPath, FileMetadata)? in 
         guard let modTime = try? fileSystem.lastModificationTime(for: inputFile.file) else { return nil }
-        guard let data = try? fileSystem.readFileContents(inputFile.file)  else { return nil }
         if incrementalFileHashes {
+            guard let data = try? fileSystem.readFileContents(inputFile.file)  else { return nil }
             let hash = SHA256().hash(data).hexadecimalRepresentation
             return (inputFile, FileMetadata(mTime: modTime, hash: hash))
         } else {
diff --git a/Sources/SwiftDriver/Execution/DriverExecutor.swift b/Sources/SwiftDriver/Execution/DriverExecutor.swift
@@ -27,6 +27,11 @@ public protocol DriverExecutor {
                forceResponseFiles: Bool,
                recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult
 
+  func execute(job: Job,
+               forceResponseFiles: Bool,
+               recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> ProcessResult
+
+
   /// Execute multiple jobs, tracking job status using the provided execution delegate.
   /// Pass in the `IncrementalCompilationState` to allow for incremental compilation.
   /// Pass in the `InterModuleDependencyGraph` to allow for module dependency tracking.
@@ -37,6 +42,13 @@ public protocol DriverExecutor {
                recordedInputMetadata: [TypedVirtualPath: FileMetadata]
   ) throws
 
+  func execute(workload: DriverExecutorWorkload,
+               delegate: JobExecutionDelegate,
+               numParallelJobs: Int,
+               forceResponseFiles: Bool,
+               recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+  ) throws
+
   /// Execute multiple jobs, tracking job status using the provided execution delegate.
   func execute(jobs: [Job],
                delegate: JobExecutionDelegate,
@@ -45,6 +57,13 @@ public protocol DriverExecutor {
                recordedInputMetadata: [TypedVirtualPath: FileMetadata]
   ) throws
 
+  func execute(jobs: [Job],
+               delegate: JobExecutionDelegate,
+               numParallelJobs: Int,
+               forceResponseFiles: Bool,
+               recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+  ) throws
+
   /// Launch a process with the given command line and report the result.
   @discardableResult
   func checkNonZeroExit(args: String..., environment: [String: String]) throws -> String
@@ -53,6 +72,34 @@ public protocol DriverExecutor {
   func description(of job: Job, forceResponseFiles: Bool) throws -> String
 }
 
+extension DriverExecutor {
+    public func execute(job: Job,
+                 forceResponseFiles: Bool,
+                 recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult
+    {
+        return try execute(job: job, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
+    }
+
+
+    public func execute(workload: DriverExecutorWorkload,
+                 delegate: JobExecutionDelegate,
+                 numParallelJobs: Int,
+                 forceResponseFiles: Bool,
+                 recordedInputMetadata: [TypedVirtualPath: FileMetadata]
+    ) throws {
+       try execute(workload: workload, delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
+    }
+
+    public func execute(jobs: [Job],
+                 delegate: JobExecutionDelegate,
+                 numParallelJobs: Int,
+                 forceResponseFiles: Bool,
+                 recordedInputMetadata: [TypedVirtualPath: FileMetadata]
+    ) throws {
+        try execute(jobs: jobs, delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
+  }
+}
+
 public struct DriverExecutorWorkload {
   public let continueBuildingAfterErrors: Bool
   public enum Kind {
@@ -121,14 +168,14 @@ extension DriverExecutor {
     delegate: JobExecutionDelegate,
     numParallelJobs: Int,
     forceResponseFiles: Bool,
-    recordedInputMetadata: [TypedVirtualPath: FileMetadata]
+    recordedInputModificationDates: [TypedVirtualPath: TimePoint]
   ) throws {
     try execute(
       workload: .all(jobs),
       delegate: delegate,
       numParallelJobs: numParallelJobs,
       forceResponseFiles: forceResponseFiles,
-      recordedInputMetadata: recordedInputMetadata)
+      recordedInputModificationDates: recordedInputModificationDates)
   }
 
   static func computeReturnCode(exitStatus: ProcessResult.ExitStatus) -> Int {
diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift
@@ -223,6 +223,8 @@ internal extension InterModuleDependencyGraph {
         reporter?.report("Unable to 'stat' \(inputPath.description)")
         return false
       }
+      // SHA256 hashes for these files from the previous build are not
+      // currently stored, so we can only check timestamps
       if inputModTime > outputModTime {
         reporter?.reportExplicitDependencyOutOfDate(moduleName,
                                                     inputPath: inputPath.description)
diff --git a/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift b/Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift
@@ -320,6 +320,8 @@ extension IncrementalCompilationState.FirstWaveComputer {
       let previousModTime = inputInfo?.previousModTime
       let previousHash = inputInfo?.hash 
 
+      assert(metadata.hash != nil || !useHashes)
+
       switch previousCompilationStatus {
       case .upToDate where metadata.mTime == previousModTime:
         reporter?.report("May skip current input:", input)
diff --git a/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift b/Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift
@@ -26,9 +26,9 @@ import struct TSCBasic.SHA256
   /// The last known modification time of this input.
   /*@_spi(Testing)*/ public let previousModTime: TimePoint
 
-  /*@_spi(Testing)*/ public let hash: String
+  /*@_spi(Testing)*/ public let hash: String?
 
-  /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint, hash: String) {
+  /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint, hash: String?) {
     self.status = status
     self.previousModTime = previousModTime
     self.hash = hash
diff --git a/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift
@@ -14,6 +14,7 @@ import SwiftOptions
 
 import protocol TSCBasic.FileSystem
 import struct TSCBasic.ByteString
+import struct TSCBasic.SHA256
 import class Dispatch.DispatchQueue
 import struct Foundation.TimeInterval
 
@@ -63,11 +64,12 @@ import struct Foundation.TimeInterval
     _ phase: Phase,
     _ internedStringTable: InternedStringTable,
     _ nodeFinder: NodeFinder,
-    _ fingerprintedExternalDependencies: Set<FingerprintedExternalDependency>
+    _ fingerprintedExternalDependencies: Set<FingerprintedExternalDependency>,
+    _ externalDependencyFileHashes: Dictionary<ExternalDependency, String>
   ) {
     self.buildRecord = buildRecord
     self.currencyCache = ExternalDependencyCurrencyCache(
-      info.fileSystem, buildStartTime: buildRecord.buildStartTime)
+      info.fileSystem, buildStartTime: buildRecord.buildStartTime, externalDependencyFileHashes: externalDependencyFileHashes, useFileHashes: info.useFileHashesInModuleDependencyGraph)
     self.info = info
     self.dotFileWriter = info.emitDependencyDotFileAfterEveryImport
       ? DependencyGraphDotFileWriter(info)
@@ -88,22 +90,24 @@ import struct Foundation.TimeInterval
            "If updating from prior, should be supplying more ingredients")
     self.init(buildRecord, info, phase, InternedStringTable(info.incrementalCompilationQueue),
               NodeFinder(),
-              Set())
+              Set(), Dictionary())
   }
 
   public static func createFromPrior(
     _ buildRecord: BuildRecord,
     _ info: IncrementalCompilationState.IncrementalDependencyAndInputSetup,
     _ internedStringTable: InternedStringTable,
     _ nodeFinder: NodeFinder,
-    _ fingerprintedExternalDependencies: Set<FingerprintedExternalDependency>
+    _ fingerprintedExternalDependencies: Set<FingerprintedExternalDependency>,
+    _ externalDependencyFileHashes: Dictionary<ExternalDependency, String>
   ) -> Self {
     self.init(buildRecord,
               info,
               .updatingFromAPrior,
               internedStringTable,
               nodeFinder,
-              fingerprintedExternalDependencies)
+              fingerprintedExternalDependencies,
+              externalDependencyFileHashes)
   }
 
   public static func createForBuildingFromSwiftDeps(
@@ -567,10 +571,24 @@ extension ModuleDependencyGraph {
     private let fileSystem: FileSystem
     private let buildStartTime: TimePoint
     private var currencyCache = [ExternalDependency: Bool]()
+    public internal(set) var externalDependencyFileHashes: [ExternalDependency: String]
+    private let useFileHashes: Bool
 
-    init(_ fileSystem: FileSystem, buildStartTime: TimePoint) {
+    init(_ fileSystem: FileSystem, buildStartTime: TimePoint, externalDependencyFileHashes: Dictionary<ExternalDependency, String>, useFileHashes: Bool) {
       self.fileSystem = fileSystem
       self.buildStartTime = buildStartTime
+      self.externalDependencyFileHashes = externalDependencyFileHashes
+      self.useFileHashes = useFileHashes
+    }
+
+    mutating func updateHash(_ externalDependency: ExternalDependency) {
+        guard useFileHashes else {
+          return
+        }
+        if let depFile = externalDependency.path, 
+        let data = try? fileSystem.readFileContents(depFile) {
+            externalDependencyFileHashes[externalDependency] = SHA256().hash(data).hexadecimalRepresentation
+        }
     }
 
     mutating func beCurrent(_ externalDependency: ExternalDependency) {
@@ -581,11 +599,31 @@ extension ModuleDependencyGraph {
       if let cachedResult = self.currencyCache[externalDependency] {
         return cachedResult
       }
-      let uncachedResult = isCurrentWRTFileSystem(externalDependency)
+      var uncachedResult = isCurrentWRTFileSystem(externalDependency)
+      if useFileHashes && !uncachedResult {
+          uncachedResult = isCurrentWRTFileHash(externalDependency)
+      }
       self.currencyCache[externalDependency] = uncachedResult
+      if !uncachedResult {
+          self.updateHash(externalDependency)
+      }
       return uncachedResult
     }
 
+    mutating func ensureHash(_ externalDependency: ExternalDependency) {
+        if (self.externalDependencyFileHashes[externalDependency] ?? "").isEmpty {
+            self.updateHash(externalDependency)
+        }
+    }
+
+    private func isCurrentWRTFileHash(_ externalDependency: ExternalDependency) -> Bool {
+      if let depFile = externalDependency.path, 
+        let data = try? fileSystem.readFileContents(depFile) {
+            return self.externalDependencyFileHashes[externalDependency] == SHA256().hash(data).hexadecimalRepresentation
+        }
+      return false
+    }
+
     private func isCurrentWRTFileSystem(_ externalDependency: ExternalDependency) -> Bool {
       if let depFile = externalDependency.path,
          let fileModTime = try? self.fileSystem.lastModificationTime(for: depFile),
@@ -753,6 +791,7 @@ extension ModuleDependencyGraph {
       private var currentDefKey: DependencyKey? = nil
       private var nodeUses: [(DependencyKey, Int)] = []
       private var fingerprintedExternalDependencies = Set<FingerprintedExternalDependency>()
+      private var externalDependencyFileHashes = Dictionary<ExternalDependency, String>()
 
       /// Deserialized nodes, in order appearing in the priors file. If `nil`, the node is for a removed source file.
       ///
@@ -789,7 +828,8 @@ extension ModuleDependencyGraph {
                                                           info,
                                                           internedStringTable,
                                                           nodeFinder,
-                                                          fingerprintedExternalDependencies)
+                                                          fingerprintedExternalDependencies,
+                                                          externalDependencyFileHashes)
         for (dependencyKey, useID) in self.nodeUses {
           guard let use = self.potentiallyUsedNodes[useID] else {
             // Don't record uses of defs of removed files.
@@ -982,16 +1022,19 @@ extension ModuleDependencyGraph {
           }
           self.nodeUses.append( (key, Int(record.fields[0])) )
         case .externalDepNode:
-          guard record.fields.count == 2
+          guard record.fields.count == 2,
+          case .blob(let hashBlob) = record.payload
           else {
             throw malformedError
           }
           let path = try internedString(field: 0)
           let fingerprint = try nonemptyInternedString(field: 1)
+          let hash = String(decoding: hashBlob, as: UTF8.self)
+          let externalDependency = ExternalDependency(fileName: path, internedStringTable)
+          
           fingerprintedExternalDependencies.insert(
-            FingerprintedExternalDependency(
-              ExternalDependency(fileName: path, internedStringTable),
-              fingerprint))
+            FingerprintedExternalDependency(externalDependency, fingerprint))
+          externalDependencyFileHashes[externalDependency] = hash
         case .identifierNode:
           guard record.fields.count == 0,
                 case .blob(let identifierBlob) = record.payload
@@ -1170,7 +1213,7 @@ extension ModuleDependencyGraph {
           $0.append(inputInfo.previousModTime)
           $0.append(inputInfo.status.code)
           $0.append(pathID)
-        }, blob: inputInfo.hash)
+        }, blob: inputInfo.hash ?? "")
       }
     }
 
@@ -1282,6 +1325,8 @@ extension ModuleDependencyGraph {
         .vbr(chunkBitWidth: 13),
         // fingerprint ID
         .vbr(chunkBitWidth: 13),
+        // file hash
+        .blob,
       ])
       self.abbreviate(.identifierNode, [
         .literal(RecordID.identifierNode.rawValue),
@@ -1356,13 +1401,15 @@ extension ModuleDependencyGraph {
           }
         }
         for fingerprintedExternalDependency in graph.fingerprintedExternalDependencies {
-          serializer.stream.writeRecord(serializer.abbreviations[.externalDepNode]!) {
+          graph.currencyCache.ensureHash(fingerprintedExternalDependency.externalDependency)
+          serializer.stream.writeRecord(serializer.abbreviations[.externalDepNode]!, {
             $0.append(RecordID.externalDepNode)
             $0.append(serializer.lookupIdentifierCode(
               for: fingerprintedExternalDependency.externalDependency.fileName))
             $0.append( serializer.lookupIdentifierCode(
               for: fingerprintedExternalDependency.fingerprint))
-          }
+          }, 
+          blob: graph.currencyCache.externalDependencyFileHashes[fingerprintedExternalDependency.externalDependency] ?? "")
         }
       }
       return ByteString(serializer.stream.data)
diff --git a/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift b/Sources/SwiftDriver/ToolingInterface/SimpleExecutor.swift
@@ -45,9 +45,45 @@ import class TSCBasic.Process
   public func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate,
                numParallelJobs: Int, forceResponseFiles: Bool,
                recordedInputMetadata: [TypedVirtualPath : FileMetadata]) throws {
-    fatalError("Unsupported operation on current executor")
+      fatalError("Unsupported operation on current executor")
   }
 
+  public func execute(job: Job,
+               forceResponseFiles: Bool,
+               recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws -> ProcessResult {
+      fatalError("Unsupported legacy operation on current executor")
+  }
+
+  public func execute(workload: DriverExecutorWorkload, delegate: JobExecutionDelegate,
+                       numParallelJobs: Int, forceResponseFiles: Bool,
+                      recordedInputModificationDates: [TypedVirtualPath : TimePoint]) throws {
+      fatalError("Unsupported operation on current executor")
+  }
+
+  public func execute(jobs: [Job],
+                      delegate: JobExecutionDelegate,
+                      numParallelJobs: Int,
+                      forceResponseFiles: Bool,
+                      recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws {
+      fatalError("Unsupported legacy operation on current executor")
+  }
+
+  public func execute(
+    jobs: [Job],
+    delegate: JobExecutionDelegate,
+    numParallelJobs: Int,
+    forceResponseFiles: Bool,
+    recordedInputMetadata: [TypedVirtualPath: FileMetadata]
+  ) throws {
+    try execute(
+      workload: .all(jobs),
+      delegate: delegate,
+      numParallelJobs: numParallelJobs,
+      forceResponseFiles: forceResponseFiles,
+      recordedInputMetadata: recordedInputMetadata)
+  }
+
+
   public func checkNonZeroExit(args: String..., environment: [String : String]) throws -> String {
     try Process.checkNonZeroExit(arguments: args, environment: environment)
   }
diff --git a/Sources/SwiftDriver/Utilities/FileMetadata.swift b/Sources/SwiftDriver/Utilities/FileMetadata.swift
diff --git a/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift
diff --git a/Tests/SwiftDriverTests/IncrementalCompilationTests.swift b/Tests/SwiftDriverTests/IncrementalCompilationTests.swift
diff --git a/Tests/SwiftDriverTests/SwiftDriverTests.swift b/Tests/SwiftDriverTests/SwiftDriverTests.swift

Original file line number	Diff line number	Diff line change
`@@ -223,6 +223,8 @@ internal extension InterModuleDependencyGraph {`
`223`	`223`	`reporter?.report("Unable to 'stat' \(inputPath.description)")`
`224`	`224`	`return false`
`225`	`225`	`}`
	`226`	`+ // SHA256 hashes for these files from the previous build are not`
	`227`	`+ // currently stored, so we can only check timestamps`
`226`	`228`	`if inputModTime > outputModTime {`
`227`	`229`	`reporter?.reportExplicitDependencyOutOfDate(moduleName,`
`228`	`230`	`inputPath: inputPath.description)`