source file hashing for incremental

Author: benb

Sources/SwiftDriver/Driver/Driver.swift

@@ -24,6 +24,7 @@ import struct TSCBasic.ByteString
 import struct TSCBasic.Diagnostic
 import struct TSCBasic.FileInfo
 import struct TSCBasic.RelativePath
+import struct TSCBasic.SHA256
 import var TSCBasic.localFileSystem
 import var TSCBasic.stderrStream
 import var TSCBasic.stdoutStream
@@ -44,6 +45,14 @@ extension Driver.ErrorDiagnostics: CustomStringConvertible {
   }
 }
 
+public struct FileMetadata {
+    public let mTime: TimePoint
+    public let hash: String
+    init(mTime: TimePoint, hash: String = "") {
+        self.mTime = mTime
+        self.hash = hash
+    }
+}
 
 /// The Swift driver.
 public struct Driver {
@@ -209,8 +218,8 @@ public struct Driver {
   /// The set of input files
   @_spi(Testing) public let inputFiles: [TypedVirtualPath]
 
-  /// The last time each input file was modified, recorded at the start of the build.
-  @_spi(Testing) public let recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+  /// The last time each input file was modified, and the file's SHA256 hash, recorded at the start of the build.
+  @_spi(Testing) public let recordedInputMetadata: [TypedVirtualPath: FileMetadata]
 
   /// The mapping from input files to output files for each kind.
   let outputFileMap: OutputFileMap?
@@ -978,11 +987,20 @@ public struct Driver {
     // Classify and collect all of the input files.
     let inputFiles = try Self.collectInputFiles(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine, fileSystem: self.fileSystem)
     self.inputFiles = inputFiles
-    self.recordedInputModificationDates = .init(uniqueKeysWithValues:
-      Set(inputFiles).compactMap {
-        guard let modTime = try? fileSystem
-          .lastModificationTime(for: $0.file) else { return nil }
-        return ($0, modTime)
+
+    let incrementalFileHashes = parsedOptions.hasFlag(positive: .enableIncrementalFileHashing,
+                                                      negative: .disableIncrementalFileHashing,
+                                                      default: false)
+    self.recordedInputMetadata = .init(uniqueKeysWithValues:
+      Set(inputFiles).compactMap { inputFile -> (TypedVirtualPath, FileMetadata)? in 
+        guard let modTime = try? fileSystem.lastModificationTime(for: inputFile.file) else { return nil }
+        guard let data = try? fileSystem.readFileContents(inputFile.file)  else { return nil }
+        if incrementalFileHashes {
+            let hash = SHA256().hash(data).hexadecimalRepresentation
+            return (inputFile, FileMetadata(mTime: modTime, hash: hash))
+        } else {
+            return (inputFile, FileMetadata(mTime: modTime))
+        }
     })
 
     do {
@@ -1101,7 +1119,7 @@ public struct Driver {
       outputFileMap: outputFileMap,
       incremental: self.shouldAttemptIncrementalCompilation,
       parsedOptions: parsedOptions,
-      recordedInputModificationDates: recordedInputModificationDates)
+      recordedInputMetadata: recordedInputMetadata)
 
     self.supportedFrontendFlags =
       try Self.computeSupportedCompilerArgs(of: self.toolchain,
@@ -1949,7 +1967,7 @@ extension Driver {
       }
       try executor.execute(job: inPlaceJob,
                            forceResponseFiles: forceResponseFiles,
-                           recordedInputModificationDates: recordedInputModificationDates)
+                           recordedInputMetadata: recordedInputMetadata)
     }
 
     // If requested, warn for options that weren't used by the driver after the build is finished.
@@ -1994,7 +2012,7 @@ extension Driver {
       delegate: jobExecutionDelegate,
       numParallelJobs: numParallelJobs ?? 1,
       forceResponseFiles: forceResponseFiles,
-      recordedInputModificationDates: recordedInputModificationDates)
+      recordedInputMetadata: recordedInputMetadata)
   }
 
   public func writeIncrementalBuildInformation(_ jobs: [Job]) {

Sources/SwiftDriver/Execution/DriverExecutor.swift

@@ -25,7 +25,7 @@ public protocol DriverExecutor {
   @discardableResult
   func execute(job: Job,
                forceResponseFiles: Bool,
-               recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> ProcessResult
+               recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult
 
   /// Execute multiple jobs, tracking job status using the provided execution delegate.
   /// Pass in the `IncrementalCompilationState` to allow for incremental compilation.
@@ -34,15 +34,15 @@ public protocol DriverExecutor {
                delegate: JobExecutionDelegate,
                numParallelJobs: Int,
                forceResponseFiles: Bool,
-               recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+               recordedInputMetadata: [TypedVirtualPath: FileMetadata]
   ) throws
 
   /// Execute multiple jobs, tracking job status using the provided execution delegate.
   func execute(jobs: [Job],
                delegate: JobExecutionDelegate,
                numParallelJobs: Int,
                forceResponseFiles: Bool,
-               recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+               recordedInputMetadata: [TypedVirtualPath: FileMetadata]
   ) throws
 
   /// Launch a process with the given command line and report the result.
@@ -96,10 +96,10 @@ extension DriverExecutor {
   func execute<T: Decodable>(job: Job,
                              capturingJSONOutputAs outputType: T.Type,
                              forceResponseFiles: Bool,
-                             recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> T {
+                             recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> T {
     let result = try execute(job: job,
                              forceResponseFiles: forceResponseFiles,
-                             recordedInputModificationDates: recordedInputModificationDates)
+                             recordedInputMetadata: recordedInputMetadata)
 
     if (result.exitStatus != .terminated(code: EXIT_SUCCESS)) {
       let returnCode = Self.computeReturnCode(exitStatus: result.exitStatus)
@@ -121,14 +121,14 @@ extension DriverExecutor {
     delegate: JobExecutionDelegate,
     numParallelJobs: Int,
     forceResponseFiles: Bool,
-    recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+    recordedInputMetadata: [TypedVirtualPath: FileMetadata]
   ) throws {
     try execute(
       workload: .all(jobs),
       delegate: delegate,
       numParallelJobs: numParallelJobs,
       forceResponseFiles: forceResponseFiles,
-      recordedInputModificationDates: recordedInputModificationDates)
+      recordedInputMetadata: recordedInputMetadata)
   }
 
   static func computeReturnCode(exitStatus: ProcessResult.ExitStatus) -> Int {

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -227,7 +227,7 @@ public extension Driver {
         try self.executor.execute(job: preScanJob,
                                   capturingJSONOutputAs: InterModuleDependencyImports.self,
                                   forceResponseFiles: forceResponseFiles,
-                                  recordedInputModificationDates: recordedInputModificationDates)
+                                  recordedInputMetadata: recordedInputMetadata)
     }
     return imports
   }
@@ -304,7 +304,7 @@ public extension Driver {
         try self.executor.execute(job: scannerJob,
                                   capturingJSONOutputAs: InterModuleDependencyGraph.self,
                                   forceResponseFiles: forceResponseFiles,
-                                  recordedInputModificationDates: recordedInputModificationDates)
+                                  recordedInputMetadata: recordedInputMetadata)
     }
     return dependencyGraph
   }

Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift

@@ -46,7 +46,7 @@ extension BuildRecord {
   init(jobs: [Job],
        finishedJobResults: [BuildRecordInfo.JobResult],
        skippedInputs: Set<TypedVirtualPath>?,
-       compilationInputModificationDates: [TypedVirtualPath: TimePoint],
+       compilationInputModificationDates: [TypedVirtualPath: FileMetadata],
        actualSwiftVersion: String,
        argsHash: String,
        timeBeforeFirstJob: TimePoint,
@@ -57,10 +57,10 @@ extension BuildRecord {
         entry.job.inputsGeneratingCode.map { ($0, entry.result) }
     })
     let inputInfosArray = compilationInputModificationDates
-      .map { input, modDate -> (VirtualPath, InputInfo) in
+      .map { input, metadata -> (VirtualPath, InputInfo) in
         let status = InputInfo.Status(  wasSkipped: skippedInputs?.contains(input),
                                         jobResult: jobResultsByInput[input])
-        return (input.file, InputInfo(status: status, previousModTime: modDate))
+        return (input.file, InputInfo(status: status, previousModTime: metadata.mTime, hash: metadata.hash))
       }
 
     self.init(

Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

@@ -48,7 +48,7 @@ import class Dispatch.DispatchQueue
   @_spi(Testing) public let actualSwiftVersion: String
   @_spi(Testing) public let timeBeforeFirstJob: TimePoint
   let diagnosticEngine: DiagnosticsEngine
-  let compilationInputModificationDates: [TypedVirtualPath: TimePoint]
+  let compilationInputModificationDates: [TypedVirtualPath: FileMetadata]
   private var explicitModuleDependencyGraph: InterModuleDependencyGraph? = nil
 
   private var finishedJobResults = [JobResult]()
@@ -64,7 +64,7 @@ import class Dispatch.DispatchQueue
     actualSwiftVersion: String,
     timeBeforeFirstJob: TimePoint,
     diagnosticEngine: DiagnosticsEngine,
-    compilationInputModificationDates: [TypedVirtualPath: TimePoint])
+    compilationInputModificationDates: [TypedVirtualPath: FileMetadata])
   {
     self.buildRecordPath = buildRecordPath
     self.fileSystem = fileSystem
@@ -85,7 +85,7 @@ import class Dispatch.DispatchQueue
     outputFileMap: OutputFileMap?,
     incremental: Bool,
     parsedOptions: ParsedOptions,
-    recordedInputModificationDates: [TypedVirtualPath: TimePoint]
+    recordedInputMetadata: [TypedVirtualPath: FileMetadata]
   ) {
     // Cannot write a buildRecord without a path.
     guard let buildRecordPath = try? Self.computeBuildRecordPath(
@@ -99,7 +99,7 @@ import class Dispatch.DispatchQueue
     }
     let currentArgsHash = BuildRecordArguments.computeHash(parsedOptions)
     let compilationInputModificationDates =
-      recordedInputModificationDates.filter { input, _ in
+      recordedInputMetadata.filter { input, _ in
         input.type.isPartOfSwiftCompilation
       }
 

Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

@@ -26,6 +26,7 @@ extension IncrementalCompilationState {
     let showJobLifecycle: Bool
     let alwaysRebuildDependents: Bool
     let interModuleDependencyGraph: InterModuleDependencyGraph?
+    let useHashes: Bool
     /// If non-null outputs information for `-driver-show-incremental` for input path
     private let reporter: Reporter?
 
@@ -45,6 +46,7 @@ extension IncrementalCompilationState {
       self.showJobLifecycle = driver.showJobLifecycle
       self.alwaysRebuildDependents = initialState.incrementalOptions.contains(
         .alwaysRebuildDependents)
+      self.useHashes = initialState.incrementalOptions.contains(.useFileHashesInModuleDependencyGraph)
       self.interModuleDependencyGraph = interModuleDependencyGraph
       self.reporter = reporter
     }
@@ -301,8 +303,9 @@ extension IncrementalCompilationState.FirstWaveComputer {
     /// The status of the input file.
     let status: InputInfo.Status
     /// If `true`, the modification time of this input matches the modification
-    /// time recorded from the prior build in the build record.
-    let datesMatch: Bool
+    /// time recorded from the prior build in the build record, or the hash of
+    /// its contents match.
+    let metadataMatch: Bool
   }
 
   // Find the inputs that have changed since last compilation, or were marked as needed a build
@@ -311,16 +314,19 @@ extension IncrementalCompilationState.FirstWaveComputer {
   ) -> [ChangedInput] {
     jobsInPhases.compileJobs.compactMap { job in
       let input = job.primaryInputs[0]
-      let modDate = buildRecordInfo.compilationInputModificationDates[input] ?? .distantFuture
+      let metadata = buildRecordInfo.compilationInputModificationDates[input] ?? FileMetadata(mTime: .distantFuture)
       let inputInfo = outOfDateBuildRecord.inputInfos[input.file]
       let previousCompilationStatus = inputInfo?.status ?? .newlyAdded
       let previousModTime = inputInfo?.previousModTime
+      let previousHash = inputInfo?.hash 
 
       switch previousCompilationStatus {
-      case .upToDate where modDate == previousModTime:
+      case .upToDate where metadata.mTime == previousModTime:
         reporter?.report("May skip current input:", input)
         return nil
-
+      case .upToDate where useHashes && (metadata.hash == previousHash):
+        reporter?.report("May skip current input (identical hash):", input)
+        return nil
       case .upToDate:
         reporter?.report("Scheduling changed input", input)
       case .newlyAdded:
@@ -330,9 +336,10 @@ extension IncrementalCompilationState.FirstWaveComputer {
       case .needsNonCascadingBuild:
         reporter?.report("Scheduling noncascading build", input)
       }
+      let metadataMatch = metadata.mTime == previousModTime || (useHashes && metadata.hash == previousHash)
       return ChangedInput(typedFile: input,
                           status: previousCompilationStatus,
-                          datesMatch: modDate == previousModTime)
+                          metadataMatch: metadataMatch )
     }
   }
 
@@ -381,7 +388,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
   ) -> [TypedVirtualPath] {
     changedInputs.compactMap { changedInput in
       let inputIsUpToDate =
-        changedInput.datesMatch && !inputsMissingOutputs.contains(changedInput.typedFile)
+        changedInput.metadataMatch && !inputsMissingOutputs.contains(changedInput.typedFile)
       let basename = changedInput.typedFile.file.basename
 
       // If we're asked to always rebuild dependents, all we need to do is

Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import struct TSCBasic.ProcessResult
+import struct TSCBasic.SHA256
 
 /// Contains information about the current status of an input to the incremental
 /// build.
@@ -25,9 +26,12 @@ import struct TSCBasic.ProcessResult
   /// The last known modification time of this input.
   /*@_spi(Testing)*/ public let previousModTime: TimePoint
 
-  /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint) {
+  /*@_spi(Testing)*/ public let hash: String
+
+  /*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint, hash: String) {
     self.status = status
     self.previousModTime = previousModTime
+    self.hash = hash
   }
 }
 

Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift

@@ -923,6 +923,7 @@ extension ModuleDependencyGraph {
         case .inputInfo:
           guard
             record.fields.count == 5,
+            case .blob(let hashBlob) = record.payload,
             let path = try nonemptyInternedString(field: 4)
           else {
             throw malformedError
@@ -931,12 +932,14 @@ extension ModuleDependencyGraph {
             lower: UInt32(record.fields[0]),
             upper: UInt32(record.fields[1]),
             nanoseconds: UInt32(record.fields[2]))
+          
           let status = try InputInfo.Status(code: UInt32(record.fields[3]))
           let pathString = path.lookup(in: internedStringTable)
           let pathHandle = try VirtualPath.intern(path: pathString)
+          let hash = String(decoding: hashBlob, as: UTF8.self)
           self.inputInfos[VirtualPath.lookup(pathHandle)] = InputInfo(
             status: status,
-            previousModTime: modTime)
+            previousModTime: modTime, hash: hash)
         case .moduleDepGraphNode:
           guard record.fields.count == 6 else {
             throw malformedError
@@ -1162,12 +1165,12 @@ extension ModuleDependencyGraph {
         let inputID = input.name.intern(in: self.internedStringTable)
         let pathID = self.lookupIdentifierCode(for: inputID)
 
-        self.stream.writeRecord(self.abbreviations[.inputInfo]!) {
+        self.stream.writeRecord(self.abbreviations[.inputInfo]!, {
           $0.append(RecordID.inputInfo)
           $0.append(inputInfo.previousModTime)
           $0.append(inputInfo.status.code)
           $0.append(pathID)
-        }
+        }, blob: inputInfo.hash)
       }
     }
 
@@ -1254,6 +1257,8 @@ extension ModuleDependencyGraph {
         .fixed(bitWidth: 3),
         // path ID
         .vbr(chunkBitWidth: 13),
+        // file hash
+        .blob,
       ])
       self.abbreviate(.moduleDepGraphNode,
         [Bitstream.Abbreviation.Operand.literal(RecordID.moduleDepGraphNode.rawValue)] +

Sources/SwiftDriver/Jobs/EmitSupportedFeaturesJob.swift

@@ -85,7 +85,7 @@ extension Driver {
       job: frontendFeaturesJob,
       capturingJSONOutputAs: SupportedCompilerFeatures.self,
       forceResponseFiles: false,
-      recordedInputModificationDates: [:]).SupportedArguments
+      recordedInputMetadata: [:]).SupportedArguments
     return Set(decodedSupportedFlagList)
   }
 

Sources/SwiftDriver/Jobs/Job.swift

@@ -152,9 +152,9 @@ extension Job {
     }
   }
 
-  public func verifyInputsNotModified(since recordedInputModificationDates: [TypedVirtualPath: TimePoint], fileSystem: FileSystem) throws {
+  public func verifyInputsNotModified(since recordedInputMetadata: [TypedVirtualPath: TimePoint], fileSystem: FileSystem) throws {
     for input in inputs {
-      if let recordedModificationTime = recordedInputModificationDates[input],
+      if let recordedModificationTime = recordedInputMetadata[input],
          try fileSystem.lastModificationTime(for: input.file) != recordedModificationTime {
         throw InputError.inputUnexpectedlyModified(input)
       }