fix cross-module-optimization builds and add the option -disable-cmo

The driver splits the compile job into two jobs: one for creating the object file, one for emitting the swiftmodule.
For the swiftmodule-job the driver adds -experimental-skip-non-inlinable-function-bodies-without-types.
This breaks CMO, because in CMO, the inlinable-decision is not derived from the AST but done in the optimizer.

As a quick hack we could disable -experimental-skip-non-inlinable-function-bodies-without-types.
But for CMO, the two-job approach is highly problematic, because it risks the two compile jobs to get out of sync, e.g. by different command line options, indeterministic behavior, etc. This can result in unresolved symbol errors, which are very hard to debug.

This change also adds a new option `-disable-cmo` to disable cross-module-optimization. It can be used to go back to the old behavior.

rdar://89223981

Author: eeckstein

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -282,6 +282,8 @@ extension Driver {
     try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs)
     // FIXME: MSVC runtime flags
 
+    addDisableCMOOption(commandLine: &commandLine)
+
     if parsedOptions.hasArgument(.parseAsLibrary, .emitLibrary) {
       commandLine.appendFlag(.parseAsLibrary)
     }

Sources/SwiftDriver/Jobs/EmitModuleJob.swift

@@ -97,6 +97,8 @@ extension Driver {
 
     addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: false)
 
+    addDisableCMOOption(commandLine: &commandLine)
+
     try commandLine.appendLast(.emitSymbolGraph, from: &parsedOptions)
     try commandLine.appendLast(.emitSymbolGraphDir, from: &parsedOptions)
     try commandLine.appendLast(.includeSpiSymbols, from: &parsedOptions)
@@ -143,6 +145,13 @@ extension Driver {
                                    default: true)
 
     case .singleCompile:
+      // Non library-evolution builds require a single job, because cross-module-optimization is enabled by default.
+      if !parsedOptions.hasArgument(.enableLibraryEvolution),
+         !parsedOptions.hasArgument(.disableCrossModuleOptimization),
+         let opt = parsedOptions.getLast(in: .O), opt.option != .Onone {
+        return false
+      }
+
       return parsedOptions.hasFlag(positive: .emitModuleSeparatelyWMO,
                                    negative: .noEmitModuleSeparatelyWMO,
                                    default: true) &&

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -300,6 +300,14 @@ extension Driver {
                                                            driver: self)
   }
 
+  /// Add options to disable cross-module-optimization.
+  mutating func addDisableCMOOption(commandLine: inout [Job.ArgTemplate]) {
+    if parsedOptions.hasArgument(.disableCrossModuleOptimization) {
+      commandLine.appendFlag(.Xllvm)
+      commandLine.appendFlag("-sil-disable-pass=cmo")
+    }
+  }
+
   mutating func addFrontendSupplementaryOutputArguments(commandLine: inout [Job.ArgTemplate],
                                                         primaryInputs: [TypedVirtualPath],
                                                         inputsGeneratingCodeCount: Int,

Sources/SwiftOptions/Options.swift

@@ -61,6 +61,7 @@ extension Option {
   public static let continueBuildingAfterErrors: Option = Option("-continue-building-after-errors", .flag, attributes: [.frontend, .doesNotAffectIncrementalBuild], helpText: "Continue building, even after errors are encountered")
   public static let coveragePrefixMap: Option = Option("-coverage-prefix-map", .separate, attributes: [.frontend], metaVar: "<prefix=replacement>", helpText: "Remap source paths in coverage info")
   public static let CrossModuleOptimization: Option = Option("-cross-module-optimization", .flag, attributes: [.helpHidden, .frontend], helpText: "Perform cross-module optimization")
+  public static let disableCrossModuleOptimization: Option = Option("-disable-cmo", .flag, attributes: [.helpHidden, .frontend], helpText: "Disable cross-module optimization")
   public static let crosscheckUnqualifiedLookup: Option = Option("-crosscheck-unqualified-lookup", .flag, attributes: [.frontend, .noDriver], helpText: "Compare legacy DeclContext- to ASTScope-based unqualified name lookup (for debugging)")
   public static let c: Option = Option("-c", .flag, alias: Option.emitObject, attributes: [.frontend, .noInteractive], group: .modes)
   public static let debugAssertAfterParse: Option = Option("-debug-assert-after-parse", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Force an assertion failure after parsing", group: .debugCrash)
@@ -704,6 +705,7 @@ extension Option {
       Option.continueBuildingAfterErrors,
       Option.coveragePrefixMap,
       Option.CrossModuleOptimization,
+      Option.disableCrossModuleOptimization,
       Option.crosscheckUnqualifiedLookup,
       Option.c,
       Option.debugAssertAfterParse,

Tests/SwiftDriverTests/SwiftDriverTests.swift

@@ -2691,12 +2691,11 @@ final class SwiftDriverTests: XCTestCase {
   func testEmitModuleSeparatelyWMO() throws {
     var envVars = ProcessEnv.vars
     envVars["SWIFT_DRIVER_LD_EXEC"] = ld.nativePathString(escaped: false)
+    let root = try localFileSystem.currentWorkingDirectory.map { AbsolutePath("/foo/bar", relativeTo: $0) }
+                  ?? AbsolutePath(validating: "/foo/bar")
 
     do {
-      let symbolGraphDir =
-          try localFileSystem.currentWorkingDirectory.map { AbsolutePath("/foo/bar", relativeTo: $0) }
-                ?? AbsolutePath(validating: "/Foo/Bar")
-      var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", rebase("Test.swiftmodule", at: symbolGraphDir), "-emit-symbol-graph", "-emit-symbol-graph-dir", symbolGraphDir.pathString, "-emit-library", "-target", "x86_64-apple-macosx10.15", "-wmo", "-emit-module-separately-wmo"],
+      var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", rebase("Test.swiftmodule", at: root), "-emit-symbol-graph", "-emit-symbol-graph-dir", root.pathString, "-emit-library", "-target", "x86_64-apple-macosx10.15", "-wmo", "-emit-module-separately-wmo"],
                                env: envVars)
 
       let abiFileCount = (driver.isFeatureSupported(.emit_abi_descriptor) && driver.targetTriple.isDarwin) ? 1 : 0
@@ -2715,21 +2714,18 @@ final class SwiftDriverTests: XCTestCase {
       let emitModuleJob = plannedJobs.first(where: {$0.kind == .emitModule})!
       XCTAssertTrue(emitModuleJob.tool.name.contains("swift"))
       XCTAssertEqual(emitModuleJob.outputs.count, 3 + abiFileCount)
-      XCTAssertEqual(1, emitModuleJob.outputs.filter({$0.file == .absolute(AbsolutePath(rebase("Test.swiftmodule", at: symbolGraphDir)))}).count)
-      XCTAssertEqual(1, emitModuleJob.outputs.filter({$0.file == .absolute(AbsolutePath(rebase("Test.swiftdoc", at: symbolGraphDir)))}).count)
-      XCTAssertEqual(1, emitModuleJob.outputs.filter({$0.file == .absolute(AbsolutePath(rebase("Test.swiftsourceinfo", at: symbolGraphDir)))}).count)
+      XCTAssertEqual(1, emitModuleJob.outputs.filter({$0.file == .absolute(AbsolutePath(rebase("Test.swiftmodule", at: root)))}).count)
+      XCTAssertEqual(1, emitModuleJob.outputs.filter({$0.file == .absolute(AbsolutePath(rebase("Test.swiftdoc", at: root)))}).count)
+      XCTAssertEqual(1, emitModuleJob.outputs.filter({$0.file == .absolute(AbsolutePath(rebase("Test.swiftsourceinfo", at: root)))}).count)
       if abiFileCount == 1 {
-        XCTAssertEqual(abiFileCount, emitModuleJob.outputs.filter({$0.file == .absolute(AbsolutePath(rebase("Test.abi.json", at: symbolGraphDir)))}).count)
+        XCTAssertEqual(abiFileCount, emitModuleJob.outputs.filter({$0.file == .absolute(AbsolutePath(rebase("Test.abi.json", at: root)))}).count)
       }
 
       // We don't know the output file of the symbol graph, just make sure the flag is passed along.
       XCTAssertTrue(emitModuleJob.commandLine.contains(.flag("-emit-symbol-graph-dir")))
     }
 
     do {
-      let root = try localFileSystem.currentWorkingDirectory.map { AbsolutePath("/foo/bar", relativeTo: $0) }
-                    ?? AbsolutePath(validating: "/foo/bar")
-
       // Ignore the `-emit-module-separately-wmo` flag when building only the module files to avoid duplicating outputs.
       var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", rebase("Test.swiftmodule", at: root), "-wmo", "-emit-module-separately-wmo"])
       let abiFileCount = (driver.isFeatureSupported(.emit_abi_descriptor) && driver.targetTriple.isDarwin) ? 1 : 0
@@ -2750,9 +2746,6 @@ final class SwiftDriverTests: XCTestCase {
     }
 
     do {
-      let root = try localFileSystem.currentWorkingDirectory.map { AbsolutePath("/foo/bar", relativeTo: $0) }
-                    ?? AbsolutePath(validating: "/foo/bar")
-
       // Specifying -no-emit-module-separately-wmo doesn't schedule the separate emit-module job.
       var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", rebase("Test.swiftmodule", at: root), "-emit-library", "-wmo", "-emit-module-separately-wmo", "-no-emit-module-separately-wmo" ])
       let abiFileCount = (driver.isFeatureSupported(.emit_abi_descriptor) && driver.targetTriple.isDarwin) ? 1 : 0
@@ -2777,6 +2770,40 @@ final class SwiftDriverTests: XCTestCase {
       }
     }
 
+    #if os(Linux) || os(Android)
+    let autoLinkExtractJob = 1
+    #else
+    let autoLinkExtractJob = 0
+    #endif
+
+    do {
+      // non library-evolution builds require a single job, because cross-module-optimization is enabled by default.
+      var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", rebase("Test.swiftmodule", at: root), "-c", "-o", rebase("test.o", at: root), "-wmo", "-O" ])
+      let plannedJobs = try driver.planBuild()
+      XCTAssertEqual(plannedJobs.count, 1 + autoLinkExtractJob)
+    }
+
+    do {
+      // library-evolution builds can emit the module in a separate job.
+      var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", rebase("Test.swiftmodule", at: root), "-c", "-o", rebase("test.o", at: root), "-wmo", "-O", "-enable-library-evolution" ])
+      let plannedJobs = try driver.planBuild()
+      XCTAssertEqual(plannedJobs.count, 2 + autoLinkExtractJob)
+    }
+
+    do {
+      // When disabling cross-module-optimization, the module can be emitted in a separate job.
+      var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", rebase("Test.swiftmodule", at: root), "-c", "-o", rebase("test.o", at: root), "-wmo", "-O", "-disable-cmo" ])
+      let plannedJobs = try driver.planBuild()
+      XCTAssertEqual(plannedJobs.count, 2 + autoLinkExtractJob)
+    }
+
+    do {
+      // non optimized builds can emit the module in a separate job.
+      var driver = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Test", "-emit-module-path", rebase("Test.swiftmodule", at: root), "-c", "-o", rebase("test.o", at: root), "-wmo" ])
+      let plannedJobs = try driver.planBuild()
+      XCTAssertEqual(plannedJobs.count, 2 + autoLinkExtractJob)
+    }
+
     do {
         // Don't use emit-module-separetely as a linker.
       var driver = try Driver(args: ["swiftc", "foo.sil", "bar.sil", "-module-name", "Test", "-emit-module-path", "/foo/bar/Test.swiftmodule", "-emit-library", "-target", "x86_64-apple-macosx10.15", "-wmo", "-emit-module-separately-wmo"],