[ClangImporter] Swift needs to pass -Xclang -fbuiltin-headers-in-system-modules for its module maps that group cstd headers

Swift has some module maps it overlays on Linux and Windows that groups all of the C standard library headers into a single module. This doesn’t allow clang and C++ headers to layer properly with the OS/SDK modules. clang will set -fbuiltin-headers-in-system-modules as necessary for Apple SDKs, but Swift will need to pass that flag itself when required by its module maps.

Author: ian-twilightcoder

include/swift/Basic/LangOptions.h

@@ -961,6 +961,10 @@ namespace swift {
     /// contains the full option set.
     bool ExtraArgsOnly = false;
 
+    /// Indicates the ClangImporter added a vfs which requires
+    /// -fbuiltin-headers-in-system-modules
+    bool RequiresBuiltinHeadersInSystemModules = false;
+
     /// When building a PCM, rely on the Swift frontend's command-line -Xcc flags
     /// to build the Clang module via Clang frontend directly,
     /// and completely bypass the Clang driver.

include/swift/ClangImporter/ClangImporter.h

@@ -657,6 +657,7 @@ struct ClangInvocationFileMapping {
 /// to allow using them from Swift.
 ClangInvocationFileMapping getClangInvocationFileMapping(
     ASTContext &ctx,
+    bool &requiresBuiltinHeadersInSystemModules,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs = nullptr);
 
 } // end namespace swift

lib/ClangImporter/ClangImporter.cpp

@@ -1178,7 +1178,7 @@ ClangImporter::create(ASTContext &ctx,
     }
   }
 
-  auto fileMapping = getClangInvocationFileMapping(ctx);
+  auto fileMapping = getClangInvocationFileMapping(ctx, importerOpts.RequiresBuiltinHeadersInSystemModules);
   // Wrap Swift's FS to allow Clang to override the working directory
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
       llvm::vfs::RedirectingFileSystem::create(
@@ -1204,6 +1204,11 @@ ClangImporter::create(ASTContext &ctx,
   // Create a new Clang compiler invocation.
   {
     importer->Impl.ClangArgs = getClangArguments(ctx);
+    if (importerOpts.RequiresBuiltinHeadersInSystemModules) {
+      if (!importerOpts.DirectClangCC1ModuleBuild)
+        importer->Impl.ClangArgs.push_back("-Xclang");
+      importer->Impl.ClangArgs.push_back("-fbuiltin-headers-in-system-modules");
+    }
     ArrayRef<std::string> invocationArgStrs = importer->Impl.ClangArgs;
     if (importerOpts.DumpClangDiagnostics) {
       llvm::errs() << "'";

lib/ClangImporter/ClangIncludePaths.cpp

@@ -420,7 +420,8 @@ GetWindowsAuxiliaryFile(StringRef modulemap, const SearchPathOptions &Options) {
 
 SmallVector<std::pair<std::string, std::string>, 2> GetWindowsFileMappings(
     ASTContext &Context,
-    const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &driverVFS) {
+    const llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &driverVFS,
+    bool &requiresBuiltinHeadersInSystemModules) {
   const llvm::Triple &Triple = Context.LangOpts.Target;
   const SearchPathOptions &SearchPathOpts = Context.SearchPathOpts;
   SmallVector<std::pair<std::string, std::string>, 2> Mappings;
@@ -468,8 +469,21 @@ SmallVector<std::pair<std::string, std::string>, 2> GetWindowsFileMappings(
     llvm::sys::path::append(UCRTInjection, "module.modulemap");
 
     AuxiliaryFile = GetWindowsAuxiliaryFile("ucrt.modulemap", SearchPathOpts);
-    if (!AuxiliaryFile.empty())
+    if (!AuxiliaryFile.empty()) {
+      // The ucrt module map has the C standard library headers all together.
+      // That leads to module cycles with the clang _Builtin_ modules. e.g.
+      // <fenv.h> on ucrt includes <float.h>. The clang builtin <float.h>
+      // include-nexts <float.h>. When both of those UCRT headers are in the
+      // ucrt module, there's a module cycle ucrt -> _Builtin_float -> ucrt
+      // (i.e. fenv.h (ucrt) -> float.h (builtin) -> float.h (ucrt)). Until the
+      // ucrt module map is updated, the builtin headers need to join the system
+      // modules. i.e. when the builtin float.h is in the ucrt module too, the
+      // cycle goes away. Note that -fbuiltin-headers-in-system-modules does
+      // nothing to fix the same problem with C++ headers, and is generally
+      // fragile.
       Mappings.emplace_back(std::string(UCRTInjection), AuxiliaryFile);
+      requiresBuiltinHeadersInSystemModules = true;
+    }
   }
 
   struct {
@@ -508,27 +522,44 @@ SmallVector<std::pair<std::string, std::string>, 2> GetWindowsFileMappings(
 } // namespace
 
 ClangInvocationFileMapping swift::getClangInvocationFileMapping(
-    ASTContext &ctx, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs) {
+    ASTContext &ctx, bool &requiresBuiltinHeadersInSystemModules,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs) {
   ClangInvocationFileMapping result;
   if (!vfs)
     vfs = llvm::vfs::getRealFileSystem();
 
   const llvm::Triple &triple = ctx.LangOpts.Target;
 
+  SmallVector<std::pair<std::string, std::string>, 2> libcFileMapping;
   if (triple.isOSWASI()) {
     // WASI Mappings
-    result.redirectedFiles.append(
-        getLibcFileMapping(ctx, "wasi-libc.modulemap", std::nullopt, vfs));
+    libcFileMapping =
+        getLibcFileMapping(ctx, "wasi-libc.modulemap", std::nullopt, vfs);
   } else {
     // Android/BSD/Linux Mappings
-    result.redirectedFiles.append(getLibcFileMapping(
-        ctx, "glibc.modulemap", StringRef("SwiftGlibc.h"), vfs));
+    libcFileMapping = getLibcFileMapping(ctx, "glibc.modulemap",
+                                         StringRef("SwiftGlibc.h"), vfs);
   }
+  result.redirectedFiles.append(libcFileMapping);
+  // Both libc module maps have the C standard library headers all together in a
+  // SwiftLibc module. That leads to module cycles with the clang _Builtin_
+  // modules. e.g. <inttypes.h> includes <stdint.h> on these platforms. The
+  // clang builtin <stdint.h> include-nexts <stdint.h>. When both of those
+  // platform headers are in the SwiftLibc module, there's a module cycle
+  // SwiftLibc -> _Builtin_stdint -> SwiftLibc (i.e. inttypes.h (platform) ->
+  // stdint.h (builtin) -> stdint.h (platform)). Until this can be fixed in
+  // these module maps, the clang builtin headers need to join the "system"
+  // modules (SwiftLibc). i.e. when the clang builtin stdint.h is in the
+  // SwiftLibc module too, the cycle goes away. Note that
+  // -fbuiltin-headers-in-system-modules does nothing to fix the same problem
+  // with C++ headers, and is generally fragile.
+  requiresBuiltinHeadersInSystemModules = !libcFileMapping.empty();
 
   if (ctx.LangOpts.EnableCXXInterop)
     getLibStdCxxFileMapping(result, ctx, vfs);
 
-  result.redirectedFiles.append(GetWindowsFileMappings(ctx, vfs));
+  result.redirectedFiles.append(GetWindowsFileMappings(
+      ctx, vfs, requiresBuiltinHeadersInSystemModules));
 
   return result;
 }

lib/DriverTool/sil_opt_main.cpp

@@ -193,6 +193,11 @@ struct SILOptOptions {
                                         "interfaces for all public declarations by "
                                         "default"));
 
+  llvm::cl::opt<bool>
+  StrictImplicitModuleContext = llvm::cl::opt<bool>("strict-implicit-module-context",
+                              llvm::cl::desc("Enable the strict forwarding of compilation "
+                                             "context to downstream implicit module dependencies"));
+
   llvm::cl::opt<bool>
   DisableSILOwnershipVerifier = llvm::cl::opt<bool>(
       "disable = llvm::cl::opt<bool> DisableSILOwnershipVerifier(-sil-ownership-verifier",
@@ -518,6 +523,10 @@ struct SILOptOptions {
                       swift::UnavailableDeclOptimization::Complete, "complete",
                       "Eliminate unavailable decls from lowered SIL/IR")),
               llvm::cl::init(swift::UnavailableDeclOptimization::None));
+
+  llvm::cl::list<std::string> ClangXCC = llvm::cl::list<std::string>(
+      "Xcc",
+      llvm::cl::desc("option to pass to clang"));
 };
 
 /// Regular expression corresponding to the value given in one of the
@@ -608,6 +617,8 @@ int sil_opt_main(ArrayRef<const char *> argv, void *MainAddr) {
     Invocation.setRuntimeResourcePath(options.ResourceDir);
   Invocation.getFrontendOptions().EnableLibraryEvolution
     = options.EnableLibraryEvolution;
+  Invocation.getFrontendOptions().StrictImplicitModuleContext
+    = options.StrictImplicitModuleContext;
   // Set the module cache path. If not passed in we use the default swift module
   // cache.
   Invocation.getClangImporterOptions().ModuleCachePath = options.ModuleCachePath;
@@ -671,6 +682,12 @@ int sil_opt_main(ArrayRef<const char *> argv, void *MainAddr) {
   Invocation.getDiagnosticOptions().VerifyMode =
     options.VerifyMode ? DiagnosticOptions::Verify : DiagnosticOptions::NoVerify;
 
+  ClangImporterOptions &clangImporterOptions =
+      Invocation.getClangImporterOptions();
+  for (const auto &xcc : options.ClangXCC) {
+    clangImporterOptions.ExtraArgs.push_back(xcc);
+  }
+
   // Setup the SIL Options.
   SILOptions &SILOpts = Invocation.getSILOptions();
   SILOpts.InlineThreshold = options.SILInlineThreshold;

lib/Frontend/ModuleInterfaceLoader.cpp

@@ -1763,10 +1763,18 @@ InterfaceSubContextDelegateImpl::InterfaceSubContextDelegateImpl(
       LoaderOpts.requestedAction == FrontendOptions::ActionType::ScanDependencies) {
     // Inherit any clang-specific state of the compilation (macros, clang flags, etc.)
     subClangImporterOpts.ExtraArgs = clangImporterOpts.ExtraArgs;
-    for (auto arg : subClangImporterOpts.ExtraArgs) {
-      GenericArgs.push_back("-Xcc");
-      GenericArgs.push_back(ArgSaver.save(arg));
-    }
+  }
+  // If the importer added a vfs that requires -fbuiltin-headers-in-system-modules,
+  // set that in ExtraArgs.
+  subClangImporterOpts.RequiresBuiltinHeadersInSystemModules = clangImporterOpts.RequiresBuiltinHeadersInSystemModules;
+  if (subClangImporterOpts.RequiresBuiltinHeadersInSystemModules) {
+    if (!subClangImporterOpts.DirectClangCC1ModuleBuild)
+      subClangImporterOpts.ExtraArgs.push_back("-Xclang");
+    subClangImporterOpts.ExtraArgs.push_back("-fbuiltin-headers-in-system-modules");
+  }
+  for (auto arg : subClangImporterOpts.ExtraArgs) {
+    GenericArgs.push_back("-Xcc");
+    GenericArgs.push_back(ArgSaver.save(arg));
   }
 
   subClangImporterOpts.EnableClangSPI = clangImporterOpts.EnableClangSPI;

stdlib/cmake/modules/AddSwiftStdlib.cmake

@@ -929,7 +929,7 @@ function(add_swift_target_library_single target name)
         -Xcc;-Xclang;-Xcc;-ivfsoverlay;-Xcc;-Xclang;-Xcc;${SWIFTLIB_SINGLE_VFS_OVERLAY})
     endif()
     list(APPEND SWIFTLIB_SINGLE_SWIFT_COMPILE_FLAGS
-      -vfsoverlay;"${SWIFT_WINDOWS_VFS_OVERLAY}")
+      -vfsoverlay;"${SWIFT_WINDOWS_VFS_OVERLAY}";-strict-implicit-module-context;-Xcc;-Xclang;-Xcc;-fbuiltin-headers-in-system-modules)
     if(NOT CMAKE_HOST_SYSTEM MATCHES Windows)
       swift_windows_include_for_arch(${SWIFTLIB_SINGLE_ARCHITECTURE} SWIFTLIB_INCLUDE)
       foreach(directory ${SWIFTLIB_INCLUDE})
@@ -2659,7 +2659,7 @@ function(_add_swift_target_executable_single name)
 
   if(SWIFTEXE_SINGLE_SDK STREQUAL "WINDOWS")
     list(APPEND SWIFTEXE_SINGLE_COMPILE_FLAGS
-      -vfsoverlay;"${SWIFT_WINDOWS_VFS_OVERLAY}")
+      -vfsoverlay;"${SWIFT_WINDOWS_VFS_OVERLAY}";-strict-implicit-module-context;-Xcc;-Xclang;-Xcc;-fbuiltin-headers-in-system-modules)
   endif()
 
   if ("${SWIFTEXE_SINGLE_SDK}" STREQUAL "LINUX")

stdlib/public/Backtracing/SymbolicatedBacktrace.swift

@@ -23,6 +23,11 @@ import Swift
 
 @_implementationOnly import OS.Libc
 @_implementationOnly import Runtime
+// Because we've turned off the OS/SDK modules, and we don't have a module for
+// stddef.h, and we sometimes build with -fbuiltin-headers-in-system-modules for
+// vfs reasons, stddef.h can be absorbed into a random module. Sometimes it's
+// SwiftOverlayShims.
+@_implementationOnly import SwiftOverlayShims
 
 /// A symbolicated backtrace
 public struct SymbolicatedBacktrace: CustomStringConvertible {

test/DebugInfo/C-typedef-Darwin.swift

@@ -9,4 +9,13 @@ let blah = size_t(1024)
 use(blah)
 // CHECK: !DIDerivedType(tag: DW_TAG_typedef,
 // CHECK-SAME:           scope: ![[DARWIN_MODULE:[0-9]+]],
-// CHECK: ![[DARWIN_MODULE]] = !DIModule({{.*}}, name: "stddef"
+
+// size_t is defined in clang, originally by <stddef.h>, and later split out to
+// <__stddef_size_t.h>. Depending on the state of clang's builtin headers and
+// modules, size_t can be in either Darwin.C.stddef or _Builtin_stddef.size_t.
+// size_t is also defined in macOS by <sys/_types/_size_t.h> which is in the
+// Darwin.POSIX._types._size_t module. Ideally Apple will remove the duplicate
+// declaration and clang will settle to _Builtin_stddef.size_t, but while it's
+// all in flux allow all three of those modules.
+// Darwin.C.stddef|_Builtin_stddef.size_t|Darwin.POSIX._types._size_t
+// CHECK: ![[DARWIN_MODULE]] = !DIModule({{.*}}, name: "{{stddef|size_t|_size_t}}"

test/Interop/lit.local.cfg

@@ -33,9 +33,12 @@ if get_target_os() in ['windows-msvc']:
     # Clang should build object files with link settings equivalent to -libc MD
     # when building for the MSVC target.
     clang_opt = clang_compile_opt + '-D_MT -D_DLL -Xclang --dependent-lib=msvcrt -Xclang --dependent-lib=oldnames '
-    config.substitutions.insert(0, ('%target-swift-flags', '-vfsoverlay {}'.format(os.path.join(config.swift_obj_root,
-                                                            'stdlib',
-                                                            'windows-vfs-overlay.yaml'))))
+    # ucrt.modulemap currently requires -fbuiltin-headers-in-system-modules. -strict-implicit-module-context
+    # is necessary for -Xcc arguments to be passed through ModuleInterfaceLoader.
+    config.substitutions.insert(0, ('%target-swift-flags', '-vfsoverlay {} -strict-implicit-module-context -Xcc -Xclang -Xcc -fbuiltin-headers-in-system-modules'.format(
+                                                            os.path.join(config.swift_obj_root,
+                                                                         'stdlib',
+                                                                         'windows-vfs-overlay.yaml'))))
 else:
     # FIXME(compnerd) do all the targets we currently support use SysV ABI?
     config.substitutions.insert(0, ('%target-abi', 'SYSV'))