From 3299f974dc3b7d58b4128e28a591f6740dd46d39 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Mon, 28 Oct 2019 14:26:45 -0700 Subject: [PATCH 01/13] [clang][clang-scan-deps] Aggregate the full dependency information. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Differential Revision: https://reviews.llvm.org/D70268 This is a recommit of f978ea498309 with a fix for the PowerPC failure. The issue was that: * `CompilerInstance::ExecuteAction` calls `getTarget().adjust(getLangOpts());`. * `PPCTargetInfo::adjust` changes `LangOptions::HasAltivec`. * This happens after the first few calls to `getModuleHash`. There’s even a FIXME saying: ``` // FIXME: We shouldn't need to do this, the target should be immutable once // created. This complexity should be lifted elsewhere. ``` This only showed up on PowerPC because it's one of the few targets that almost always changes a hashed langopt. I looked into addressing the fixme, but that would be a much larger change, and it's not the only thing that happens in `ExecuteAction` that can change the module context hash. Instead I changed the code to not call `getModuleHash` until after it has been modified in `ExecuteAction`. (cherry picked from commit 356a4b433bf7224666fd7152a8ad2560fb286ef4) --- .../DependencyScanningTool.h | 73 +++++- .../DependencyScanning/ModuleDepCollector.h | 87 ++++++- .../DependencyScanningTool.cpp | 127 +++++------ .../DependencyScanningWorker.cpp | 11 +- .../DependencyScanning/ModuleDepCollector.cpp | 91 ++++++-- .../ClangScanDeps/Inputs/modules_cdb.json | 13 +- clang/test/ClangScanDeps/modules-full.cpp | 192 +++++++++++----- clang/tools/clang-scan-deps/ClangScanDeps.cpp | 215 +++++++++++++++++- 8 files changed, 653 insertions(+), 156 deletions(-) diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index a0c1900f7ed98..1c106ed4b765a 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -11,13 +11,69 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" #include "clang/Tooling/JSONCompilationDatabase.h" +#include "llvm/ADT/StringSet.h" #include namespace clang{ namespace tooling{ namespace dependencies{ +/// The full dependencies and module graph for a specific input. +struct FullDependencies { + /// The name of the C++20 module this translation unit exports. This may + /// include `:` for C++20 module partitons. + /// + /// If the translation unit is not a module then this will be empty. + std::string ExportedModuleName; + + /// The context hash represents the set of compiler options that may make one + /// version of a module incompatible with another. This includes things like + /// language mode, predefined macros, header search paths, etc... + /// + /// Modules with the same name but a different \c ContextHash should be + /// treated as separate modules for the purpose of a build. + std::string ContextHash; + + /// A collection of absolute paths to files that this translation unit + /// directly depends on, not including transitive dependencies. + std::vector FileDeps; + + /// A list of modules this translation unit directly depends on, not including + /// transitive dependencies. + /// + /// This may include modules with a different context hash when it can be + /// determined that the differences are benign for this compilation. + std::vector ClangModuleDeps; + + /// A partial addtional set of command line arguments that can be used to + /// build this translation unit. + /// + /// Call \c getFullAdditionalCommandLine() to get a command line suitable for + /// appending to the original command line to pass to clang. + std::vector AdditionalNonPathCommandLine; + + /// Gets the full addtional command line suitable for appending to the + /// original command line to pass to clang. + /// + /// \param LookupPCMPath this function is called to fill in `-fmodule-file=` + /// flags and for the `-o` flag. It needs to return a + /// path for where the PCM for the given module is to + /// be located. + /// \param LookupModuleDeps this fucntion is called to collect the full + /// transitive set of dependencies for this + /// compilation. + std::vector getAdditionalCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const; +}; + +struct FullDependenciesResult { + FullDependencies FullDeps; + std::vector DiscoveredModules; +}; + /// The high-level implementation of the dependency discovery tool that runs on /// an individual worker thread. class DependencyScanningTool { @@ -35,8 +91,23 @@ class DependencyScanningTool { getDependencyFile(const tooling::CompilationDatabase &Compilations, StringRef CWD); + /// Collect the full module depenedency graph for the input, ignoring any + /// modules which have already been seen. + /// + /// \param AlreadySeen this is used to not report modules that have previously + /// been reported. Use the same `llvm::StringSet<>` for all + /// calls to `getFullDependencies` for a single + /// `DependencyScanningTool` for a single build. Use a + /// different one for different tools, and clear it between + /// builds. + /// + /// \returns a \c StringError with the diagnostic output if clang errors + /// occurred, \c FullDependencies otherwise. + llvm::Expected + getFullDependencies(const tooling::CompilationDatabase &Compilations, + StringRef CWD, const llvm::StringSet<> &AlreadySeen); + private: - const ScanningOutputFormat Format; DependencyScanningWorker Worker; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index 7a9fc276fcaa7..c5d12fc73e1a9 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -28,16 +28,82 @@ namespace dependencies { class DependencyConsumer; +/// This is used to refer to a specific module. +/// +/// See \c ModuleDeps for details about what these members mean. +struct ClangModuleDep { + std::string ModuleName; + std::string ContextHash; +}; + struct ModuleDeps { + /// The name of the module. This may include `:` for C++20 module partitons, + /// or a header-name for C++20 header units. std::string ModuleName; - std::string ClangModuleMapFile; - std::string ModulePCMPath; + + /// The context hash of a module represents the set of compiler options that + /// may make one version of a module incompatible with another. This includes + /// things like language mode, predefined macros, header search paths, etc... + /// + /// Modules with the same name but a different \c ContextHash should be + /// treated as separate modules for the purpose of a build. std::string ContextHash; + + /// The path to the modulemap file which defines this module. + /// + /// This can be used to explicitly build this module. This file will + /// additionally appear in \c FileDeps as a dependency. + std::string ClangModuleMapFile; + + /// The path to where an implicit build would put the PCM for this module. + std::string ImplicitModulePCMPath; + + /// A collection of absolute paths to files that this module directly depends + /// on, not including transitive dependencies. llvm::StringSet<> FileDeps; - llvm::StringSet<> ClangModuleDeps; + + /// A list of modules this module directly depends on, not including + /// transitive dependencies. + /// + /// This may include modules with a different context hash when it can be + /// determined that the differences are benign for this compilation. + std::vector ClangModuleDeps; + + /// A partial command line that can be used to build this module. + /// + /// Call \c getFullCommandLine() to get a command line suitable for passing to + /// clang. + std::vector NonPathCommandLine; + + // Used to track which modules that were discovered were directly imported by + // the primary TU. bool ImportedByMainFile = false; + + /// Gets the full command line suitable for passing to clang. + /// + /// \param LookupPCMPath this function is called to fill in `-fmodule-file=` + /// flags and for the `-o` flag. It needs to return a + /// path for where the PCM for the given module is to + /// be located. + /// \param LookupModuleDeps this fucntion is called to collect the full + /// transitive set of dependencies for this + /// compilation. + std::vector getFullCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const; }; +namespace detail { +/// Append the `-fmodule-file=` and `-fmodule-map-file=` arguments for the +/// modules in \c Modules transitively, along with other needed arguments to +/// use explicitly built modules. +void appendCommonModuleArguments( + llvm::ArrayRef Modules, + std::function LookupPCMPath, + std::function LookupModuleDeps, + std::vector &Result); +} // namespace detail + class ModuleDepCollector; class ModuleDepCollectorPP final : public PPCallbacks { @@ -54,6 +120,8 @@ class ModuleDepCollectorPP final : public PPCallbacks { StringRef SearchPath, StringRef RelativePath, const Module *Imported, SrcMgr::CharacteristicKind FileType) override; + void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path, + const Module *Imported) override; void EndOfMainFile() override; @@ -62,16 +130,18 @@ class ModuleDepCollectorPP final : public PPCallbacks { ModuleDepCollector &MDC; llvm::DenseSet DirectDeps; + void handleImport(const Module *Imported); void handleTopLevelModule(const Module *M); - void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD); - void addModuleDep(const Module *M, ModuleDeps &MD); - - void addDirectDependencies(const Module *Mod); + void addAllSubmoduleDeps(const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules); + void addModuleDep(const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules); }; class ModuleDepCollector final : public DependencyCollector { public: - ModuleDepCollector(CompilerInstance &I, DependencyConsumer &C); + ModuleDepCollector(std::unique_ptr Opts, + CompilerInstance &I, DependencyConsumer &C); void attachToPreprocessor(Preprocessor &PP) override; void attachToASTReader(ASTReader &R) override; @@ -85,6 +155,7 @@ class ModuleDepCollector final : public DependencyCollector { std::string ContextHash; std::vector MainDeps; std::unordered_map Deps; + std::unique_ptr Opts; }; } // end namespace dependencies diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index f643c538f8f9a..31b8346b4efb3 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -8,24 +8,25 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Frontend/Utils.h" -#include "llvm/Support/JSON.h" - -static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { - std::vector Strings; - for (auto &&I : Set) - Strings.push_back(I.getKey()); - std::sort(Strings.begin(), Strings.end()); - return llvm::json::Array(Strings); -} namespace clang{ namespace tooling{ namespace dependencies{ +std::vector FullDependencies::getAdditionalCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const { + std::vector Ret = AdditionalNonPathCommandLine; + + dependencies::detail::appendCommonModuleArguments( + ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret); + + return Ret; +} + DependencyScanningTool::DependencyScanningTool( DependencyScanningService &Service) - : Format(Service.getFormat()), Worker(Service) { -} + : Worker(Service) {} llvm::Expected DependencyScanningTool::getDependencyFile( const tooling::CompilationDatabase &Compilations, StringRef CWD) { @@ -75,8 +76,33 @@ llvm::Expected DependencyScanningTool::getDependencyFile( std::vector Dependencies; }; + // We expect a single command here because if a source file occurs multiple + // times in the original CDB, then `computeDependencies` would run the + // `DependencyScanningAction` once for every time the input occured in the + // CDB. Instead we split up the CDB into single command chunks to avoid this + // behavior. + assert(Compilations.getAllCompileCommands().size() == 1 && + "Expected a compilation database with a single command!"); + std::string Input = Compilations.getAllCompileCommands().front().Filename; + + MakeDependencyPrinterConsumer Consumer; + auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + std::string Output; + Consumer.printDependencies(Output); + return Output; +} + +llvm::Expected +DependencyScanningTool::getFullDependencies( + const tooling::CompilationDatabase &Compilations, StringRef CWD, + const llvm::StringSet<> &AlreadySeen) { class FullDependencyPrinterConsumer : public DependencyConsumer { public: + FullDependencyPrinterConsumer(const llvm::StringSet<> &AlreadySeen) + : AlreadySeen(AlreadySeen) {} + void handleFileDependency(const DependencyOutputOptions &Opts, StringRef File) override { Dependencies.push_back(File); @@ -90,55 +116,41 @@ llvm::Expected DependencyScanningTool::getDependencyFile( ContextHash = std::move(Hash); } - void printDependencies(std::string &S, StringRef MainFile) { - // Sort the modules by name to get a deterministic order. - std::vector Modules; - for (auto &&Dep : ClangModuleDeps) - Modules.push_back(Dep.first); - std::sort(Modules.begin(), Modules.end()); + FullDependenciesResult getFullDependencies() const { + FullDependencies FD; - llvm::raw_string_ostream OS(S); + FD.ContextHash = std::move(ContextHash); - using namespace llvm::json; + FD.FileDeps.assign(Dependencies.begin(), Dependencies.end()); - Array Imports; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; + for (auto &&M : ClangModuleDeps) { + auto &MD = M.second; if (MD.ImportedByMainFile) - Imports.push_back(MD.ModuleName); + FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash}); } - Array Mods; - for (auto &&ModName : Modules) { - auto &MD = ClangModuleDeps[ModName]; - Object Mod{ - {"name", MD.ModuleName}, - {"file-deps", toJSONSorted(MD.FileDeps)}, - {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, - {"clang-modulemap-file", MD.ClangModuleMapFile}, - }; - Mods.push_back(std::move(Mod)); - } + FullDependenciesResult FDR; - Object O{ - {"input-file", MainFile}, - {"clang-context-hash", ContextHash}, - {"file-deps", Dependencies}, - {"clang-module-deps", std::move(Imports)}, - {"clang-modules", std::move(Mods)}, - }; + for (auto &&M : ClangModuleDeps) { + // TODO: Avoid handleModuleDependency even being called for modules + // we've already seen. + if (AlreadySeen.count(M.first)) + continue; + FDR.DiscoveredModules.push_back(std::move(M.second)); + } - S = llvm::formatv("{0:2},\n", Value(std::move(O))).str(); - return; + FDR.FullDeps = std::move(FD); + return FDR; } private: std::vector Dependencies; std::unordered_map ClangModuleDeps; std::string ContextHash; + std::vector OutputPaths; + const llvm::StringSet<> &AlreadySeen; }; - // We expect a single command here because if a source file occurs multiple // times in the original CDB, then `computeDependencies` would run the // `DependencyScanningAction` once for every time the input occured in the @@ -147,26 +159,13 @@ llvm::Expected DependencyScanningTool::getDependencyFile( assert(Compilations.getAllCompileCommands().size() == 1 && "Expected a compilation database with a single command!"); std::string Input = Compilations.getAllCompileCommands().front().Filename; - - if (Format == ScanningOutputFormat::Make) { - MakeDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output); - return Output; - } else { - FullDependencyPrinterConsumer Consumer; - auto Result = - Worker.computeDependencies(Input, CWD, Compilations, Consumer); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output, Input); - return Output; - } + + FullDependencyPrinterConsumer Consumer(AlreadySeen); + llvm::Error Result = + Worker.computeDependencies(Input, CWD, Compilations, Consumer); + if (Result) + return std::move(Result); + return Consumer.getFullDependencies(); } } // end namespace dependencies diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index d9e078d462f6e..73135dd2ff58d 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -142,12 +142,17 @@ class DependencyScanningAction : public tooling::ToolAction { Consumer)); break; case ScanningOutputFormat::Full: - Compiler.addDependencyCollector( - std::make_shared(Compiler, Consumer)); + Compiler.addDependencyCollector(std::make_shared( + std::move(Opts), Compiler, Consumer)); break; } - Consumer.handleContextHash(Compiler.getInvocation().getModuleHash(Compiler.getDiagnostics())); + // Consider different header search and diagnostic options to create + // different modules. This avoids the unsound aliasing of module PCMs. + // + // TODO: Implement diagnostic bucketing and header search pruning to reduce + // the impact of strict context hashing. + Compiler.getHeaderSearchOpts().ModulesStrictContextHash = true; auto Action = std::make_unique(); const bool Result = Compiler.ExecuteAction(*Action); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index d53ff25a0ce48..9d2d2a187da3c 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -17,12 +17,60 @@ using namespace clang; using namespace tooling; using namespace dependencies; +std::vector ModuleDeps::getFullCommandLine( + std::function LookupPCMPath, + std::function LookupModuleDeps) const { + std::vector Ret = NonPathCommandLine; + + // TODO: Build full command line. That also means capturing the original + // command line into NonPathCommandLine. + + dependencies::detail::appendCommonModuleArguments( + ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret); + + return Ret; +} + +void dependencies::detail::appendCommonModuleArguments( + llvm::ArrayRef Modules, + std::function LookupPCMPath, + std::function LookupModuleDeps, + std::vector &Result) { + llvm::StringSet<> AlreadyAdded; + + std::function)> AddArgs = + [&](llvm::ArrayRef Modules) { + for (const ClangModuleDep &CMD : Modules) { + if (!AlreadyAdded.insert(CMD.ModuleName + CMD.ContextHash).second) + continue; + const ModuleDeps &M = LookupModuleDeps(CMD); + // Depth first traversal. + AddArgs(M.ClangModuleDeps); + Result.push_back(("-fmodule-file=" + LookupPCMPath(CMD)).str()); + if (!M.ClangModuleMapFile.empty()) { + Result.push_back("-fmodule-map-file=" + M.ClangModuleMapFile); + } + } + }; + + Result.push_back("-fno-implicit-modules"); + Result.push_back("-fno-implicit-module-maps"); + AddArgs(Modules); +} + void ModuleDepCollectorPP::FileChanged(SourceLocation Loc, FileChangeReason Reason, SrcMgr::CharacteristicKind FileType, FileID PrevFID) { if (Reason != PPCallbacks::EnterFile) return; + + // This has to be delayed as the context hash can change at the start of + // `CompilerInstance::ExecuteAction`. + if (MDC.ContextHash.empty()) { + MDC.ContextHash = Instance.getInvocation().getModuleHash(); + MDC.Consumer.handleContextHash(MDC.ContextHash); + } SourceManager &SM = Instance.getSourceManager(); @@ -50,7 +98,16 @@ void ModuleDepCollectorPP::InclusionDirective( // here as `FileChanged` will never see it. MDC.MainDeps.push_back(FileName); } + handleImport(Imported); +} +void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc, + ModuleIdPath Path, + const Module *Imported) { + handleImport(Imported); +} + +void ModuleDepCollectorPP::handleImport(const Module *Imported) { if (!Imported) return; @@ -71,9 +128,8 @@ void ModuleDepCollectorPP::EndOfMainFile() { for (auto &&I : MDC.Deps) MDC.Consumer.handleModuleDependency(I.second); - DependencyOutputOptions Opts; for (auto &&I : MDC.MainDeps) - MDC.Consumer.handleFileDependency(Opts, I); + MDC.Consumer.handleFileDependency(*MDC.Opts, I); } void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { @@ -94,7 +150,7 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : ""; MD.ModuleName = M->getFullModuleName(); - MD.ModulePCMPath = M->getASTFile()->getName(); + MD.ImplicitModulePCMPath = M->getASTFile()->getName(); MD.ContextHash = MDC.ContextHash; serialization::ModuleFile *MF = MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile()); @@ -103,30 +159,37 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.FileDeps.insert(IF.getFile()->getName()); }); - addAllSubmoduleDeps(M, MD); + llvm::DenseSet AddedModules; + addAllSubmoduleDeps(M, MD, AddedModules); } -void ModuleDepCollectorPP::addAllSubmoduleDeps(const Module *M, - ModuleDeps &MD) { - addModuleDep(M, MD); +void ModuleDepCollectorPP::addAllSubmoduleDeps( + const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules) { + addModuleDep(M, MD, AddedModules); for (const Module *SubM : M->submodules()) - addAllSubmoduleDeps(SubM, MD); + addAllSubmoduleDeps(SubM, MD, AddedModules); } -void ModuleDepCollectorPP::addModuleDep(const Module *M, ModuleDeps &MD) { +void ModuleDepCollectorPP::addModuleDep( + const Module *M, ModuleDeps &MD, + llvm::DenseSet &AddedModules) { for (const Module *Import : M->Imports) { if (Import->getTopLevelModule() != M->getTopLevelModule()) { - MD.ClangModuleDeps.insert(Import->getTopLevelModuleName()); + if (AddedModules.insert(Import->getTopLevelModule()).second) + MD.ClangModuleDeps.push_back( + {Import->getTopLevelModuleName(), + Instance.getInvocation().getModuleHash()}); handleTopLevelModule(Import->getTopLevelModule()); } } } -ModuleDepCollector::ModuleDepCollector(CompilerInstance &I, - DependencyConsumer &C) - : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash(I.getDiagnostics())) { -} +ModuleDepCollector::ModuleDepCollector( + std::unique_ptr Opts, CompilerInstance &I, + DependencyConsumer &C) + : Instance(I), Consumer(C), Opts(std::move(Opts)) {} void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) { PP.addPPCallbacks(std::make_unique(Instance, *this)); diff --git a/clang/test/ClangScanDeps/Inputs/modules_cdb.json b/clang/test/ClangScanDeps/Inputs/modules_cdb.json index da5f9bc6a522f..a0c5123cd2124 100644 --- a/clang/test/ClangScanDeps/Inputs/modules_cdb.json +++ b/clang/test/ClangScanDeps/Inputs/modules_cdb.json @@ -1,13 +1,22 @@ [ { "directory": "DIR", - "command": "clang -E -fsyntax-only DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", + "command": "clang -E DIR/modules_cdb_input2.cpp -IInputs -D INCLUDE_HEADER2 -MD -MF DIR/modules_cdb2.d -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", "file": "DIR/modules_cdb_input2.cpp" }, { "directory": "DIR", "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o a.o", + "file": "DIR/modules_cdb_input.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -IInputs -fmodules -fcxx-modules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -o b.o", + "file": "DIR/modules_cdb_input.cpp" } ] - diff --git a/clang/test/ClangScanDeps/modules-full.cpp b/clang/test/ClangScanDeps/modules-full.cpp index 693dffeecbf76..1e6a740c27392 100644 --- a/clang/test/ClangScanDeps/modules-full.cpp +++ b/clang/test/ClangScanDeps/modules-full.cpp @@ -1,6 +1,5 @@ // RUN: rm -rf %t.dir // RUN: rm -rf %t.cdb -// RUN: rm -rf %t.module-cache // RUN: mkdir -p %t.dir // RUN: cp %s %t.dir/modules_cdb_input.cpp // RUN: cp %s %t.dir/modules_cdb_input2.cpp @@ -11,67 +10,146 @@ // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/modules_cdb.json > %t.cdb // // RUN: echo %t.dir > %t.result -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 4 -full-command-line \ // RUN: -mode preprocess-minimized-sources -format experimental-full >> %t.result -// RUN: cat %t.result | FileCheck --check-prefixes=CHECK %s +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK %s // FIXME: Backslash issues. // XFAIL: system-windows #include "header.h" -// CHECK: [[PREFIX:(.*[/\\])+[a-zA-Z0-9.-]+]] +// CHECK: [[PREFIX:.*]] +// CHECK-NEXT: { +// CHECK-NEXT: "modules": [ // CHECK-NEXT: { -// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH:[A-Z0-9]+]]", -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header1" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modules": [ -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header2" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header1" -// CHECK-NEXT: }, -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header2.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header2" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp" -// CHECK-NEXT: ], -// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input2.cpp" -// CHECK-NEXT:}, -// CHECK-NEXT:{ -// CHECK-NOT: "clang-context-hash": "[[CONTEXT_HASH]]", -// CHECK-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", -// CHECK-NEXT: "clang-module-deps": [ -// CHECK-NEXT: "header1" -// CHECK-NEXT: ], -// CHECK-NEXT: "clang-modules": [ -// CHECK-NEXT: { -// CHECK-NEXT: "clang-module-deps": [], -// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap", -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}header.h", -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}Inputs{{[/\\]}}module.modulemap" -// CHECK-NEXT: ], -// CHECK-NEXT: "name": "header1" -// CHECK-NEXT: } -// CHECK-NEXT: ], -// CHECK-NEXT: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp" -// CHECK-NEXT: ], -// CHECK-NEXT: "input-file": "[[PREFIX]]{{[/\\]}}modules_cdb_input.cpp" -// CHECK-NEXT:}, +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1:[A-Z0-9]+]]", +// CHECK-NEXT: "module-name": "header2" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps" +// CHECK-NEXT: ], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2:[A-Z0-9]+]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps" +// CHECK-NEXT: ], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Inputs/header2.h", +// CHECK-NEXT: "[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "header2" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H2]]/header1-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "module-name": "header1" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header1-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input2.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input2.cpp" +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 1294e6682841d..70e546b3416b8 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -15,6 +15,7 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileUtilities.h" #include "llvm/Support/InitLLVM.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Threading.h" @@ -129,6 +130,11 @@ static llvm::cl::opt Format( llvm::cl::init(ScanningOutputFormat::Make), llvm::cl::cat(DependencyScannerCategory)); +static llvm::cl::opt FullCommandLine( + "full-command-line", + llvm::cl::desc("Include the full command lines to use to build modules"), + llvm::cl::init(false), llvm::cl::cat(DependencyScannerCategory)); + llvm::cl::opt NumThreads("j", llvm::cl::Optional, llvm::cl::desc("Number of worker threads to use (default: use " @@ -189,9 +195,10 @@ class SingleCommandCompilationDatabase : public tooling::CompilationDatabase { /// based on the result. /// /// \returns True on error. -static bool handleDependencyToolResult(const std::string &Input, - llvm::Expected &MaybeFile, - SharedStream &OS, SharedStream &Errs) { +static bool +handleMakeDependencyToolResult(const std::string &Input, + llvm::Expected &MaybeFile, + SharedStream &OS, SharedStream &Errs) { if (!MaybeFile) { llvm::handleAllErrors( MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) { @@ -206,6 +213,184 @@ static bool handleDependencyToolResult(const std::string &Input, return false; } +static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) { + std::vector Strings; + for (auto &&I : Set) + Strings.push_back(I.getKey()); + std::sort(Strings.begin(), Strings.end()); + return llvm::json::Array(Strings); +} + +static llvm::json::Array toJSONSorted(std::vector V) { + std::sort(V.begin(), V.end(), + [](const ClangModuleDep &A, const ClangModuleDep &B) { + return std::tie(A.ModuleName, A.ContextHash) < + std::tie(B.ModuleName, B.ContextHash); + }); + + llvm::json::Array Ret; + for (const ClangModuleDep &CMD : V) + Ret.push_back(llvm::json::Object( + {{"module-name", CMD.ModuleName}, {"context-hash", CMD.ContextHash}})); + return Ret; +} + +// Thread safe. +class FullDeps { +public: + void mergeDeps(StringRef Input, FullDependenciesResult FDR, + size_t InputIndex) { + const FullDependencies &FD = FDR.FullDeps; + + InputDeps ID; + ID.FileName = Input; + ID.ContextHash = std::move(FD.ContextHash); + ID.FileDeps = std::move(FD.FileDeps); + ID.ModuleDeps = std::move(FD.ClangModuleDeps); + + std::unique_lock ul(Lock); + for (const ModuleDeps &MD : FDR.DiscoveredModules) { + auto I = Modules.find({MD.ContextHash, MD.ModuleName, 0}); + if (I != Modules.end()) { + I->first.InputIndex = std::min(I->first.InputIndex, InputIndex); + continue; + } + Modules.insert( + I, {{MD.ContextHash, MD.ModuleName, InputIndex}, std::move(MD)}); + } + + if (FullCommandLine) + ID.AdditonalCommandLine = FD.getAdditionalCommandLine( + [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); }, + [&](ClangModuleDep CMD) -> const ModuleDeps & { + return lookupModuleDeps(CMD); + }); + + Inputs.push_back(std::move(ID)); + } + + void printFullOutput(raw_ostream &OS) { + // Sort the modules by name to get a deterministic order. + std::vector ModuleNames; + for (auto &&M : Modules) + ModuleNames.push_back(M.first); + std::sort(ModuleNames.begin(), ModuleNames.end(), + [](const ContextModulePair &A, const ContextModulePair &B) { + return std::tie(A.ModuleName, A.InputIndex) < + std::tie(B.ModuleName, B.InputIndex); + }); + + std::sort(Inputs.begin(), Inputs.end(), + [](const InputDeps &A, const InputDeps &B) { + return A.FileName < B.FileName; + }); + + using namespace llvm::json; + + Array OutModules; + for (auto &&ModName : ModuleNames) { + auto &MD = Modules[ModName]; + Object O{ + {"name", MD.ModuleName}, + {"context-hash", MD.ContextHash}, + {"file-deps", toJSONSorted(MD.FileDeps)}, + {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)}, + {"clang-modulemap-file", MD.ClangModuleMapFile}, + {"command-line", + FullCommandLine + ? MD.getFullCommandLine( + [&](ClangModuleDep CMD) { return lookupPCMPath(CMD); }, + [&](ClangModuleDep CMD) -> const ModuleDeps & { + return lookupModuleDeps(CMD); + }) + : MD.NonPathCommandLine}, + }; + OutModules.push_back(std::move(O)); + } + + Array TUs; + for (auto &&I : Inputs) { + Object O{ + {"input-file", I.FileName}, + {"clang-context-hash", I.ContextHash}, + {"file-deps", I.FileDeps}, + {"clang-module-deps", toJSONSorted(I.ModuleDeps)}, + {"command-line", I.AdditonalCommandLine}, + }; + TUs.push_back(std::move(O)); + } + + Object Output{ + {"modules", std::move(OutModules)}, + {"translation-units", std::move(TUs)}, + }; + + OS << llvm::formatv("{0:2}\n", Value(std::move(Output))); + } + +private: + StringRef lookupPCMPath(ClangModuleDep CMD) { + return Modules[ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}] + .ImplicitModulePCMPath; + } + + const ModuleDeps &lookupModuleDeps(ClangModuleDep CMD) { + auto I = + Modules.find(ContextModulePair{CMD.ContextHash, CMD.ModuleName, 0}); + assert(I != Modules.end()); + return I->second; + }; + + struct ContextModulePair { + std::string ContextHash; + std::string ModuleName; + mutable size_t InputIndex; + + bool operator==(const ContextModulePair &Other) const { + return ContextHash == Other.ContextHash && ModuleName == Other.ModuleName; + } + }; + + struct ContextModulePairHasher { + std::size_t operator()(const ContextModulePair &CMP) const { + using llvm::hash_combine; + + return hash_combine(CMP.ContextHash, CMP.ModuleName); + } + }; + + struct InputDeps { + std::string FileName; + std::string ContextHash; + std::vector FileDeps; + std::vector ModuleDeps; + std::vector AdditonalCommandLine; + }; + + std::mutex Lock; + std::unordered_map + Modules; + std::vector Inputs; +}; + +static bool handleFullDependencyToolResult( + const std::string &Input, + llvm::Expected &MaybeFullDeps, FullDeps &FD, + size_t InputIndex, SharedStream &OS, SharedStream &Errs) { + if (!MaybeFullDeps) { + llvm::handleAllErrors( + MaybeFullDeps.takeError(), [&Input, &Errs](llvm::StringError &Err) { + Errs.applyLocked([&](raw_ostream &OS) { + OS << "Error while scanning dependencies for " << Input << ":\n"; + OS << Err.getMessage(); + }); + }); + return true; + } + FD.mergeDeps(Input, std::move(*MaybeFullDeps), InputIndex); + return false; +} + int main(int argc, const char **argv) { llvm::InitLLVM X(argc, argv); llvm::cl::HideUnrelatedOptions(DependencyScannerCategory); @@ -316,6 +501,7 @@ int main(int argc, const char **argv) { std::vector WorkerThreads; std::atomic HadErrors(false); + FullDeps FD; std::mutex Lock; size_t Index = 0; @@ -324,26 +510,38 @@ int main(int argc, const char **argv) { << " files using " << NumWorkers << " workers\n"; } for (unsigned I = 0; I < NumWorkers; ++I) { - auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools, + auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools, &DependencyOS, &Errs]() { + llvm::StringSet<> AlreadySeenModules; while (true) { const SingleCommandCompilationDatabase *Input; std::string Filename; std::string CWD; + size_t LocalIndex; // Take the next input. { std::unique_lock LockGuard(Lock); if (Index >= Inputs.size()) return; + LocalIndex = Index; Input = &Inputs[Index++]; tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0]; Filename = std::move(Cmd.Filename); CWD = std::move(Cmd.Directory); } // Run the tool on it. - auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); - if (handleDependencyToolResult(Filename, MaybeFile, DependencyOS, Errs)) - HadErrors = true; + if (Format == ScanningOutputFormat::Make) { + auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD); + if (handleMakeDependencyToolResult(Filename, MaybeFile, DependencyOS, + Errs)) + HadErrors = true; + } else { + auto MaybeFullDeps = WorkerTools[I]->getFullDependencies( + *Input, CWD, AlreadySeenModules); + if (handleFullDependencyToolResult(Filename, MaybeFullDeps, FD, + LocalIndex, DependencyOS, Errs)) + HadErrors = true; + } } }; #if LLVM_ENABLE_THREADS @@ -356,5 +554,8 @@ int main(int argc, const char **argv) { for (auto &W : WorkerThreads) W.join(); + if (Format == ScanningOutputFormat::Full) + FD.printFullOutput(llvm::outs()); + return HadErrors; } From cd0f9c81c08e85e9a73aedf8d531552ae2f8c318 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Fri, 24 Jan 2020 12:59:46 -0800 Subject: [PATCH 02/13] [Clang][ScanDeps] Pass diagnostics to `getModuleHash`. rdar://58857443 (cherry picked from commit 74b01d9b60b697d5dfb121ae4843bab5a4d378ef) --- .../Tooling/DependencyScanning/ModuleDepCollector.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 9d2d2a187da3c..38249582f8623 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -68,7 +68,8 @@ void ModuleDepCollectorPP::FileChanged(SourceLocation Loc, // This has to be delayed as the context hash can change at the start of // `CompilerInstance::ExecuteAction`. if (MDC.ContextHash.empty()) { - MDC.ContextHash = Instance.getInvocation().getModuleHash(); + MDC.ContextHash = + Instance.getInvocation().getModuleHash(Instance.getDiagnostics()); MDC.Consumer.handleContextHash(MDC.ContextHash); } @@ -178,9 +179,9 @@ void ModuleDepCollectorPP::addModuleDep( for (const Module *Import : M->Imports) { if (Import->getTopLevelModule() != M->getTopLevelModule()) { if (AddedModules.insert(Import->getTopLevelModule()).second) - MD.ClangModuleDeps.push_back( - {Import->getTopLevelModuleName(), - Instance.getInvocation().getModuleHash()}); + MD.ClangModuleDeps.push_back({Import->getTopLevelModuleName(), + Instance.getInvocation().getModuleHash( + Instance.getDiagnostics())}); handleTopLevelModule(Import->getTopLevelModule()); } } From e23a09ebb8d7db36b08b10075f50a1d80b1b3e44 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Fri, 15 Nov 2019 17:38:10 -0800 Subject: [PATCH 03/13] [clang][clang-scan-deps] Add an experimental C API. This adds an experimental C API for clang-scan-deps. It provides both the full module dependency graph along with a flattened list of dependencies. See clang/include/clang-c/Dependencies.h for the API and documentation. You can test it out using: c-index-test core --scan-deps -- clang --cc1 This will output a list of modules and then the direct dependencies of the main translation unit. (cherry picked from commit 3e60b065115d4a544b8872a3ea2782780796a824) --- clang/include/clang-c/Dependencies.h | 223 ++++++++++++++++++ .../DependencyScanningWorker.h | 9 + .../DependencyScanningWorker.cpp | 26 ++ clang/test/Index/Core/scan-deps.m | 29 +++ clang/tools/c-index-test/CMakeLists.txt | 1 + clang/tools/c-index-test/core_main.cpp | 86 ++++++- clang/tools/libclang/CDependencies.cpp | 222 +++++++++++++++++ clang/tools/libclang/CMakeLists.txt | 2 + clang/tools/libclang/CXString.cpp | 17 ++ clang/tools/libclang/CXString.h | 3 + clang/tools/libclang/libclang.exports | 7 + llvm/include/llvm/ADT/FunctionExtras.h | 31 +++ 12 files changed, 655 insertions(+), 1 deletion(-) create mode 100644 clang/include/clang-c/Dependencies.h create mode 100644 clang/test/Index/Core/scan-deps.m create mode 100644 clang/tools/libclang/CDependencies.cpp diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h new file mode 100644 index 0000000000000..5ea6e791aa1fd --- /dev/null +++ b/clang/include/clang-c/Dependencies.h @@ -0,0 +1,223 @@ +/*==-- clang-c/Dependencies.h - Dependency Discovery C Interface --*- C -*-===*\ +|* *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header provides a dependency discovery interface similar to *| +|* clang-scan-deps. *| +|* *| +|* An example of its usage is available in c-index-test/core_main.cpp. *| +|* *| +|* EXPERIMENTAL: These interfaces are experimental and will change. If you *| +|* use these be prepared for them to change without notice on any commit. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_CLANG_C_DEPENDENCIES_H +#define LLVM_CLANG_C_DEPENDENCIES_H + +#include "clang-c/BuildSystem.h" +#include "clang-c/CXErrorCode.h" +#include "clang-c/CXString.h" +#include "clang-c/Platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \defgroup SCAN_DEPS Dependency scanning service. + * @{ + */ + +typedef struct { + CXString Name; + /** + * The context hash of a module represents the set of compiler options that + * may make one version of a module incompatible from another. This includes + * things like language mode, predefined macros, header search paths, etc... + * + * Modules with the same name but a different \c ContextHash should be treated + * as separate modules for the purpose of a build. + */ + CXString ContextHash; + + /** + * The path to the modulemap file which defines this module. + * + * This can be used to explicitly build this module. This file will + * additionally appear in \c FileDeps as a dependency. + */ + CXString ModuleMapPath; + + /** + * The list of files which this module directly depends on. + * + * If any of these change then the module needs to be rebuilt. + */ + CXStringSet *FileDeps; + + /** + * The list of modules which this module direct depends on. + * + * This does include the context hash. The format is + * `:` + */ + CXStringSet *ModuleDeps; + + /** + * The full command line needed to build this module. + * + * Not including `-fmodule-file=` or `-o`. + */ + CXStringSet *BuildArguments; +} CXModuleDependency; + +typedef struct { + int Count; + CXModuleDependency *Modules; +} CXModuleDependencySet; + +/** + * See \c CXModuleDependency for the meaning of these fields, with the addition + * that they represent only the direct dependencies for \c CXDependencyMode_Full + * mode. + */ +typedef struct { + CXString ContextHash; + CXStringSet *FileDeps; + CXStringSet *ModuleDeps; + + /** + * Additional arguments to append to the build of this file. + * + * This contains things like disabling implicit modules. This does not include + * the `-fmodule-file=` arguments that are needed. + */ + CXStringSet *AdditionalArguments; +} CXFileDependencies; + +CINDEX_LINKAGE void +clang_experimental_ModuleDependencySet_dispose(CXModuleDependencySet *MD); + +CINDEX_LINKAGE void +clang_experimental_FileDependencies_dispose(CXFileDependencies *ID); + +/** + * Object encapsulating instance of a dependency scanner service. + * + * The dependency scanner service is a global instance that owns the + * global cache and other global state that's shared between the dependency + * scanner workers. The service APIs are thread safe. + */ +typedef struct CXOpaqueDependencyScannerService *CXDependencyScannerService; + +/** + * The mode to report module dependencies in. + */ +typedef enum { + /** + * Flatten all module dependencies. This reports the full transitive set of + * header and module map dependencies needed to do an implicit module build. + */ + CXDependencyMode_Flat, + + /** + * Report the full module graph. This reports only the direct dependencies of + * each file, and calls a callback for each module that is discovered. + */ + CXDependencyMode_Full, +} CXDependencyMode; + +/** + * Create a \c CXDependencyScannerService object. + * Must be disposed with \c clang_DependencyScannerService_dispose(). + */ +CINDEX_LINKAGE CXDependencyScannerService +clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format); + +/** + * Dispose of a \c CXDependencyScannerService object. + * + * The service object must be disposed of after the workers are disposed of. + */ +CINDEX_LINKAGE void clang_experimental_DependencyScannerService_dispose_v0( + CXDependencyScannerService); + +/** + * Object encapsulating instance of a dependency scanner worker. + * + * The dependency scanner workers are expected to be used in separate worker + * threads. An individual worker is not thread safe. + * + * Operations on a worker are not thread-safe and should only be used from a + * single thread at a time. They are intended to be used by a single dedicated + * thread in a thread pool, but they are not inherently pinned to a thread. + */ +typedef struct CXOpaqueDependencyScannerWorker *CXDependencyScannerWorker; + +/** + * Create a \c CXDependencyScannerWorker object. + * Must be disposed with + * \c clang_experimental_DependencyScannerWorker_dispose_v0(). + */ +CINDEX_LINKAGE CXDependencyScannerWorker + clang_experimental_DependencyScannerWorker_create_v0( + CXDependencyScannerService); + +CINDEX_LINKAGE void clang_experimental_DependencyScannerWorker_dispose_v0( + CXDependencyScannerWorker); + +/** + * A callback that is called whenever a module is discovered when in + * \c CXDependencyMode_Full mode. + * + * \param Context the context that was passed to + * \c clang_experimental_DependencyScannerWorker_getFileDependencies_v0. + * \param MDS the list of discovered modules. Must be freed by calling + * \c clang_experimental_ModuleDependencySet_dispose. + */ +typedef void CXModuleDiscoveredCallback(void *Context, + CXModuleDependencySet *MDS); + +/** + * Returns the list of file dependencies for a particular compiler invocation. + * + * \param argc the number of compiler invocation arguments (including argv[0]). + * \param argv the compiler invocation arguments (including argv[0]). + * the invocation may be a -cc1 clang invocation or a driver + * invocation. + * \param WorkingDirectory the directory in which the invocation runs. + * \param MDC a callback that is called whenever a new module is discovered. + * This may receive the same module on different workers. This should + * be NULL if + * \c clang_experimental_DependencyScannerService_create_v0 was + * called with \c CXDependencyMode_Flat. This callback will be called + * on the same thread that called this function. + * \param Context the context that will be passed to \c MDC each time it is + * called. + * \param [out] error the error string to pass back to client (if any). + * + * \returns A pointer to a CXFileDependencies on success, NULL otherwise. The + * CXFileDependencies must be freed by calling + * \c clang_experimental_FileDependencies_dispose. + */ +CINDEX_LINKAGE CXFileDependencies * +clang_experimental_DependencyScannerWorker_getFileDependencies_v0( + CXDependencyScannerWorker Worker, int argc, const char *const *argv, + const char *WorkingDirectory, CXModuleDiscoveredCallback *MDC, + void *Context, CXString *error); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif // LLVM_CLANG_C_DEPENDENCIES_H diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 689119330c412..22c84e032fd31 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -63,6 +63,15 @@ class DependencyScanningWorker { const CompilationDatabase &CDB, DependencyConsumer &Consumer); + llvm::Error + computeDependenciesForClangInvocation(StringRef WorkingDirectory, + ArrayRef Arguments, + DependencyConsumer &Consumer); + + ScanningOutputFormat getFormat() const { return Format; } + + llvm::StringSet<> AlreadySeen; + private: IntrusiveRefCntPtr DiagOpts; std::shared_ptr PCHContainerOps; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 73135dd2ff58d..6d513b1286ab0 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -220,3 +220,29 @@ llvm::Error DependencyScanningWorker::computeDependencies( return !Tool.run(&Action); }); } + +llvm::Error DependencyScanningWorker::computeDependenciesForClangInvocation( + StringRef WorkingDirectory, ArrayRef Arguments, + DependencyConsumer &Consumer) { + RealFS->setCurrentWorkingDirectory(WorkingDirectory); + return runWithDiags(DiagOpts.get(), [&](DiagnosticConsumer &DC) { + IntrusiveRefCntPtr DiagID = new DiagnosticIDs(); + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + DiagnosticsEngine Diags(DiagID, &*DiagOpts, &DC, /*ShouldOwnClient=*/false); + + llvm::opt::ArgStringList CC1Args; + for (const auto &Arg : Arguments) + CC1Args.push_back(Arg.c_str()); + std::unique_ptr Invocation( + newInvocation(&Diags, CC1Args)); + + DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS, + PPSkipMappings.get(), Format); + + llvm::IntrusiveRefCntPtr FM = Files; + if (!FM) + FM = new FileManager(FileSystemOptions(), RealFS); + return Action.runInvocation(std::move(Invocation), FM.get(), + PCHContainerOps, &DC); + }); +} diff --git a/clang/test/Index/Core/scan-deps.m b/clang/test/Index/Core/scan-deps.m new file mode 100644 index 0000000000000..2453ca5dfe1ef --- /dev/null +++ b/clang/test/Index/Core/scan-deps.m @@ -0,0 +1,29 @@ +// RUN: rm -rf %t.mcp +// RUN: echo %S > %t.result +// RUN: c-index-test core --scan-deps %S -- %clang -cc1 -I %S/Inputs/module \ +// RUN: -fmodules -fmodules-cache-path=%t.mcp -fimplicit-module-maps \ +// RUN: -o FoE.o -x objective-c %s >> %t.result +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s + +@import ModA; + +// CHECK: [[PREFIX:.*]] +// CHECK-NEXT: modules: +// CHECK-NEXT: module: +// CHECK-NEXT: name: ModA +// CHECK-NEXT: context-hash: [[CONTEXT_HASH:[A-Z0-9]+]] +// CHECK-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap +// CHECK-NEXT: module-deps: +// CHECK-NEXT: file-deps: +// CHECK-NEXT: [[PREFIX]]/Inputs/module/ModA.h +// CHECK-NEXT: [[PREFIX]]/Inputs/module/SubModA.h +// CHECK-NEXT: [[PREFIX]]/Inputs/module/SubSubModA.h +// CHECK-NEXT: [[PREFIX]]/Inputs/module/module.modulemap +// CHECK-NEXT: build-args: +// CHECK-NEXT: dependencies: +// CHECK-NEXT: context-hash: [[CONTEXT_HASH]] +// CHECK-NEXT: module-deps: +// CHECK-NEXT: ModA:[[CONTEXT_HASH]] +// CHECK-NEXT: file-deps: +// CHECK-NEXT: [[PREFIX]]/scan-deps.m +// CHECK-NEXT: additional-build-args: diff --git a/clang/tools/c-index-test/CMakeLists.txt b/clang/tools/c-index-test/CMakeLists.txt index dd76de05826af..dbca85e20cffb 100644 --- a/clang/tools/c-index-test/CMakeLists.txt +++ b/clang/tools/c-index-test/CMakeLists.txt @@ -37,6 +37,7 @@ else() clangAST clangBasic clangCodeGen + clangDependencyScanning clangDirectoryWatcher clangFrontend clangIndex diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 4a99c12c530f9..b68b525bacb9a 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -8,6 +8,7 @@ #include "JSONAggregation.h" #include "indexstore/IndexStoreCXX.h" +#include "clang-c/Dependencies.h" #include "clang/DirectoryWatcher/DirectoryWatcher.h" #include "clang/AST/Mangle.h" #include "clang/Basic/LangOptions.h" @@ -24,12 +25,13 @@ #include "clang/Index/USRGeneration.h" #include "clang/Lex/Preprocessor.h" #include "clang/Serialization/ASTReader.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/Signals.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Support/PrettyStackTrace.h" #include using namespace clang; @@ -47,6 +49,7 @@ enum class ActionType { PrintUnit, PrintStoreFormatVersion, AggregateAsJSON, + ScanDeps, WatchDir, }; @@ -67,6 +70,8 @@ Action(cl::desc("Action:"), cl::init(ActionType::None), "print-store-format-version", "Print store format version"), clEnumValN(ActionType::AggregateAsJSON, "aggregate-json", "Aggregate index data in JSON format"), + clEnumValN(ActionType::ScanDeps, "scan-deps", + "Get file dependencies"), clEnumValN(ActionType::WatchDir, "watch-dir", "Watch directory for file events")), cl::cat(IndexTestCoreCategory)); @@ -631,6 +636,77 @@ static void printSymbolNameAndUSR(const clang::Module *Mod, raw_ostream &OS) { generateFullUSRForModule(Mod, OS); } +static int scanDeps(ArrayRef Args, std::string WorkingDirectory) { + CXDependencyScannerService Service = + clang_experimental_DependencyScannerService_create_v0( + CXDependencyMode_Full); + CXDependencyScannerWorker Worker = + clang_experimental_DependencyScannerWorker_create_v0(Service); + CXString Error; + + auto Callback = [&](CXModuleDependencySet *MDS) { + llvm::outs() << "modules:\n"; + for (const auto &M : llvm::makeArrayRef(MDS->Modules, MDS->Count)) { + llvm::outs() << " module:\n" + << " name: " << clang_getCString(M.Name) << "\n" + << " context-hash: " << clang_getCString(M.ContextHash) + << "\n" + << " module-map-path: " + << clang_getCString(M.ModuleMapPath) << "\n" + << " module-deps:\n"; + for (const auto &ModuleName : + llvm::makeArrayRef(M.ModuleDeps->Strings, M.ModuleDeps->Count)) + llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; + llvm::outs() << " file-deps:\n"; + for (const auto &FileName : + llvm::makeArrayRef(M.FileDeps->Strings, M.FileDeps->Count)) + llvm::outs() << " " << clang_getCString(FileName) << "\n"; + llvm::outs() << " build-args:"; + for (const auto &Arg : llvm::makeArrayRef(M.BuildArguments->Strings, + M.BuildArguments->Count)) + llvm::outs() << " " << clang_getCString(Arg); + llvm::outs() << "\n"; + } + clang_experimental_ModuleDependencySet_dispose(MDS); + }; + + auto CB = + functionObjectToCCallbackRef(Callback); + + CXFileDependencies *Result = + clang_experimental_DependencyScannerWorker_getFileDependencies_v0( + Worker, Args.size(), Args.data(), WorkingDirectory.c_str(), + CB.Callback, CB.Context, &Error); + if (!Result) { + llvm::errs() << "error: failed to get dependencies\n"; + llvm::errs() << clang_getCString(Error) << "\n"; + clang_disposeString(Error); + return 1; + } + llvm::outs() << "dependencies:\n"; + llvm::outs() << " context-hash: " << clang_getCString(Result->ContextHash) + << "\n" + << " module-deps:\n"; + for (const auto &ModuleName : llvm::makeArrayRef(Result->ModuleDeps->Strings, + Result->ModuleDeps->Count)) + llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; + llvm::outs() << " file-deps:\n"; + for (const auto &FileName : + llvm::makeArrayRef(Result->FileDeps->Strings, Result->FileDeps->Count)) + llvm::outs() << " " << clang_getCString(FileName) << "\n"; + llvm::outs() << " additional-build-args:"; + for (const auto &Arg : + llvm::makeArrayRef(Result->AdditionalArguments->Strings, + Result->AdditionalArguments->Count)) + llvm::outs() << " " << clang_getCString(Arg); + llvm::outs() << "\n"; + + clang_experimental_FileDependencies_dispose(Result); + clang_experimental_DependencyScannerWorker_dispose_v0(Worker); + clang_experimental_DependencyScannerService_dispose_v0(Service); + return 0; +} + static void printSymbol(const IndexRecordDecl &Rec, raw_ostream &OS) { printSymbolInfo(Rec.SymInfo, OS); OS << " | "; @@ -919,6 +995,14 @@ int indextest_core_main(int argc, const char **argv) { } return aggregateDataAsJSON(storePath, OS); } + + if (options::Action == ActionType::ScanDeps) { + if (options::InputFiles.empty()) { + errs() << "error: missing working directory\n"; + return 1; + } + return scanDeps(CompArgs, options::InputFiles[0]); + } if (options::Action == ActionType::WatchDir) { if (options::InputFiles.empty()) { diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp new file mode 100644 index 0000000000000..9366505cc8398 --- /dev/null +++ b/clang/tools/libclang/CDependencies.cpp @@ -0,0 +1,222 @@ +//===- CDependencies.cpp - Dependency Discovery C Interface ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements the dependency discovery interface. It provides a C library for +// the functionality that clang-scan-deps provides. +// +//===----------------------------------------------------------------------===// + +#include "CXString.h" + +#include "clang-c/Dependencies.h" + +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" + +using namespace clang; +using namespace clang::tooling::dependencies; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningService, + CXDependencyScannerService) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScanningWorker, + CXDependencyScannerWorker) + +inline ScanningOutputFormat unwrap(CXDependencyMode Format) { + switch (Format) { + case CXDependencyMode_Flat: + return ScanningOutputFormat::Make; + case CXDependencyMode_Full: + return ScanningOutputFormat::Full; + } +} + +void clang_experimental_ModuleDependencySet_dispose( + CXModuleDependencySet *MDS) { + for (int I = 0; I < MDS->Count; ++I) { + CXModuleDependency &MD = MDS->Modules[I]; + clang_disposeString(MD.Name); + clang_disposeString(MD.ContextHash); + clang_disposeString(MD.ModuleMapPath); + clang_disposeStringSet(MD.FileDeps); + clang_disposeStringSet(MD.ModuleDeps); + } + delete[] MDS->Modules; + delete MDS; +} + +CXDependencyScannerService +clang_experimental_DependencyScannerService_create_v0(CXDependencyMode Format) { + return wrap(new DependencyScanningService( + ScanningMode::MinimizedSourcePreprocessing, unwrap(Format), + /*ReuseFilemanager=*/false)); +} + +void clang_experimental_DependencyScannerService_dispose_v0( + CXDependencyScannerService Service) { + delete unwrap(Service); +} + +void clang_experimental_FileDependencies_dispose(CXFileDependencies *ID) { + clang_disposeString(ID->ContextHash); + clang_disposeStringSet(ID->FileDeps); + clang_disposeStringSet(ID->ModuleDeps); + delete ID; +} + +CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0( + CXDependencyScannerService Service) { + return wrap(new DependencyScanningWorker(*unwrap(Service))); +} + +void clang_experimental_DependencyScannerWorker_dispose_v0( + CXDependencyScannerWorker Worker) { + delete unwrap(Worker); +} + +static CXFileDependencies * +getFlatDependencies(DependencyScanningWorker *Worker, + ArrayRef Compilation, + const char *WorkingDirectory, CXString *error) { + // TODO: Implement flat deps. + return nullptr; +} + +namespace { +class FullDependencyConsumer : public DependencyConsumer { +public: + FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen) + : AlreadySeen(AlreadySeen) {} + + void handleFileDependency(const DependencyOutputOptions &Opts, + StringRef File) override { + if (OutputPaths.empty()) + OutputPaths = Opts.Targets; + Dependencies.push_back(std::string(File)); + } + + void handleModuleDependency(ModuleDeps MD) override { + ClangModuleDeps[MD.ContextHash + MD.ModuleName] = std::move(MD); + } + + void handleContextHash(std::string Hash) override { + ContextHash = std::move(Hash); + } + + FullDependenciesResult getFullDependencies() const { + FullDependencies FD; + + FD.ContextHash = std::move(ContextHash); + + FD.FileDeps.assign(Dependencies.begin(), Dependencies.end()); + + for (auto &&M : ClangModuleDeps) { + auto &MD = M.second; + if (MD.ImportedByMainFile) + FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash}); + } + + FullDependenciesResult FDR; + + for (auto &&M : ClangModuleDeps) { + // TODO: Avoid handleModuleDependency even being called for modules + // we've already seen. + if (AlreadySeen.count(M.first)) + continue; + FDR.DiscoveredModules.push_back(std::move(M.second)); + } + + FDR.FullDeps = std::move(FD); + return FDR; + } + +private: + std::vector Dependencies; + std::unordered_map ClangModuleDeps; + std::string ContextHash; + std::vector OutputPaths; + const llvm::StringSet<> &AlreadySeen; +}; +} // namespace + +static CXFileDependencies *getFullDependencies( + DependencyScanningWorker *Worker, ArrayRef Compilation, + const char *WorkingDirectory, CXModuleDiscoveredCallback *MDC, + void *Context, CXString *error) { + FullDependencyConsumer Consumer(Worker->AlreadySeen); + llvm::Error Result = Worker->computeDependenciesForClangInvocation( + WorkingDirectory, Compilation, Consumer); + + if (Result) { + std::string Str; + llvm::raw_string_ostream OS(Str); + llvm::handleAllErrors(std::move(Result), + [&](const llvm::ErrorInfoBase &EI) { EI.log(OS); }); + *error = cxstring::createDup(OS.str()); + return nullptr; + } + + FullDependenciesResult FDR = Consumer.getFullDependencies(); + + if (!FDR.DiscoveredModules.empty()) { + CXModuleDependencySet *MDS = new CXModuleDependencySet; + MDS->Count = FDR.DiscoveredModules.size(); + MDS->Modules = new CXModuleDependency[MDS->Count]; + for (int I = 0; I < MDS->Count; ++I) { + CXModuleDependency &M = MDS->Modules[I]; + const ModuleDeps &MD = FDR.DiscoveredModules[I]; + M.Name = cxstring::createDup(MD.ModuleName); + M.ContextHash = cxstring::createDup(MD.ContextHash); + M.ModuleMapPath = cxstring::createDup(MD.ClangModuleMapFile); + M.FileDeps = cxstring::createSet(MD.FileDeps); + std::vector Modules; + for (const ClangModuleDep &CMD : MD.ClangModuleDeps) + Modules.push_back(CMD.ModuleName + ":" + CMD.ContextHash); + M.ModuleDeps = cxstring::createSet(Modules); + M.BuildArguments = cxstring::createSet(std::vector{}); + } + MDC(Context, MDS); + } + + const FullDependencies &FD = FDR.FullDeps; + CXFileDependencies *FDeps = new CXFileDependencies; + FDeps->ContextHash = cxstring::createDup(FD.ContextHash); + FDeps->FileDeps = cxstring::createSet(FD.FileDeps); + std::vector Modules; + for (const ClangModuleDep &CMD : FD.ClangModuleDeps) + Modules.push_back(CMD.ModuleName + ":" + CMD.ContextHash); + FDeps->ModuleDeps = cxstring::createSet(Modules); + FDeps->AdditionalArguments = cxstring::createSet(std::vector{}); + return FDeps; +} + +CXFileDependencies * +clang_experimental_DependencyScannerWorker_getFileDependencies_v0( + CXDependencyScannerWorker W, int argc, const char *const *argv, + const char *WorkingDirectory, CXModuleDiscoveredCallback *MDC, + void *Context, CXString *error) { + if (!W || argc < 2) + return nullptr; + if (error) + *error = cxstring::createEmpty(); + + DependencyScanningWorker *Worker = unwrap(W); + + std::vector Compilation; + if (StringRef(argv[1]) == "-cc1") + for (int i = 2; i < argc; ++i) + Compilation.push_back(argv[i]); + else { + return nullptr; // TODO: Run the driver to get -cc1 args. + } + + if (Worker->getFormat() == ScanningOutputFormat::Full) + return getFullDependencies(Worker, Compilation, WorkingDirectory, MDC, + Context, error); + return getFlatDependencies(Worker, Compilation, WorkingDirectory, error); +} diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt index 192bad104d523..54b6607df7a7f 100644 --- a/clang/tools/libclang/CMakeLists.txt +++ b/clang/tools/libclang/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCES ARCMigrate.cpp BuildSystem.cpp + CDependencies.cpp CIndex.cpp CIndexCXX.cpp CIndexCodeCompletion.cpp @@ -40,6 +41,7 @@ set(LIBS clangASTMatchers clangAPINotes clangBasic + clangDependencyScanning clangDriver clangFrontend clangIndex diff --git a/clang/tools/libclang/CXString.cpp b/clang/tools/libclang/CXString.cpp index 2754795f4a647..ce4964df9fca9 100644 --- a/clang/tools/libclang/CXString.cpp +++ b/clang/tools/libclang/CXString.cpp @@ -119,6 +119,23 @@ CXStringSet *createSet(const std::vector &Strings) { return Set; } +CXStringSet *createSet(const llvm::StringSet<> &StringsUnordered) { + std::vector Strings; + + for (auto SI = StringsUnordered.begin(), + SE = StringsUnordered.end(); SI != SE; ++SI) + Strings.push_back(SI->getKey()); + + llvm::sort(Strings); + + CXStringSet *Set = new CXStringSet; + Set->Count = Strings.size(); + Set->Strings = new CXString[Set->Count]; + int I = 0; + for (auto SI = Strings.begin(), SE = Strings.end(); SI != SE; ++SI) + Set->Strings[I++] = createDup(*SI); + return Set; +} //===----------------------------------------------------------------------===// // String pools. diff --git a/clang/tools/libclang/CXString.h b/clang/tools/libclang/CXString.h index 809bdec3d677f..5f9351c0a7c45 100644 --- a/clang/tools/libclang/CXString.h +++ b/clang/tools/libclang/CXString.h @@ -17,6 +17,7 @@ #include "clang/Basic/LLVM.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/Support/Compiler.h" #include #include @@ -69,6 +70,8 @@ CXString createCXString(CXStringBuf *buf); CXStringSet *createSet(const std::vector &Strings); +CXStringSet *createSet(const llvm::StringSet<> &Strings); + /// A string pool used for fast allocation/deallocation of strings. class CXStringPool { public: diff --git a/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports index 2bbc8dca6cbb8..035bfb91b8e1e 100644 --- a/clang/tools/libclang/libclang.exports +++ b/clang/tools/libclang/libclang.exports @@ -157,6 +157,13 @@ clang_equalLocations clang_equalRanges clang_equalTypes clang_executeOnThread +clang_experimental_DependencyScannerService_create_v0 +clang_experimental_DependencyScannerService_dispose_v0 +clang_experimental_DependencyScannerWorker_create_v0 +clang_experimental_DependencyScannerWorker_dispose_v0 +clang_experimental_DependencyScannerWorker_getFileDependencies_v0 +clang_experimental_FileDependencies_dispose +clang_experimental_ModuleDependencySet_dispose clang_findIncludesInFile clang_findIncludesInFileWithBlock clang_findReferencesInFile diff --git a/llvm/include/llvm/ADT/FunctionExtras.h b/llvm/include/llvm/ADT/FunctionExtras.h index 121aa527a5dac..69df9352bf5cd 100644 --- a/llvm/include/llvm/ADT/FunctionExtras.h +++ b/llvm/include/llvm/ADT/FunctionExtras.h @@ -287,6 +287,37 @@ class unique_function { } }; +template struct FunctionObjectCallback { + void *Context; + CallTy *Callback; +}; + +namespace detail { +template +struct functionObjectToCCallbackRefImpl; + +template +struct functionObjectToCCallbackRefImpl { + static FunctionObjectCallback impl(FuncTy &F) { + auto Func = +[](void *C, Args... V) -> Ret { + return (*reinterpret_cast *>(C))( + std::forward(V)...); + }; + + return {&F, Func}; + } +}; +} // namespace detail + +/// Returns a function pointer and context pair suitable for use as a C +/// callback. +/// +/// \param F the function object to turn into a C callback. The returned +/// callback has the same lifetime as F. +template +auto functionObjectToCCallbackRef(FuncTy &F) { + return detail::functionObjectToCCallbackRefImpl::impl(F); +} } // end namespace llvm #endif // LLVM_ADT_FUNCTION_H From ced3ced67cd55dcc0854433a449e6446c6995608 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Mon, 16 Dec 2019 18:26:33 -0800 Subject: [PATCH 04/13] [libclang] Add clang_Driver_getExternalActionsForCommand_v0 This function returns the equivalent of -### (cherry picked from commit 1fdae62265b1603bdc97491c5b653962be356d80) --- clang/include/clang-c/Driver.h | 78 +++++++++++++++ clang/tools/libclang/CMakeLists.txt | 1 + clang/tools/libclang/Driver.cpp | 125 ++++++++++++++++++++++++ clang/tools/libclang/libclang.exports | 2 + clang/unittests/libclang/CMakeLists.txt | 1 + clang/unittests/libclang/DriverTest.cpp | 75 ++++++++++++++ 6 files changed, 282 insertions(+) create mode 100644 clang/include/clang-c/Driver.h create mode 100644 clang/tools/libclang/Driver.cpp create mode 100644 clang/unittests/libclang/DriverTest.cpp diff --git a/clang/include/clang-c/Driver.h b/clang/include/clang-c/Driver.h new file mode 100644 index 0000000000000..54dd1b0b043a3 --- /dev/null +++ b/clang/include/clang-c/Driver.h @@ -0,0 +1,78 @@ +/*==-- clang-c/Driver.h - A C Interface for the Clang Driver ------*- C -*-===*\ +|* *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header provides a C API for extracting information from the clang *| +|* driver. *| +|* *| +\*===----------------------------------------------------------------------===*/ + + +#ifndef CLANG_CLANG_C_DRIVER +#define CLANG_CLANG_C_DRIVER + +#include "clang-c/Index.h" +#include "clang-c/Platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Contains the command line arguments for an external action. Same format as + * provided to main. + */ +typedef struct { + /* Number of arguments in ArgV */ + int ArgC; + /* Null terminated array of pointers to null terminated argument strings */ + const char **ArgV; +} CXExternalAction; + +/** + * Contains the list of external actions clang would invoke. + */ +typedef struct { + int Count; + CXExternalAction **Actions; +} CXExternalActionList; + +/** + * Get the external actions that the clang driver will invoke for the given + * command line. + * + * \param ArgC number of arguments in \p ArgV. + * \param ArgV array of null terminated arguments. Doesn't need to be null + * terminated. + * \param Environment must be null. + * \param WorkingDirectory a null terminated path to the working directory to + * use for this invocation. `nullptr` to use the current working directory of + * the process. + * \param OutDiags will be set to a \c CXDiagnosticSet if there's an error. + * Must be freed by calling \c clang_disposeDiagnosticSet . + * \returns A pointer to a \c CXExternalActionList on success, null on failure. + * The returned \c CXExternalActionList must be freed by calling + * \c clang_Driver_ExternalActionList_dispose . + */ +CINDEX_LINKAGE CXExternalActionList * +clang_Driver_getExternalActionsForCommand_v0(int ArgC, const char **ArgV, + const char **Environment, + const char *WorkingDirectory, + CXDiagnosticSet *OutDiags); + +/** + * Deallocate a \c CXExternalActionList + */ +CINDEX_LINKAGE void +clang_Driver_ExternalActionList_dispose(CXExternalActionList *EAL); + +#ifdef __cplusplus +} +#endif + +#endif // CLANG_CLANG_C_DRIVER diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt index 54b6607df7a7f..bcb5d2fc3bb68 100644 --- a/clang/tools/libclang/CMakeLists.txt +++ b/clang/tools/libclang/CMakeLists.txt @@ -20,6 +20,7 @@ set(SOURCES CXStoredDiagnostic.cpp CXString.cpp CXType.cpp + Driver.cpp Indexing.cpp FatalErrorHandler.cpp diff --git a/clang/tools/libclang/Driver.cpp b/clang/tools/libclang/Driver.cpp new file mode 100644 index 0000000000000..1a5294a54138c --- /dev/null +++ b/clang/tools/libclang/Driver.cpp @@ -0,0 +1,125 @@ +//===- Driver.cpp - A C Interface for the Clang Driver --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a C API for extracting information from the clang driver. +// +//===----------------------------------------------------------------------===// + +#include "clang-c/Driver.h" + +#include "CIndexDiagnostic.h" + +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Frontend/CompilerInstance.h" +#include "llvm/ADT/ArrayRef.h" + +using namespace clang; + +class CXDiagnosticSetDiagnosticConsumer : public DiagnosticConsumer { + SmallVector Errors; +public: + + void HandleDiagnostic(DiagnosticsEngine::Level level, + const Diagnostic &Info) override { + if (level >= DiagnosticsEngine::Error) + Errors.push_back(StoredDiagnostic(level, Info)); + } + + CXDiagnosticSet getDiagnosticSet() { + return cxdiag::createStoredDiags(Errors, LangOptions()); + } +}; + +CXExternalActionList * +clang_Driver_getExternalActionsForCommand_v0(int ArgC, const char **ArgV, + const char **Environment, + const char *WorkingDirectory, + CXDiagnosticSet *OutDiags) { + if (OutDiags) + *OutDiags = nullptr; + + // Non empty environments are not currently supported. + if (Environment) + return nullptr; + + // ArgV must at least include the compiler executable name. + if (ArgC < 1) + return nullptr; + + CXDiagnosticSetDiagnosticConsumer DiagConsumer; + auto Diags = CompilerInstance::createDiagnostics(new DiagnosticOptions, + &DiagConsumer, false); + + // Use createPhysicalFileSystem instead of getRealFileSystem so that + // setCurrentWorkingDirectory doesn't change the working directory of the + // process. + std::unique_ptr VFS = + llvm::vfs::createPhysicalFileSystem(); + if (WorkingDirectory) + if (std::error_code EC = + VFS->setCurrentWorkingDirectory(WorkingDirectory)) { + Diags->Report(diag::err_drv_unable_to_set_working_directory) << + WorkingDirectory; + if (OutDiags) + *OutDiags = DiagConsumer.getDiagnosticSet(); + return nullptr; + } + + driver::Driver TheDriver(ArgV[0], llvm::sys::getDefaultTargetTriple(), *Diags, + VFS.release()); + TheDriver.setCheckInputsExist(false); + std::unique_ptr C( + TheDriver.BuildCompilation(llvm::makeArrayRef(ArgV, ArgC))); + if (!C) { + if (OutDiags) + *OutDiags = DiagConsumer.getDiagnosticSet(); + return nullptr; + } + + const driver::JobList &Jobs = C->getJobs(); + CXExternalAction **Actions = new CXExternalAction *[Jobs.size()]; + int AI = 0; + for (auto &&J : Jobs) { + // First calculate the total space we'll need for this action's arguments. + llvm::opt::ArgStringList Args = J.getArguments(); + Args.insert(Args.begin(), J.getExecutable()); + int ArgSpace = (Args.size() + 1) * sizeof(const char *); + for (auto &&Arg : Args) + ArgSpace += strlen(Arg) + 1; // Null terminator + + // Tail allocate the space for the strings. + auto Action = + new ((CXExternalAction *)malloc(sizeof(CXExternalAction) + ArgSpace)) + CXExternalAction; + Action->ArgC = Args.size(); + Action->ArgV = reinterpret_cast(Action + 1); + Action->ArgV[Args.size()] = nullptr; + char *StrTable = ((char *)Action) + sizeof(CXExternalAction) + + (Args.size() + 1) * sizeof(const char *); + int I = 0; + for (auto &&Arg : Args) { + Action->ArgV[I++] = strcpy(StrTable, Arg); + StrTable += strlen(Arg) + 1; + } + Actions[AI++] = Action; + } + + return new CXExternalActionList{(int)Jobs.size(), Actions}; +} + +void clang_Driver_ExternalActionList_dispose(CXExternalActionList *EAL) { + if (!EAL) + return; + + for (int I = 0; I < EAL->Count; ++I) + free(EAL->Actions[I]); + delete[] EAL->Actions; + delete EAL; +} diff --git a/clang/tools/libclang/libclang.exports b/clang/tools/libclang/libclang.exports index 035bfb91b8e1e..b3f4c7b0f36f6 100644 --- a/clang/tools/libclang/libclang.exports +++ b/clang/tools/libclang/libclang.exports @@ -49,6 +49,8 @@ clang_Cursor_isObjCOptional clang_Cursor_isVariadic clang_Cursor_getModule clang_Cursor_getStorageClass +clang_Driver_ExternalActionList_dispose +clang_Driver_getExternalActionsForCommand_v0 clang_File_isEqual clang_File_tryGetRealPathName clang_Module_getASTFile diff --git a/clang/unittests/libclang/CMakeLists.txt b/clang/unittests/libclang/CMakeLists.txt index b3644a0e710e1..51288a8b91deb 100644 --- a/clang/unittests/libclang/CMakeLists.txt +++ b/clang/unittests/libclang/CMakeLists.txt @@ -1,5 +1,6 @@ add_clang_unittest(libclangTests LibclangTest.cpp + DriverTest.cpp ) target_link_libraries(libclangTests diff --git a/clang/unittests/libclang/DriverTest.cpp b/clang/unittests/libclang/DriverTest.cpp new file mode 100644 index 0000000000000..6a2713fb9a775 --- /dev/null +++ b/clang/unittests/libclang/DriverTest.cpp @@ -0,0 +1,75 @@ +//===---- DriverTest.cpp --------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang-c/Driver.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Debug.h" +#include "gtest/gtest.h" + +#define DEBUG_TYPE "driver-test" + +TEST(DriverTests, Basic) { + const char *ArgV[] = {"clang", "-w", "t.cpp", "-o", "t.ll"}; + + CXExternalActionList *EAL = clang_Driver_getExternalActionsForCommand_v0( + GTEST_ARRAY_SIZE_(ArgV), ArgV, nullptr, nullptr, nullptr); + ASSERT_NE(EAL, nullptr); + ASSERT_EQ(EAL->Count, 2); + auto *CompileAction = EAL->Actions[0]; + ASSERT_GE(CompileAction->ArgC, 2); + EXPECT_STREQ(CompileAction->ArgV[0], "clang"); + EXPECT_STREQ(CompileAction->ArgV[1], "-cc1"); + + clang_Driver_ExternalActionList_dispose(EAL); +} + +TEST(DriverTests, WorkingDirectory) { + const char *ArgV[] = {"clang", "-c", "t.cpp", "-o", "t.o"}; + + CXExternalActionList *EAL = clang_Driver_getExternalActionsForCommand_v0( + GTEST_ARRAY_SIZE_(ArgV), ArgV, nullptr, "/", nullptr); + ASSERT_NE(EAL, nullptr); + ASSERT_EQ(EAL->Count, 1); + auto *CompileAction = EAL->Actions[0]; + + const char **FDCD = std::find(CompileAction->ArgV, CompileAction->ArgV + + CompileAction->ArgC, + llvm::StringRef("-fdebug-compilation-dir")); + ASSERT_NE(FDCD, CompileAction->ArgV + CompileAction->ArgC); + ASSERT_NE(FDCD + 1, CompileAction->ArgV + CompileAction->ArgC); + EXPECT_STREQ(*(FDCD + 1), "/"); + + clang_Driver_ExternalActionList_dispose(EAL); +} + +TEST(DriverTests, Diagnostics) { + const char *ArgV[] = {"clang", "-c", "nosuchfile.cpp", "-o", "t.o"}; + + CXExternalActionList *EAL = clang_Driver_getExternalActionsForCommand_v0( + GTEST_ARRAY_SIZE_(ArgV), ArgV, nullptr, "/no/such/working/dir", nullptr); + EXPECT_EQ(nullptr, EAL); + clang_Driver_ExternalActionList_dispose(EAL); + + CXDiagnosticSet Diags; + EAL = clang_Driver_getExternalActionsForCommand_v0( + GTEST_ARRAY_SIZE_(ArgV), ArgV, nullptr, "/no/such/working/dir", &Diags); + EXPECT_EQ(nullptr, EAL); + ASSERT_NE(nullptr, Diags); + + unsigned NumDiags = clang_getNumDiagnosticsInSet(Diags); + ASSERT_EQ(1u, NumDiags); + CXDiagnostic Diag = clang_getDiagnosticInSet(Diags, 0); + CXString Str = clang_formatDiagnostic(Diag, 0); + EXPECT_STREQ(clang_getCString(Str), + "error: unable to set working directory: /no/such/working/dir"); + clang_disposeString(Str); + + clang_disposeDiagnosticSet(Diags); + clang_Driver_ExternalActionList_dispose(EAL); +} From cf2048b954f8ae045e36d4d96ce5685e639a86f7 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Wed, 8 Jan 2020 16:41:17 -0800 Subject: [PATCH 05/13] [Clang][cc1] Add -remove-preceeding-explicit-module-build-incompatible-options Removes any arguments before this one that would be incompatible with explicitly building a module. This includes things like -o and input files. This option can be used to append arguments to convert a build of a translation unit with implicit modules into an explicit build of a specific module. (cherry picked from commit 5286f371be86e391cff0c6dfd2bf36dad1c4c6e0) --- clang/include/clang/Driver/CC1Options.td | 12 +++++++++ clang/lib/Frontend/CompilerInvocation.cpp | 25 +++++++++++++++++++ ...plicit-module-build-incompatible-options.c | 6 +++++ llvm/include/llvm/Option/ArgList.h | 12 +++++++++ 4 files changed, 55 insertions(+) create mode 100644 clang/test/Modules/remove-preceeding-explicit-module-build-incompatible-options.c diff --git a/clang/include/clang/Driver/CC1Options.td b/clang/include/clang/Driver/CC1Options.td index 3471a2523d559..71ec975fb1fcd 100644 --- a/clang/include/clang/Driver/CC1Options.td +++ b/clang/include/clang/Driver/CC1Options.td @@ -12,6 +12,18 @@ let Flags = [CC1Option, NoDriverOption] in { +//===----------------------------------------------------------------------===// +// Option Options +//===----------------------------------------------------------------------===// + +def remove_preceeding_explicit_module_build_incompatible_options : + Flag<["-"], "remove-preceeding-explicit-module-build-incompatible-options">, + HelpText<"Removes any arguments before this one that would be incompatible " + "with explicitly building a module. This includes things like -o " + "and input files. This option can be used to append arguments to " + "convert a build of a translation unit with implicit modules " + "into an explicit build of a specific module.">; + //===----------------------------------------------------------------------===// // Target Options //===----------------------------------------------------------------------===// diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 92f0c642e8009..3a06c238e601b 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -3616,6 +3616,28 @@ static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args, } } +static void removeExplicitModuleBuildIncompatibleOptions(InputArgList &Args) { + auto REMBIO = llvm::find_if(Args, [](const Arg *A){ + return A->getOption().getID() == + OPT_remove_preceeding_explicit_module_build_incompatible_options; + }); + if (REMBIO == Args.end()) + return; + + llvm::SmallPtrSet BeforeREMBIO; + for (auto I = Args.begin(); I != REMBIO; ++I) + BeforeREMBIO.insert(*I); + + Args.eraseArgIf([&](const Arg *A) { + if (!BeforeREMBIO.count(A)) + return false; + const Option &O = A->getOption(); + return O.matches(OPT_INPUT) || + O.matches(OPT_Action_Group) || + O.matches(OPT__output); + }); +} + bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res, ArrayRef CommandLineArgs, DiagnosticsEngine &Diags) { @@ -3627,6 +3649,9 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res, unsigned MissingArgIndex, MissingArgCount; InputArgList Args = Opts.ParseArgs(CommandLineArgs, MissingArgIndex, MissingArgCount, IncludedFlagsBitmask); + + removeExplicitModuleBuildIncompatibleOptions(Args); + LangOptions &LangOpts = *Res.getLangOpts(); // Check for missing argument error. diff --git a/clang/test/Modules/remove-preceeding-explicit-module-build-incompatible-options.c b/clang/test/Modules/remove-preceeding-explicit-module-build-incompatible-options.c new file mode 100644 index 0000000000000..8754dd2ee78e6 --- /dev/null +++ b/clang/test/Modules/remove-preceeding-explicit-module-build-incompatible-options.c @@ -0,0 +1,6 @@ +// RUN: rm -rf %t +// RUN: %clang_cc1 -emit-obj -o %t.o input-that-doesnt-exist.c \ +// RUN: -remove-preceeding-explicit-module-build-incompatible-options \ +// RUN: -emit-llvm-bc -o %t.bc %s +// RUN: not ls %t.o +// RUN: llvm-dis %t.bc diff --git a/llvm/include/llvm/Option/ArgList.h b/llvm/include/llvm/Option/ArgList.h index 74bfadcba7267..af83a9024a67d 100644 --- a/llvm/include/llvm/Option/ArgList.h +++ b/llvm/include/llvm/Option/ArgList.h @@ -228,6 +228,18 @@ class ArgList { /// eraseArg - Remove any option matching \p Id. void eraseArg(OptSpecifier Id); + + /// eraseArgIf - Remove every `const Arg *A` for which P(A) is true. + template + void eraseArgIf(Pred P) { + for (Arg *const &A : Args) { + if (P(A)) { + Arg **ArgsBegin = Args.data(); + ArgsBegin[&A - ArgsBegin] = nullptr; + // Don't update OptRanges as it's not required and would be slow to do. + } + } + } /// @} /// @name Arg Access From 611489822fcec05b3d93d52bf6de423b25aac215 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Wed, 8 Jan 2020 16:43:14 -0800 Subject: [PATCH 06/13] [Clang][ScanDeps] Actually pass the non path command line modifications. (cherry picked from commit b05c4d38f0bf9bd1458a170b7e32275ea7f1d0ed) --- .../DependencyScanning/DependencyScanningTool.cpp | 5 +++++ .../DependencyScanning/ModuleDepCollector.cpp | 6 ++++-- clang/test/ClangScanDeps/modules-full.cpp | 12 +++++++++--- clang/test/Index/Core/scan-deps.m | 4 ++-- clang/tools/libclang/CDependencies.cpp | 10 ++++++++-- 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 31b8346b4efb3..52ec074e458fb 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -128,6 +128,11 @@ DependencyScanningTool::getFullDependencies( if (MD.ImportedByMainFile) FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash}); } + + FD.AdditionalNonPathCommandLine = { + "-fno-implicit-modules", + "-fno-implicit-module-maps", + }; FullDependenciesResult FDR; diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 38249582f8623..43317c768d88a 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -53,8 +53,6 @@ void dependencies::detail::appendCommonModuleArguments( } }; - Result.push_back("-fno-implicit-modules"); - Result.push_back("-fno-implicit-module-maps"); AddArgs(Modules); } @@ -159,6 +157,10 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) { MD.FileDeps.insert(IF.getFile()->getName()); }); + MD.NonPathCommandLine = { + "-remove-preceeding-explicit-module-build-incompatible-options", + "-fno-implicit-modules", "-emit-module", "-fmodule-name=" + MD.ModuleName, + }; llvm::DenseSet AddedModules; addAllSubmoduleDeps(M, MD, AddedModules); diff --git a/clang/test/ClangScanDeps/modules-full.cpp b/clang/test/ClangScanDeps/modules-full.cpp index 1e6a740c27392..599001743dcdc 100644 --- a/clang/test/ClangScanDeps/modules-full.cpp +++ b/clang/test/ClangScanDeps/modules-full.cpp @@ -31,8 +31,10 @@ // CHECK-NEXT: ], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-remove-preceeding-explicit-module-build-incompatible-options", // CHECK-NEXT: "-fno-implicit-modules", -// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-emit-module", +// CHECK-NEXT: "-fmodule-name=header1", // CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/header2-{{[A-Z0-9]+}}.pcm", // CHECK-NEXT: "-fmodule-map-file=[[PREFIX]]/Inputs/module.modulemap" // CHECK-NEXT: ], @@ -47,8 +49,10 @@ // CHECK-NEXT: "clang-module-deps": [], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-remove-preceeding-explicit-module-build-incompatible-options", // CHECK-NEXT: "-fno-implicit-modules", -// CHECK-NEXT: "-fno-implicit-module-maps" +// CHECK-NEXT: "-emit-module", +// CHECK-NEXT: "-fmodule-name=header1" // CHECK-NEXT: ], // CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H2:[A-Z0-9]+]]", // CHECK-NEXT: "file-deps": [ @@ -61,8 +65,10 @@ // CHECK-NEXT: "clang-module-deps": [], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/Inputs/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-remove-preceeding-explicit-module-build-incompatible-options", // CHECK-NEXT: "-fno-implicit-modules", -// CHECK-NEXT: "-fno-implicit-module-maps" +// CHECK-NEXT: "-emit-module", +// CHECK-NEXT: "-fmodule-name=header2" // CHECK-NEXT: ], // CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", // CHECK-NEXT: "file-deps": [ diff --git a/clang/test/Index/Core/scan-deps.m b/clang/test/Index/Core/scan-deps.m index 2453ca5dfe1ef..6bb38db04b81b 100644 --- a/clang/test/Index/Core/scan-deps.m +++ b/clang/test/Index/Core/scan-deps.m @@ -19,11 +19,11 @@ // CHECK-NEXT: [[PREFIX]]/Inputs/module/SubModA.h // CHECK-NEXT: [[PREFIX]]/Inputs/module/SubSubModA.h // CHECK-NEXT: [[PREFIX]]/Inputs/module/module.modulemap -// CHECK-NEXT: build-args: +// CHECK-NEXT: build-args: -remove-preceeding-explicit-module-build-incompatible-options -fno-implicit-modules -emit-module -fmodule-name=ModA // CHECK-NEXT: dependencies: // CHECK-NEXT: context-hash: [[CONTEXT_HASH]] // CHECK-NEXT: module-deps: // CHECK-NEXT: ModA:[[CONTEXT_HASH]] // CHECK-NEXT: file-deps: // CHECK-NEXT: [[PREFIX]]/scan-deps.m -// CHECK-NEXT: additional-build-args: +// CHECK-NEXT: additional-build-args: -fno-implicit-modules -fno-implicit-module-maps diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 9366505cc8398..86c1a5f7864de 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -121,6 +121,11 @@ class FullDependencyConsumer : public DependencyConsumer { FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash}); } + FD.AdditionalNonPathCommandLine = { + "-fno-implicit-modules", + "-fno-implicit-module-maps", + }; + FullDependenciesResult FDR; for (auto &&M : ClangModuleDeps) { @@ -178,7 +183,7 @@ static CXFileDependencies *getFullDependencies( for (const ClangModuleDep &CMD : MD.ClangModuleDeps) Modules.push_back(CMD.ModuleName + ":" + CMD.ContextHash); M.ModuleDeps = cxstring::createSet(Modules); - M.BuildArguments = cxstring::createSet(std::vector{}); + M.BuildArguments = cxstring::createSet(MD.NonPathCommandLine); } MDC(Context, MDS); } @@ -191,7 +196,8 @@ static CXFileDependencies *getFullDependencies( for (const ClangModuleDep &CMD : FD.ClangModuleDeps) Modules.push_back(CMD.ModuleName + ":" + CMD.ContextHash); FDeps->ModuleDeps = cxstring::createSet(Modules); - FDeps->AdditionalArguments = cxstring::createSet(std::vector{}); + FDeps->AdditionalArguments = + cxstring::createSet(FD.AdditionalNonPathCommandLine); return FDeps; } From 4e1cda68107ac0e5494b92f74a8cebd854123f03 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Thu, 9 Jan 2020 15:31:36 -0800 Subject: [PATCH 07/13] [Clang][ScanDeps] Use the module map a module was inferred from for inferred modules. (cherry picked from commit 3eae6a83be0c351a6fe19f665ca6cd3690775288) --- .../DependencyScanning/ModuleDepCollector.cpp | 2 +- .../Frameworks/Sub.framework/Headers/Sub.h | 0 .../Inferred.framework/Headers/Inferred.h | 1 + .../Inputs/frameworks/module.modulemap | 1 + .../Inputs/modules_inferred_cdb.json | 7 +++ clang/test/ClangScanDeps/modules-inferred.m | 61 +++++++++++++++++++ 6 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 clang/test/ClangScanDeps/Inputs/frameworks/Inferred.framework/Frameworks/Sub.framework/Headers/Sub.h create mode 100644 clang/test/ClangScanDeps/Inputs/frameworks/Inferred.framework/Headers/Inferred.h create mode 100644 clang/test/ClangScanDeps/Inputs/frameworks/module.modulemap create mode 100644 clang/test/ClangScanDeps/Inputs/modules_inferred_cdb.json create mode 100644 clang/test/ClangScanDeps/modules-inferred.m diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 43317c768d88a..07a7cd03ef8fd 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -145,7 +145,7 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { const FileEntry *ModuleMap = Instance.getPreprocessor() .getHeaderSearchInfo() .getModuleMap() - .getContainingModuleMapFile(M); + .getModuleMapFileForUniquing(M); MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : ""; MD.ModuleName = M->getFullModuleName(); diff --git a/clang/test/ClangScanDeps/Inputs/frameworks/Inferred.framework/Frameworks/Sub.framework/Headers/Sub.h b/clang/test/ClangScanDeps/Inputs/frameworks/Inferred.framework/Frameworks/Sub.framework/Headers/Sub.h new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/clang/test/ClangScanDeps/Inputs/frameworks/Inferred.framework/Headers/Inferred.h b/clang/test/ClangScanDeps/Inputs/frameworks/Inferred.framework/Headers/Inferred.h new file mode 100644 index 0000000000000..1855e4fad5f8d --- /dev/null +++ b/clang/test/ClangScanDeps/Inputs/frameworks/Inferred.framework/Headers/Inferred.h @@ -0,0 +1 @@ +typedef int inferred; diff --git a/clang/test/ClangScanDeps/Inputs/frameworks/module.modulemap b/clang/test/ClangScanDeps/Inputs/frameworks/module.modulemap new file mode 100644 index 0000000000000..e3bad873c7e7b --- /dev/null +++ b/clang/test/ClangScanDeps/Inputs/frameworks/module.modulemap @@ -0,0 +1 @@ +framework module * {} diff --git a/clang/test/ClangScanDeps/Inputs/modules_inferred_cdb.json b/clang/test/ClangScanDeps/Inputs/modules_inferred_cdb.json new file mode 100644 index 0000000000000..c6123e30994ee --- /dev/null +++ b/clang/test/ClangScanDeps/Inputs/modules_inferred_cdb.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/modules_cdb_input.cpp -FFRAMEWORKS -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps", + "file": "DIR/modules_cdb_input.cpp" +} +] diff --git a/clang/test/ClangScanDeps/modules-inferred.m b/clang/test/ClangScanDeps/modules-inferred.m new file mode 100644 index 0000000000000..3e41e7c01ac14 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-inferred.m @@ -0,0 +1,61 @@ +// RUN: rm -rf %t.dir +// RUN: rm -rf %t.cdb +// RUN: mkdir -p %t.dir +// RUN: cp %s %t.dir/modules_cdb_input.cpp +// RUN: sed -e "s|DIR|%/t.dir|g" -e "s|FRAMEWORKS|%S/Inputs/frameworks|g" \ +// RUN: %S/Inputs/modules_inferred_cdb.json > %t.cdb +// +// RUN: echo -%t.dir > %t.result +// RUN: echo -%S >> %t.result +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -full-command-line \ +// RUN: -mode preprocess-minimized-sources -format experimental-full >> %t.result +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck --check-prefixes=CHECK %s + +#include + +inferred a = 0; + +// CHECK: -[[PREFIX:.*]] +// CHECK-NEXT: -[[SOURCEDIR:.*]] +// CHECK-NEXT: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[SOURCEDIR]]/Inputs/frameworks/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-remove-preceeding-explicit-module-build-incompatible-options", +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-emit-module", +// CHECK-NEXT: "-fmodule-name=Inferred" +// CHECK-NEXT: ], +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1:[A-Z0-9]+]]", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[SOURCEDIR]]/Inputs/frameworks/Inferred.framework/Frameworks/Sub.framework/Headers/Sub.h", +// CHECK-NEXT: "[[SOURCEDIR]]/Inputs/frameworks/Inferred.framework/Headers/Inferred.h", +// CHECK-NEXT: "[[SOURCEDIR]]/Inputs/frameworks/Inferred.framework/__inferred_module.map" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "Inferred" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "[[CONTEXT_HASH_H1]]", +// CHECK-NEXT: "module-name": "Inferred" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "command-line": [ +// CHECK-NEXT: "-fno-implicit-modules", +// CHECK-NEXT: "-fno-implicit-module-maps", +// CHECK-NEXT: "-fmodule-file=[[PREFIX]]/module-cache/[[CONTEXT_HASH_H1]]/Inferred-{{[A-Z0-9]+}}.pcm", +// CHECK-NEXT: "-fmodule-map-file=[[SOURCEDIR]]/Inputs/frameworks/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: ], +// CHECK-NEXT: "input-file": "[[PREFIX]]/modules_cdb_input.cpp" +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } From 61c21390a14662f40b380d28be993962caf4630e Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Fri, 17 Jan 2020 17:45:53 -0800 Subject: [PATCH 08/13] [Clang] Temporarily comment out assert. This assert fires for ObjectiveC id, SEL, and Class declarations when the module is explicitly built. For implicit builds it works fine. This reproduces with: typedef struct objc_class Class; struct objc_class; As a module imported into another module, and both the above and that module imported into a third TU. Changing the name (`Class`) or reordering the declarations does not assert. rdar://58552906 (cherry picked from commit 9327c60da885b5c1295d4237375f40e033912ac7) --- clang/lib/Sema/IdentifierResolver.cpp | 6 +++++- clang/test/Modules/Inputs/module.map | 5 +++++ clang/test/Modules/Inputs/objc_redef_indirect.h | 1 + clang/test/Modules/Inputs/weird_objc.h | 2 ++ clang/test/Modules/objc_redef.m | 8 ++++++++ 5 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 clang/test/Modules/Inputs/objc_redef_indirect.h diff --git a/clang/lib/Sema/IdentifierResolver.cpp b/clang/lib/Sema/IdentifierResolver.cpp index c1bc8f79a309e..a6555ffa88f6c 100644 --- a/clang/lib/Sema/IdentifierResolver.cpp +++ b/clang/lib/Sema/IdentifierResolver.cpp @@ -217,7 +217,11 @@ void IdentifierResolver::RemoveDecl(NamedDecl *D) { assert(Ptr && "Didn't find this decl on its identifier's chain!"); if (isDeclPtr(Ptr)) { - assert(D == Ptr && "Didn't find this decl on its identifier's chain!"); + // FIXME: The following assert fires for ObjectiveC id, SEL, and Class + // declarations when the module is explicitly built. For implicit builds + // it works fine. rdar://58552906 + + // assert(D == Ptr && "Didn't find this decl on its identifier's chain!"); Name.setFETokenInfo(nullptr); return; } diff --git a/clang/test/Modules/Inputs/module.map b/clang/test/Modules/Inputs/module.map index 3f128c0bb0e0f..d78e42cc99672 100644 --- a/clang/test/Modules/Inputs/module.map +++ b/clang/test/Modules/Inputs/module.map @@ -475,3 +475,8 @@ module template_nontrivial1 { header "template-nontrivial1.h" export * } + +module objc_redef_indirect { + header "objc_redef_indirect.h" + export * +} diff --git a/clang/test/Modules/Inputs/objc_redef_indirect.h b/clang/test/Modules/Inputs/objc_redef_indirect.h new file mode 100644 index 0000000000000..2bdd1898ffbf8 --- /dev/null +++ b/clang/test/Modules/Inputs/objc_redef_indirect.h @@ -0,0 +1 @@ +@import weird_objc; diff --git a/clang/test/Modules/Inputs/weird_objc.h b/clang/test/Modules/Inputs/weird_objc.h index 8acaf746e855c..e2ca9a5103120 100644 --- a/clang/test/Modules/Inputs/weird_objc.h +++ b/clang/test/Modules/Inputs/weird_objc.h @@ -1 +1,3 @@ typedef struct objc_object { void *super; int wibble; } *id; +typedef struct A *Class; +struct A; diff --git a/clang/test/Modules/objc_redef.m b/clang/test/Modules/objc_redef.m index 2e57f41bc6624..ab3a77ae68a97 100644 --- a/clang/test/Modules/objc_redef.m +++ b/clang/test/Modules/objc_redef.m @@ -1,5 +1,6 @@ @import redeclarations_left; @import weird_objc; +@import objc_redef_indirect; int test(id x) { return x->wibble; @@ -9,5 +10,12 @@ int test(id x) { // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -x objective-c -fmodules-cache-path=%t -emit-module -fmodule-name=redeclarations_left %S/Inputs/module.map // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -x objective-c -fmodules-cache-path=%t -emit-module -fmodule-name=weird_objc %S/Inputs/module.map // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -I %S/Inputs %s -verify + +// Try explicit too. + +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -x objective-c -emit-module -fmodule-name=redeclarations_left %S/Inputs/module.map -o %t/redeclarations_left.pcm +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -x objective-c -emit-module -fmodule-name=weird_objc %S/Inputs/module.map -o %t/weird_objc.pcm +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -x objective-c -emit-module -fmodule-file=%t/weird_objc.pcm -fmodule-name=objc_redef_indirect %S/Inputs/module.map -o %t/objc_redef_indirect.pcm -I %S/Inputs +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodule-file=%t/redeclarations_left.pcm -fmodule-file=%t/weird_objc.pcm -fmodule-file=%t/objc_redef_indirect.pcm -I %S/Inputs %s -verify // expected-no-diagnostics From 4876ab38323fbec6e9d808be51b28f7e3fd55cce Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Wed, 8 Jan 2020 16:39:04 -0800 Subject: [PATCH 09/13] [Clang][cc1] Support -fno-implicit-module-maps in -cc1. Gives the last of -f{no-}implicit-module-maps precedence. rdar://58883354 (cherry picked from commit 2d28507121d64dbe9971eb859fd37cc0022f9fd0) --- clang/include/clang/Driver/Options.td | 2 +- clang/lib/Frontend/CompilerInvocation.cpp | 3 ++- clang/test/Modules/no-implicit-maps.cpp | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 871b215a68f03..1b1aefdfb58fa 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1538,7 +1538,7 @@ def fno_merge_all_constants : Flag<["-"], "fno-merge-all-constants">, Group, Group, Flags<[DriverOption]>; def fno_implicit_module_maps : Flag <["-"], "fno-implicit-module-maps">, Group, - Flags<[DriverOption]>; + Flags<[DriverOption, CC1Option]>; def fno_module_maps : Flag <["-"], "fno-module-maps">, Alias; def fno_modules_decluse : Flag <["-"], "fno-modules-decluse">, Group, Flags<[DriverOption]>; diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 3a06c238e601b..29108d54463a7 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -2193,7 +2193,8 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args, Opts.ModulesStrictContextHash = Args.hasArg(OPT_fmodules_strict_context_hash); Opts.ModulesValidateDiagnosticOptions = !Args.hasArg(OPT_fmodules_disable_diagnostic_validation); - Opts.ImplicitModuleMaps = Args.hasArg(OPT_fimplicit_module_maps); + Opts.ImplicitModuleMaps = Args.hasFlag(OPT_fimplicit_module_maps, + OPT_fno_implicit_module_maps, false); Opts.ModuleMapFileHomeIsCwd = Args.hasArg(OPT_fmodule_map_file_home_is_cwd); Opts.ModuleCachePruneInterval = getLastArgIntValue(Args, OPT_fmodules_prune_interval, 7 * 24 * 60 * 60); diff --git a/clang/test/Modules/no-implicit-maps.cpp b/clang/test/Modules/no-implicit-maps.cpp index ddcad5df1e6f1..3ee7728048367 100644 --- a/clang/test/Modules/no-implicit-maps.cpp +++ b/clang/test/Modules/no-implicit-maps.cpp @@ -1,3 +1,6 @@ // RUN: rm -rf %t // RUN: %clang_cc1 -x objective-c -fmodules-cache-path=%t -fmodules -I %S/Inputs/private %s -verify +// RUN: %clang_cc1 -x objective-c -fmodules-cache-path=%t -fmodules \ +// RUN: -I %S/Inputs/private %s -verify -fimplicit-module-maps \ +// RUN: -fno-implicit-module-maps @import libPrivate1; // expected-error {{not found}} From 56ff1f1096fa844fcc851f6438657bb7d56c544d Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Fri, 17 Jan 2020 17:47:28 -0800 Subject: [PATCH 10/13] [Clang] Allow explicitly building an inferred module. Building the actual module still fails, but make sure it fails for the right reason. (cherry picked from commit 1815e2418e91d0c394e951b054ecbca68de1dc90) --- clang/lib/Frontend/FrontendAction.cpp | 2 +- .../test/ClangScanDeps/module-deps-to-rsp.py | 76 +++++++++++++++++++ .../modules-inferred-explicit-build.m | 17 +++++ 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100755 clang/test/ClangScanDeps/module-deps-to-rsp.py create mode 100644 clang/test/ClangScanDeps/modules-inferred-explicit-build.m diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 58f939e1abb49..ce99474cce3ba 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -460,7 +460,7 @@ static Module *prepareToBuildModule(CompilerInstance &CI, // Dig out the module definition. HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); Module *M = HS.lookupModule(CI.getLangOpts().CurrentModule, - /*AllowSearch=*/false); + /*AllowSearch=*/true); if (!M) { CI.getDiagnostics().Report(diag::err_missing_module) << CI.getLangOpts().CurrentModule << ModuleMapFilename; diff --git a/clang/test/ClangScanDeps/module-deps-to-rsp.py b/clang/test/ClangScanDeps/module-deps-to-rsp.py new file mode 100755 index 0000000000000..45a6bcc90af1f --- /dev/null +++ b/clang/test/ClangScanDeps/module-deps-to-rsp.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +import argparse +import json +import sys + +class ModuleNotFoundError(Exception): + def __init__(self, module_name): + self.module_name = module_name + +class FullDeps: + def __init__(self): + self.modules = dict() + self.translation_units = str() + +def getModulePathArgs(modules, full_deps): + cmd = [] + for md in modules: + m = full_deps.modules[md['module-name'] + '-' + md['context-hash']] + cmd += [u'-fmodule-map-file=' + m['clang-modulemap-file']] + cmd += [u'-fmodule-file=' + md['module-name'] + '-' + md['context-hash'] + '.pcm'] + return cmd + +def getCommandLineForModule(module_name, full_deps): + for m in full_deps.modules.values(): + if m['name'] == module_name: + module = m + break + else: + raise ModuleNotFoundError(module_name) + + cmd = m['command-line'] + cmd += getModulePathArgs(m['clang-module-deps'], full_deps) + cmd += [u'-o', m['name'] + '-' + m['context-hash'] + '.pcm'] + cmd += [m['clang-modulemap-file']] + + return cmd + +def getCommandLineForTU(tu, full_deps): + cmd = tu['command-line'] + cmd = [a for a in cmd if not (a.startswith('-fmodule-map-file=') or a.startswith('-fmodule-file='))] + cmd += getModulePathArgs(tu['clang-module-deps'], full_deps) + return cmd + +def parseFullDeps(json): + ret = FullDeps() + for m in json['modules']: + ret.modules[m['name'] + '-' + m['context-hash']] = m + ret.translation_units = json['translation-units'] + return ret + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("full_deps_file", help="Path to the full dependencies json file", + type=str) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--module-name", help="The name of the module to get arguments for", + type=str) + action.add_argument("--tu-index", help="The index of the translation unit to get arguments for", + type=int) + args = parser.parse_args() + + full_deps = parseFullDeps(json.load(open(args.full_deps_file, 'r'))) + + try: + if args.module_name: + print(" ".join(getCommandLineForModule(args.module_name, full_deps))) + + elif args.tu_index != None: + print(" ".join(getCommandLineForTU(full_deps.translation_units[args.tu_index], full_deps))) + except: + print("Unexpected error:", sys.exc_info()[0]) + raise + +if __name__ == '__main__': + main() diff --git a/clang/test/ClangScanDeps/modules-inferred-explicit-build.m b/clang/test/ClangScanDeps/modules-inferred-explicit-build.m new file mode 100644 index 0000000000000..8915a77cc1abf --- /dev/null +++ b/clang/test/ClangScanDeps/modules-inferred-explicit-build.m @@ -0,0 +1,17 @@ +// RUN: rm -rf %t.dir +// RUN: rm -rf %t.cdb +// RUN: mkdir -p %t.dir +// RUN: cp %s %t.dir/modules_cdb_input.cpp +// RUN: sed -e "s|DIR|%/t.dir|g" -e "s|FRAMEWORKS|%/S/Inputs/frameworks|g" \ +// RUN: %S/Inputs/modules_inferred_cdb.json > %t.cdb +// +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -full-command-line \ +// RUN: -mode preprocess-minimized-sources -format experimental-full > %t.db +// RUN: %S/module-deps-to-rsp.py %t.db --module-name=Inferred > %t.inferred.rsp +// RUN: %S/module-deps-to-rsp.py %t.db --tu-index=0 > %t.tu.rsp +// RUN: not %clang_cc1 -E %t.dir/modules_cdb_input.cpp -F%S/Inputs/frameworks -fmodules -fimplicit-module-maps @%t.inferred.rsp 2>&1 | grep "'Inferred.h' file not found" +// RUN: not %clang_cc1 -E %t.dir/modules_cdb_input.cpp -F%S/Inputs/frameworks -fmodules -fimplicit-module-maps @%t.tu.rsp + +#include + +inferred a = 0; From 3ddc2d4b122855d4db2a022e4679e8b8ab7c75d0 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Fri, 17 Jan 2020 17:58:13 -0800 Subject: [PATCH 11/13] [Clang] Fix the header paths in `clang::Module` for inferred modules. The `UmbrellaAsWritten` and `NameAsWritten` fields in `clang::Module` are a lie for framework modules. For those they actually are the path to the header or umbrella relative to the `clang::Module::Directory`. The exception to this case is for inferred modules. Here it actually is the name as written, because we print out the module and read it back in when implicitly building modules. This causes a problem when explicitly building an inferred module, as we skip the printing out step. In order to fix this issue this patch adds a new field for the path we want to use in `getInputBufferForModule`. It also makes `NameAsWritten` actually be the name written in the module map file (or that would be, in the case of an inferred module). rdar://58619519 (cherry picked from commit 0672a6a4937e4aad9a5f56eed5279a5740c32c88) --- clang/include/clang/Basic/Module.h | 8 ++++- clang/include/clang/Lex/ModuleMap.h | 6 ++-- clang/lib/Basic/Module.cpp | 5 +-- clang/lib/Frontend/FrontendAction.cpp | 14 ++++---- clang/lib/Frontend/FrontendActions.cpp | 2 +- clang/lib/Lex/ModuleMap.cpp | 36 ++++++++++++------- clang/lib/Serialization/ASTReader.cpp | 8 +++-- .../modules-inferred-explicit-build.m | 7 ++-- 8 files changed, 57 insertions(+), 29 deletions(-) diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index e0399b82a47c7..7ac048f918a4e 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -109,6 +109,9 @@ class Module { /// The name of the umbrella entry, as written in the module map. std::string UmbrellaAsWritten; + // The path to the umbrella entry relative to the root module's \c Directory. + std::string UmbrellaRelativeToRootModuleDirectory; + /// The module through which entities defined in this module will /// eventually be exposed, for use in "private" modules. std::string ExportAsModule; @@ -156,6 +159,7 @@ class Module { /// file. struct Header { std::string NameAsWritten; + std::string PathRelativeToRootModuleDirectory; const FileEntry *Entry; explicit operator bool() { return Entry; } @@ -165,6 +169,7 @@ class Module { /// file. struct DirectoryName { std::string NameAsWritten; + std::string PathRelativeToRootModuleDirectory; const DirectoryEntry *Entry; explicit operator bool() { return Entry; } @@ -491,7 +496,8 @@ class Module { /// module. Header getUmbrellaHeader() const { if (auto *E = Umbrella.dyn_cast()) - return Header{UmbrellaAsWritten, E}; + return Header{UmbrellaAsWritten, UmbrellaRelativeToRootModuleDirectory, + E}; return Header{}; } diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h index 6bd36cd83d979..380f44d6ab991 100644 --- a/clang/include/clang/Lex/ModuleMap.h +++ b/clang/include/clang/Lex/ModuleMap.h @@ -653,12 +653,14 @@ class ModuleMap { /// Sets the umbrella header of the given module to the given /// header. void setUmbrellaHeader(Module *Mod, const FileEntry *UmbrellaHeader, - Twine NameAsWritten); + Twine NameAsWritten, + Twine PathRelativeToRootModuleDirectory); /// Sets the umbrella directory of the given module to the given /// directory. void setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir, - Twine NameAsWritten); + Twine NameAsWritten, + Twine PathRelativeToRootModuleDirectory); /// Adds this header to the given module. /// \param Role The role of the header wrt the module. diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp index eb18b66ca5422..dd3e8cba0625c 100644 --- a/clang/lib/Basic/Module.cpp +++ b/clang/lib/Basic/Module.cpp @@ -243,9 +243,10 @@ bool Module::fullModuleNameIs(ArrayRef nameParts) const { Module::DirectoryName Module::getUmbrellaDir() const { if (Header U = getUmbrellaHeader()) - return {"", U.Entry->getDir()}; + return {"", "", U.Entry->getDir()}; - return {UmbrellaAsWritten, Umbrella.dyn_cast()}; + return {UmbrellaAsWritten, UmbrellaRelativeToRootModuleDirectory, + Umbrella.dyn_cast()}; } ArrayRef Module::getTopHeaders(FileManager &FileMgr) { diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index ce99474cce3ba..5195f7c60dc51 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -345,7 +345,8 @@ static std::error_code collectModuleHeaderIncludes( // file relative to the module build directory (the directory containing // the module map file) so this will find the same file that we found // while parsing the module map. - addHeaderInclude(H.NameAsWritten, Includes, LangOpts, Module->IsExternC); + addHeaderInclude(H.PathRelativeToRootModuleDirectory, Includes, LangOpts, + Module->IsExternC); } } // Note that Module->PrivateHeaders will not be a TopHeader. @@ -354,8 +355,8 @@ static std::error_code collectModuleHeaderIncludes( Module->addTopHeader(UmbrellaHeader.Entry); if (Module->Parent) // Include the umbrella header for submodules. - addHeaderInclude(UmbrellaHeader.NameAsWritten, Includes, LangOpts, - Module->IsExternC); + addHeaderInclude(UmbrellaHeader.PathRelativeToRootModuleDirectory, + Includes, LangOpts, Module->IsExternC); } else if (Module::DirectoryName UmbrellaDir = Module->getUmbrellaDir()) { // Add all of the headers we find in this subdirectory. std::error_code EC; @@ -388,7 +389,8 @@ static std::error_code collectModuleHeaderIncludes( auto PathIt = llvm::sys::path::rbegin(Dir->path()); for (int I = 0; I != Dir.level() + 1; ++I, ++PathIt) Components.push_back(*PathIt); - SmallString<128> RelativeHeader(UmbrellaDir.NameAsWritten); + SmallString<128> RelativeHeader( + UmbrellaDir.PathRelativeToRootModuleDirectory); for (auto It = Components.rbegin(), End = Components.rend(); It != End; ++It) llvm::sys::path::append(RelativeHeader, *It); @@ -518,8 +520,8 @@ getInputBufferForModule(CompilerInstance &CI, Module *M) { SmallString<256> HeaderContents; std::error_code Err = std::error_code(); if (Module::Header UmbrellaHeader = M->getUmbrellaHeader()) - addHeaderInclude(UmbrellaHeader.NameAsWritten, HeaderContents, - CI.getLangOpts(), M->IsExternC); + addHeaderInclude(UmbrellaHeader.PathRelativeToRootModuleDirectory, + HeaderContents, CI.getLangOpts(), M->IsExternC); Err = collectModuleHeaderIncludes( CI.getLangOpts(), FileMgr, CI.getDiagnostics(), CI.getPreprocessor().getHeaderSearchInfo().getModuleMap(), M, diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 8574d0a7e8132..09efb64b53ebc 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -295,7 +295,7 @@ bool GenerateHeaderModuleAction::BeginSourceFileAction( << Name; continue; } - Headers.push_back({Name, &FE->getFileEntry()}); + Headers.push_back({std::string(Name), std::string(Name), &FE->getFileEntry()}); } HS.getModuleMap().createHeaderModule(CI.getLangOpts().CurrentModule, Headers); diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index 3056b0c28594d..8b63200167b9b 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -260,9 +260,9 @@ void ModuleMap::resolveHeader(Module *Mod, << UmbrellaMod->getFullModuleName(); else // Record this umbrella header. - setUmbrellaHeader(Mod, File, RelativePathName.str()); + setUmbrellaHeader(Mod, File, Header.FileName, RelativePathName.str()); } else { - Module::Header H = {RelativePathName.str(), File}; + Module::Header H = {Header.FileName, std::string(RelativePathName.str()), File}; if (Header.Kind == Module::HK_Excluded) excludeHeader(Mod, H); else @@ -305,7 +305,7 @@ bool ModuleMap::resolveAsBuiltinHeader( return false; auto Role = headerKindToRole(Header.Kind); - Module::Header H = {Path.str(), *File}; + Module::Header H = {Header.FileName, std::string(Path.str()), *File}; addHeader(Mod, H, Role); return true; } @@ -1027,11 +1027,14 @@ Module *ModuleMap::inferFrameworkModule(const DirectoryEntry *FrameworkDir, Result->NoUndeclaredIncludes |= Attrs.NoUndeclaredIncludes; Result->Directory = FrameworkDir; + // Chop off the first framework bit, as that is implied. + StringRef RelativePath = + UmbrellaName.str().substr( + Result->getTopLevelModule()->Directory->getName().size()); + RelativePath = llvm::sys::path::relative_path(RelativePath); + // umbrella header "umbrella-header-name" - // - // The "Headers/" component of the name is implied because this is - // a framework module. - setUmbrellaHeader(Result, *UmbrellaHeader, ModuleName + ".h"); + setUmbrellaHeader(Result, *UmbrellaHeader, ModuleName + ".h", RelativePath); // export * Result->Exports.push_back(Module::ExportDecl(nullptr, true)); @@ -1091,6 +1094,9 @@ Module *ModuleMap::inferFrameworkModule(const DirectoryEntry *FrameworkDir, if (!Result->isSubFramework()) { inferFrameworkLink(Result, FrameworkDir, FileMgr); } + + Result->dump(); + llvm::errs() << Result->Directory->getName() << "\n"; return Result; } @@ -1111,10 +1117,13 @@ Module *ModuleMap::createShadowedModule(StringRef Name, bool IsFramework, } void ModuleMap::setUmbrellaHeader(Module *Mod, const FileEntry *UmbrellaHeader, - Twine NameAsWritten) { + Twine NameAsWritten, + Twine PathRelativeToRootModuleDirectory) { Headers[UmbrellaHeader].push_back(KnownHeader(Mod, NormalHeader)); Mod->Umbrella = UmbrellaHeader; Mod->UmbrellaAsWritten = NameAsWritten.str(); + Mod->UmbrellaRelativeToRootModuleDirectory = + PathRelativeToRootModuleDirectory.str(); UmbrellaDirs[UmbrellaHeader->getDir()] = Mod; // Notify callbacks that we just added a new header. @@ -1123,9 +1132,12 @@ void ModuleMap::setUmbrellaHeader(Module *Mod, const FileEntry *UmbrellaHeader, } void ModuleMap::setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir, - Twine NameAsWritten) { + Twine NameAsWritten, + Twine PathRelativeToRootModuleDirectory) { Mod->Umbrella = UmbrellaDir; Mod->UmbrellaAsWritten = NameAsWritten.str(); + Mod->UmbrellaRelativeToRootModuleDirectory = + PathRelativeToRootModuleDirectory.str(); UmbrellaDirs[UmbrellaDir] = Mod; } @@ -2393,6 +2405,7 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) { } std::string DirName = Tok.getString(); + std::string DirNameAsWritten = DirName; SourceLocation DirNameLoc = consumeToken(); // Check whether we already have an umbrella. @@ -2434,8 +2447,7 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) { for (llvm::vfs::recursive_directory_iterator I(FS, Dir->getName(), EC), E; I != E && !EC; I.increment(EC)) { if (auto FE = SourceMgr.getFileManager().getFile(I->path())) { - - Module::Header Header = {I->path(), *FE}; + Module::Header Header = {"", std::string(I->path()), *FE}; Headers.push_back(std::move(Header)); } } @@ -2456,7 +2468,7 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) { } // Record this umbrella directory. - Map.setUmbrellaDir(ActiveModule, Dir, DirName); + Map.setUmbrellaDir(ActiveModule, Dir, DirNameAsWritten, DirName); } /// Parse a module export declaration. diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index fef1b04b86fc2..84167de2aeb53 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1919,7 +1919,7 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d, // FIXME: This is not always the right filename-as-written, but we're not // going to use this information to rebuild the module, so it doesn't make // a lot of difference. - Module::Header H = { key.Filename, *FileMgr.getFile(Filename) }; + Module::Header H = {std::string(key.Filename), "", *FileMgr.getFile(Filename)}; ModMap.addHeader(Mod, H, HeaderRole, /*Imported*/true); HFI.isModuleHeader |= !(HeaderRole & ModuleMap::TextualHeader); } @@ -5549,7 +5549,8 @@ ASTReader::ReadSubmoduleBlock(ModuleFile &F, unsigned ClientLoadCapabilities) { ResolveImportedPath(F, Filename); if (auto Umbrella = PP.getFileManager().getFile(Filename)) { if (!CurrentModule->getUmbrellaHeader()) - ModMap.setUmbrellaHeader(CurrentModule, *Umbrella, Blob); + // FIXME: NameAsWritten + ModMap.setUmbrellaHeader(CurrentModule, *Umbrella, Blob, ""); else if (CurrentModule->getUmbrellaHeader().Entry != *Umbrella) { if ((ClientLoadCapabilities & ARR_OutOfDate) == 0) Error("mismatched umbrella headers in submodule"); @@ -5582,7 +5583,8 @@ ASTReader::ReadSubmoduleBlock(ModuleFile &F, unsigned ClientLoadCapabilities) { ResolveImportedPath(F, Dirname); if (auto Umbrella = PP.getFileManager().getDirectory(Dirname)) { if (!CurrentModule->getUmbrellaDir()) - ModMap.setUmbrellaDir(CurrentModule, *Umbrella, Blob); + // FIXME: NameAsWritten + ModMap.setUmbrellaDir(CurrentModule, *Umbrella, Blob, ""); else if (CurrentModule->getUmbrellaDir().Entry != *Umbrella) { if ((ClientLoadCapabilities & ARR_OutOfDate) == 0) Error("mismatched umbrella directories in submodule"); diff --git a/clang/test/ClangScanDeps/modules-inferred-explicit-build.m b/clang/test/ClangScanDeps/modules-inferred-explicit-build.m index 8915a77cc1abf..18a0c3bd441c2 100644 --- a/clang/test/ClangScanDeps/modules-inferred-explicit-build.m +++ b/clang/test/ClangScanDeps/modules-inferred-explicit-build.m @@ -9,8 +9,11 @@ // RUN: -mode preprocess-minimized-sources -format experimental-full > %t.db // RUN: %S/module-deps-to-rsp.py %t.db --module-name=Inferred > %t.inferred.rsp // RUN: %S/module-deps-to-rsp.py %t.db --tu-index=0 > %t.tu.rsp -// RUN: not %clang_cc1 -E %t.dir/modules_cdb_input.cpp -F%S/Inputs/frameworks -fmodules -fimplicit-module-maps @%t.inferred.rsp 2>&1 | grep "'Inferred.h' file not found" -// RUN: not %clang_cc1 -E %t.dir/modules_cdb_input.cpp -F%S/Inputs/frameworks -fmodules -fimplicit-module-maps @%t.tu.rsp +// RUN: %clang_cc1 -x objective-c -E %t.dir/modules_cdb_input.cpp \ +// RUN: -F%S/Inputs/frameworks -fmodules -fimplicit-module-maps \ +// RUN: @%t.inferred.rsp +// RUN: %clang_cc1 -x objective-c -E %t.dir/modules_cdb_input.cpp \ +// RUN: -F%S/Inputs/frameworks -fmodules -fimplicit-module-maps @%t.tu.rsp #include From 05bfc81f9d0abd503958c465194cf3026b7e6b27 Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Fri, 24 Jan 2020 12:52:25 -0800 Subject: [PATCH 12/13] [Clang][ScanDeps] Ignore __inferred_module.map dependency. This shows up with inferred modules, but it doesn't exist on disk, so don't report it as a dependency. (cherry picked from commit 21c4c103bf65398150532ae1ec2ca79ef6bcace9) --- .../Tooling/DependencyScanning/ModuleDepCollector.cpp | 9 +++++++++ clang/test/ClangScanDeps/modules-inferred.m | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 07a7cd03ef8fd..889872997b949 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -155,6 +155,15 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile()); MDC.Instance.getASTReader()->visitInputFiles( *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) { + // __inferred_module.map is the result of the way in which an implicit + // module build handles inferred modules. It adds an overlay VFS with + // this file in the proper directory and relies on the rest of Clang to + // handle it like normal. With explicitly built modules we don't need + // to play VFS tricks, so replace it with the correct module map. + if (IF.getFile()->getName().endswith("__inferred_module.map")) { + MD.FileDeps.insert(ModuleMap->getName()); + return; + } MD.FileDeps.insert(IF.getFile()->getName()); }); MD.NonPathCommandLine = { diff --git a/clang/test/ClangScanDeps/modules-inferred.m b/clang/test/ClangScanDeps/modules-inferred.m index 3e41e7c01ac14..cca2306b50aca 100644 --- a/clang/test/ClangScanDeps/modules-inferred.m +++ b/clang/test/ClangScanDeps/modules-inferred.m @@ -32,7 +32,7 @@ // CHECK-NEXT: "file-deps": [ // CHECK-NEXT: "[[SOURCEDIR]]/Inputs/frameworks/Inferred.framework/Frameworks/Sub.framework/Headers/Sub.h", // CHECK-NEXT: "[[SOURCEDIR]]/Inputs/frameworks/Inferred.framework/Headers/Inferred.h", -// CHECK-NEXT: "[[SOURCEDIR]]/Inputs/frameworks/Inferred.framework/__inferred_module.map" +// CHECK-NEXT: "[[SOURCEDIR]]/Inputs/frameworks/module.modulemap" // CHECK-NEXT: ], // CHECK-NEXT: "name": "Inferred" // CHECK-NEXT: } From b429e1cf63dcfda4325403870d0cf59ecc013a4a Mon Sep 17 00:00:00 2001 From: Michael Spencer Date: Tue, 4 Feb 2020 18:09:33 -0800 Subject: [PATCH 13/13] [Clang][Lex] Remove debugging code. (cherry picked from commit 499404ce8e8792ae5fc6abfdac750ef1d3eee75d) --- clang/lib/Lex/ModuleMap.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index 8b63200167b9b..6b61c79751a1c 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -1094,9 +1094,6 @@ Module *ModuleMap::inferFrameworkModule(const DirectoryEntry *FrameworkDir, if (!Result->isSubFramework()) { inferFrameworkLink(Result, FrameworkDir, FileMgr); } - - Result->dump(); - llvm::errs() << Result->Directory->getName() << "\n"; return Result; }