From 8593e97b775c64828767d3981db3ba0759576c92 Mon Sep 17 00:00:00 2001 From: Blue Gaston Date: Tue, 21 Mar 2023 08:48:22 -0700 Subject: [PATCH 01/88] [TSAN][Darwin] Forward declare spinlock functions on darwin for TSAN interceptors Spinlock symbols are removed from headers in MacOS version 10.12 and greater. Even though they are deprecated, the symbols remain available on the system. The TSAN interceptors currently cause a build failure after this version because of the change in availability of the symbol. We want to continue intercepting the symbols available on the OS. So we add forward declarations so that the TSAN interceptors can build. This is tested with the existing osspinlock_norace test. Differential Revision: https://reviews.llvm.org/D146537 (cherry picked from commit ae484c21c05668f84b13304c28bc39f753e493de) --- .../lib/tsan/rtl/tsan_interceptors_mac.cpp | 1 + .../lib/tsan/rtl/tsan_spinlock_defs_mac.h | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 compiler-rt/lib/tsan/rtl/tsan_spinlock_defs_mac.h diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp index 1ee47bcd1237e..e4f9e2915ced2 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_mac.cpp @@ -18,6 +18,7 @@ #include "tsan_interceptors.h" #include "tsan_interface.h" #include "tsan_interface_ann.h" +#include "tsan_spinlock_defs_mac.h" #include "sanitizer_common/sanitizer_addrhashmap.h" #include diff --git a/compiler-rt/lib/tsan/rtl/tsan_spinlock_defs_mac.h b/compiler-rt/lib/tsan/rtl/tsan_spinlock_defs_mac.h new file mode 100644 index 0000000000000..1a99a81c03023 --- /dev/null +++ b/compiler-rt/lib/tsan/rtl/tsan_spinlock_defs_mac.h @@ -0,0 +1,45 @@ +//===-- tsan_spinlock_defs_mac.h -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Mac-specific forward-declared function defintions that may be +// deprecated in later versions of the OS. +// These are needed for interceptors. +// +//===----------------------------------------------------------------------===// + +#if SANITIZER_APPLE + +#ifndef TSAN_SPINLOCK_DEFS_MAC_H +#define TSAN_SPINLOCK_DEFS_MAC_H + +#include + +extern "C" { + +/* +Provides forward declarations related to OSSpinLocks on Darwin. These functions are +deprecated on macOS version 10.12 and later, +and are no longer included in the system headers. + +However, the symbols are still available on the system, so we provide these forward +declarations to prevent compilation errors in tsan_interceptors_mac.cpp, which +references these functions when defining TSAN interceptor functions. +*/ + +typedef int32_t OSSpinLock; + +void OSSpinLockLock(volatile OSSpinLock *__lock); +void OSSpinLockUnlock(volatile OSSpinLock *__lock); +bool OSSpinLockTry(volatile OSSpinLock *__lock); + +} + +#endif //TSAN_SPINLOCK_DEFS_MAC_H +#endif // SANITIZER_APPLE From 86b7ccb0c0c736a1828d25d0230573564510dd63 Mon Sep 17 00:00:00 2001 From: Pavel Labath Date: Wed, 1 Mar 2023 13:51:06 +0100 Subject: [PATCH 02/88] [lldb/test] Update error message in debug-types-signature-loop.s The error message changed in D144664. (cherry picked from commit fad60105ace546deb219adb14e0eef01a2b6c9d6) --- .../Shell/SymbolFile/DWARF/x86/debug-types-signature-loop.s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-signature-loop.s b/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-signature-loop.s index d0d0fd5705a45..64b835353ed4e 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-signature-loop.s +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-signature-loop.s @@ -4,7 +4,7 @@ # RUN: ld.lld %t.o -o %t # RUN: %lldb %t -o "target variable e" -b | FileCheck %s -# CHECK: e = +# CHECK: Error: 'Unable to determine byte size.' .type e,@object # @e .section .rodata,"a",@progbits From 3600f9fa9f92bd6609c40950578b5fe5d2cf9879 Mon Sep 17 00:00:00 2001 From: Jim Ingham Date: Fri, 24 Mar 2023 10:38:56 -0700 Subject: [PATCH 03/88] Fix backtick handling in parsed commands. https://reviews.llvm.org/D146779 (cherry picked from commit 75ca15fcbb2e1b3671e41f971a000c6d59f5e5ae) --- .../lldb/Interpreter/CommandInterpreter.h | 5 +- .../source/Interpreter/CommandInterpreter.cpp | 208 +++++++++--------- lldb/source/Interpreter/CommandObject.cpp | 12 +- .../command/backticks/TestBackticksInAlias.py | 49 +++++ 4 files changed, 170 insertions(+), 104 deletions(-) diff --git a/lldb/include/lldb/Interpreter/CommandInterpreter.h b/lldb/include/lldb/Interpreter/CommandInterpreter.h index 091a384af0f8e..9e03f51e75de2 100644 --- a/lldb/include/lldb/Interpreter/CommandInterpreter.h +++ b/lldb/include/lldb/Interpreter/CommandInterpreter.h @@ -636,6 +636,9 @@ class CommandInterpreter : public Broadcaster, bool IOHandlerInterrupt(IOHandler &io_handler) override; + Status PreprocessCommand(std::string &command); + Status PreprocessToken(std::string &token); + protected: friend class Debugger; @@ -670,8 +673,6 @@ class CommandInterpreter : public Broadcaster, void RestoreExecutionContext(); - Status PreprocessCommand(std::string &command); - void SourceInitFile(FileSpec file, CommandReturnObject &result); // Completely resolves aliases and abbreviations, returning a pointer to the diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp index 8c5b359417510..3382fc6b31d3f 100644 --- a/lldb/source/Interpreter/CommandInterpreter.cpp +++ b/lldb/source/Interpreter/CommandInterpreter.cpp @@ -1761,112 +1761,124 @@ Status CommandInterpreter::PreprocessCommand(std::string &command) { std::string expr_str(command, expr_content_start, end_backtick - expr_content_start); + error = PreprocessToken(expr_str); + // We always stop at the first error: + if (error.Fail()) + break; - ExecutionContext exe_ctx(GetExecutionContext()); + command.erase(start_backtick, end_backtick - start_backtick + 1); + command.insert(start_backtick, std::string(expr_str)); + pos = start_backtick + expr_str.size(); + } + return error; +} - // Get a dummy target to allow for calculator mode while processing - // backticks. This also helps break the infinite loop caused when target is - // null. - Target *exe_target = exe_ctx.GetTargetPtr(); - Target &target = exe_target ? *exe_target : m_debugger.GetDummyTarget(); - - ValueObjectSP expr_result_valobj_sp; - - EvaluateExpressionOptions options; - options.SetCoerceToId(false); - options.SetUnwindOnError(true); - options.SetIgnoreBreakpoints(true); - options.SetKeepInMemory(false); - options.SetTryAllThreads(true); - options.SetTimeout(llvm::None); - - ExpressionResults expr_result = - target.EvaluateExpression(expr_str.c_str(), exe_ctx.GetFramePtr(), - expr_result_valobj_sp, options); - - if (expr_result == eExpressionCompleted) { - Scalar scalar; - if (expr_result_valobj_sp) - expr_result_valobj_sp = - expr_result_valobj_sp->GetQualifiedRepresentationIfAvailable( - expr_result_valobj_sp->GetDynamicValueType(), true); - if (expr_result_valobj_sp->ResolveValue(scalar)) { - command.erase(start_backtick, end_backtick - start_backtick + 1); - StreamString value_strm; - const bool show_type = false; - scalar.GetValue(&value_strm, show_type); - size_t value_string_size = value_strm.GetSize(); - if (value_string_size) { - command.insert(start_backtick, std::string(value_strm.GetString())); - pos = start_backtick + value_string_size; - continue; - } else { - error.SetErrorStringWithFormat("expression value didn't result " - "in a scalar value for the " - "expression '%s'", - expr_str.c_str()); - break; - } - } else { - error.SetErrorStringWithFormat("expression value didn't result " - "in a scalar value for the " - "expression '%s'", - expr_str.c_str()); - break; - } +Status +CommandInterpreter::PreprocessToken(std::string &expr_str) { + Status error; + ExecutionContext exe_ctx(GetExecutionContext()); - continue; - } + // Get a dummy target to allow for calculator mode while processing + // backticks. This also helps break the infinite loop caused when target is + // null. + Target *exe_target = exe_ctx.GetTargetPtr(); + Target &target = exe_target ? *exe_target : m_debugger.GetDummyTarget(); - if (expr_result_valobj_sp) - error = expr_result_valobj_sp->GetError(); + ValueObjectSP expr_result_valobj_sp; - if (error.Success()) { - switch (expr_result) { - case eExpressionSetupError: - error.SetErrorStringWithFormat( - "expression setup error for the expression '%s'", expr_str.c_str()); - break; - case eExpressionParseError: - error.SetErrorStringWithFormat( - "expression parse error for the expression '%s'", expr_str.c_str()); - break; - case eExpressionResultUnavailable: - error.SetErrorStringWithFormat( - "expression error fetching result for the expression '%s'", - expr_str.c_str()); - break; - case eExpressionCompleted: - break; - case eExpressionDiscarded: - error.SetErrorStringWithFormat( - "expression discarded for the expression '%s'", expr_str.c_str()); - break; - case eExpressionInterrupted: - error.SetErrorStringWithFormat( - "expression interrupted for the expression '%s'", expr_str.c_str()); - break; - case eExpressionHitBreakpoint: - error.SetErrorStringWithFormat( - "expression hit breakpoint for the expression '%s'", - expr_str.c_str()); - break; - case eExpressionTimedOut: - error.SetErrorStringWithFormat( - "expression timed out for the expression '%s'", expr_str.c_str()); - break; - case eExpressionStoppedForDebug: - error.SetErrorStringWithFormat("expression stop at entry point " - "for debugging for the " + EvaluateExpressionOptions options; + options.SetCoerceToId(false); + options.SetUnwindOnError(true); + options.SetIgnoreBreakpoints(true); + options.SetKeepInMemory(false); + options.SetTryAllThreads(true); + options.SetTimeout(std::nullopt); + + ExpressionResults expr_result = + target.EvaluateExpression(expr_str.c_str(), exe_ctx.GetFramePtr(), + expr_result_valobj_sp, options); + + if (expr_result == eExpressionCompleted) { + Scalar scalar; + if (expr_result_valobj_sp) + expr_result_valobj_sp = + expr_result_valobj_sp->GetQualifiedRepresentationIfAvailable( + expr_result_valobj_sp->GetDynamicValueType(), true); + if (expr_result_valobj_sp->ResolveValue(scalar)) { + + StreamString value_strm; + const bool show_type = false; + scalar.GetValue(&value_strm, show_type); + size_t value_string_size = value_strm.GetSize(); + if (value_string_size) { + expr_str = value_strm.GetData(); + } else { + error.SetErrorStringWithFormat("expression value didn't result " + "in a scalar value for the " "expression '%s'", expr_str.c_str()); - break; - case eExpressionThreadVanished: - error.SetErrorStringWithFormat( - "expression thread vanished for the expression '%s'", - expr_str.c_str()); - break; } + } else { + error.SetErrorStringWithFormat("expression value didn't result " + "in a scalar value for the " + "expression '%s'", + expr_str.c_str()); + } + return error; + } + + // If we have an error from the expression evaluation it will be in the + // ValueObject error, which won't be success and we will just report it. + // But if for some reason we didn't get a value object at all, then we will + // make up some helpful errors from the expression result. + if (expr_result_valobj_sp) + error = expr_result_valobj_sp->GetError(); + + if (error.Success()) { + switch (expr_result) { + case eExpressionSetupError: + error.SetErrorStringWithFormat( + "expression setup error for the expression '%s'", expr_str.c_str()); + break; + case eExpressionParseError: + error.SetErrorStringWithFormat( + "expression parse error for the expression '%s'", expr_str.c_str()); + break; + case eExpressionResultUnavailable: + error.SetErrorStringWithFormat( + "expression error fetching result for the expression '%s'", + expr_str.c_str()); + break; + case eExpressionCompleted: + break; + case eExpressionDiscarded: + error.SetErrorStringWithFormat( + "expression discarded for the expression '%s'", expr_str.c_str()); + break; + case eExpressionInterrupted: + error.SetErrorStringWithFormat( + "expression interrupted for the expression '%s'", expr_str.c_str()); + break; + case eExpressionHitBreakpoint: + error.SetErrorStringWithFormat( + "expression hit breakpoint for the expression '%s'", + expr_str.c_str()); + break; + case eExpressionTimedOut: + error.SetErrorStringWithFormat( + "expression timed out for the expression '%s'", expr_str.c_str()); + break; + case eExpressionStoppedForDebug: + error.SetErrorStringWithFormat("expression stop at entry point " + "for debugging for the " + "expression '%s'", + expr_str.c_str()); + break; + case eExpressionThreadVanished: + error.SetErrorStringWithFormat( + "expression thread vanished for the expression '%s'", + expr_str.c_str()); + break; } } return error; diff --git a/lldb/source/Interpreter/CommandObject.cpp b/lldb/source/Interpreter/CommandObject.cpp index 4500378c62298..c13ce4ece4297 100644 --- a/lldb/source/Interpreter/CommandObject.cpp +++ b/lldb/source/Interpreter/CommandObject.cpp @@ -727,10 +727,14 @@ bool CommandObjectParsed::Execute(const char *args_string, } if (!handled) { for (auto entry : llvm::enumerate(cmd_args.entries())) { - if (!entry.value().ref().empty() && entry.value().GetQuoteChar() == '`') { - cmd_args.ReplaceArgumentAtIndex( - entry.index(), - m_interpreter.ProcessEmbeddedScriptCommands(entry.value().c_str())); + const Args::ArgEntry &value = entry.value(); + if (!value.ref().empty() && value.GetQuoteChar() == '`') { + // We have to put the backtick back in place for PreprocessCommand. + std::string opt_string = value.c_str(); + Status error; + error = m_interpreter.PreprocessToken(opt_string); + if (error.Success()) + cmd_args.ReplaceArgumentAtIndex(entry.index(), opt_string); } } diff --git a/lldb/test/API/commands/command/backticks/TestBackticksInAlias.py b/lldb/test/API/commands/command/backticks/TestBackticksInAlias.py index 495e52db53aa3..4a95cff21e713 100644 --- a/lldb/test/API/commands/command/backticks/TestBackticksInAlias.py +++ b/lldb/test/API/commands/command/backticks/TestBackticksInAlias.py @@ -29,4 +29,53 @@ def test_backticks_in_alias(self): interp.HandleCommand("_test-argv-parray-cmd", result) self.assertFalse(result.Succeeded(), "CommandAlias::Desugar currently fails if a alias substitutes %N arguments in another alias") + def test_backticks_in_parsed_cmd_argument(self): + """ break list is a parsed command, use a variable for the breakpoint number + and make sure that and the direct use of the ID get the same result. """ + self.build() + target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c")) + # Make a second breakpoint so that if the backtick part -> nothing we'll print too much: + # It doesn't need to resolve to anything. + dummy_bkpt = target.BreakpointCreateByName("dont_really_care_if_this_exists") + + bkpt_id = bkpt.GetID() + self.runCmd(f"expr int $number = {bkpt_id}") + direct_result = lldb.SBCommandReturnObject() + backtick_result = lldb.SBCommandReturnObject() + interp = self.dbg.GetCommandInterpreter() + interp.HandleCommand(f"break list {bkpt_id}", direct_result) + self.assertTrue(direct_result.Succeeded(), "Break list with id works") + interp.HandleCommand("break list `$number`", backtick_result) + self.assertTrue(direct_result.Succeeded(), "Break list with backtick works") + self.assertEqual(direct_result.GetOutput(), backtick_result.GetOutput(), "Output is the same") + + def test_backticks_in_parsed_cmd_option(self): + # The script interpreter is a raw command, so try that one: + self.build() + target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c")) + + self.runCmd(f"expr int $number = 2") + direct_result = lldb.SBCommandReturnObject() + backtick_result = lldb.SBCommandReturnObject() + interp = self.dbg.GetCommandInterpreter() + interp.HandleCommand(f"memory read --count 2 argv", direct_result) + self.assertTrue(direct_result.Succeeded(), "memory read with direct count works") + interp.HandleCommand("memory read --count `$number` argv", backtick_result) + self.assertTrue(direct_result.Succeeded(), "memory read with backtick works") + self.assertEqual(direct_result.GetOutput(), backtick_result.GetOutput(), "Output is the same") + + def test_backticks_in_raw_cmd(self): + # The script interpreter is a raw command, so try that one: + self.build() + target, process, thread, bkpt = lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.c")) + argc_valobj = thread.frames[0].FindVariable("argc") + self.assertTrue(argc_valobj.GetError().Success(), "Made argc valobj") + argc_value = argc_valobj.GetValueAsUnsigned(0) + self.assertNotEqual(argc_value, 0, "Got a value for argc") + result = lldb.SBCommandReturnObject() + interp = self.dbg.GetCommandInterpreter() + interp.HandleCommand(f"script {argc_value} - `argc`", result) + self.assertTrue(result.Succeeded(), "Command succeeded") + self.assertEqual("0\n", result.GetOutput(), "Substitution worked") + From cb34fb7456a54d5c5a9fcdbabf48bf3c58326b80 Mon Sep 17 00:00:00 2001 From: Alex Langford Date: Thu, 23 Mar 2023 17:38:38 -0700 Subject: [PATCH 04/88] [lldb] Annotate remaining swift tests with @swiftTest --- .../test/API/commands/dwim-print/swift/TestDWIMPrintSwift.py | 3 ++- .../swift_exception/TestExpressionErrorBreakpoint.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lldb/test/API/commands/dwim-print/swift/TestDWIMPrintSwift.py b/lldb/test/API/commands/dwim-print/swift/TestDWIMPrintSwift.py index 07222e64b8a3c..a2245c4c59ddc 100644 --- a/lldb/test/API/commands/dwim-print/swift/TestDWIMPrintSwift.py +++ b/lldb/test/API/commands/dwim-print/swift/TestDWIMPrintSwift.py @@ -7,8 +7,8 @@ from lldbsuite.test.decorators import * import lldbsuite.test.lldbutil as lldbutil - class TestCase(TestBase): + @swiftTest def test_swift_po_address(self): self.build() _, _, thread, _ = lldbutil.run_to_source_breakpoint( @@ -20,6 +20,7 @@ def test_swift_po_address(self): self.expect(f"dwim-print -O -- 0x{hex_addr}", patterns=[f"Object@0x0*{hex_addr}"]) self.expect(f"dwim-print -O -- {addr}", patterns=[f"Object@0x0*{hex_addr}"]) + @swiftTest def test_swift_po_non_address_hex(self): """No special handling of non-memory integer values.""" self.build() diff --git a/lldb/test/API/functionalities/breakpoint/swift_exception/TestExpressionErrorBreakpoint.py b/lldb/test/API/functionalities/breakpoint/swift_exception/TestExpressionErrorBreakpoint.py index 1da102f13211b..dbfade98dca1c 100644 --- a/lldb/test/API/functionalities/breakpoint/swift_exception/TestExpressionErrorBreakpoint.py +++ b/lldb/test/API/functionalities/breakpoint/swift_exception/TestExpressionErrorBreakpoint.py @@ -28,26 +28,29 @@ class TestSwiftErrorBreakpoint(TestBase): @swiftTest def test_swift_error_no_typename(self): """Tests that swift error throws are correctly caught by the Swift Error breakpoint""" + self.build() self.do_tests(None, True) @swiftTest def test_swift_error_matching_base_typename(self): """Tests that swift error throws are correctly caught by the Swift Error breakpoint""" + self.build() self.do_tests("EnumError", True) @swiftTest def test_swift_error_matching_full_typename(self): """Tests that swift error throws are correctly caught by the Swift Error breakpoint""" + self.build() self.do_tests("a.EnumError", True) @swiftTest def test_swift_error_bogus_typename(self): """Tests that swift error throws are correctly caught by the Swift Error breakpoint""" + self.build() self.do_tests("NoSuchErrorHere", False) def setUp(self): TestBase.setUp(self) - self.build() def do_tests(self, typename, should_stop): self.do_test(typename, should_stop, self.create_breakpoint_with_api) From bf9465898912f008274a40e26cec3e3a8f367e1d Mon Sep 17 00:00:00 2001 From: Alex Langford Date: Thu, 23 Mar 2023 17:39:21 -0700 Subject: [PATCH 05/88] [lldb][CMake] Further gate swift-specific CMake code behind LLDB_ENABLE_SWIFT_SUPPORT --- lldb/CMakeLists.txt | 40 +++++++++++++++---------- lldb/cmake/modules/LLDBConfig.cmake | 32 +++++++++++++------- lldb/cmake/modules/LLDBStandalone.cmake | 15 +++++++--- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/lldb/CMakeLists.txt b/lldb/CMakeLists.txt index f4114494d951c..86a75977ab5e1 100644 --- a/lldb/CMakeLists.txt +++ b/lldb/CMakeLists.txt @@ -17,6 +17,10 @@ endif() # Must go below project(..) include(GNUInstallDirs) +# BEGIN SWIFT MOD +option(LLDB_ENABLE_SWIFT_SUPPORT "Enable swift support" ON) +# END SWIFT MOD + if(LLDB_BUILT_STANDALONE) include(LLDBStandalone) @@ -33,25 +37,29 @@ include(LLDBConfig) include(AddLLDB) # BEGIN - Swift Mods -if(EXISTS "${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}") - list(APPEND CMAKE_MODULE_PATH - "${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}/cmake" - "${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}/cmake/modules") -elseif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/../swift) - list(APPEND CMAKE_MODULE_PATH - "${CMAKE_CURRENT_SOURCE_DIR}/../swift/cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/../swift/cmake/modules") +if (LLDB_ENABLE_SWIFT_SUPPORT) + if(EXISTS "${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}") + list(APPEND CMAKE_MODULE_PATH + "${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}/cmake" + "${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}/cmake/modules") + elseif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/../swift) + list(APPEND CMAKE_MODULE_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/../swift/cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/../swift/cmake/modules") + endif() endif() # When we have the early SwiftSyntax build, we can include its parser. -if(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR) - set(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS - ${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/cmake/SwiftSyntaxTargets.cmake) - if(NOT EXISTS "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS}") - message(STATUS "Skipping Swift Swift parser integration due to missing early SwiftSyntax") - else() - set(SWIFT_SWIFT_PARSER TRUE) - include(${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS}) +if (LLDB_ENABLE_SWIFT_SUPPORT) + if(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR) + set(SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS + ${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/cmake/SwiftSyntaxTargets.cmake) + if(NOT EXISTS "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS}") + message(STATUS "Skipping Swift Swift parser integration due to missing early SwiftSyntax") + else() + set(SWIFT_SWIFT_PARSER TRUE) + include(${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_TARGETS}) + endif() endif() endif() # END - Swift Mods diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake index 97ee382203c4d..048a6db104553 100644 --- a/lldb/cmake/modules/LLDBConfig.cmake +++ b/lldb/cmake/modules/LLDBConfig.cmake @@ -86,10 +86,18 @@ option(LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS "Fail to configure if certain requirements are not met for testing." OFF) # BEGIN SWIFT MOD -option(LLDB_ENABLE_SWIFT_SUPPORT "Enable swift support" ON) option(LLDB_ENABLE_WERROR "Fail and stop if a warning is triggered." ${LLVM_ENABLE_WERROR}) if(LLDB_ENABLE_SWIFT_SUPPORT) add_definitions( -DLLDB_ENABLE_SWIFT ) +else() + # LLVM_DISTRIBUTION_COMPONENTS may have swift-specific things in them (e.g. + # repl_swift). This may be set in a cache where LLDB_ENABLE_SWIFT_SUPPORT does + # not yet have a value. We have to touch up LLVM_DISTRIBUTION_COMPONENTS after + # the fact. + if(LLVM_DISTRIBUTION_COMPONENTS) + list(REMOVE_ITEM LLVM_DISTRIBUTION_COMPONENTS repl_swift) + set(LLVM_DISTRIBUTION_COMPONENTS ${LLVM_DISTRIBUTION_COMPONENTS} CACHE STRING "" FORCE) + endif() endif() # END SWIFT CODE @@ -210,17 +218,19 @@ else () endif () include_directories("${CMAKE_CURRENT_BINARY_DIR}/../clang/include") -if(NOT LLDB_BUILT_STANDALONE) - if (LLVM_EXTERNAL_SWIFT_SOURCE_DIR) - include_directories(${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}/include) - include_directories(${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}/stdlib/public/SwiftShims) +if(LLDB_ENABLE_SWIFT_SUPPORT) + if(NOT LLDB_BUILT_STANDALONE) + if (LLVM_EXTERNAL_SWIFT_SOURCE_DIR) + include_directories(${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}/include) + include_directories(${LLVM_EXTERNAL_SWIFT_SOURCE_DIR}/stdlib/public/SwiftShims) + else () + include_directories(${CMAKE_SOURCE_DIR}/tools/swift/include) + include_directories(${CMAKE_SOURCE_DIR}/tools/swift/stdlib/public/SwiftShims) + endif () + include_directories("${CMAKE_CURRENT_BINARY_DIR}/../swift/include") else () - include_directories(${CMAKE_SOURCE_DIR}/tools/swift/include) - include_directories(${CMAKE_SOURCE_DIR}/tools/swift/stdlib/public/SwiftShims) - endif () - include_directories("${CMAKE_CURRENT_BINARY_DIR}/../swift/include") -else () - include_directories("${SWIFT_INCLUDE_DIRS}") + include_directories("${SWIFT_INCLUDE_DIRS}") + endif() endif() # GCC silently accepts any -Wno- option, but warns about those options diff --git a/lldb/cmake/modules/LLDBStandalone.cmake b/lldb/cmake/modules/LLDBStandalone.cmake index f6e2bd11cf252..d64791f194172 100644 --- a/lldb/cmake/modules/LLDBStandalone.cmake +++ b/lldb/cmake/modules/LLDBStandalone.cmake @@ -14,7 +14,9 @@ option(LLVM_INSTALL_TOOLCHAIN_ONLY "Only include toolchain files in the 'install find_package(LLVM REQUIRED CONFIG HINTS ${LLVM_DIR} NO_CMAKE_FIND_ROOT_PATH) find_package(Clang REQUIRED CONFIG HINTS ${Clang_DIR} ${LLVM_DIR}/../clang NO_CMAKE_FIND_ROOT_PATH) -find_package(Swift REQUIRED CONFIG HINTS "${Swift_DIR}" NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) +if(LLDB_ENABLE_SWIFT_SUPPORT) + find_package(Swift REQUIRED CONFIG HINTS "${Swift_DIR}" NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) +endif() # We set LLVM_CMAKE_DIR so that GetSVN.cmake is found correctly when building SVNVersion.inc set(LLVM_CMAKE_DIR ${LLVM_CMAKE_DIR} CACHE PATH "Path to LLVM CMake modules") @@ -88,7 +90,9 @@ endif() # CMake modules to be in that directory as well. file(TO_CMAKE_PATH ${LLVM_DIR} LLVM_DIR) list(APPEND CMAKE_MODULE_PATH "${LLVM_DIR}") -list(APPEND CMAKE_MODULE_PATH "${SWIFT_CMAKE_DIR}") +if(LLDB_ENABLE_SWIFT_SUPPORT) + list(APPEND CMAKE_MODULE_PATH "${SWIFT_CMAKE_DIR}") +endif() include(AddLLVM) include(TableGen) @@ -124,9 +128,12 @@ include_directories( "${CMAKE_BINARY_DIR}/include" "${LLVM_INCLUDE_DIRS}" "${CLANG_INCLUDE_DIRS}" - "${SWIFT_INCLUDE_DIRS}" - "${SWIFT_MAIN_SRC_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/source") +if(LLDB_ENABLE_SWIFT_SUPPORT) + include_directories( + "${SWIFT_INCLUDE_DIRS}" + "${SWIFT_MAIN_SRC_DIR}/include") +endif() if(NOT DEFINED LLVM_COMMON_CMAKE_UTILS) set(LLVM_COMMON_CMAKE_UTILS ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) From 8800893f9f7729a711278f70ce75de85caeaa602 Mon Sep 17 00:00:00 2001 From: Jim Ingham Date: Fri, 24 Mar 2023 12:27:33 -0700 Subject: [PATCH 06/88] Don't expect what newlines look like - never works on Windows. (cherry picked from commit 0999996f68186183eaf065e4a05a3e73170a19ae) --- .../API/commands/command/backticks/TestBackticksInAlias.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lldb/test/API/commands/command/backticks/TestBackticksInAlias.py b/lldb/test/API/commands/command/backticks/TestBackticksInAlias.py index 4a95cff21e713..74c33c66cdd08 100644 --- a/lldb/test/API/commands/command/backticks/TestBackticksInAlias.py +++ b/lldb/test/API/commands/command/backticks/TestBackticksInAlias.py @@ -76,6 +76,7 @@ def test_backticks_in_raw_cmd(self): interp = self.dbg.GetCommandInterpreter() interp.HandleCommand(f"script {argc_value} - `argc`", result) self.assertTrue(result.Succeeded(), "Command succeeded") - self.assertEqual("0\n", result.GetOutput(), "Substitution worked") + fixed_output = result.GetOutput().rstrip() + self.assertEqual("0", fixed_output, "Substitution worked") From 65ba74295a19722777f6a188da3b04feca703848 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Fri, 24 Mar 2023 17:31:39 -0700 Subject: [PATCH 07/88] [lldb] Remove errant call to SBReproducer.SetWorkingDirectory The old reproducer functionality has been removed. Remove this call as it's now just a NO-OP. (cherry picked from commit 0016f476ab1f9013d336491924495ec01f43a045) --- lldb/packages/Python/lldbsuite/test/lldbtest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py index 92bd52083296e..bf9cc1073959c 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbtest.py +++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py @@ -553,7 +553,6 @@ def setUpClass(cls): if traceAlways: print("Change dir to:", full_dir, file=sys.stderr) os.chdir(full_dir) - lldb.SBReproducer.SetWorkingDirectory(full_dir) # Set platform context. cls.platformContext = lldbplatformutil.createPlatformContext() From d7024c4e94c0d6fb9a1f03705ca3ba3414b3f7e5 Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Mon, 27 Mar 2023 09:44:28 -0700 Subject: [PATCH 08/88] TypeSystem: early exit if there are 0 imports This is a minor u-opt but more importantly the `Progress` type requires that the count of imports being >=1. This avoids an assertion being triggered. --- lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp index 2cec760726388..4862848250398 100644 --- a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp +++ b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp @@ -8340,6 +8340,8 @@ bool SwiftASTContext::GetCompileUnitImportsImpl( return true; auto cu_imports = compile_unit->GetImportedModules(); + if (cu_imports.size() == 0) + return true; Progress progress( llvm::formatv("Getting Swift compile unit imports for '{0}'", From ed3ceb97411b81ee9cba6c55e9b8b3ece90d77fa Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 15 Mar 2023 11:16:55 -0700 Subject: [PATCH 09/88] [clang][cas] Factor IncludeTreeBuilder out of IncludeTreeActionConsumer The IncludeTreeBuilder is responsible for building the include-tree for a specific TU. Factor it out of IncludeTreeActionConsumer to prepare for building modules, which require their own builder. (cherry picked from commit 062b023007b3b28e707a74c3bbdffb27b3d8b528) (cherry picked from commit 30c89b9f767156267c9973fd928069150a935498) --- .../IncludeTreeActionController.cpp | 192 ++++++++++++------ 1 file changed, 132 insertions(+), 60 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 5d8a2ee36b334..3182cf4ce41e9 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -20,6 +20,8 @@ using namespace dependencies; using llvm::Error; namespace { +class IncludeTreeBuilder; + class IncludeTreeActionController : public CallbackActionController { public: IncludeTreeActionController(cas::ObjectStore &DB, @@ -47,12 +49,41 @@ class IncludeTreeActionController : public CallbackActionController { Error finalize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; - Expected getObjectForFile(Preprocessor &PP, FileID FID); - Expected - getObjectForFileNonCached(FileManager &FM, const SrcMgr::FileInfo &FI); - Expected getObjectForBuffer(const SrcMgr::FileInfo &FI); - Expected addToFileList(FileManager &FM, const FileEntry *FE); +private: + IncludeTreeBuilder ¤t() { + assert(!BuilderStack.empty()); + return *BuilderStack.back(); + } + +private: + cas::ObjectStore &DB; + CASOptions CASOpts; + DepscanPrefixMapping PrefixMapping; + llvm::PrefixMapper PrefixMapper; + SmallVector> BuilderStack; + std::optional IncludeTreeResult; +}; +/// Callbacks for building an include-tree for a given translation unit or +/// module. The \c IncludeTreeActionController is responsiblee for pushing and +/// popping builders from the stack as modules are required. +class IncludeTreeBuilder { +public: + IncludeTreeBuilder(cas::ObjectStore &DB, llvm::PrefixMapper &PrefixMapper) + : DB(DB), PrefixMapper(PrefixMapper) {} + + Expected + finishIncludeTree(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation); + + void enteredInclude(Preprocessor &PP, FileID FID); + + void exitedInclude(Preprocessor &PP, FileID IncludedBy, FileID Include, + SourceLocation ExitLoc); + + void handleHasIncludeCheck(Preprocessor &PP, bool Result); + +private: struct FilePPState { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; @@ -60,8 +91,12 @@ class IncludeTreeActionController : public CallbackActionController { llvm::SmallBitVector HasIncludeChecks; }; + Expected getObjectForFile(Preprocessor &PP, FileID FID); + Expected + getObjectForFileNonCached(FileManager &FM, const SrcMgr::FileInfo &FI); + Expected getObjectForBuffer(const SrcMgr::FileInfo &FI); + Expected addToFileList(FileManager &FM, const FileEntry *FE); Expected getCASTreeForFileIncludes(FilePPState &&PPState); - Expected createIncludeFile(StringRef Filename, cas::ObjectRef Contents); @@ -75,10 +110,10 @@ class IncludeTreeActionController : public CallbackActionController { return *E; } +private: cas::ObjectStore &DB; - CASOptions CASOpts; - DepscanPrefixMapping PrefixMapping; - llvm::PrefixMapper PrefixMapper; + llvm::PrefixMapper &PrefixMapper; + Optional PCHRef; bool StartedEnteringIncludes = false; // When a PCH is used this lists the filenames of the included files as they @@ -87,6 +122,10 @@ class IncludeTreeActionController : public CallbackActionController { llvm::BitVector SeenIncludeFiles; SmallVector IncludedFiles; Optional PredefinesBufferRef; + Optional ModuleIncludesBufferRef; + Optional ModuleMapFileRef; + /// When the builder is created from an existing tree, the main include tree. + Optional MainIncludeTreeRef; SmallVector IncludeStack; llvm::DenseMap> ObjectForFile; Optional ErrorToReport; @@ -152,6 +191,13 @@ void dependencies::addReversePrefixMappingFileSystem( ScanInstance.getFileManager().setVirtualFileSystem(std::move(FS)); } +Expected IncludeTreeActionController::getIncludeTree() { + if (IncludeTreeResult) + return *IncludeTreeResult; + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "failed to produce include-tree"); +} + Error IncludeTreeActionController::initialize( CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { if (Error E = @@ -184,10 +230,50 @@ Error IncludeTreeActionController::initialize( CASOpts = ScanInstance.getCASOpts(); + BuilderStack.push_back( + std::make_unique(DB, PrefixMapper)); + return Error::success(); } void IncludeTreeActionController::enteredInclude(Preprocessor &PP, FileID FID) { + current().enteredInclude(PP, FID); +} + +void IncludeTreeActionController::exitedInclude(Preprocessor &PP, + FileID IncludedBy, + FileID Include, + SourceLocation ExitLoc) { + current().exitedInclude(PP, IncludedBy, Include, ExitLoc); +} + +void IncludeTreeActionController::handleHasIncludeCheck(Preprocessor &PP, + bool Result) { + current().handleHasIncludeCheck(PP, Result); +} + +Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) { + assert(!IncludeTreeResult); + assert(BuilderStack.size() == 1); + auto Builder = BuilderStack.pop_back_val(); + Error E = Builder->finishIncludeTree(ScanInstance, NewInvocation) + .moveInto(IncludeTreeResult); + if (E) + return E; + + configureInvocationForCaching(NewInvocation, CASOpts, + IncludeTreeResult->getID().toString(), + // FIXME: working dir? + /*CASFSWorkingDir=*/"", + /*ProduceIncludeTree=*/true); + + DepscanPrefixMapping::remapInvocationPaths(NewInvocation, PrefixMapper); + + return Error::success(); +} + +void IncludeTreeBuilder::enteredInclude(Preprocessor &PP, FileID FID) { if (hasErrorOccurred()) return; @@ -212,10 +298,8 @@ void IncludeTreeActionController::enteredInclude(Preprocessor &PP, FileID FID) { IncludeStack.push_back({FI.getFileCharacteristic(), *FileRef, {}, {}}); } -void IncludeTreeActionController::exitedInclude(Preprocessor &PP, - FileID IncludedBy, - FileID Include, - SourceLocation ExitLoc) { +void IncludeTreeBuilder::exitedInclude(Preprocessor &PP, FileID IncludedBy, + FileID Include, SourceLocation ExitLoc) { if (hasErrorOccurred()) return; @@ -231,16 +315,19 @@ void IncludeTreeActionController::exitedInclude(Preprocessor &PP, {IncludeTree->getRef(), LocInfo.second}); } -void IncludeTreeActionController::handleHasIncludeCheck(Preprocessor &PP, - bool Result) { +void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { if (hasErrorOccurred()) return; IncludeStack.back().HasIncludeChecks.push_back(Result); } -Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, - CompilerInvocation &NewInvocation) { +Expected +IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) { + if (ErrorToReport) + return std::move(*ErrorToReport); + FileManager &FM = ScanInstance.getFileManager(); auto addFile = [&](StringRef FilePath, @@ -257,7 +344,7 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, for (StringRef FilePath : NewInvocation.getLangOpts()->NoSanitizeFiles) { if (Error E = addFile(FilePath)) - return E; + return std::move(E); } // Add profile files. // FIXME: Do not have the logic here to determine which path should be set @@ -265,13 +352,13 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, // checked the file needed exists. Just try load and ignore errors. if (Error E = addFile(NewInvocation.getCodeGenOpts().ProfileInstrumentUsePath, /*IgnoreFileError=*/true)) - return E; + return std::move(E); if (Error E = addFile(NewInvocation.getCodeGenOpts().SampleProfileFile, /*IgnoreFileError=*/true)) - return E; + return std::move(E); if (Error E = addFile(NewInvocation.getCodeGenOpts().ProfileRemappingFile, /*IgnoreFileError=*/true)) - return E; + return std::move(E); StringRef Sysroot = NewInvocation.getHeaderSearchOpts().Sysroot; if (!Sysroot.empty()) { @@ -280,7 +367,7 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, llvm::SmallString<256> FilePath = Sysroot; llvm::sys::path::append(FilePath, "SDKSettings.json"); if (Error E = addFile(FilePath, /*IgnoreFileError*/ true)) - return E; + return std::move(E); } auto FinishIncludeTree = [&]() -> Error { @@ -319,24 +406,26 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, }; if (Error E = FinishIncludeTree()) - return E; - - auto IncludeTreeRoot = getIncludeTree(); - if (!IncludeTreeRoot) - return IncludeTreeRoot.takeError(); + return std::move(E); - configureInvocationForCaching(NewInvocation, CASOpts, - IncludeTreeRoot->getID().toString(), - /*CASFSWorkingDir=*/"", - /*ProduceIncludeTree=*/true); + if (ErrorToReport) + return std::move(*ErrorToReport); - DepscanPrefixMapping::remapInvocationPaths(NewInvocation, PrefixMapper); + assert(IncludeStack.size() == 1); + Expected MainIncludeTree = + getCASTreeForFileIncludes(IncludeStack.pop_back_val()); + if (!MainIncludeTree) + return MainIncludeTree.takeError(); + auto FileList = cas::IncludeFileList::create(DB, IncludedFiles); + if (!FileList) + return FileList.takeError(); - return Error::success(); + return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), + FileList->getRef(), PCHRef); } -Expected -IncludeTreeActionController::getObjectForFile(Preprocessor &PP, FileID FID) { +Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, + FileID FID) { SourceManager &SM = PP.getSourceManager(); const SrcMgr::FileInfo &FI = SM.getSLocEntry(FID).getFile(); if (PP.getPredefinesFileID() == FID) { @@ -359,8 +448,9 @@ IncludeTreeActionController::getObjectForFile(Preprocessor &PP, FileID FID) { return *FileRef; } -Expected IncludeTreeActionController::getObjectForFileNonCached( - FileManager &FM, const SrcMgr::FileInfo &FI) { +Expected +IncludeTreeBuilder::getObjectForFileNonCached(FileManager &FM, + const SrcMgr::FileInfo &FI) { const FileEntry *FE = FI.getContentCache().OrigEntry; assert(FE); @@ -373,7 +463,7 @@ Expected IncludeTreeActionController::getObjectForFileNonCached( } Expected -IncludeTreeActionController::getObjectForBuffer(const SrcMgr::FileInfo &FI) { +IncludeTreeBuilder::getObjectForBuffer(const SrcMgr::FileInfo &FI) { // This is a non-file buffer, like the predefines. auto Ref = DB.storeFromString( {}, FI.getContentCache().getBufferIfLoaded()->getBuffer()); @@ -386,8 +476,7 @@ IncludeTreeActionController::getObjectForBuffer(const SrcMgr::FileInfo &FI) { } Expected -IncludeTreeActionController::addToFileList(FileManager &FM, - const FileEntry *FE) { +IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { StringRef Filename = FE->getName(); llvm::ErrorOr> CASContents = FM.getObjectRefForFileContent(Filename); @@ -420,14 +509,14 @@ IncludeTreeActionController::addToFileList(FileManager &FM, } Expected -IncludeTreeActionController::getCASTreeForFileIncludes(FilePPState &&PPState) { +IncludeTreeBuilder::getCASTreeForFileIncludes(FilePPState &&PPState) { return cas::IncludeTree::create(DB, PPState.FileCharacteristic, PPState.File, PPState.Includes, PPState.HasIncludeChecks); } Expected -IncludeTreeActionController::createIncludeFile(StringRef Filename, - cas::ObjectRef Contents) { +IncludeTreeBuilder::createIncludeFile(StringRef Filename, + cas::ObjectRef Contents) { SmallString<256> MappedPath; if (!PrefixMapper.empty()) { PrefixMapper.map(Filename, MappedPath); @@ -436,23 +525,6 @@ IncludeTreeActionController::createIncludeFile(StringRef Filename, return cas::IncludeFile::create(DB, Filename, std::move(Contents)); } -Expected IncludeTreeActionController::getIncludeTree() { - if (ErrorToReport) - return std::move(*ErrorToReport); - - assert(IncludeStack.size() == 1); - Expected MainIncludeTree = - getCASTreeForFileIncludes(IncludeStack.pop_back_val()); - if (!MainIncludeTree) - return MainIncludeTree.takeError(); - auto FileList = cas::IncludeFileList::create(DB, IncludedFiles); - if (!FileList) - return FileList.takeError(); - - return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), - FileList->getRef(), PCHRef); -} - std::unique_ptr dependencies::createIncludeTreeActionController( cas::ObjectStore &DB, DepscanPrefixMapping PrefixMapping) { From d90d38ed1233bcbe353c2c38483a053802c8f0ed Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 15 Mar 2023 11:25:45 -0700 Subject: [PATCH 10/88] [clang][cas] Remove PPCallbacks from DependencyActionController These were only used by include-tree, and we now have a mechanism to inject PPCallbacks without going through DependencyActionController. Remove them and simplify the IncludeTreeActionController. (cherry picked from commit c603c65426e0fafe33cfdf3d062f927cad442cdd) (cherry picked from commit ca4eed9ef66a23c49df71b33dc588b42ebce41c6) --- .../DependencyScanningWorker.h | 7 --- .../IncludeTreeActionController.cpp | 48 +++++-------------- 2 files changed, 13 insertions(+), 42 deletions(-) diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 1c2182811da40..a0df3970696c7 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -97,13 +97,6 @@ class DependencyActionController { return llvm::Error::success(); } - virtual void enteredInclude(Preprocessor &PP, FileID FID) {} - - virtual void exitedInclude(Preprocessor &PP, FileID IncludedBy, - FileID Include, SourceLocation ExitLoc) {} - - virtual void handleHasIncludeCheck(Preprocessor &PP, bool Result) {} - /// FIXME: This is temporary until we eliminate the split between consumers in /// \p DependencyScanningTool and collectors in \p DependencyScanningWorker /// and have them both in the same file. see FIXME in \p diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 3182cf4ce41e9..9800692475901 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -35,13 +35,6 @@ class IncludeTreeActionController : public CallbackActionController { Error initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; - void enteredInclude(Preprocessor &PP, FileID FID) override; - - void exitedInclude(Preprocessor &PP, FileID IncludedBy, FileID Include, - SourceLocation ExitLoc) override; - - void handleHasIncludeCheck(Preprocessor &PP, bool Result) override; - const DepscanPrefixMapping *getPrefixMapping() override { return &PrefixMapping; } @@ -60,6 +53,8 @@ class IncludeTreeActionController : public CallbackActionController { CASOptions CASOpts; DepscanPrefixMapping PrefixMapping; llvm::PrefixMapper PrefixMapper; + // IncludeTreePPCallbacks keeps a pointer to the current builder, so use a + // pointer so the builder cannot move when resizing. SmallVector> BuilderStack; std::optional IncludeTreeResult; }; @@ -146,23 +141,22 @@ struct PPCallbacksDependencyCollector : public DependencyCollector { }; struct IncludeTreePPCallbacks : public PPCallbacks { - DependencyActionController &Controller; + IncludeTreeBuilder &Builder; Preprocessor &PP; public: - IncludeTreePPCallbacks(DependencyActionController &Controller, - Preprocessor &PP) - : Controller(Controller), PP(PP) {} + IncludeTreePPCallbacks(IncludeTreeBuilder &Builder, Preprocessor &PP) + : Builder(Builder), PP(PP) {} void LexedFileChanged(FileID FID, LexedFileChangeReason Reason, SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) override { switch (Reason) { case LexedFileChangeReason::EnterFile: - Controller.enteredInclude(PP, FID); + Builder.enteredInclude(PP, FID); break; case LexedFileChangeReason::ExitFile: { - Controller.exitedInclude(PP, FID, PrevFID, Loc); + Builder.exitedInclude(PP, FID, PrevFID, Loc); break; } } @@ -171,7 +165,7 @@ struct IncludeTreePPCallbacks : public PPCallbacks { void HasInclude(SourceLocation Loc, StringRef FileName, bool IsAngled, Optional File, SrcMgr::CharacteristicKind FileType) override { - Controller.handleHasIncludeCheck(PP, File.has_value()); + Builder.handleHasIncludeCheck(PP, File.has_value()); } }; } // namespace @@ -220,38 +214,22 @@ Error IncludeTreeActionController::initialize( }; ensurePathRemapping(); + BuilderStack.push_back( + std::make_unique(DB, PrefixMapper)); + // Attach callbacks for the IncludeTree of the TU. The preprocessor // does not exist yet, so we need to indirect this via DependencyCollector. auto DC = std::make_shared( - [this](Preprocessor &PP) { - return std::make_unique(*this, PP); + [&Builder = current()](Preprocessor &PP) { + return std::make_unique(Builder, PP); }); ScanInstance.addDependencyCollector(std::move(DC)); CASOpts = ScanInstance.getCASOpts(); - BuilderStack.push_back( - std::make_unique(DB, PrefixMapper)); - return Error::success(); } -void IncludeTreeActionController::enteredInclude(Preprocessor &PP, FileID FID) { - current().enteredInclude(PP, FID); -} - -void IncludeTreeActionController::exitedInclude(Preprocessor &PP, - FileID IncludedBy, - FileID Include, - SourceLocation ExitLoc) { - current().exitedInclude(PP, IncludedBy, Include, ExitLoc); -} - -void IncludeTreeActionController::handleHasIncludeCheck(Preprocessor &PP, - bool Result) { - current().handleHasIncludeCheck(PP, Result); -} - Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { assert(!IncludeTreeResult); From 2e15f4feba3090df785976de6e9ab40b24a18ea0 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 15 Mar 2023 15:53:41 -0700 Subject: [PATCH 11/88] [clang][cas] Scaffolding for modules with include-tree This adds the scaffolding needed for include-tree to push IncludeTreeBuilder stacks, to compute an include-tree for a module and to get fmodule-file-cache-key options for dependencies. Note: this does not yet support building or importing modules, but it lets us compute dependencies in the right way. (cherry picked from commit 3b87a2091b1055f6a7acf55cd4fa237d794ac65b) (cherry picked from commit 322105695fc18d0ce00dd0542be0fb64a3565a05) --- .../include/clang/Basic/DiagnosticCASKinds.td | 4 + clang/include/clang/Basic/Module.h | 13 ++ .../include/clang/Serialization/ASTBitCodes.h | 6 +- clang/include/clang/Serialization/ASTReader.h | 6 + .../include/clang/Serialization/ModuleFile.h | 4 + .../DependencyScanningTool.h | 12 +- .../DependencyScanning/ModuleDepCollector.h | 3 + clang/lib/Frontend/CompilerInstance.cpp | 31 ++- clang/lib/Serialization/ASTReader.cpp | 16 +- clang/lib/Serialization/ASTWriter.cpp | 10 + .../DependencyScanning/CachingActions.h | 3 +- .../DependencyScanningTool.cpp | 18 +- .../DependencyScanningWorker.cpp | 15 +- .../IncludeTreeActionController.cpp | 76 ++++++- .../DependencyScanning/ModuleDepCollector.cpp | 1 + .../DependencyScanning/ScanAndUpdateArgs.cpp | 13 +- .../test/ClangScanDeps/modules-include-tree.c | 202 ++++++++++++++++++ clang/tools/clang-scan-deps/ClangScanDeps.cpp | 12 +- clang/unittests/CAS/IncludeTreeTest.cpp | 3 +- 19 files changed, 407 insertions(+), 41 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree.c diff --git a/clang/include/clang/Basic/DiagnosticCASKinds.td b/clang/include/clang/Basic/DiagnosticCASKinds.td index c173f434be6fe..d08b034b98d6f 100644 --- a/clang/include/clang/Basic/DiagnosticCASKinds.td +++ b/clang/include/clang/Basic/DiagnosticCASKinds.td @@ -36,6 +36,10 @@ def err_cas_missing_root_id : Error< "CAS missing expected root-id '%0'">, DefaultFatal; def err_cas_cannot_parse_root_id_for_module : Error< "CAS cannot parse root-id '%0' for module '%1'">, DefaultFatal; +def err_cas_cannot_parse_include_tree_id : Error< + "CAS cannot parse include-tree-id '%0'">, DefaultFatal; +def err_cas_missing_include_tree_id : Error< + "CAS missing expected include-tree '%0'">, DefaultFatal; def warn_clang_cache_disabled_caching: Warning< "caching disabled because %0">, diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index 3218e33bb88ff..8b9a93c6b0e7d 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -196,6 +196,10 @@ class alignas(8) Module { /// scanner, if any. Optional CASFileSystemRootID; + /// The include-tree root ID for implicit modules built with the dependency + /// scanner, if any. + Optional IncludeTreeID; + /// The top-level headers associated with this module. llvm::SmallSetVector TopHeaders; @@ -645,6 +649,15 @@ class alignas(8) Module { getTopLevelModule()->CASFileSystemRootID = std::move(ID); } + Optional getIncludeTreeID() const { + return getTopLevelModule()->IncludeTreeID; + } + + void setIncludeTreeID(std::string ID) { + assert(!getIncludeTreeID() || *getIncludeTreeID() == ID); + getTopLevelModule()->IncludeTreeID = std::move(ID); + } + /// Retrieve the directory for which this module serves as the /// umbrella. DirectoryName getUmbrellaDir() const; diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index 34792dae0e36e..1cfeaf5bc81c5 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -41,7 +41,7 @@ namespace serialization { /// Version 4 of AST files also requires that the version control branch and /// revision match exactly, since there is no backward compatibility of /// AST files at this time. -const unsigned VERSION_MAJOR = 25; +const unsigned VERSION_MAJOR = 26; /// AST file minor version number supported by this version of /// Clang. @@ -367,6 +367,10 @@ enum ControlRecordTypes { /// Record code for the (optional) CAS filesystem root ID for implicit modules /// built with the dependency scanner. CASFS_ROOT_ID, + + /// Record code for the (optional) include-tree ID for implicit modules + /// built with the dependency scanner. + CAS_INCLUDE_TREE_ID, }; /// Record types that occur within the options block inside diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index cad25a360c553..191372471c5ff 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -246,6 +246,11 @@ class ASTReaderListener { return false; } + /// Called for each CAS include-tree root ID. + /// + /// \returns true to indicate \p RootID is invalid, or false otherwise. + virtual bool readIncludeTreeID(StringRef ID, bool Complain) { return false; } + /// Called for each module cache key. /// /// \returns true to indicate the key cannot be loaded. @@ -300,6 +305,7 @@ class ChainedASTReaderListener : public ASTReaderListener { bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden, bool isExplicitModule) override; bool readCASFileSystemRootID(StringRef RootID, bool Complain) override; + bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; void readModuleFileExtension( diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 2d0be293d93d3..c94b089bc7936 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -143,6 +143,10 @@ class ModuleFile { /// scanner, or empty. std::string CASFileSystemRootID; + /// The include-tree root ID for implicit modules built with the dependency + /// scanner, or empty. + std::string IncludeTreeID; + /// The name of the module. std::string ModuleName; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index c9f96805f5a4e..87d56443a9e33 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -71,6 +71,9 @@ struct TranslationUnitDeps { /// The CASID for input file dependency tree. llvm::Optional CASFileSystemRootID; + /// The include-tree for input file dependency tree. + llvm::Optional IncludeTreeID; + /// The sequence of commands required to build the translation unit. Commands /// should be executed in order. /// @@ -120,6 +123,7 @@ class DependencyScanningTool { Expected getIncludeTree(cas::ObjectStore &DB, const std::vector &CommandLine, StringRef CWD, + LookupModuleOutputCallback LookupModuleOutput, const DepscanPrefixMapping &PrefixMapping); /// If \p DiagGenerationAsCompilation is true it will generate error @@ -128,7 +132,8 @@ class DependencyScanningTool { /// \p DiagOpts.DiagnosticSerializationFile setting is set for the invocation. Expected getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping, + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, + const DepscanPrefixMapping &PrefixMapping, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation); @@ -225,6 +230,10 @@ class FullDependencyConsumer : public DependencyConsumer { CASFileSystemRootID = std::move(ID); } + void handleIncludeTreeID(std::string ID) override { + IncludeTreeID = std::move(ID); + } + TranslationUnitDeps takeTranslationUnitDeps(); ModuleDepsGraph takeModuleGraphDeps(); @@ -236,6 +245,7 @@ class FullDependencyConsumer : public DependencyConsumer { std::vector Commands; std::string ContextHash; Optional CASFileSystemRootID; + Optional IncludeTreeID; std::vector OutputPaths; const llvm::StringSet<> &AlreadySeen; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index b4b8b1206098c..809abf722e5ec 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -121,6 +121,9 @@ struct ModuleDeps { /// The CASID for the module input dependency tree, if any. llvm::Optional CASFileSystemRootID; + /// The CASID for the module include-tree, if any. + llvm::Optional IncludeTreeID; + /// The \c ActionCache key for this module, if any. llvm::Optional ModuleCacheKey; diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 2ee4509d7eb42..f53c34d9bff87 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -640,10 +640,14 @@ class CompileCacheASTReaderHelper : public ASTReaderListener { : CAS(CAS), Cache(Cache), ModuleCache(ModuleCache), Diags(Diags) {} bool readCASFileSystemRootID(StringRef RootID, bool Complain) override; + bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; private: + bool checkCASID(bool Complain, StringRef RootID, unsigned ParseDiagID, + unsigned MissingDiagID); + cas::ObjectStore &CAS; cas::ActionCache &Cache; InMemoryModuleCache &ModuleCache; @@ -2431,21 +2435,36 @@ bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName, Filename, CacheKey, "imported module", CAS, Cache, ModuleCache, Diags); } -bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID, - bool Complain) { - // Verify that RootID is in the CAS. Otherwise the module cache probably was - // created with a different CAS. +/// Verify that ID is in the CAS. Otherwise the module cache probably was +/// created with a different CAS. +bool CompileCacheASTReaderHelper::checkCASID(bool Complain, StringRef RootID, + unsigned ParseDiagID, + unsigned MissingDiagID) { std::optional ID; if (errorToBool(CAS.parseID(RootID).moveInto(ID))) { if (Complain) - Diags.Report(diag::err_cas_cannot_parse_root_id) << RootID; + Diags.Report(ParseDiagID) << RootID; return true; } if (errorToBool(CAS.getProxy(*ID).takeError())) { if (Complain) { - Diags.Report(diag::err_cas_missing_root_id) << RootID; + Diags.Report(MissingDiagID) << RootID; } return true; } return false; } + +bool CompileCacheASTReaderHelper::readCASFileSystemRootID(StringRef RootID, + bool Complain) { + return checkCASID(Complain, RootID, diag::err_cas_cannot_parse_root_id, + diag::err_cas_missing_root_id); +} + +bool CompileCacheASTReaderHelper::readIncludeTreeID(StringRef ID, + bool Complain) { + // Verify that ID is in the CAS. Otherwise the module cache probably was + // created with a different CAS. + return checkCASID(Complain, ID, diag::err_cas_cannot_parse_include_tree_id, + diag::err_cas_missing_include_tree_id); +} diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index eb7333b167a72..61e17c51af70f 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -257,7 +257,10 @@ bool ChainedASTReaderListener::readCASFileSystemRootID(StringRef RootID, return First->readCASFileSystemRootID(RootID, Complain) || Second->readCASFileSystemRootID(RootID, Complain); } - +bool ChainedASTReaderListener::readIncludeTreeID(StringRef ID, bool Complain) { + return First->readIncludeTreeID(ID, Complain) || + Second->readIncludeTreeID(ID, Complain); +} bool ChainedASTReaderListener::readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) { @@ -2984,6 +2987,15 @@ ASTReader::ReadControlBlock(ModuleFile &F, return OutOfDate; } break; + case CAS_INCLUDE_TREE_ID: + F.IncludeTreeID = Blob.str(); + if (Listener) { + bool Complain = + !canRecoverFromOutOfDate(F.FileName, ClientLoadCapabilities); + if (Listener->readIncludeTreeID(F.IncludeTreeID, Complain)) + return OutOfDate; + } + break; } } } @@ -5626,6 +5638,8 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, CurrentModule->setModuleCacheKey(F.ModuleCacheKey); if (!F.CASFileSystemRootID.empty()) CurrentModule->setCASFileSystemRootID(F.CASFileSystemRootID); + if (!F.IncludeTreeID.empty()) + CurrentModule->setIncludeTreeID(F.IncludeTreeID); } CurrentModule->Kind = Kind; diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 419b7c109ecfd..a79ae19a10317 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -797,6 +797,7 @@ void ASTWriter::WriteBlockInfoBlock() { RECORD(INPUT_FILE_OFFSETS); RECORD(MODULE_CACHE_KEY); RECORD(CASFS_ROOT_ID); + RECORD(CAS_INCLUDE_TREE_ID); BLOCK(OPTIONS_BLOCK); RECORD(LANGUAGE_OPTIONS); @@ -1361,6 +1362,15 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context, RecordData::value_type Record[] = {CASFS_ROOT_ID}; Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); } + // CAS include-tree id, for the scanner. + if (auto ID = WritingModule->getIncludeTreeID()) { + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID}; + Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); + } } // Imports diff --git a/clang/lib/Tooling/DependencyScanning/CachingActions.h b/clang/lib/Tooling/DependencyScanning/CachingActions.h index 4cbcdd87ddbdc..8dc3e33b61553 100644 --- a/clang/lib/Tooling/DependencyScanning/CachingActions.h +++ b/clang/lib/Tooling/DependencyScanning/CachingActions.h @@ -19,7 +19,8 @@ class CachingOnDiskFileSystem; namespace clang::tooling::dependencies { std::unique_ptr -createIncludeTreeActionController(cas::ObjectStore &DB, +createIncludeTreeActionController(LookupModuleOutputCallback LookupModuleOutput, + cas::ObjectStore &DB, DepscanPrefixMapping PrefixMapping); std::unique_ptr diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 5845ca5bd3f96..90b92d3790cc2 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -190,10 +190,11 @@ DependencyScanningTool::getDependencyTreeFromCompilerInvocation( Expected DependencyScanningTool::getIncludeTree( cas::ObjectStore &DB, const std::vector &CommandLine, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping) { + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, + const DepscanPrefixMapping &PrefixMapping) { GetIncludeTree Consumer(DB); - auto Controller = - createIncludeTreeActionController(DB, std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB, + std::move(PrefixMapping)); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -204,12 +205,13 @@ Expected DependencyScanningTool::getIncludeTree( Expected DependencyScanningTool::getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, - StringRef CWD, const DepscanPrefixMapping &PrefixMapping, + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, + const DepscanPrefixMapping &PrefixMapping, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation) { GetIncludeTree Consumer(DB); - auto Controller = - createIncludeTreeActionController(DB, std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB, + std::move(PrefixMapping)); Worker.computeDependenciesFromCompilerInvocation( std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer, VerboseOS, DiagGenerationAsCompilation); @@ -255,6 +257,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps); TU.Commands = std::move(Commands); TU.CASFileSystemRootID = std::move(CASFileSystemRootID); + TU.IncludeTreeID = std::move(IncludeTreeID); for (auto &&M : ClangModuleDeps) { auto &MD = M.second; @@ -293,7 +296,8 @@ DependencyScanningTool::createActionController( LookupModuleOutputCallback LookupModuleOutput, DepscanPrefixMapping PrefixMapping) { if (Worker.getScanningFormat() == ScanningOutputFormat::FullIncludeTree) - return createIncludeTreeActionController(*Worker.getCAS(), std::move(PrefixMapping)); + return createIncludeTreeActionController( + LookupModuleOutput, *Worker.getCAS(), std::move(PrefixMapping)); if (auto CacheFS = Worker.getCASFS()) return createCASFSActionController(LookupModuleOutput, *CacheFS, std::move(PrefixMapping)); diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index d08e41e4b2fc8..5d5cd6c8326f5 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -435,15 +435,12 @@ class DependencyScanningAction : public tooling::ToolAction { std::move(Opts), ScanInstance, Consumer, Controller, OriginalInvocation, OptimizeArgs, EagerLoadModules); ScanInstance.addDependencyCollector(MDC); - if (CacheFS) { - ScanInstance.setGenModuleActionWrapper( - [CacheFS = CacheFS, &Controller = Controller]( - const FrontendOptions &Opts, - std::unique_ptr Wrapped) { - return std::make_unique( - std::move(Wrapped), Controller); - }); - } + ScanInstance.setGenModuleActionWrapper( + [&Controller = Controller](const FrontendOptions &Opts, + std::unique_ptr Wrapped) { + return std::make_unique( + std::move(Wrapped), Controller); + }); break; } diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 9800692475901..14a860a7fe587 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -25,8 +25,9 @@ class IncludeTreeBuilder; class IncludeTreeActionController : public CallbackActionController { public: IncludeTreeActionController(cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping) - : CallbackActionController(nullptr), DB(DB), + DepscanPrefixMapping PrefixMapping, + LookupModuleOutputCallback LookupOutput) + : CallbackActionController(LookupOutput), DB(DB), PrefixMapping(std::move(PrefixMapping)) {} Expected getIncludeTree(); @@ -34,14 +35,18 @@ class IncludeTreeActionController : public CallbackActionController { private: Error initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; + Error finalize(CompilerInstance &ScanInstance, + CompilerInvocation &NewInvocation) override; + + Error initializeModuleBuild(CompilerInstance &ModuleScanInstance) override; + Error finalizeModuleBuild(CompilerInstance &ModuleScanInstance) override; + Error finalizeModuleInvocation(CompilerInvocation &CI, + const ModuleDeps &MD) override; const DepscanPrefixMapping *getPrefixMapping() override { return &PrefixMapping; } - Error finalize(CompilerInstance &ScanInstance, - CompilerInvocation &NewInvocation) override; - private: IncludeTreeBuilder ¤t() { assert(!BuilderStack.empty()); @@ -225,6 +230,8 @@ Error IncludeTreeActionController::initialize( }); ScanInstance.addDependencyCollector(std::move(DC)); + // Enable caching in the resulting commands. + ScanInstance.getFrontendOpts().CacheCompileJob = true; CASOpts = ScanInstance.getCASOpts(); return Error::success(); @@ -251,6 +258,51 @@ Error IncludeTreeActionController::finalize(CompilerInstance &ScanInstance, return Error::success(); } +Error IncludeTreeActionController::initializeModuleBuild( + CompilerInstance &ModuleScanInstance) { + BuilderStack.push_back( + std::make_unique(DB, PrefixMapper)); + + // Attach callbacks for the IncludeTree of the module. The preprocessor + // does not exist yet, so we need to indirect this via DependencyCollector. + auto DC = std::make_shared( + [&Builder = current()](Preprocessor &PP) { + return std::make_unique(Builder, PP); + }); + ModuleScanInstance.addDependencyCollector(std::move(DC)); + + return Error::success(); +} + +Error IncludeTreeActionController::finalizeModuleBuild( + CompilerInstance &ModuleScanInstance) { + // FIXME: the scan invocation is incorrect here; we need the `NewInvocation` + // from `finalizeModuleInvocation` to finish the tree. + auto Builder = BuilderStack.pop_back_val(); + auto Tree = Builder->finishIncludeTree(ModuleScanInstance, + ModuleScanInstance.getInvocation()); + if (!Tree) + return Tree.takeError(); + + Module *M = ModuleScanInstance.getPreprocessor().getCurrentModule(); + assert(M && "finalizing without a module"); + M->setIncludeTreeID(Tree->getID().toString()); + + return Error::success(); +} + +Error IncludeTreeActionController::finalizeModuleInvocation( + CompilerInvocation &CI, const ModuleDeps &MD) { + if (auto ID = MD.IncludeTreeID) { + configureInvocationForCaching(CI, CASOpts, std::move(*ID), + /*CASFSWorkingDir=*/"", + /*ProduceIncludeTree=*/true); + } + + DepscanPrefixMapping::remapInvocationPaths(CI, PrefixMapper); + return Error::success(); +} + void IncludeTreeBuilder::enteredInclude(Preprocessor &PP, FileID FID) { if (hasErrorOccurred()) return; @@ -415,6 +467,15 @@ Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, } return *PredefinesBufferRef; } + if (!FI.getContentCache().OrigEntry && + FI.getName() == Module::getModuleInputBufferName()) { + // Virtual buffer + if (!ModuleIncludesBufferRef) { + if (Error E = getObjectForBuffer(FI).moveInto(ModuleIncludesBufferRef)) + return std::move(E); + } + return *ModuleIncludesBufferRef; + } assert(FI.getContentCache().OrigEntry); auto &FileRef = ObjectForFile[FI.getContentCache().OrigEntry]; if (!FileRef) { @@ -505,7 +566,8 @@ IncludeTreeBuilder::createIncludeFile(StringRef Filename, std::unique_ptr dependencies::createIncludeTreeActionController( - cas::ObjectStore &DB, DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput, cas::ObjectStore &DB, + DepscanPrefixMapping PrefixMapping) { return std::make_unique( - DB, std::move(PrefixMapping)); + DB, std::move(PrefixMapping), LookupModuleOutput); } diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 984582c08350d..b3e692cec9962 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -449,6 +449,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { << *ID << MD.ID.ModuleName; } } + MD.IncludeTreeID = M->getIncludeTreeID(); ModuleMap &ModMapInfo = MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 43272f80290e1..e1234095a95e0 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -36,10 +36,12 @@ void tooling::dependencies::configureInvocationForCaching( FrontendOpts.CASIncludeTreeID = std::move(RootID); FrontendOpts.Inputs.clear(); // Preserve sysroot path to accommodate lookup for 'SDKSettings.json' during - // availability checking. - std::string OriginalSysroot = CI.getHeaderSearchOpts().Sysroot; - CI.getHeaderSearchOpts() = HeaderSearchOptions(); - CI.getHeaderSearchOpts().Sysroot = OriginalSysroot; + // availability checking, and module files. + HeaderSearchOptions &HSOpts = CI.getHeaderSearchOpts(); + HeaderSearchOptions OriginalHSOpts; + std::swap(HSOpts, OriginalHSOpts); + HSOpts.Sysroot = std::move(OriginalHSOpts.Sysroot); + HSOpts.PrebuiltModuleFiles = std::move(OriginalHSOpts.PrebuiltModuleFiles); auto &PPOpts = CI.getPreprocessorOpts(); // We don't need this because we save the contents of the PCH file in the // include tree root. @@ -254,7 +256,8 @@ Expected clang::scanAndUpdateCC1InlineWithTool( if (ProduceIncludeTree) { if (Error E = Tool.getIncludeTreeFromCompilerInvocation( DB, std::move(ScanInvocation), WorkingDirectory, - PrefixMapping, DiagsConsumer, VerboseOS, + /*LookupModuleOutput=*/nullptr, PrefixMapping, + DiagsConsumer, VerboseOS, /*DiagGenerationAsCompilation*/ true) .moveInto(Root)) return std::move(E); diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c new file mode 100644 index 0000000000000..26792c21724df --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -0,0 +1,202 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t + +// CHECK: { +// CHECK-NEXT "modules": [ +// CHECK-NEXT { +// CHECK: "cas-include-tree-id": "[[LEFT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[LEFT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Left" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Left.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK: ] +// CHECK: "name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Right.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TOP_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TOP_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Top" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Top.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Top" +// CHECK: } +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TU_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TU_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Left=[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { + int x; +}; +void top(void); + +//--- Left.h +#pragma once +#include "Top.h" +void left(void); + +//--- Right.h +#pragma once +#include "Top.h" +void right(void); + +//--- tu.c +#include "Left.h" +#include "Right.h" + +void tu(void) { + left(); + right(); + top(); +} diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index cfe6d22aef6f1..ecb3d04f80adf 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -544,6 +544,7 @@ class FullDeps { ID.FileDeps = std::move(TUDeps.FileDeps); ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps); ID.CASFileSystemRootID = std::move(TUDeps.CASFileSystemRootID); + ID.IncludeTreeID = std::move(TUDeps.IncludeTreeID); ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine); ID.Commands = std::move(TUDeps.Commands); @@ -624,6 +625,8 @@ class FullDeps { }; if (MD.CASFileSystemRootID) O.try_emplace("casfs-root-id", MD.CASFileSystemRootID->toString()); + if (MD.IncludeTreeID) + O.try_emplace("cas-include-tree-id", MD.IncludeTreeID); OutModules.push_back(std::move(O)); } @@ -642,6 +645,8 @@ class FullDeps { }; if (I.CASFileSystemRootID) O.try_emplace("casfs-root-id", I.CASFileSystemRootID); + if (I.IncludeTreeID) + O.try_emplace("cas-include-tree-id", I.IncludeTreeID); Commands.push_back(std::move(O)); } } else { @@ -655,6 +660,8 @@ class FullDeps { }; if (I.CASFileSystemRootID) O.try_emplace("casfs-root-id", I.CASFileSystemRootID); + if (I.IncludeTreeID) + O.try_emplace("cas-include-tree-id", I.IncludeTreeID); Commands.push_back(std::move(O)); } TUs.push_back(Object{ @@ -694,7 +701,8 @@ class FullDeps { std::string ContextHash; std::vector FileDeps; std::vector ModuleDeps; - llvm::Optional CASFileSystemRootID; + Optional CASFileSystemRootID; + Optional IncludeTreeID; std::vector DriverCommandLine; std::vector Commands; }; @@ -995,7 +1003,7 @@ int main(int argc, const char **argv) { std::move(MaybeTree)); } else if (Format == ScanningOutputFormat::IncludeTree) { auto MaybeTree = WorkerTools[I]->getIncludeTree( - *CAS, Input->CommandLine, CWD, PrefixMapping); + *CAS, Input->CommandLine, CWD, LookupOutput, PrefixMapping); std::unique_lock LockGuard(Lock); TreeResults.emplace_back(LocalIndex, std::move(Filename), std::move(MaybeTree)); diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index 6ddba05cde1ca..1c82d5abb40fa 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -55,7 +55,8 @@ TEST(IncludeTree, IncludeTreeScan) { Optional Root; DepscanPrefixMapping PrefixMapping; ASSERT_THAT_ERROR( - ScanTool.getIncludeTree(*DB, CommandLine, /*CWD*/ "", PrefixMapping) + ScanTool + .getIncludeTree(*DB, CommandLine, /*CWD*/ "", nullptr, PrefixMapping) .moveInto(Root), llvm::Succeeded()); From fb59fd3f4ab9faf00edc2143b5e9df3c78d0cbaf Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 22 Mar 2023 10:23:20 -0700 Subject: [PATCH 12/88] [clang][cas] Move IncludeFile after IncludeTree in the header NFC Moves IncludeFile after IncludeTree in the header file in preparation for changing it to IncludeTree::File. Also moves FileInfo to IncludeTree to simplify declaration. (cherry picked from commit f36469e06536b6d08aaf384bfacfcadf01df77a0) (cherry picked from commit fa13d21f32dd79b28041c8567ce06ca5e40a9790) --- clang/include/clang/CAS/IncludeTree.h | 140 +++++++++++------------- clang/lib/CAS/IncludeTree.cpp | 14 +++ clang/unittests/CAS/IncludeTreeTest.cpp | 10 +- 3 files changed, 85 insertions(+), 79 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 5519010160937..47f2d13dacd60 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -45,68 +45,7 @@ template class IncludeTreeBase : public ObjectProxy { friend class IncludeTreeRoot; }; -/// Represents a \p SourceManager file (or buffer in the case of preprocessor -/// predefines) that got included by the preprocessor. -class IncludeFile : public IncludeTreeBase { -public: - static constexpr StringRef getNodeKind() { return "File"; } - - ObjectRef getFilenameRef() const { return getReference(0); } - ObjectRef getContentsRef() const { return getReference(1); } - - Expected getFilename() { - return getCAS().getProxy(getFilenameRef()); - } - - Expected getContents() { - return getCAS().getProxy(getContentsRef()); - } - - struct FileInfo { - StringRef Filename; - StringRef Contents; - }; - - Expected getFileInfo() { - auto Filename = getFilename(); - if (!Filename) - return Filename.takeError(); - auto Contents = getContents(); - if (!Contents) - return Contents.takeError(); - return FileInfo{Filename->getData(), Contents->getData()}; - } - - static Expected create(ObjectStore &DB, StringRef Filename, - ObjectRef Contents); - - llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); - - static bool isValid(const ObjectProxy &Node) { - if (!IncludeTreeBase::isValid(Node)) - return false; - IncludeTreeBase Base(Node); - return Base.getNumReferences() == 2 && Base.getData().empty(); - } - static bool isValid(ObjectStore &DB, ObjectRef Ref) { - auto Node = DB.getProxy(Ref); - if (!Node) { - llvm::consumeError(Node.takeError()); - return false; - } - return isValid(*Node); - } - -private: - friend class IncludeTreeBase; - friend class IncludeFileList; - friend class IncludeTree; - friend class IncludeTreeRoot; - - explicit IncludeFile(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { - assert(isValid(*this)); - } -}; +class IncludeFile; /// Represents a DAG of included files by the preprocessor. /// Each node in the DAG represents a particular inclusion of a file that @@ -117,22 +56,17 @@ class IncludeTree : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "Tree"; } - Expected getBaseFile() { - auto Node = getCAS().getProxy(getBaseFileRef()); - if (!Node) - return Node.takeError(); - return IncludeFile(std::move(*Node)); - } + Expected getBaseFile(); /// The include file that resulted in this include-tree. ObjectRef getBaseFileRef() const { return getReference(0); } - Expected getBaseFileInfo() { - auto File = getBaseFile(); - if (!File) - return File.takeError(); - return File->getFileInfo(); - } + struct FileInfo { + StringRef Filename; + StringRef Contents; + }; + + Expected getBaseFileInfo(); SrcMgr::CharacteristicKind getFileCharacteristic() const { return (SrcMgr::CharacteristicKind)dataSkippingIncludes().front(); @@ -216,6 +150,64 @@ class IncludeTree : public IncludeTreeBase { } }; +/// Represents a \p SourceManager file (or buffer in the case of preprocessor +/// predefines) that got included by the preprocessor. +class IncludeFile : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "File"; } + + ObjectRef getFilenameRef() const { return getReference(0); } + ObjectRef getContentsRef() const { return getReference(1); } + + Expected getFilename() { + return getCAS().getProxy(getFilenameRef()); + } + + Expected getContents() { + return getCAS().getProxy(getContentsRef()); + } + + Expected getFileInfo() { + auto Filename = getFilename(); + if (!Filename) + return Filename.takeError(); + auto Contents = getContents(); + if (!Contents) + return Contents.takeError(); + return IncludeTree::FileInfo{Filename->getData(), Contents->getData()}; + } + + static Expected create(ObjectStore &DB, StringRef Filename, + ObjectRef Contents); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getNumReferences() == 2 && Base.getData().empty(); + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + friend class IncludeTreeBase; + friend class IncludeFileList; + friend class IncludeTree; + friend class IncludeTreeRoot; + + explicit IncludeFile(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } +}; + /// A flat list of \p IncludeFile entries. This is used along with a simple /// implementation of a \p vfs::FileSystem produced via /// \p createIncludeTreeFileSystem(). diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 435522869d7c8..b507661bc6a88 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -42,6 +42,20 @@ Expected IncludeFile::create(ObjectStore &DB, StringRef Filename, return IncludeTreeBase::create(DB, Refs, {}); } +Expected IncludeTree::getBaseFile() { + auto Node = getCAS().getProxy(getBaseFileRef()); + if (!Node) + return Node.takeError(); + return IncludeFile(std::move(*Node)); +} + +Expected IncludeTree::getBaseFileInfo() { + auto File = getBaseFile(); + if (!File) + return File.takeError(); + return File->getFileInfo(); +} + llvm::Error IncludeTree::forEachInclude( llvm::function_ref)> Callback) { diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index 1c82d5abb40fa..ba422a1e99006 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -71,7 +71,7 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_THAT_ERROR(Main->getBaseFile().moveInto(MainFile), llvm::Succeeded()); EXPECT_EQ(Main->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(MainFile->getFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "t.cpp"); EXPECT_EQ(FI.Contents, MainContents); @@ -83,7 +83,7 @@ TEST(IncludeTree, IncludeTreeScan) { EXPECT_EQ(Main->getIncludeOffset(0), uint32_t(0)); { EXPECT_EQ(Predef->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(Predef->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, ""); @@ -95,7 +95,7 @@ TEST(IncludeTree, IncludeTreeScan) { { ASSERT_THAT_ERROR(A1->getBaseFile().moveInto(A1File), llvm::Succeeded()); EXPECT_EQ(A1->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(A1File->getFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "./a1.h"); EXPECT_EQ(FI.Contents, A1Contents); @@ -109,7 +109,7 @@ TEST(IncludeTree, IncludeTreeScan) { { ASSERT_THAT_ERROR(B1->getBaseFile().moveInto(B1File), llvm::Succeeded()); EXPECT_EQ(B1->getFileCharacteristic(), SrcMgr::C_User); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(B1->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "./b1.h"); EXPECT_EQ(FI.Contents, ""); @@ -124,7 +124,7 @@ TEST(IncludeTree, IncludeTreeScan) { { ASSERT_THAT_ERROR(Sys->getBaseFile().moveInto(SysFile), llvm::Succeeded()); EXPECT_EQ(Sys->getFileCharacteristic(), SrcMgr::C_System); - IncludeFile::FileInfo FI; + IncludeTree::FileInfo FI; ASSERT_THAT_ERROR(Sys->getBaseFileInfo().moveInto(FI), llvm::Succeeded()); EXPECT_EQ(FI.Filename, "sys/sys.h"); EXPECT_EQ(FI.Contents, ""); From 5fe54315d88c3b223f156254ccca0c892c3e711b Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 22 Mar 2023 10:42:40 -0700 Subject: [PATCH 13/88] [clang][cas] Move IncludeFile[List] to IncludeTree::File[List] NFC (cherry picked from commit 7caa9d961f963f8875af76424108cd309931332d) (cherry picked from commit 2c3a9496492cf40629283293eab2f301e7c58212) --- clang/include/clang/CAS/IncludeTree.h | 56 +++++++++---------- clang/lib/CAS/IncludeTree.cpp | 47 +++++++++------- .../IncludeTreeActionController.cpp | 17 +++--- clang/unittests/CAS/IncludeTreeTest.cpp | 22 ++++---- 4 files changed, 72 insertions(+), 70 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 47f2d13dacd60..a2b9051b59220 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -39,14 +39,9 @@ template class IncludeTreeBase : public ObjectProxy { return Node.getData().startswith(NodeT::getNodeKind()); } - friend class IncludeFile; - friend class IncludeFileList; - friend class IncludeTree; - friend class IncludeTreeRoot; + friend NodeT; }; -class IncludeFile; - /// Represents a DAG of included files by the preprocessor. /// Each node in the DAG represents a particular inclusion of a file that /// encompasses inclusions of other files as sub-trees, along with all the @@ -56,7 +51,10 @@ class IncludeTree : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "Tree"; } - Expected getBaseFile(); + class File; + class FileList; + + Expected getBaseFile(); /// The include file that resulted in this include-tree. ObjectRef getBaseFileRef() const { return getReference(0); } @@ -152,7 +150,7 @@ class IncludeTree : public IncludeTreeBase { /// Represents a \p SourceManager file (or buffer in the case of preprocessor /// predefines) that got included by the preprocessor. -class IncludeFile : public IncludeTreeBase { +class IncludeTree::File : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "File"; } @@ -167,7 +165,7 @@ class IncludeFile : public IncludeTreeBase { return getCAS().getProxy(getContentsRef()); } - Expected getFileInfo() { + Expected getFileInfo() { auto Filename = getFilename(); if (!Filename) return Filename.takeError(); @@ -177,8 +175,8 @@ class IncludeFile : public IncludeTreeBase { return IncludeTree::FileInfo{Filename->getData(), Contents->getData()}; } - static Expected create(ObjectStore &DB, StringRef Filename, - ObjectRef Contents); + static Expected create(ObjectStore &DB, StringRef Filename, + ObjectRef Contents); llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); @@ -198,20 +196,20 @@ class IncludeFile : public IncludeTreeBase { } private: - friend class IncludeTreeBase; - friend class IncludeFileList; + friend class IncludeTreeBase; + friend class FileList; friend class IncludeTree; friend class IncludeTreeRoot; - explicit IncludeFile(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + explicit File(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); } }; -/// A flat list of \p IncludeFile entries. This is used along with a simple +/// A flat list of \c File entries. This is used along with a simple /// implementation of a \p vfs::FileSystem produced via /// \p createIncludeTreeFileSystem(). -class IncludeFileList : public IncludeTreeBase { +class IncludeTree::FileList : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "List"; } @@ -224,12 +222,12 @@ class IncludeFileList : public IncludeTreeBase { return getReference(I); } - Expected getFile(size_t I) { return getFile(getFileRef(I)); } + Expected getFile(size_t I) { return getFile(getFileRef(I)); } FileSizeTy getFileSize(size_t I) const; - /// \returns each \p IncludeFile entry along with its file size. - llvm::Error forEachFile( - llvm::function_ref Callback); + /// \returns each \c File entry along with its file size. + llvm::Error + forEachFile(llvm::function_ref Callback); /// We record the file size as well to avoid needing to materialize the /// underlying buffer for the \p IncludeTreeFileSystem::status() @@ -238,27 +236,25 @@ class IncludeFileList : public IncludeTreeBase { ObjectRef FileRef; FileSizeTy Size; }; - static Expected create(ObjectStore &DB, - ArrayRef Files); + static Expected create(ObjectStore &DB, ArrayRef Files); - static Expected get(ObjectStore &CAS, ObjectRef Ref); + static Expected get(ObjectStore &CAS, ObjectRef Ref); llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); private: - friend class IncludeTreeBase; + friend class IncludeTreeBase; friend class IncludeTreeRoot; - explicit IncludeFileList(ObjectProxy Node) - : IncludeTreeBase(std::move(Node)) { + explicit FileList(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); } - Expected getFile(ObjectRef Ref) { + Expected getFile(ObjectRef Ref) { auto Node = getCAS().getProxy(Ref); if (!Node) return Node.takeError(); - return IncludeFile(std::move(*Node)); + return File(std::move(*Node)); } static bool isValid(const ObjectProxy &Node); @@ -294,11 +290,11 @@ class IncludeTreeRoot : public IncludeTreeBase { return IncludeTree(std::move(*Node)); } - Expected getFileList() { + Expected getFileList() { auto Node = getCAS().getProxy(getFileListRef()); if (!Node) return Node.takeError(); - return IncludeFileList(std::move(*Node)); + return IncludeTree::FileList(std::move(*Node)); } Expected> getPCHBuffer() { diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index b507661bc6a88..4ab886eee568c 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -33,8 +33,9 @@ Expected IncludeTreeBase::create(ObjectStore &DB, return NodeT(*Proxy); } -Expected IncludeFile::create(ObjectStore &DB, StringRef Filename, - ObjectRef Contents) { +Expected IncludeTree::File::create(ObjectStore &DB, + StringRef Filename, + ObjectRef Contents) { auto PathRef = DB.storeFromString({}, Filename); if (!PathRef) return PathRef.takeError(); @@ -42,11 +43,11 @@ Expected IncludeFile::create(ObjectStore &DB, StringRef Filename, return IncludeTreeBase::create(DB, Refs, {}); } -Expected IncludeTree::getBaseFile() { +Expected IncludeTree::getBaseFile() { auto Node = getCAS().getProxy(getBaseFileRef()); if (!Node) return Node.takeError(); - return IncludeFile(std::move(*Node)); + return File(std::move(*Node)); } Expected IncludeTree::getBaseFileInfo() { @@ -85,7 +86,7 @@ Expected IncludeTree::create( char Kind = FileCharacteristic; assert(Kind == FileCharacteristic && "SrcMgr::CharacteristicKind too big!"); - assert(IncludeFile::isValid(DB, BaseFile)); + assert(File::isValid(DB, BaseFile)); SmallVector Refs; Refs.reserve(Includes.size() + 1); Refs.push_back(BaseFile); @@ -169,7 +170,8 @@ bool IncludeTree::isValid(const ObjectProxy &Node) { return Base.getData().size() >= NumIncludes * sizeof(uint32_t) + 1; } -IncludeFileList::FileSizeTy IncludeFileList::getFileSize(size_t I) const { +IncludeTree::FileList::FileSizeTy +IncludeTree::FileList::getFileSize(size_t I) const { assert(I < getNumFiles()); StringRef Data = getData(); assert(Data.size() >= (I + 1) * sizeof(FileSizeTy)); @@ -177,8 +179,8 @@ IncludeFileList::FileSizeTy IncludeFileList::getFileSize(size_t I) const { Data.data() + I * sizeof(FileSizeTy)); } -llvm::Error IncludeFileList::forEachFile( - llvm::function_ref Callback) { +llvm::Error IncludeTree::FileList::forEachFile( + llvm::function_ref Callback) { size_t I = 0; return forEachReference([&](ObjectRef Ref) -> llvm::Error { auto Include = getFile(Ref); @@ -188,8 +190,8 @@ llvm::Error IncludeFileList::forEachFile( }); } -Expected IncludeFileList::create(ObjectStore &DB, - ArrayRef Files) { +Expected +IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files) { SmallVector Refs; Refs.reserve(Files.size()); SmallString<256> Buffer; @@ -199,24 +201,25 @@ Expected IncludeFileList::create(ObjectStore &DB, llvm::support::endian::Writer Writer(BufOS, llvm::support::little); for (const FileEntry &Entry : Files) { - assert(IncludeFile::isValid(DB, Entry.FileRef)); + assert(File::isValid(DB, Entry.FileRef)); Refs.push_back(Entry.FileRef); Writer.write(Entry.Size); } return IncludeTreeBase::create(DB, Refs, Buffer); } -Expected IncludeFileList::get(ObjectStore &DB, ObjectRef Ref) { +Expected IncludeTree::FileList::get(ObjectStore &DB, + ObjectRef Ref) { auto Node = DB.getProxy(Ref); if (!Node) return Node.takeError(); if (!isValid(*Node)) return llvm::createStringError(llvm::inconvertibleErrorCode(), "not a IncludeFileList node kind"); - return IncludeFileList(std::move(*Node)); + return FileList(std::move(*Node)); } -bool IncludeFileList::isValid(const ObjectProxy &Node) { +bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); @@ -230,7 +233,7 @@ Expected IncludeTreeRoot::create(ObjectStore &DB, ObjectRef FileList, Optional PCHRef) { assert(IncludeTree::isValid(DB, MainFileTree)); - assert(IncludeFileList::isValid(DB, FileList)); + assert(IncludeTree::FileList::isValid(DB, FileList)); if (PCHRef) { return IncludeTreeBase::create(DB, {MainFileTree, FileList, *PCHRef}, {}); } else { @@ -248,7 +251,7 @@ Expected IncludeTreeRoot::get(ObjectStore &DB, ObjectRef Ref) { return IncludeTreeRoot(std::move(*Node)); } -llvm::Error IncludeFile::print(llvm::raw_ostream &OS, unsigned Indent) { +llvm::Error IncludeTree::File::print(llvm::raw_ostream &OS, unsigned Indent) { auto Filename = getFilename(); if (!Filename) return Filename.takeError(); @@ -282,8 +285,9 @@ llvm::Error IncludeTree::print(llvm::raw_ostream &OS, unsigned Indent) { }); } -llvm::Error IncludeFileList::print(llvm::raw_ostream &OS, unsigned Indent) { - return forEachFile([&](cas::IncludeFile File, FileSizeTy) -> llvm::Error { +llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachFile([&](File File, FileSizeTy) -> llvm::Error { return File.print(OS, Indent); }); } @@ -300,7 +304,7 @@ llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { if (llvm::Error E = MainTree->print(OS.indent(Indent), Indent)) return E; OS.indent(Indent) << "Files:\n"; - Optional List; + Optional List; if (llvm::Error E = getFileList().moveInto(List)) return E; return List->print(OS, Indent); @@ -347,7 +351,7 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { struct FileEntry { cas::ObjectRef ContentsRef; - IncludeFileList::FileSizeTy Size; + IncludeTree::FileList::FileSizeTy Size; llvm::sys::fs::UniqueID UniqueID; }; @@ -450,7 +454,8 @@ cas::createIncludeTreeFileSystem(IncludeTreeRoot &Root) { IntrusiveRefCntPtr IncludeTreeFS = new IncludeTreeFileSystem(Root.getCAS()); llvm::Error E = FileList->forEachFile( - [&](IncludeFile File, IncludeFileList::FileSizeTy Size) -> llvm::Error { + [&](IncludeTree::File File, + IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { auto FilenameBlob = File.getFilename(); if (!FilenameBlob) return FilenameBlob.takeError(); diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 14a860a7fe587..32ac0595f06e8 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -97,8 +97,8 @@ class IncludeTreeBuilder { Expected getObjectForBuffer(const SrcMgr::FileInfo &FI); Expected addToFileList(FileManager &FM, const FileEntry *FE); Expected getCASTreeForFileIncludes(FilePPState &&PPState); - Expected createIncludeFile(StringRef Filename, - cas::ObjectRef Contents); + Expected createIncludeFile(StringRef Filename, + cas::ObjectRef Contents); bool hasErrorOccurred() const { return ErrorToReport.has_value(); } @@ -120,7 +120,7 @@ class IncludeTreeBuilder { // are recorded in the PCH, ordered by \p FileEntry::UID index. SmallVector PreIncludedFileNames; llvm::BitVector SeenIncludeFiles; - SmallVector IncludedFiles; + SmallVector IncludedFiles; Optional PredefinesBufferRef; Optional ModuleIncludesBufferRef; Optional ModuleMapFileRef; @@ -446,7 +446,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, getCASTreeForFileIncludes(IncludeStack.pop_back_val()); if (!MainIncludeTree) return MainIncludeTree.takeError(); - auto FileList = cas::IncludeFileList::create(DB, IncludedFiles); + auto FileList = cas::IncludeTree::FileList::create(DB, IncludedFiles); if (!FileList) return FileList.takeError(); @@ -508,7 +508,8 @@ IncludeTreeBuilder::getObjectForBuffer(const SrcMgr::FileInfo &FI) { {}, FI.getContentCache().getBufferIfLoaded()->getBuffer()); if (!Ref) return Ref.takeError(); - Expected FileNode = createIncludeFile(FI.getName(), *Ref); + Expected FileNode = + createIncludeFile(FI.getName(), *Ref); if (!FileNode) return FileNode.takeError(); return FileNode->getRef(); @@ -530,7 +531,7 @@ IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { return FileNode.takeError(); IncludedFiles.push_back( {FileNode->getRef(), - static_cast(FE->getSize())}); + static_cast(FE->getSize())}); return FileNode->getRef(); }; @@ -553,7 +554,7 @@ IncludeTreeBuilder::getCASTreeForFileIncludes(FilePPState &&PPState) { PPState.Includes, PPState.HasIncludeChecks); } -Expected +Expected IncludeTreeBuilder::createIncludeFile(StringRef Filename, cas::ObjectRef Contents) { SmallString<256> MappedPath; @@ -561,7 +562,7 @@ IncludeTreeBuilder::createIncludeFile(StringRef Filename, PrefixMapper.map(Filename, MappedPath); Filename = MappedPath; } - return cas::IncludeFile::create(DB, Filename, std::move(Contents)); + return cas::IncludeTree::File::create(DB, Filename, std::move(Contents)); } std::unique_ptr diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index ba422a1e99006..f687ef57762d8 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -60,10 +60,10 @@ TEST(IncludeTree, IncludeTreeScan) { .moveInto(Root), llvm::Succeeded()); - Optional MainFile; - Optional A1File; - Optional B1File; - Optional SysFile; + Optional MainFile; + Optional A1File; + Optional B1File; + Optional SysFile; Optional Main; ASSERT_THAT_ERROR(Root->getMainFileTree().moveInto(Main), llvm::Succeeded()); @@ -132,31 +132,31 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(Sys->getNumIncludes(), uint32_t(0)); } - Optional FileList; + Optional FileList; ASSERT_THAT_ERROR(Root->getFileList().moveInto(FileList), llvm::Succeeded()); ASSERT_EQ(FileList->getNumFiles(), size_t(4)); { - Optional File; + Optional File; ASSERT_THAT_ERROR(FileList->getFile(0).moveInto(File), llvm::Succeeded()); EXPECT_EQ(File->getRef(), MainFile->getRef()); EXPECT_EQ(FileList->getFileSize(0), MainContents.size()); } { - Optional File; + Optional File; ASSERT_THAT_ERROR(FileList->getFile(1).moveInto(File), llvm::Succeeded()); EXPECT_EQ(File->getRef(), A1File->getRef()); EXPECT_EQ(FileList->getFileSize(1), A1Contents.size()); } { - Optional File; + Optional File; ASSERT_THAT_ERROR(FileList->getFile(2).moveInto(File), llvm::Succeeded()); EXPECT_EQ(File->getRef(), B1File->getRef()); - EXPECT_EQ(FileList->getFileSize(2), IncludeFileList::FileSizeTy(0)); + EXPECT_EQ(FileList->getFileSize(2), IncludeTree::FileList::FileSizeTy(0)); } { - Optional File; + Optional File; ASSERT_THAT_ERROR(FileList->getFile(3).moveInto(File), llvm::Succeeded()); EXPECT_EQ(File->getRef(), SysFile->getRef()); - EXPECT_EQ(FileList->getFileSize(3), IncludeFileList::FileSizeTy(0)); + EXPECT_EQ(FileList->getFileSize(3), IncludeTree::FileList::FileSizeTy(0)); } } From a21d74f10664e5bcad67d62428436ab33056dbef Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 15 Mar 2023 16:30:23 -0700 Subject: [PATCH 14/88] [clang][cas] Basic building and importing modules with include-tree This handles building and importing simple modules and PCH with modules using include-tree, when modules are found via header path. * Extend IncludeTree to represent module imports and modulemap mainfile * Make IncludeTreeBuilder add the new nodes for #include * Make PPDirectives import modules from PPCachedActions in #include * Teach FrontendAction/CompilerInvocation how to setup the main input file modulemap and module includes buffer from IncludeTreeRoot. There are still lots of things missing, including @import syntax and submodule semantics. (cherry picked from commit 55b697db03fd0a78ba4e41371772460c939800bc) (cherry picked from commit a9db0557b4226710616892af5ec33be1dfc97e90) --- clang/include/clang/CAS/IncludeTree.h | 147 ++++++++++++++--- clang/include/clang/Lex/PPCachedActions.h | 18 +- clang/lib/CAS/IncludeTree.cpp | 139 ++++++++++++---- clang/lib/Frontend/CompilerInvocation.cpp | 23 ++- clang/lib/Frontend/FrontendAction.cpp | 52 ++++-- clang/lib/Frontend/IncludeTreePPActions.cpp | 44 +++-- clang/lib/Lex/PPDirectives.cpp | 58 ++++--- .../IncludeTreeActionController.cpp | 91 ++++++++-- .../DependencyScanning/ScanAndUpdateArgs.cpp | 8 +- .../modules-include-tree-with-pch.c | 156 ++++++++++++++++++ .../test/ClangScanDeps/modules-include-tree.c | 95 ++++++++++- clang/unittests/CAS/IncludeTreeTest.cpp | 9 +- 12 files changed, 701 insertions(+), 139 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-with-pch.c diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index a2b9051b59220..1917f6f1c54bb 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -42,9 +42,9 @@ template class IncludeTreeBase : public ObjectProxy { friend NodeT; }; -/// Represents a DAG of included files by the preprocessor. -/// Each node in the DAG represents a particular inclusion of a file that -/// encompasses inclusions of other files as sub-trees, along with all the +/// Represents a DAG of included files by the preprocessor and module imports. +/// Each node in the DAG represents a particular inclusion of a file or module +/// that encompasses inclusions of other files as sub-trees, along with all the /// \p __has_include() preprocessor checks that occurred during preprocessing /// of that file. class IncludeTree : public IncludeTreeBase { @@ -53,6 +53,8 @@ class IncludeTree : public IncludeTreeBase { class File; class FileList; + class Node; + class ModuleImport; Expected getBaseFile(); @@ -77,9 +79,21 @@ class IncludeTree : public IncludeTreeBase { return getReference(I + 1); } + enum class NodeKind : uint8_t { + Tree, + ModuleImport, + }; + + /// The kind of node included at the given index. + NodeKind getIncludeKind(size_t I) const; + + /// The sub-include-tree or module import for the given index. + Expected getIncludeNode(size_t I); + /// The sub-include-trees of included files, in the order that they occurred. - Expected getInclude(size_t I) { - return getInclude(getIncludeRef(I)); + Expected getIncludeTree(size_t I) { + assert(getIncludeKind(I) == NodeKind::Tree); + return getIncludeTree(getIncludeRef(I)); } /// The source byte offset for a particular include, pointing to the beginning @@ -104,14 +118,19 @@ class IncludeTree : public IncludeTreeBase { /// occur in the same order. bool getCheckResult(size_t I) const; - /// Passes pairs of (IncludeTree, include offset) to \p Callback. + /// Passes pairs of (Node, include offset) to \p Callback. llvm::Error forEachInclude( - llvm::function_ref)> - Callback); + llvm::function_ref)> Callback); + + struct IncludeInfo { + ObjectRef Ref; ///< IncludeTree or IncludeTreeModuleImport + uint32_t Offset; + NodeKind Kind; + }; static Expected create(ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, - ObjectRef BaseFile, ArrayRef> Includes, + ObjectRef BaseFile, ArrayRef Includes, llvm::SmallBitVector Checks); static Expected get(ObjectStore &DB, ObjectRef Ref); @@ -126,15 +145,17 @@ class IncludeTree : public IncludeTreeBase { assert(isValid(*this)); } - Expected getInclude(ObjectRef Ref) { + Expected getIncludeTree(ObjectRef Ref) { auto Node = getCAS().getProxy(Ref); if (!Node) return Node.takeError(); return IncludeTree(std::move(*Node)); } + Expected getIncludeNode(ObjectRef Ref, NodeKind Kind); + StringRef dataSkippingIncludes() const { - return getData().drop_front(getNumIncludes() * sizeof(uint32_t)); + return getData().drop_front(getNumIncludes() * (sizeof(uint32_t) + 1)); } static bool isValid(const ObjectProxy &Node); @@ -175,6 +196,16 @@ class IncludeTree::File : public IncludeTreeBase { return IncludeTree::FileInfo{Filename->getData(), Contents->getData()}; } + Expected> getMemoryBuffer() { + auto Filename = getFilename(); + if (!Filename) + return Filename.takeError(); + auto Contents = getContents(); + if (!Contents) + return Contents.takeError(); + return Contents->getMemoryBuffer(Filename->getData()); + } + static Expected create(ObjectStore &DB, StringRef Filename, ObjectRef Contents); @@ -268,6 +299,63 @@ class IncludeTree::FileList : public IncludeTreeBase { } }; +/// Represents a module imported by an IncludeTree. +class IncludeTree::ModuleImport : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ModI"; } + + static Expected create(ObjectStore &DB, StringRef ModuleName); + + StringRef getModuleName() { return getData(); } + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getNumReferences() == 0 && !Base.getData().empty(); + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + friend class IncludeTreeBase; + friend class Node; + + explicit ModuleImport(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } +}; + +/// Represents an \c IncludeTree or \c ModuleImport. +class IncludeTree::Node { +public: + IncludeTree getIncludeTree() const { + assert(K == NodeKind::Tree); + return IncludeTree(N); + } + ModuleImport getModuleImport() const { + assert(K == NodeKind::ModuleImport); + return ModuleImport(N); + } + NodeKind getKind() const { return K; } + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + +private: + friend class IncludeTree; + Node(ObjectProxy N, NodeKind K) : N(std::move(N)), K(K) {} + ObjectProxy N; + NodeKind K; +}; + /// Represents the include-tree result for a translation unit. class IncludeTreeRoot : public IncludeTreeBase { public: @@ -278,9 +366,15 @@ class IncludeTreeRoot : public IncludeTreeBase { ObjectRef getFileListRef() const { return getReference(1); } Optional getPCHRef() const { - if (getNumReferences() > 2) - return getReference(2); - return None; + if (auto Index = getPCHRefIndex()) + return getReference(*Index); + return std::nullopt; + } + + Optional getModuleMapRef() const { + if (auto Index = getModuleMapRefIndex()) + return getReference(*Index); + return std::nullopt; } Expected getMainFileTree() { @@ -307,10 +401,20 @@ class IncludeTreeRoot : public IncludeTreeBase { return None; } - static Expected create(ObjectStore &DB, - ObjectRef MainFileTree, - ObjectRef FileList, - Optional PCHRef); + Expected> getModuleMapFile() { + if (Optional Ref = getModuleMapRef()) { + auto Node = getCAS().getProxy(*Ref); + if (!Node) + return Node.takeError(); + return IncludeTree::File(*Node); + } + return std::nullopt; + } + + static Expected + create(ObjectStore &DB, ObjectRef MainFileTree, ObjectRef FileList, + Optional PCHRef, + Optional ModuleMapRef); static Expected get(ObjectStore &DB, ObjectRef Ref); @@ -320,8 +424,8 @@ class IncludeTreeRoot : public IncludeTreeBase { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - return (Base.getNumReferences() == 2 || Base.getNumReferences() == 3) && - Base.getData().empty(); + return (Base.getNumReferences() >= 2 && Base.getNumReferences() <= 4) && + Base.getData().size() == 1; } static bool isValid(ObjectStore &DB, ObjectRef Ref) { auto Node = DB.getProxy(Ref); @@ -335,6 +439,9 @@ class IncludeTreeRoot : public IncludeTreeBase { private: friend class IncludeTreeBase; + std::optional getPCHRefIndex() const; + std::optional getModuleMapRefIndex() const; + explicit IncludeTreeRoot(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index d9ae64030d621..da00f7f41939c 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -15,9 +15,11 @@ #define LLVM_CLANG_LEX_PPCACHEDACTIONS_H #include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/SmallVector.h" namespace clang { +class IdentifierInfo; class Preprocessor; /// This interface provides a way to override the actions of the preprocessor as @@ -28,6 +30,15 @@ class PPCachedActions { virtual void anchor(); public: + /// The file that is included by an \c #include directive. + struct IncludeFile { + FileID FID; + }; + /// The module that is imported by an \c #include directive or \c @import. + struct IncludeModule { + SmallVector, 2> ImportPath; + }; + virtual ~PPCachedActions() = default; /// \returns the \p FileID that should be used for predefines. @@ -37,9 +48,10 @@ class PPCachedActions { virtual bool evaluateHasInclude(Preprocessor &PP, SourceLocation Loc, bool IsIncludeNext) = 0; - /// \returns the \p FileID that should be entered for an include directive. - /// \p None indicates that the directive should be skipped. - virtual Optional + /// \returns the file that should be entered or module that should be imported + /// for an \c #include directive. \c {} indicates that the directive + /// should be skipped. + virtual std::variant handleIncludeDirective(Preprocessor &PP, SourceLocation IncludeLoc, SourceLocation AfterDirectiveLoc) = 0; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 4ab886eee568c..ed8ce36a76fda 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -58,8 +58,7 @@ Expected IncludeTree::getBaseFileInfo() { } llvm::Error IncludeTree::forEachInclude( - llvm::function_ref)> - Callback) { + llvm::function_ref)> Callback) { size_t RefI = 0; return forEachReference([&](ObjectRef Ref) -> llvm::Error { if (RefI == 0) { @@ -68,19 +67,20 @@ llvm::Error IncludeTree::forEachInclude( } size_t IncludeI = RefI - 1; ++RefI; - auto Include = getInclude(Ref); + auto Include = getIncludeNode(Ref, getIncludeKind(IncludeI)); if (!Include) return Include.takeError(); return Callback({*Include, getIncludeOffset(IncludeI)}); }); } -Expected IncludeTree::create( - ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, - ObjectRef BaseFile, ArrayRef> Includes, - llvm::SmallBitVector Checks) { +Expected +IncludeTree::create(ObjectStore &DB, + SrcMgr::CharacteristicKind FileCharacteristic, + ObjectRef BaseFile, ArrayRef Includes, + llvm::SmallBitVector Checks) { // The data buffer is composed of - // 1. `uint32_t` offsets of includes + // 1. `uint32_t` offset and `uint8_t` kind for each includes // 2. 1 byte for `CharacteristicKind` // 3. variable number of bitset bytes for `Checks`. @@ -97,11 +97,14 @@ Expected IncludeTree::create( llvm::support::endian::Writer Writer(BufOS, llvm::support::little); for (const auto &Include : Includes) { - ObjectRef FileRef = Include.first; - uint32_t Offset = Include.second; - assert(IncludeTree::isValid(DB, FileRef)); - Refs.push_back(FileRef); - Writer.write(Offset); + assert((Include.Kind == NodeKind::Tree && + IncludeTree::isValid(DB, Include.Ref)) || + (Include.Kind == NodeKind::ModuleImport && + ModuleImport::isValid(DB, Include.Ref))); + Refs.push_back(Include.Ref); + Writer.write(Include.Offset); + static_assert(sizeof(uint8_t) == sizeof(Kind)); + Writer.write(static_cast(Include.Kind)); } Buffer += Kind; @@ -141,13 +144,21 @@ Expected IncludeTree::get(ObjectStore &DB, ObjectRef Ref) { return IncludeTree(std::move(*Node)); } +IncludeTree::NodeKind IncludeTree::getIncludeKind(size_t I) const { + assert(I < getNumIncludes()); + StringRef Data = getData(); + assert(Data.size() >= (I + 1) * (sizeof(uint32_t) + 1)); + uint8_t K = *(Data.data() + I * (sizeof(uint32_t) + 1) + sizeof(uint32_t)); + return NodeKind(K); +} + uint32_t IncludeTree::getIncludeOffset(size_t I) const { assert(I < getNumIncludes()); StringRef Data = getData(); assert(Data.size() >= (I + 1) * sizeof(uint32_t)); uint32_t Offset = llvm::support::endian::read( - Data.data() + I * sizeof(uint32_t)); + Data.data() + I * (sizeof(uint32_t) + 1)); return Offset; } @@ -160,6 +171,18 @@ bool IncludeTree::getCheckResult(size_t I) const { return Bits & (1 << RemainingIndex); } +Expected IncludeTree::getIncludeNode(size_t I) { + return getIncludeNode(getIncludeRef(I), getIncludeKind(I)); +} + +Expected IncludeTree::getIncludeNode(ObjectRef Ref, + NodeKind K) { + auto N = getCAS().getProxy(Ref); + if (!N) + return N.takeError(); + return Node(std::move(*N), K); +} + bool IncludeTree::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; @@ -167,7 +190,13 @@ bool IncludeTree::isValid(const ObjectProxy &Node) { if (Base.getNumReferences() == 0) return false; unsigned NumIncludes = Base.getNumReferences() - 1; - return Base.getData().size() >= NumIncludes * sizeof(uint32_t) + 1; + return Base.getData().size() >= NumIncludes * (sizeof(uint32_t) + 1) + 1; +} + +Expected +IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName) { + return IncludeTreeBase::create(DB, {}, + llvm::arrayRefFromStringRef(ModuleName)); } IncludeTree::FileList::FileSizeTy @@ -228,17 +257,30 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { Base.getData().size() == NumFiles * sizeof(FileSizeTy); } -Expected IncludeTreeRoot::create(ObjectStore &DB, - ObjectRef MainFileTree, - ObjectRef FileList, - Optional PCHRef) { +static constexpr char HasPCH = 0x01; +static constexpr char HasModuleMap = 0x02; + +Expected +IncludeTreeRoot::create(ObjectStore &DB, ObjectRef MainFileTree, + ObjectRef FileList, Optional PCHRef, + Optional ModuleMapRef) { assert(IncludeTree::isValid(DB, MainFileTree)); assert(IncludeTree::FileList::isValid(DB, FileList)); - if (PCHRef) { - return IncludeTreeBase::create(DB, {MainFileTree, FileList, *PCHRef}, {}); - } else { - return IncludeTreeBase::create(DB, {MainFileTree, FileList}, {}); - } + assert(!ModuleMapRef || IncludeTree::File::isValid(DB, *ModuleMapRef)); + + std::array Data = {0}; + if (PCHRef) + Data[0] |= HasPCH; + if (ModuleMapRef) + Data[0] |= HasModuleMap; + + SmallVector Refs = {MainFileTree, FileList}; + if (PCHRef) + Refs.push_back(*PCHRef); + if (ModuleMapRef) + Refs.push_back(*ModuleMapRef); + + return IncludeTreeBase::create(DB, Refs, Data); } Expected IncludeTreeRoot::get(ObjectStore &DB, ObjectRef Ref) { @@ -251,6 +293,17 @@ Expected IncludeTreeRoot::get(ObjectStore &DB, ObjectRef Ref) { return IncludeTreeRoot(std::move(*Node)); } +std::optional IncludeTreeRoot::getPCHRefIndex() const { + if (getData()[0] & HasPCH) + return 2; + return std::nullopt; +} +std::optional IncludeTreeRoot::getModuleMapRefIndex() const { + if (getData()[0] & HasModuleMap) + return (getData()[0] & HasPCH) ? 3u : 2u; + return std::nullopt; +} + llvm::Error IncludeTree::File::print(llvm::raw_ostream &OS, unsigned Indent) { auto Filename = getFilename(); if (!Filename) @@ -275,14 +328,13 @@ llvm::Error IncludeTree::print(llvm::raw_ostream &OS, unsigned Indent) { auto MemBuf = llvm::MemoryBuffer::getMemBuffer(Blob->getData()); unsigned BufID = SM.AddNewSourceBuffer(std::move(MemBuf), llvm::SMLoc()); - return forEachInclude( - [&](std::pair Include) -> llvm::Error { - llvm::SMLoc Loc = llvm::SMLoc::getFromPointer( - SM.getMemoryBuffer(BufID)->getBufferStart() + Include.second); - auto LineCol = SM.getLineAndColumn(Loc); - OS.indent(Indent) << LineCol.first << ':' << LineCol.second << ' '; - return Include.first.print(OS, Indent + 2); - }); + return forEachInclude([&](std::pair Include) -> llvm::Error { + llvm::SMLoc Loc = llvm::SMLoc::getFromPointer( + SM.getMemoryBuffer(BufID)->getBufferStart() + Include.second); + auto LineCol = SM.getLineAndColumn(Loc); + OS.indent(Indent) << LineCol.first << ':' << LineCol.second << ' '; + return Include.first.print(OS, Indent + 2); + }); } llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, @@ -292,6 +344,21 @@ llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, }); } +llvm::Error IncludeTree::ModuleImport::print(llvm::raw_ostream &OS, + unsigned Indent) { + OS << "(Module) " << getModuleName() << '\n'; + return llvm::Error::success(); +} + +llvm::Error IncludeTree::Node::print(llvm::raw_ostream &OS, unsigned Indent) { + switch (K) { + case NodeKind::Tree: + return getIncludeTree().print(OS, Indent); + case NodeKind::ModuleImport: + return getModuleImport().print(OS, Indent); + } +} + llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { if (Optional PCHRef = getPCHRef()) { OS.indent(Indent) << "(PCH) "; @@ -303,6 +370,14 @@ llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { return E; if (llvm::Error E = MainTree->print(OS.indent(Indent), Indent)) return E; + Optional ModuleMap; + if (llvm::Error E = getModuleMapFile().moveInto(ModuleMap)) + return E; + if (ModuleMap) { + OS.indent(Indent) << "Module Map: "; + if (llvm::Error E = ModuleMap->print(OS, Indent)) + return E; + } OS.indent(Indent) << "Files:\n"; Optional List; if (llvm::Error E = getFileList().moveInto(List)) diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 1502cfef57d1b..ecf1848f86738 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -3037,12 +3037,23 @@ static void determineInputFromIncludeTree( auto Root = cas::IncludeTreeRoot::get(*CAS, *Object); if (!Root) return reportError(Root.takeError()); - auto MainTree = Root->getMainFileTree(); - if (!MainTree) - return reportError(MainTree.takeError()); - auto BaseFile = MainTree->getBaseFile(); - if (!BaseFile) - return reportError(BaseFile.takeError()); + + Optional BaseFile; + + auto MaybeModuleMap = Root->getModuleMapFile(); + if (!MaybeModuleMap) + return reportError(MaybeModuleMap.takeError()); + if (*MaybeModuleMap) { + // Building a module from a modulemap, the modulemap is the primary input. + BaseFile = *MaybeModuleMap; + } else { + auto MainTree = Root->getMainFileTree(); + if (!MainTree) + return reportError(MainTree.takeError()); + if (llvm::Error E = MainTree->getBaseFile().moveInto(BaseFile)) + return reportError(std::move(E)); + } + auto FilenameBlob = BaseFile->getFilename(); if (!FilenameBlob) return reportError(FilenameBlob.takeError()); diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 2a03e521bd704..7694664195708 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -856,28 +856,31 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, CI.getLangOpts().CurrentModule = CI.getLangOpts().ModuleName; } + auto reportError = [&](llvm::Error &&E) -> bool { + std::string IncludeTreeID = + CI.getOrCreateObjectStore().getID(Input.getIncludeTree()).toString(); + CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) + << IncludeTreeID << llvm::toString(std::move(E)); + return false; + }; + + Optional IncludeTreeRoot; Optional IncludeTreePCHBuffer; if (Input.isIncludeTree()) { - auto reportError = [&](llvm::Error &&E) -> bool { - std::string IncludeTreeID = - CI.getOrCreateObjectStore().getID(Input.getIncludeTree()).toString(); - CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) - << IncludeTreeID << llvm::toString(std::move(E)); - return false; - }; - auto Root = cas::IncludeTreeRoot::get(CI.getOrCreateObjectStore(), - Input.getIncludeTree()); - if (!Root) - return reportError(Root.takeError()); + if (llvm::Error E = cas::IncludeTreeRoot::get(CI.getOrCreateObjectStore(), + Input.getIncludeTree()) + .moveInto(IncludeTreeRoot)) + return reportError(std::move(E)); Expected> PPCachedAct = - createPPActionsFromIncludeTree(*Root); + createPPActionsFromIncludeTree(*IncludeTreeRoot); if (!PPCachedAct) return reportError(PPCachedAct.takeError()); CI.getPreprocessor().setPPCachedActions(std::move(*PPCachedAct)); CI.getFrontendOpts().IncludeTimestamps = false; - if (llvm::Error E = Root->getPCHBuffer().moveInto(IncludeTreePCHBuffer)) + if (llvm::Error E = + IncludeTreeRoot->getPCHBuffer().moveInto(IncludeTreePCHBuffer)) return reportError(std::move(E)); } @@ -921,10 +924,25 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // If the module contents are in the same file, skip to them. CI.getPreprocessor().setSkipMainFilePreamble(OffsetToContents, true); else { - // Otherwise, convert the module description to a suitable input buffer. - auto Buffer = getInputBufferForModule(CI, CurrentModule); - if (!Buffer) - return false; + std::unique_ptr Buffer; + if (Input.isIncludeTree()) { + // Get the existing module include buffer from the include-tree. + assert(IncludeTreeRoot); + auto MainTree = IncludeTreeRoot->getMainFileTree(); + if (!MainTree) + return reportError(MainTree.takeError()); + auto BaseFile = MainTree->getBaseFile(); + if (!BaseFile) + return reportError(BaseFile.takeError()); + if (auto E = BaseFile->getMemoryBuffer().moveInto(Buffer)) + return reportError(std::move(E)); + assert(Buffer); + } else { + // Otherwise, convert the module description to a suitable input buffer. + Buffer = getInputBufferForModule(CI, CurrentModule); + if (!Buffer) + return false; + } // Reinitialize the main file entry to refer to the new input. auto Kind = CurrentModule->IsSystem ? SrcMgr::C_System : SrcMgr::C_User; diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index d18908577677f..8c6c1cdad2838 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -61,7 +61,7 @@ class IncludeTreePPActions final : public PPCachedActions { IncludeStackInfo &IncludeInfo = IncludeStack.back(); Expected EnteredTree = - IncludeInfo.Tree.getInclude(IncludeInfo.CurIncludeIndex++); + IncludeInfo.Tree.getIncludeTree(IncludeInfo.CurIncludeIndex++); if (!EnteredTree) return reportError(EnteredTree.takeError()); auto FileInfo = EnteredTree->getBaseFileInfo(); @@ -84,33 +84,47 @@ class IncludeTreePPActions final : public PPCachedActions { return IncludeInfo.Tree.getCheckResult(Index); } - Optional + std::variant handleIncludeDirective(Preprocessor &PP, SourceLocation IncludeLoc, SourceLocation AfterDirectiveLoc) override { if (HasCASErrorOccurred) - return None; + return {}; IncludeStackInfo &IncludeInfo = IncludeStack.back(); if (IncludeInfo.CurIncludeIndex >= IncludeInfo.Tree.getNumIncludes()) - return None; + return {}; unsigned ExpectedOffset = IncludeInfo.Tree.getIncludeOffset(IncludeInfo.CurIncludeIndex); SourceLocation ExpectedLoc = IncludeInfo.FileStartLoc.getLocWithOffset(ExpectedOffset); if (ExpectedLoc != AfterDirectiveLoc) - return None; + return {}; - auto reportError = [&](llvm::Error &&E) -> Optional { + auto reportError = [&](llvm::Error &&E) -> std::monostate { this->reportError(PP, std::move(E)); - return None; + return {}; }; - Expected EnteredTree = - IncludeInfo.Tree.getInclude(IncludeInfo.CurIncludeIndex++); - if (!EnteredTree) - return reportError(EnteredTree.takeError()); - auto File = EnteredTree->getBaseFile(); + Expected Node = + IncludeInfo.Tree.getIncludeNode(IncludeInfo.CurIncludeIndex++); + if (!Node) + return reportError(Node.takeError()); + + if (Node->getKind() == cas::IncludeTree::NodeKind::ModuleImport) { + cas::IncludeTree::ModuleImport Import = Node->getModuleImport(); + SmallVector, 2> Path; + SmallVector ModuleComponents; + Import.getModuleName().split(ModuleComponents, '.'); + for (StringRef Component : ModuleComponents) + Path.emplace_back(PP.getIdentifierInfo(Component), IncludeLoc); + return IncludeModule{std::move(Path)}; + } + + assert(Node->getKind() == cas::IncludeTree::NodeKind::Tree); + + cas::IncludeTree EnteredTree = Node->getIncludeTree(); + auto File = EnteredTree.getBaseFile(); if (!File) return reportError(File.takeError()); auto FilenameBlob = File->getFilename(); @@ -124,11 +138,11 @@ class IncludeTreePPActions final : public PPCachedActions { if (!FE) return reportError(FE.takeError()); FileID FID = - SM.createFileID(*FE, IncludeLoc, EnteredTree->getFileCharacteristic()); + SM.createFileID(*FE, IncludeLoc, EnteredTree.getFileCharacteristic()); PP.markIncluded(*FE); IncludeStack.push_back( - {std::move(*EnteredTree), SM.getLocForStartOfFile(FID)}); - return FID; + {std::move(EnteredTree), SM.getLocForStartOfFile(FID)}); + return IncludeFile{FID}; } void exitedFile(Preprocessor &PP, FileID FID) override { diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 29f3790325c19..5a4530f49339c 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1973,39 +1973,59 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, return; } + // Verify that there is nothing after the filename, other than EOD. Note + // that we allow macros that expand to nothing after the filename, because + // this falls into the category of "#include pp-tokens new-line" specified + // in C99 6.10.2p4. + SourceLocation EndLoc = + CheckEndOfDirective(IncludeTok.getIdentifierInfo()->getNameStart(), true); + if (auto *CActions = getPPCachedActions()) { - DiscardUntilEndOfDirective(); SourceLocation IncludePos = FilenameTok.getLocation(); // If the filename string was the result of macro expansions, set the // include position on the file where it will be included and after the // expansions. if (IncludePos.isMacroID()) IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd(); - Optional FID = CActions->handleIncludeDirective( + auto Include = CActions->handleIncludeDirective( *this, IncludePos, CurLexer->getSourceLocation()); - if (!FID) { - // FIXME: Report \p Callbacks->FileSkipped? Note that it currently - // requires the resolved FileEntry for this particular #include. + + if (auto *File = std::get_if(&Include)) { + const FileEntry *FE = SourceMgr.getFileEntryForID(File->FID); + bool IsImport = + IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import; + if (FE && IsImport) { + HeaderInfo.getFileInfo(FE).isImport = true; + } + EnterSourceFile(File->FID, nullptr, FilenameTok.getLocation(), + /*IsFirstIncludeOfFile*/ true); return; } - const FileEntry *FE = SourceMgr.getFileEntryForID(*FID); - bool IsImport = - IncludeTok.getIdentifierInfo()->getPPKeywordID() == tok::pp_import; - if (FE && IsImport) { - HeaderInfo.getFileInfo(FE).isImport = true; + if (auto *Import = std::get_if(&Include)) { + ModuleLoadResult Imported = TheModuleLoader.loadModule( + IncludeTok.getLocation(), Import->ImportPath, Module::Hidden, + /*IsIncludeDirective=*/true); + if (!Imported) { + assert(hadModuleLoaderFatalFailure() && "unexpected failure kind"); + if (hadModuleLoaderFatalFailure()) { + IncludeTok.setKind(tok::eof); + CurLexer->cutOffLexing(); + } + return; + } + makeModuleVisible(Imported, EndLoc); + if (IncludeTok.getIdentifierInfo()->getPPKeywordID() != + tok::pp___include_macros) + EnterAnnotationToken(SourceRange(HashLoc, EndLoc), + tok::annot_module_include, Imported); + return; } - EnterSourceFile(*FID, nullptr, FilenameTok.getLocation(), - /*IsFirstIncludeOfFile*/ true); + assert(std::holds_alternative(Include)); + // FIXME: Report \p Callbacks->FileSkipped? Note that it currently + // requires the resolved FileEntry for this particular #include. return; } - // Verify that there is nothing after the filename, other than EOD. Note - // that we allow macros that expand to nothing after the filename, because - // this falls into the category of "#include pp-tokens new-line" specified - // in C99 6.10.2p4. - SourceLocation EndLoc = - CheckEndOfDirective(IncludeTok.getIdentifierInfo()->getNameStart(), true); - auto Action = HandleHeaderIncludeOrImport(HashLoc, IncludeTok, FilenameTok, EndLoc, LookupFrom, LookupFromFile); switch (Action.Kind) { diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 32ac0595f06e8..909b9b7d56084 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -83,11 +83,13 @@ class IncludeTreeBuilder { void handleHasIncludeCheck(Preprocessor &PP, bool Result); + void moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc); + private: struct FilePPState { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; - SmallVector, 6> Includes; + SmallVector Includes; llvm::SmallBitVector HasIncludeChecks; }; @@ -172,6 +174,32 @@ struct IncludeTreePPCallbacks : public PPCallbacks { SrcMgr::CharacteristicKind FileType) override { Builder.handleHasIncludeCheck(PP, File.has_value()); } + + void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok, + StringRef FileName, bool IsAngled, + CharSourceRange FilenameRange, + Optional File, StringRef SearchPath, + StringRef RelativePath, const Module *Imported, + SrcMgr::CharacteristicKind FileType) override { + if (!Imported) + return; // File includes handled by LexedFileChanged. + + // Calculate EndLoc for the directive + // FIXME: pass EndLoc through PPCallbacks; it is already calculated + SourceManager &SM = PP.getSourceManager(); + std::pair LocInfo = SM.getDecomposedExpansionLoc(HashLoc); + StringRef Buffer = SM.getBufferData(LocInfo.first); + Lexer L(SM.getLocForStartOfFile(LocInfo.first), PP.getLangOpts(), + Buffer.begin(), Buffer.begin() + LocInfo.second, Buffer.end()); + L.setParsingPreprocessorDirective(true); + Token Tok; + do { + L.LexFromRawLexer(Tok); + } while (!Tok.isOneOf(tok::eod, tok::eof)); + SourceLocation EndLoc = L.getSourceLocation(); + + Builder.moduleImport(PP, Imported, EndLoc); + } }; } // namespace @@ -341,8 +369,8 @@ void IncludeTreeBuilder::exitedInclude(Preprocessor &PP, FileID IncludedBy, assert(*check(getObjectForFile(PP, IncludedBy)) == IncludeStack.back().File); SourceManager &SM = PP.getSourceManager(); std::pair LocInfo = SM.getDecomposedExpansionLoc(ExitLoc); - IncludeStack.back().Includes.push_back( - {IncludeTree->getRef(), LocInfo.second}); + IncludeStack.back().Includes.push_back({IncludeTree->getRef(), LocInfo.second, + cas::IncludeTree::NodeKind::Tree}); } void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { @@ -352,6 +380,20 @@ void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { IncludeStack.back().HasIncludeChecks.push_back(Result); } +void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, + SourceLocation EndLoc) { + auto Import = + check(cas::IncludeTree::ModuleImport::create(DB, M->getFullModuleName())); + if (!Import) + return; + + std::pair EndLocInfo = + PP.getSourceManager().getDecomposedExpansionLoc(EndLoc); + IncludeStack.back().Includes.push_back( + {Import->getRef(), EndLocInfo.second, + cas::IncludeTree::NodeKind::ModuleImport}); +} + Expected IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { @@ -400,20 +442,36 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return std::move(E); } + auto &FrontendOpts = NewInvocation.getFrontendOpts(); + if (!FrontendOpts.Inputs.empty() && + FrontendOpts.Inputs[0].getKind().getFormat() == InputKind::ModuleMap) { + // FIXME: handle inferred module maps + Expected FE = FM.getFileRef(FrontendOpts.Inputs[0].getFile()); + if (!FE) + return FE.takeError(); + if (Error E = addToFileList(FM, *FE).moveInto(ModuleMapFileRef)) + return std::move(E); + } + auto FinishIncludeTree = [&]() -> Error { - PreprocessorOptions &PPOpts = NewInvocation.getPreprocessorOpts(); - if (PPOpts.ImplicitPCHInclude.empty()) + IntrusiveRefCntPtr Reader = ScanInstance.getASTReader(); + if (!Reader) return Error::success(); // no need for additional work. - // Go through all the recorded included files; we'll get additional files - // from the PCH that we need to include in the file list, in case they are - // referenced while replaying the include-tree. + // Go through all the recorded input files. SmallVector NotSeenIncludes; - for (const FileEntry *FE : - ScanInstance.getPreprocessor().getIncludedFiles()) { - if (FE->getUID() >= SeenIncludeFiles.size() || - !SeenIncludeFiles[FE->getUID()]) - NotSeenIncludes.push_back(FE); + for (serialization::ModuleFile &MF : Reader->getModuleManager()) { + if (hasErrorOccurred()) + break; + Reader->visitInputFiles( + MF, /*IncludeSystem=*/true, /*Complain=*/false, + [&](const serialization::InputFile &IF, bool isSystem) { + Optional FE = IF.getFile(); + assert(FE); + if (FE->getUID() >= SeenIncludeFiles.size() || + !SeenIncludeFiles[FE->getUID()]) + NotSeenIncludes.push_back(*FE); + }); } // Sort so we can visit the files in deterministic order. llvm::sort(NotSeenIncludes, [](const FileEntry *LHS, const FileEntry *RHS) { @@ -426,6 +484,10 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return FileNode.takeError(); } + PreprocessorOptions &PPOpts = NewInvocation.getPreprocessorOpts(); + if (PPOpts.ImplicitPCHInclude.empty()) + return Error::success(); // no need for additional work. + llvm::ErrorOr> CASContents = FM.getObjectRefForFileContent(PPOpts.ImplicitPCHInclude); if (!CASContents) @@ -451,7 +513,8 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return FileList.takeError(); return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), - FileList->getRef(), PCHRef); + FileList->getRef(), PCHRef, + ModuleMapFileRef); } Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index e1234095a95e0..af851c979f89c 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -35,12 +35,16 @@ void tooling::dependencies::configureInvocationForCaching( if (ProduceIncludeTree) { FrontendOpts.CASIncludeTreeID = std::move(RootID); FrontendOpts.Inputs.clear(); - // Preserve sysroot path to accommodate lookup for 'SDKSettings.json' during - // availability checking, and module files. HeaderSearchOptions &HSOpts = CI.getHeaderSearchOpts(); HeaderSearchOptions OriginalHSOpts; std::swap(HSOpts, OriginalHSOpts); + // Preserve sysroot path to accommodate lookup for 'SDKSettings.json' during + // availability checking. HSOpts.Sysroot = std::move(OriginalHSOpts.Sysroot); + // Preserve resource-dir, which is added back by cc1_main if missing, and + // affects the cache key. + HSOpts.ResourceDir = std::move(OriginalHSOpts.ResourceDir); + // Preserve fmodule-file options. HSOpts.PrebuiltModuleFiles = std::move(OriginalHSOpts.PrebuiltModuleFiles); auto &PPOpts = CI.getPreprocessorOpts(); // We don't need this because we save the contents of the PCH file in the diff --git a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c new file mode 100644 index 0000000000000..97168db219320 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c @@ -0,0 +1,156 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// Scan PCH +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// Build PCH +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Top.rsp +// RUN: %clang @%t/Left.rsp +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/pch.rsp +// RUN: rm -rf %t/outputs + +// Scan TU with PCH +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Build TU +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Right.rsp +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/tu.rsp + +// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/Right.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NOT: "clang-modulemap-file" +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { int x; }; + +//--- Left.h +#pragma once +#include "Top.h" +struct Left { struct Top top; }; + +//--- Right.h +#pragma once +#include "Top.h" +struct Right { struct Top top; }; + +//--- prefix.h +#include "Left.h" + +//--- tu.c +#include "Right.h" + +void tu(void) { + struct Left _left; + struct Right _right; + struct Top _top; +} diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index 26792c21724df..75a45400f63f0 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -9,7 +9,71 @@ // RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ // RUN: > %t/deps.json -// RUN: FileCheck %s -input-file %t/deps.json -DPREFIX=%/t +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Top.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Top.casid +// RUN: cat %t/Left.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Left.casid +// RUN: cat %t/Right.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Right.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Top" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Top.casid >> %t/result.txt +// RUN: echo "MODULE Left" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Left.casid >> %t/result.txt +// RUN: echo "MODULE Right" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Right.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt +// RUN: cat %t/deps.json >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Top +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Files: +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: MODULE Left +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Files: +// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: MODULE Right +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Files: +// CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Left +// CHECK: 3:1 (Module) Right +// CHECK: Files: +// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} // CHECK: { // CHECK-NEXT "modules": [ @@ -162,11 +226,30 @@ // CHECK-NEXT: ] // CHECK-NEXT: } +// Build the include-tree commands +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// Ensure the pcm comes from the action cache +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS + +// Check cache hits +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT + +// CACHE_MISS: compile job cache miss +// CACHE_HIT: compile job cache hit + //--- cdb.json.template [{ "file": "DIR/tu.c", "directory": "DIR", - "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" + "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" }] //--- module.modulemap @@ -182,21 +265,19 @@ struct Top { void top(void); //--- Left.h -#pragma once #include "Top.h" void left(void); //--- Right.h -#pragma once #include "Top.h" void right(void); //--- tu.c -#include "Left.h" -#include "Right.h" +#import "Left.h" +#import void tu(void) { + top(); left(); right(); - top(); } diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index f687ef57762d8..c7739611bfce3 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -79,7 +79,8 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(Main->getNumIncludes(), uint32_t(3)); Optional Predef; - ASSERT_THAT_ERROR(Main->getInclude(0).moveInto(Predef), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(0).moveInto(Predef), + llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(0), uint32_t(0)); { EXPECT_EQ(Predef->getFileCharacteristic(), SrcMgr::C_User); @@ -90,7 +91,7 @@ TEST(IncludeTree, IncludeTreeScan) { } Optional A1; - ASSERT_THAT_ERROR(Main->getInclude(1).moveInto(A1), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(1).moveInto(A1), llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(1), uint32_t(21)); { ASSERT_THAT_ERROR(A1->getBaseFile().moveInto(A1File), llvm::Succeeded()); @@ -104,7 +105,7 @@ TEST(IncludeTree, IncludeTreeScan) { ASSERT_EQ(A1->getNumIncludes(), uint32_t(1)); Optional B1; - ASSERT_THAT_ERROR(A1->getInclude(0).moveInto(B1), llvm::Succeeded()); + ASSERT_THAT_ERROR(A1->getIncludeTree(0).moveInto(B1), llvm::Succeeded()); EXPECT_EQ(A1->getIncludeOffset(0), uint32_t(122)); { ASSERT_THAT_ERROR(B1->getBaseFile().moveInto(B1File), llvm::Succeeded()); @@ -119,7 +120,7 @@ TEST(IncludeTree, IncludeTreeScan) { } Optional Sys; - ASSERT_THAT_ERROR(Main->getInclude(2).moveInto(Sys), llvm::Succeeded()); + ASSERT_THAT_ERROR(Main->getIncludeTree(2).moveInto(Sys), llvm::Succeeded()); EXPECT_EQ(Main->getIncludeOffset(2), uint32_t(42)); { ASSERT_THAT_ERROR(Sys->getBaseFile().moveInto(SysFile), llvm::Succeeded()); From a82079c65ca837eb1cc3d1a7f611a895bfdd182d Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 16 Mar 2023 13:50:54 -0700 Subject: [PATCH 15/88] [clang][cas] include-tree support for submodule visibility Track the submodule of each modular header and enter the submodule via PPCachedActions. This fixes declaration and macro header visibility. (cherry picked from commit bd61c2c82c59f703f157d4129f05aa36b5837ab9) (cherry picked from commit d926cee9331681166732cd90f36363b28a79486e) --- clang/include/clang/CAS/IncludeTree.h | 33 +++- clang/include/clang/Lex/PPCachedActions.h | 2 + clang/lib/CAS/IncludeTree.cpp | 45 ++++-- clang/lib/Frontend/IncludeTreePPActions.cpp | 29 +++- clang/lib/Lex/PPDirectives.cpp | 9 ++ .../IncludeTreeActionController.cpp | 37 ++++- .../modules-include-tree-submodules.c | 147 ++++++++++++++++++ 7 files changed, 281 insertions(+), 21 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-submodules.c diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 1917f6f1c54bb..3d136304a5989 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -69,10 +69,31 @@ class IncludeTree : public IncludeTreeBase { Expected getBaseFileInfo(); SrcMgr::CharacteristicKind getFileCharacteristic() const { - return (SrcMgr::CharacteristicKind)dataSkippingIncludes().front(); + return (SrcMgr::CharacteristicKind)(getData().front() & ~IsSubmoduleBit); } - size_t getNumIncludes() const { return getNumReferences() - 1; } + bool isSubmodule() const { return (getData().front() & IsSubmoduleBit) != 0; } + + std::optional getSubmoduleNameRef() const { + if (isSubmodule()) + return getReference(getNumReferences() - 1); + return std::nullopt; + } + + /// If \c getBaseFile() is a modular header, get its submodule name. + Expected> getSubmoduleName() { + auto Ref = getSubmoduleNameRef(); + if (!Ref) + return std::nullopt; + auto Node = getCAS().getProxy(*Ref); + if (!Node) + return Node.takeError(); + return Node->getData(); + } + + size_t getNumIncludes() const { + return getNumReferences() - (isSubmodule() ? 2 : 1); + }; ObjectRef getIncludeRef(size_t I) const { assert(I < getNumIncludes()); @@ -131,7 +152,7 @@ class IncludeTree : public IncludeTreeBase { static Expected create(ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, ObjectRef BaseFile, ArrayRef Includes, - llvm::SmallBitVector Checks); + std::optional SubmoduleName, llvm::SmallBitVector Checks); static Expected get(ObjectStore &DB, ObjectRef Ref); @@ -141,6 +162,8 @@ class IncludeTree : public IncludeTreeBase { friend class IncludeTreeBase; friend class IncludeTreeRoot; + static constexpr char IsSubmoduleBit = 0x80; + explicit IncludeTree(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { assert(isValid(*this)); } @@ -154,8 +177,10 @@ class IncludeTree : public IncludeTreeBase { Expected getIncludeNode(ObjectRef Ref, NodeKind Kind); + StringRef dataSkippingFlags() const { return getData().drop_front(); } StringRef dataSkippingIncludes() const { - return getData().drop_front(getNumIncludes() * (sizeof(uint32_t) + 1)); + return dataSkippingFlags().drop_front(getNumIncludes() * + (sizeof(uint32_t) + 1)); } static bool isValid(const ObjectProxy &Node); diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index da00f7f41939c..dde2c6f30ed93 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -20,6 +20,7 @@ namespace clang { class IdentifierInfo; +class Module; class Preprocessor; /// This interface provides a way to override the actions of the preprocessor as @@ -33,6 +34,7 @@ class PPCachedActions { /// The file that is included by an \c #include directive. struct IncludeFile { FileID FID; + Module *Submodule; }; /// The module that is imported by an \c #include directive or \c @import. struct IncludeModule { diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index ed8ce36a76fda..956c53dc9e546 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -60,12 +60,15 @@ Expected IncludeTree::getBaseFileInfo() { llvm::Error IncludeTree::forEachInclude( llvm::function_ref)> Callback) { size_t RefI = 0; + const size_t IncludeEnd = getNumIncludes(); return forEachReference([&](ObjectRef Ref) -> llvm::Error { if (RefI == 0) { ++RefI; return llvm::Error::success(); } size_t IncludeI = RefI - 1; + if (IncludeI >= IncludeEnd) + return llvm::Error::success(); ++RefI; auto Include = getIncludeNode(Ref, getIncludeKind(IncludeI)); if (!Include) @@ -74,25 +77,30 @@ llvm::Error IncludeTree::forEachInclude( }); } -Expected -IncludeTree::create(ObjectStore &DB, - SrcMgr::CharacteristicKind FileCharacteristic, - ObjectRef BaseFile, ArrayRef Includes, - llvm::SmallBitVector Checks) { +Expected IncludeTree::create( + ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, + ObjectRef BaseFile, ArrayRef Includes, + std::optional SubmoduleName, llvm::SmallBitVector Checks) { // The data buffer is composed of - // 1. `uint32_t` offset and `uint8_t` kind for each includes - // 2. 1 byte for `CharacteristicKind` + // 1. 1 byte for `CharacteristicKind` and IsSubmodule + // 2. `uint32_t` offset and `uint8_t` kind for each includes // 3. variable number of bitset bytes for `Checks`. char Kind = FileCharacteristic; - assert(Kind == FileCharacteristic && "SrcMgr::CharacteristicKind too big!"); + assert(Kind == FileCharacteristic && (Kind & IsSubmoduleBit) == 0 && + "SrcMgr::CharacteristicKind too big!"); + if (SubmoduleName) + Kind |= IsSubmoduleBit; + assert(File::isValid(DB, BaseFile)); SmallVector Refs; - Refs.reserve(Includes.size() + 1); + Refs.reserve(Includes.size() + 2); Refs.push_back(BaseFile); SmallString<64> Buffer; Buffer.reserve(Includes.size() * sizeof(uint32_t) + 1); + Buffer += Kind; + llvm::raw_svector_ostream BufOS(Buffer); llvm::support::endian::Writer Writer(BufOS, llvm::support::little); @@ -107,7 +115,8 @@ IncludeTree::create(ObjectStore &DB, Writer.write(static_cast(Include.Kind)); } - Buffer += Kind; + if (SubmoduleName) + Refs.push_back(*SubmoduleName); uintptr_t Store; ArrayRef BitWords = Checks.getData(Store); @@ -146,7 +155,7 @@ Expected IncludeTree::get(ObjectStore &DB, ObjectRef Ref) { IncludeTree::NodeKind IncludeTree::getIncludeKind(size_t I) const { assert(I < getNumIncludes()); - StringRef Data = getData(); + StringRef Data = dataSkippingFlags(); assert(Data.size() >= (I + 1) * (sizeof(uint32_t) + 1)); uint8_t K = *(Data.data() + I * (sizeof(uint32_t) + 1) + sizeof(uint32_t)); return NodeKind(K); @@ -154,7 +163,7 @@ IncludeTree::NodeKind IncludeTree::getIncludeKind(size_t I) const { uint32_t IncludeTree::getIncludeOffset(size_t I) const { assert(I < getNumIncludes()); - StringRef Data = getData(); + StringRef Data = dataSkippingFlags(); assert(Data.size() >= (I + 1) * sizeof(uint32_t)); uint32_t Offset = llvm::support::endian::read( @@ -164,7 +173,7 @@ uint32_t IncludeTree::getIncludeOffset(size_t I) const { bool IncludeTree::getCheckResult(size_t I) const { // Skip include offsets and CharacteristicKind. - StringRef Data = dataSkippingIncludes().drop_front(); + StringRef Data = dataSkippingIncludes(); unsigned ByteIndex = I / CHAR_BIT; size_t RemainingIndex = I % CHAR_BIT; uint8_t Bits = Data[ByteIndex]; @@ -187,9 +196,11 @@ bool IncludeTree::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - if (Base.getNumReferences() == 0) + if (Base.getNumReferences() == 0 || Base.getData().empty()) return false; unsigned NumIncludes = Base.getNumReferences() - 1; + if (Base.getData().front() & IsSubmoduleBit) + NumIncludes -= 1; return Base.getData().size() >= NumIncludes * (sizeof(uint32_t) + 1) + 1; } @@ -321,6 +332,12 @@ llvm::Error IncludeTree::print(llvm::raw_ostream &OS, unsigned Indent) { if (llvm::Error E = IncludeBy->print(OS)) return E; + auto Submodule = getSubmoduleName(); + if (!Submodule) + return Submodule.takeError(); + if (*Submodule) + OS.indent(Indent) << "Submodule: " << **Submodule << '\n'; + llvm::SourceMgr SM; auto Blob = IncludeBy->getContents(); if (!Blob) diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index 8c6c1cdad2838..3531f94c6d737 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -106,6 +106,11 @@ class IncludeTreePPActions final : public PPCachedActions { return {}; }; + auto reportErrorTwine = [&](const llvm::Twine &T) -> std::monostate { + return reportError( + llvm::createStringError(llvm::inconvertibleErrorCode(), T)); + }; + Expected Node = IncludeInfo.Tree.getIncludeNode(IncludeInfo.CurIncludeIndex++); if (!Node) @@ -142,7 +147,29 @@ class IncludeTreePPActions final : public PPCachedActions { PP.markIncluded(*FE); IncludeStack.push_back( {std::move(EnteredTree), SM.getLocForStartOfFile(FID)}); - return IncludeFile{FID}; + + Module *M = nullptr; + auto SubmoduleName = EnteredTree.getSubmoduleName(); + if (!SubmoduleName) + return reportError(SubmoduleName.takeError()); + if (*SubmoduleName) { + SmallVector ModuleComponents; + (*SubmoduleName)->split(ModuleComponents, '.'); + M = PP.getHeaderSearchInfo().lookupModule( + ModuleComponents[0], IncludeLoc, + /*AllowSearch=*/false, /*AllowExtraModuleMapSearch=*/false); + if (!M) + return reportErrorTwine(llvm::Twine("failed to find module '") + + ModuleComponents[0] + "'"); + for (StringRef Sub : ArrayRef(ModuleComponents).drop_front()) { + M = M->findOrInferSubmodule(Sub); + if (!M) + return reportErrorTwine( + llvm::Twine("failed to find or infer submodule '") + Sub + "'"); + } + } + + return IncludeFile{FID, M}; } void exitedFile(Preprocessor &PP, FileID FID) override { diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 5a4530f49339c..92b68ddf10a9c 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1999,6 +1999,15 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, } EnterSourceFile(File->FID, nullptr, FilenameTok.getLocation(), /*IsFirstIncludeOfFile*/ true); + + if (Module *SM = File->Submodule) { + assert(!CurLexerSubmodule && + "should not have marked this as a module yet"); + CurLexerSubmodule = SM; + EnterSubmodule(SM, EndLoc, /*ForPragma=*/false); + EnterAnnotationToken(SourceRange(HashLoc, EndLoc), + tok::annot_module_begin, SM); + } return; } if (auto *Import = std::get_if(&Include)) { diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 909b9b7d56084..65813116730d6 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -85,11 +85,17 @@ class IncludeTreeBuilder { void moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc); + void enteredSubmodule(Preprocessor &PP, Module *M, SourceLocation ImportLoc, + bool ForPragma); + void exitedSubmodule(Preprocessor &PP, Module *M, SourceLocation ImportLoc, + bool ForPragma); + private: struct FilePPState { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; SmallVector Includes; + std::optional SubmoduleName; llvm::SmallBitVector HasIncludeChecks; }; @@ -200,6 +206,15 @@ struct IncludeTreePPCallbacks : public PPCallbacks { Builder.moduleImport(PP, Imported, EndLoc); } + + void EnteredSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) override { + Builder.enteredSubmodule(PP, M, ImportLoc, ForPragma); + } + void LeftSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) override { + Builder.exitedSubmodule(PP, M, ImportLoc, ForPragma); + } }; } // namespace @@ -353,7 +368,7 @@ void IncludeTreeBuilder::enteredInclude(Preprocessor &PP, FileID FID) { return; const SrcMgr::FileInfo &FI = PP.getSourceManager().getSLocEntry(FID).getFile(); - IncludeStack.push_back({FI.getFileCharacteristic(), *FileRef, {}, {}}); + IncludeStack.push_back({FI.getFileCharacteristic(), *FileRef, {}, {}, {}}); } void IncludeTreeBuilder::exitedInclude(Preprocessor &PP, FileID IncludedBy, @@ -394,6 +409,23 @@ void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, cas::IncludeTree::NodeKind::ModuleImport}); } +void IncludeTreeBuilder::enteredSubmodule(Preprocessor &PP, Module *M, + SourceLocation ImportLoc, + bool ForPragma) { + if (ForPragma) + return; // Will be parsed as normal. + if (hasErrorOccurred()) + return; + assert(!IncludeStack.back().SubmoduleName && "repeated enteredSubmodule"); + auto Ref = check(DB.storeFromString({}, M->getFullModuleName())); + IncludeStack.back().SubmoduleName = Ref; +} +void IncludeTreeBuilder::exitedSubmodule(Preprocessor &PP, Module *M, + SourceLocation ImportLoc, + bool ForPragma) { + // Submodule exit is handled automatically when leaving a modular file. +} + Expected IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { @@ -614,7 +646,8 @@ IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { Expected IncludeTreeBuilder::getCASTreeForFileIncludes(FilePPState &&PPState) { return cas::IncludeTree::create(DB, PPState.FileCharacteristic, PPState.File, - PPState.Includes, PPState.HasIncludeChecks); + PPState.Includes, PPState.SubmoduleName, + PPState.HasIncludeChecks); } Expected diff --git a/clang/test/ClangScanDeps/modules-include-tree-submodules.c b/clang/test/ClangScanDeps/modules-include-tree-submodules.c new file mode 100644 index 0000000000000..91b92443258f3 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-submodules.c @@ -0,0 +1,147 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name TwoSubs > %t/TwoSubs.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name WithExplicit > %t/WithExplicit.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu1.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu2.rsp + +// Extract include-tree casids +// RUN: cat %t/TwoSubs.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/TwoSubs.casid +// RUN: cat %t/WithExplicit.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/WithExplicit.casid +// RUN: cat %t/tu1.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu1.casid +// RUN: cat %t/tu2.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu2.casid + +// RUN: echo "MODULE TwoSubs" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/TwoSubs.casid >> %t/result.txt +// RUN: echo "MODULE WithExplicit" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/WithExplicit.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT 1" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu1.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT 2" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu2.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// Build the include-tree commands +// RUN: %clang @%t/TwoSubs.rsp +// RUN: %clang @%t/WithExplicit.rsp +// RUN: not %clang @%t/tu1.rsp 2>&1 | FileCheck %s -check-prefix=TU1 +// RUN: not %clang @%t/tu2.rsp 2>&1 | FileCheck %s -check-prefix=TU2 + +// CHECK: MODULE TwoSubs +// CHECK: 2:1 [[PREFIX]]/Sub1.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: TwoSubs.Sub1 +// CHECK: 3:1 [[PREFIX]]/Sub2.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: TwoSubs.Sub2 + +// CHECK: MODULE WithExplicit +// CHECK: 2:1 [[PREFIX]]/TopLevel.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit +// CHECK: 3:1 [[PREFIX]]/Implicit.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit.Implicit +// CHECK: 4:1 [[PREFIX]]/Explicit.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: WithExplicit.Explicit + +// CHECK: TRANSLATION UNIT 1 +// CHECK: [[PREFIX]]/tu1.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) TwoSubs.Sub1 +// CHECK: 3:1 (Module) WithExplicit + +// CHECK: TRANSLATION UNIT 2 +// CHECK: [[PREFIX]]/tu2.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) TwoSubs.Sub2 +// CHECK: 3:1 (Module) WithExplicit.Explicit + +//--- cdb.json.template +[ +{ + "file": "DIR/tu1.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu1.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu2.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu2.c -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +] + +//--- module.modulemap +module TwoSubs { + module Sub1 { header "Sub1.h" } + module Sub2 { header "Sub2.h" } +} + +module WithExplicit { + header "TopLevel.h" + module Implicit { header "Implicit.h" } + explicit module Explicit { header "Explicit.h" } +} + +//--- Sub1.h +void sub1(void); + +//--- Sub2.h +void sub2(void); + +//--- TopLevel.h +void top(void); + +//--- Implicit.h +void implicit(void); + +//--- Explicit.h +void explicit(void); + +//--- tu1.c +#include "Sub1.h" +#include "TopLevel.h" + +void tu1(void) { + top(); + sub1(); + implicit(); + sub2(); +// TU1-NOT: error: +// TU1: error: call to undeclared function 'sub2' +// TU1: error: missing '#include "Sub2.h"' +// TU1: note: declaration here is not visible + explicit(); +// TU1: error: call to undeclared function 'explicit' +// TU1: error: missing '#include "Explicit.h"' +// TU1: Explicit.h:1:6: note: declaration here is not visible +} + +//--- tu2.c +#include "Sub2.h" +#include "Explicit.h" + +void tu2(void) { + sub2(); + explicit(); +// TU2-NOT: error: + top(); +// TU2: error: call to undeclared function 'top' +// TU2: error: missing '#include "TopLevel.h"' +// TU2: note: declaration here is not visible + sub1(); +// TU2: error: call to undeclared function 'sub1' +// TU2: error: missing '#include "Sub1.h"' +// TU2: note: declaration here is not visible + implicit(); +// TU2: error: call to undeclared function 'implicit' +// TU2: error: missing '#include "Implicit.h"' +// TU2: note: declaration here is not visible +} From c9d3aae5d78180267b4b6920c3ff37730348543d Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 16 Mar 2023 15:37:00 -0700 Subject: [PATCH 16/88] [clang][cas] Handle include-tree for a file considered part of the module When building a non-header file with -fmodule-name set, add the requisite modulemap file even though it is not imported as a module, and test that we are getting the right semantics for the header include. (cherry picked from commit 43df61c6ba7fc0014bd72354502bb45ac4edbcb7) (cherry picked from commit 145905544e3a071b65de4cfa6aa5ed404d43e7d5) --- .../IncludeTreeActionController.cpp | 4 ++ .../modules-include-tree-implementation.c | 58 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-implementation.c diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 65813116730d6..14bce2cde1e8b 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -485,6 +485,10 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return std::move(E); } + for (StringRef ModuleMap : FrontendOpts.ModuleMapFiles) + if (Error E = addFile(ModuleMap)) + return std::move(E); + auto FinishIncludeTree = [&]() -> Error { IntrusiveRefCntPtr Reader = ScanInstance.getASTReader(); if (!Reader) diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c new file mode 100644 index 0000000000000..8d879d21236ed --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -0,0 +1,58 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -check-prefix=NO_MODULES +// NO_MODULES: "modules": [] + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/tu.rsp + +// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} + +// Note: this is surprising, but correct: when building the implementation files +// of a module, the first include is textual but still uses the submodule +// machinery. The second include is treated as a module import (unless in a PCH) +// but will not actually import the module only trigger visibility changes. + +// CHECK: 2:1 [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Submodule: Mod +// CHECK: 3:1 (Module) Mod + +// CHECK: Files: +// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} + +// Despite not importing the module, we need its modulemap for submodule info. +// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodule-name=Mod -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Mod { header "Mod.h" } + +//--- Mod.h +#pragma once +void top(void); + +//--- tu.c +#include "Mod.h" +#include "Mod.h" +void tu(void) { + top(); +} From 0c76a3ac11816704f1393f4f835fd1607495730a Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 16 Mar 2023 15:57:06 -0700 Subject: [PATCH 17/88] [clang][cas] Add a test for include-tree with explicit imports With explicit imports, we do not record the module import in the include tree, because the only thing that happens is to lookup the module by name, which works already via the command-line options that provide the module via the action cache. (cherry picked from commit a13b79f23e54a598003fcd9c3cf848f474c82c22) (cherry picked from commit 232e6623d4a440ae3c4872cdd6c0b6caa14aed48) --- .../test/ClangScanDeps/modules-include-tree.c | 103 +++++++++++++++--- 1 file changed, 89 insertions(+), 14 deletions(-) diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index 75a45400f63f0..8ab36ff66ab02 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -13,6 +13,8 @@ // RUN: %deps-to-rsp %t/deps.json --module-name Top > %t/Top.rsp // RUN: %deps-to-rsp %t/deps.json --module-name Left > %t/Left.rsp // RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ZAtImport > %t/ZAtImport.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ZPragmaImport > %t/ZPragmaImport.rsp // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp // Extract include-tree casids @@ -65,10 +67,15 @@ // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK-LABEL: TRANSLATION UNIT -// CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/tu.m llvmcas://{{[[:xdigit:]]+}} // CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 (Module) Left // CHECK: 3:1 (Module) Right + +// Note: the modules with explicit imports are imported via parser and are not +// recorded in the include-tree; it's handled entirely by fmodule-map-file, +// fmodule-file, and fmodule-file-cache-key options. + // CHECK: Files: // CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} @@ -101,8 +108,6 @@ // CHECK: "-fmodule-file-cache-key" // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-x" -// CHECK-NEXT: "c" // CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Left" @@ -138,8 +143,6 @@ // CHECK: "-fmodule-file-cache-key // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-x" -// CHECK-NEXT: "c" // CHECK: "-fmodule-file=Top=[[PREFIX]]/outputs/{{.*}}/Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Right" @@ -167,8 +170,6 @@ // CHECK-NEXT: "[[TOP_TREE]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-x" -// CHECK-NEXT: "c" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Top" // CHECK: "-fno-implicit-modules" @@ -179,6 +180,58 @@ // CHECK-NEXT: ] // CHECK: "name": "Top" // CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[AT_IMPORT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/ZAtImport-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[AT_IMPORT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=ZAtImport" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/AtImport.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "ZAtImport" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[PRAGMA_IMPORT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/ZPragmaImport-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[PRAGMA_IMPORT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=ZPragmaImport" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/PragmaImport.h" +// CHECK-NEXT: "[[PREFIX]]/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "ZPragmaImport" +// CHECK: } // CHECK: ] // CHECK-NEXT: "translation-units": [ // CHECK-NEXT: { @@ -192,6 +245,12 @@ // CHECK-NEXT: { // CHECK: "module-name": "Right" // CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "ZAtImport" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "ZPragmaImport" +// CHECK: } // CHECK-NEXT: ] // CHECK: "command-line": [ // CHECK-NEXT: "-cc1" @@ -209,17 +268,15 @@ // CHECK: "-fmodule-file-cache-key" // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-x" -// CHECK-NEXT: "c" // CHECK: "-fmodule-file=Left=[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" // CHECK: "-fmodule-file=Right=[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] // CHECK: "file-deps": [ -// CHECK-NEXT: "[[PREFIX]]/tu.c" +// CHECK-NEXT: "[[PREFIX]]/tu.m" // CHECK-NEXT: ] -// CHECK: "input-file": "[[PREFIX]]/tu.c" +// CHECK: "input-file": "[[PREFIX]]/tu.m" // CHECK: } // CHECK-NEXT: ] // CHECK-NEXT: } @@ -234,12 +291,18 @@ // RUN: rm -rf %t/outputs // RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS // RUN: rm -rf %t/outputs +// RUN: %clang @%t/ZAtImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs +// RUN: %clang @%t/ZPragmaImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: rm -rf %t/outputs // RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS // Check cache hits // RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT // RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT // RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/ZAtImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/ZPragmaImport.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT // RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT // CACHE_MISS: compile job cache miss @@ -247,15 +310,17 @@ //--- cdb.json.template [{ - "file": "DIR/tu.c", + "file": "DIR/tu.m", "directory": "DIR", - "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" + "command": "clang -fsyntax-only DIR/tu.m -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" }] //--- module.modulemap module Top { header "Top.h" export *} module Left { header "Left.h" export *} module Right { header "Right.h" export *} +module ZAtImport { header "AtImport.h" } +module ZPragmaImport { header "PragmaImport.h" } //--- Top.h #pragma once @@ -272,12 +337,22 @@ void left(void); #include "Top.h" void right(void); -//--- tu.c +//--- AtImport.h +void at_import(void); + +//--- PragmaImport.h +void pragma_import(void); + +//--- tu.m #import "Left.h" #import +@import ZAtImport; +#pragma clang module import ZPragmaImport void tu(void) { top(); left(); right(); + at_import(); + pragma_import(); } From 3b02d04ef8a858a9a3aa2028d3b63a8eb6b827da Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Mon, 27 Mar 2023 09:25:21 -0700 Subject: [PATCH 18/88] [dsymutil] Add the ability to generate universal binaries with a fat64 header Add the ability to generate universal binaries with a fat64 header. rdar://107223939 Differential revision: https://reviews.llvm.org/D146879 (cherry picked from commit 4d683f7fa7d49fa5837cea7a55902dd565d32838) --- llvm/docs/CommandGuide/dsymutil.rst | 4 ++ llvm/test/tools/dsymutil/cmdline.test | 1 + llvm/test/tools/dsymutil/fat-header.test | 10 ++++ llvm/tools/dsymutil/LinkUtils.h | 3 + llvm/tools/dsymutil/MachOUtils.cpp | 9 ++- llvm/tools/dsymutil/MachOUtils.h | 2 +- llvm/tools/dsymutil/Options.td | 4 ++ llvm/tools/dsymutil/dsymutil.cpp | 72 +++++++++++++----------- 8 files changed, 70 insertions(+), 35 deletions(-) create mode 100644 llvm/test/tools/dsymutil/fat-header.test diff --git a/llvm/docs/CommandGuide/dsymutil.rst b/llvm/docs/CommandGuide/dsymutil.rst index 7ee6566cad421..d3ae16d03679e 100644 --- a/llvm/docs/CommandGuide/dsymutil.rst +++ b/llvm/docs/CommandGuide/dsymutil.rst @@ -37,6 +37,10 @@ OPTIONS Dump the *executable*'s debug-map (the list of the object files containing the debug information) in YAML format and exit. No DWARF link will take place. +.. option:: --fat64 + + Use a 64-bit header when emitting universal binaries. + .. option:: --flat, -f Produce a flat dSYM file. A ``.dwarf`` extension will be appended to the diff --git a/llvm/test/tools/dsymutil/cmdline.test b/llvm/test/tools/dsymutil/cmdline.test index 03a57f62eedbc..bf256bd4a9b70 100644 --- a/llvm/test/tools/dsymutil/cmdline.test +++ b/llvm/test/tools/dsymutil/cmdline.test @@ -8,6 +8,7 @@ HELP: Dsymutil Options: CHECK: -accelerator CHECK: -arch CHECK: -dump-debug-map +CHECK: -fat64 CHECK: -flat CHECK: -gen-reproducer CHECK: -help diff --git a/llvm/test/tools/dsymutil/fat-header.test b/llvm/test/tools/dsymutil/fat-header.test new file mode 100644 index 0000000000000..91192b2a412f7 --- /dev/null +++ b/llvm/test/tools/dsymutil/fat-header.test @@ -0,0 +1,10 @@ +REQUIRES: system-darwin + +RUN: dsymutil -oso-prepend-path %p %p/Inputs/fat-test.arm.dylib -o %t.fat32.dSYM +RUN: llvm-objdump -m --universal-headers %t.fat32.dSYM/Contents/Resources/DWARF/fat-test.arm.dylib | FileCheck %s -check-prefixes=FAT32 + +RUN: dsymutil -oso-prepend-path %p %p/Inputs/fat-test.arm.dylib -o %t.fat64.dSYM -fat64 +RUN: llvm-objdump -m --universal-headers %t.fat64.dSYM/Contents/Resources/DWARF/fat-test.arm.dylib | FileCheck %s -check-prefixes=FAT64 + +FAT32: fat_magic FAT_MAGIC +FAT64: fat_magic FAT_MAGIC_64 diff --git a/llvm/tools/dsymutil/LinkUtils.h b/llvm/tools/dsymutil/LinkUtils.h index 03b9a64b6bec5..ac7ccd6c7e529 100644 --- a/llvm/tools/dsymutil/LinkUtils.h +++ b/llvm/tools/dsymutil/LinkUtils.h @@ -49,6 +49,9 @@ struct LinkOptions { /// function. bool KeepFunctionForStatic = false; + /// Use a 64-bit header when emitting universal binaries. + bool Fat64 = false; + /// Number of threads. unsigned Threads = 1; diff --git a/llvm/tools/dsymutil/MachOUtils.cpp b/llvm/tools/dsymutil/MachOUtils.cpp index 3bf455eb4eae6..6fb01f38f5446 100644 --- a/llvm/tools/dsymutil/MachOUtils.cpp +++ b/llvm/tools/dsymutil/MachOUtils.cpp @@ -77,7 +77,8 @@ static bool runLipo(StringRef SDKPath, SmallVectorImpl &Args) { bool generateUniversalBinary(SmallVectorImpl &ArchFiles, StringRef OutputFileName, - const LinkOptions &Options, StringRef SDKPath) { + const LinkOptions &Options, StringRef SDKPath, + bool Fat64) { // No need to merge one file into a universal fat binary. if (ArchFiles.size() == 1) { if (auto E = ArchFiles.front().File->keep(OutputFileName)) { @@ -96,7 +97,7 @@ bool generateUniversalBinary(SmallVectorImpl &ArchFiles, for (auto &Thin : ArchFiles) Args.push_back(Thin.path()); - // Align segments to match dsymutil-classic alignment + // Align segments to match dsymutil-classic alignment. for (auto &Thin : ArchFiles) { Thin.Arch = getArchName(Thin.Arch); Args.push_back("-segalign"); @@ -104,6 +105,10 @@ bool generateUniversalBinary(SmallVectorImpl &ArchFiles, Args.push_back("20"); } + // Use a 64-bit fat header if requested. + if (Fat64) + Args.push_back("-fat64"); + Args.push_back("-output"); Args.push_back(OutputFileName.data()); diff --git a/llvm/tools/dsymutil/MachOUtils.h b/llvm/tools/dsymutil/MachOUtils.h index 8cc4e73e59f14..9d3581348853a 100644 --- a/llvm/tools/dsymutil/MachOUtils.h +++ b/llvm/tools/dsymutil/MachOUtils.h @@ -54,7 +54,7 @@ struct DwarfRelocationApplicationInfo { bool generateUniversalBinary(SmallVectorImpl &ArchFiles, StringRef OutputFileName, const LinkOptions &, - StringRef SDKPath); + StringRef SDKPath, bool Fat64 = false); bool generateDsymCompanion( llvm::IntrusiveRefCntPtr VFS, const DebugMap &DM, SymbolMapTranslator &Translator, MCStreamer &MS, raw_fd_ostream &OutFile, diff --git a/llvm/tools/dsymutil/Options.td b/llvm/tools/dsymutil/Options.td index d500762fc830b..8af6dbef0c3d5 100644 --- a/llvm/tools/dsymutil/Options.td +++ b/llvm/tools/dsymutil/Options.td @@ -91,6 +91,10 @@ def: Flag<["-"], "f">, HelpText<"Alias for --flat">, Group; +def fat64: F<"fat64">, + HelpText<"Use a 64-bit header when emitting universal binaries.">, + Group; + def update: F<"update">, HelpText<"Updates existing dSYM files to contain the latest accelerator tables and other DWARF optimizations.">, Group; diff --git a/llvm/tools/dsymutil/dsymutil.cpp b/llvm/tools/dsymutil/dsymutil.cpp index c54287c1e8e9a..9ea8aa0c4512f 100644 --- a/llvm/tools/dsymutil/dsymutil.cpp +++ b/llvm/tools/dsymutil/dsymutil.cpp @@ -297,6 +297,7 @@ static Expected getOptions(opt::InputArgList &Args) { Options.LinkOpts.Update = Args.hasArg(OPT_update); Options.LinkOpts.Verbose = Args.hasArg(OPT_verbose); Options.LinkOpts.Statistics = Args.hasArg(OPT_statistics); + Options.LinkOpts.Fat64 = Args.hasArg(OPT_fat64); Options.LinkOpts.KeepFunctionForStatic = Args.hasArg(OPT_keep_func_for_static); @@ -775,40 +776,47 @@ int dsymutil_main(int argc, char **argv) { return EXIT_FAILURE; if (NeedsTempFiles) { - // Universal Mach-O files can't have an archicture slice that starts - // beyond the 4GB boundary. "lipo" can creeate a 64 bit universal header, - // but not all tools can parse these files so we want to return an error - // if the file can't be encoded as a file with a 32 bit universal header. - // To detect this, we check the size of each architecture's skinny Mach-O - // file and add up the offsets. If they exceed 4GB, then we return an - // error. - - // First we compute the right offset where the first architecture will fit - // followin the 32 bit universal header. The 32 bit universal header - // starts with a uint32_t magic and a uint32_t number of architecture - // infos. Then it is followed by 5 uint32_t values for each architecture. - // So we set the start offset to the right value so we can calculate the - // exact offset that the first architecture slice can start at. - constexpr uint64_t MagicAndCountSize = 2 * 4; - constexpr uint64_t UniversalArchInfoSize = 5 * 4; - uint64_t FileOffset = MagicAndCountSize + - UniversalArchInfoSize * TempFiles.size(); - for (const auto &File: TempFiles) { - ErrorOr stat = Options.LinkOpts.VFS->status(File.path()); - if (!stat) - break; - if (FileOffset > UINT32_MAX) { - WithColor::error() << formatv( - "the universal binary has a slice with a starting offset ({0:x}) " - "that exceeds 4GB and will produce an invalid Mach-O file.", - FileOffset); - return EXIT_FAILURE; + const bool Fat64 = Options.LinkOpts.Fat64; + if (!Fat64) { + // Universal Mach-O files can't have an archicture slice that starts + // beyond the 4GB boundary. "lipo" can create a 64 bit universal + // header, but not all tools can parse these files so we want to return + // an error if the file can't be encoded as a file with a 32 bit + // universal header. To detect this, we check the size of each + // architecture's skinny Mach-O file and add up the offsets. If they + // exceed 4GB, then we return an error. + + // First we compute the right offset where the first architecture will + // fit followin the 32 bit universal header. The 32 bit universal header + // starts with a uint32_t magic and a uint32_t number of architecture + // infos. Then it is followed by 5 uint32_t values for each + // architecture. So we set the start offset to the right value so we can + // calculate the exact offset that the first architecture slice can + // start at. + constexpr uint64_t MagicAndCountSize = 2 * 4; + constexpr uint64_t UniversalArchInfoSize = 5 * 4; + uint64_t FileOffset = + MagicAndCountSize + UniversalArchInfoSize * TempFiles.size(); + for (const auto &File : TempFiles) { + ErrorOr stat = Options.LinkOpts.VFS->status(File.path()); + if (!stat) + break; + if (FileOffset > UINT32_MAX) { + WithColor::error() + << formatv("the universal binary has a slice with a starting " + "offset ({0:x}) that exceeds 4GB and will produce " + "an invalid Mach-O file. Use the -fat64 flag to " + "generate a universal binary with a 64-bit header " + "but note that not all tools support this format.", + FileOffset); + return EXIT_FAILURE; + } + FileOffset += stat->getSize(); } - FileOffset += stat->getSize(); } - if (!MachOUtils::generateUniversalBinary(TempFiles, - OutputLocationOrErr->DWARFFile, - Options.LinkOpts, SDKPath)) + if (!MachOUtils::generateUniversalBinary( + TempFiles, OutputLocationOrErr->DWARFFile, Options.LinkOpts, + SDKPath, Fat64)) return EXIT_FAILURE; } } From 4bb0e89caff37e8a2e6c5f05768117039b80e4d2 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 11:44:51 -0700 Subject: [PATCH 19/88] [clang][cas] Move include-tree and cas-fs IDs out of Module into ASTContext NFC Unlike the cache key string, these are not needed for multiple modules at once, and this allows us to store the include-tree ID for a PCH, which will be useful in the next commit. (cherry picked from commit d64e27abdd95a8d978a374ae82b14b7020eb2c4f) (cherry picked from commit 843daa725a29ba9ec45bd872e619e6276a764b4d) --- clang/include/clang/AST/ASTContext.h | 22 +++++++++++ clang/include/clang/Basic/Module.h | 25 ------------- clang/include/clang/CAS/IncludeTree.h | 2 +- clang/lib/CAS/IncludeTree.cpp | 2 +- clang/lib/Frontend/FrontendAction.cpp | 4 ++ clang/lib/Serialization/ASTReader.cpp | 4 -- clang/lib/Serialization/ASTWriter.cpp | 37 ++++++++++--------- .../CASFSActionController.cpp | 2 +- .../IncludeTreeActionController.cpp | 7 ++-- .../DependencyScanning/ModuleDepCollector.cpp | 23 +++++++----- 10 files changed, 64 insertions(+), 64 deletions(-) diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index e6f4fece8a4ad..4f62b11a05db4 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -477,6 +477,14 @@ class ASTContext : public RefCountedBase { /// For module code-gen cases, this is the top-level module we are building. Module *TopLevelModule = nullptr; + /// The include tree that is being built, if any. + /// See \c FrontendOptions::CASIncludeTreeID. + std::optional CASIncludeTreeID; + + /// The cas-fs tree that is being built, if any. + /// See \c FileSystemOptions::CASFileSystemRootID. + std::optional CASFileSystemRootID; + static constexpr unsigned ConstantArrayTypesLog2InitSize = 8; static constexpr unsigned GeneralTypesLog2InitSize = 9; static constexpr unsigned FunctionProtoTypesLog2InitSize = 12; @@ -1086,6 +1094,20 @@ class ASTContext : public RefCountedBase { /// Get module under construction, nullptr if this is not a C++20 module. Module *getModuleForCodeGen() const { return TopLevelModule; } + std::optional getCASIncludeTreeID() const { + return CASIncludeTreeID; + } + void setCASIncludeTreeID(std::string ID) { + CASIncludeTreeID = std::move(ID); + } + + std::optional getCASFileSystemRootID() const { + return CASFileSystemRootID; + } + void setCASFileSystemRootID(std::string ID) { + CASFileSystemRootID = std::move(ID); + } + TranslationUnitDecl *getTranslationUnitDecl() const { return TUDecl->getMostRecentDecl(); } diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index 8b9a93c6b0e7d..e216117194974 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -192,14 +192,6 @@ class alignas(8) Module { /// The \c ActionCache key for this module, if any. Optional ModuleCacheKey; - /// The CAS filesystem root ID for implicit modules built with the dependency - /// scanner, if any. - Optional CASFileSystemRootID; - - /// The include-tree root ID for implicit modules built with the dependency - /// scanner, if any. - Optional IncludeTreeID; - /// The top-level headers associated with this module. llvm::SmallSetVector TopHeaders; @@ -640,23 +632,6 @@ class alignas(8) Module { getTopLevelModule()->ModuleCacheKey = std::move(Key); } - Optional getCASFileSystemRootID() const { - return getTopLevelModule()->CASFileSystemRootID; - } - - void setCASFileSystemRootID(std::string ID) { - assert(!getCASFileSystemRootID() || *getCASFileSystemRootID() == ID); - getTopLevelModule()->CASFileSystemRootID = std::move(ID); - } - - Optional getIncludeTreeID() const { - return getTopLevelModule()->IncludeTreeID; - } - - void setIncludeTreeID(std::string ID) { - assert(!getIncludeTreeID() || *getIncludeTreeID() == ID); - getTopLevelModule()->IncludeTreeID = std::move(ID); - } /// Retrieve the directory for which this module serves as the /// umbrella. diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 3d136304a5989..977ff57bbcb5a 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -152,7 +152,7 @@ class IncludeTree : public IncludeTreeBase { static Expected create(ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, ObjectRef BaseFile, ArrayRef Includes, - std::optional SubmoduleName, llvm::SmallBitVector Checks); + Optional SubmoduleName, llvm::SmallBitVector Checks); static Expected get(ObjectStore &DB, ObjectRef Ref); diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 956c53dc9e546..ae6deffe2052d 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -80,7 +80,7 @@ llvm::Error IncludeTree::forEachInclude( Expected IncludeTree::create( ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, ObjectRef BaseFile, ArrayRef Includes, - std::optional SubmoduleName, llvm::SmallBitVector Checks) { + Optional SubmoduleName, llvm::SmallBitVector Checks) { // The data buffer is composed of // 1. 1 byte for `CharacteristicKind` and IsSubmodule // 2. `uint32_t` offset and `uint8_t` kind for each includes diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index 7694664195708..b0e8a724310bb 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -1067,6 +1067,10 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, } } + if (!CI.getFrontendOpts().CASIncludeTreeID.empty()) + CI.getASTContext().setCASIncludeTreeID( + CI.getFrontendOpts().CASIncludeTreeID); + CI.setASTConsumer(std::move(Consumer)); if (!CI.hasASTConsumer()) return false; diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 61e17c51af70f..cc53bbdfe2ada 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -5636,10 +5636,6 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; if (!F.ModuleCacheKey.empty()) CurrentModule->setModuleCacheKey(F.ModuleCacheKey); - if (!F.CASFileSystemRootID.empty()) - CurrentModule->setCASFileSystemRootID(F.CASFileSystemRootID); - if (!F.IncludeTreeID.empty()) - CurrentModule->setIncludeTreeID(F.IncludeTreeID); } CurrentModule->Kind = Kind; diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index a79ae19a10317..fbb0f90804752 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1353,24 +1353,25 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context, RecordData::value_type Record[] = {MODULE_CACHE_KEY}; Stream.EmitRecordWithBlob(AbbrevCode, Record, *Key); } - // CAS filesystem root id, for the scanner. - if (auto ID = WritingModule->getCASFileSystemRootID()) { - auto Abbrev = std::make_shared(); - Abbrev->Add(BitCodeAbbrevOp(CASFS_ROOT_ID)); - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); - unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); - RecordData::value_type Record[] = {CASFS_ROOT_ID}; - Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); - } - // CAS include-tree id, for the scanner. - if (auto ID = WritingModule->getIncludeTreeID()) { - auto Abbrev = std::make_shared(); - Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID)); - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); - unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); - RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID}; - Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); - } + } + + // CAS include-tree id, for the scanner. + if (auto ID = Context.getCASIncludeTreeID()) { + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(CAS_INCLUDE_TREE_ID)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {CAS_INCLUDE_TREE_ID}; + Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); + } + // CAS filesystem root id, for the scanner. + if (auto ID = Context.getCASFileSystemRootID()) { + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(CASFS_ROOT_ID)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {CASFS_ROOT_ID}; + Stream.EmitRecordWithBlob(AbbrevCode, Record, *ID); } // Imports diff --git a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp index e6cd88115ceb8..e1a6167923be3 100644 --- a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp @@ -176,7 +176,7 @@ Error CASFSActionController::finalizeModuleBuild( Module *M = ModuleScanInstance.getPreprocessor().getCurrentModule(); assert(M && "finalizing without a module"); - M->setCASFileSystemRootID(RootID->toString()); + ModuleScanInstance.getASTContext().setCASFileSystemRootID(RootID->toString()); return Error::success(); } diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 14bce2cde1e8b..ced5e93d70cec 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -95,7 +95,7 @@ class IncludeTreeBuilder { SrcMgr::CharacteristicKind FileCharacteristic; cas::ObjectRef File; SmallVector Includes; - std::optional SubmoduleName; + Optional SubmoduleName; llvm::SmallBitVector HasIncludeChecks; }; @@ -327,9 +327,8 @@ Error IncludeTreeActionController::finalizeModuleBuild( if (!Tree) return Tree.takeError(); - Module *M = ModuleScanInstance.getPreprocessor().getCurrentModule(); - assert(M && "finalizing without a module"); - M->setIncludeTreeID(Tree->getID().toString()); + ModuleScanInstance.getASTContext().setCASIncludeTreeID( + Tree->getID().toString()); return Error::success(); } diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index b3e692cec9962..d157350603a91 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -441,16 +441,6 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ImplicitModulePCMPath = std::string(M->getASTFile()->getName()); MD.IsSystem = M->IsSystem; - if (auto ID = M->getCASFileSystemRootID()) { - auto &CAS = MDC.ScanInstance.getOrCreateObjectStore(); - if (auto Err = CAS.parseID(*ID).moveInto(MD.CASFileSystemRootID)) { - MDC.ScanInstance.getDiagnostics().Report( - diag::err_cas_cannot_parse_root_id_for_module) - << *ID << MD.ID.ModuleName; - } - } - MD.IncludeTreeID = M->getIncludeTreeID(); - ModuleMap &ModMapInfo = MDC.ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); @@ -491,6 +481,19 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ModuleMapFileDeps.emplace_back(FE.getNameAsRequested()); }); + if (!MF->IncludeTreeID.empty()) + MD.IncludeTreeID = MF->IncludeTreeID; + + if (!MF->CASFileSystemRootID.empty()) { + auto &CAS = MDC.ScanInstance.getOrCreateObjectStore(); + if (auto Err = CAS.parseID(MF->CASFileSystemRootID) + .moveInto(MD.CASFileSystemRootID)) { + MDC.ScanInstance.getDiagnostics().Report( + diag::err_cas_cannot_parse_root_id_for_module) + << MF->CASFileSystemRootID << MD.ID.ModuleName; + } + } + CompilerInvocation CI = MDC.makeInvocationForModuleBuildWithoutOutputs( MD, [&](CompilerInvocation &BuildInvocation) { if (MDC.OptimizeArgs) From 734531f35dd2a716218317941092a35a03a9e045 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 11:46:33 -0700 Subject: [PATCH 20/88] [clang][cas] IncludeTree filesystem merges from imported PCH/modules Instead of looking through the PCH or modules for input files and loading them into the current FileManager while scanning, merge them directly from the include-tree that was used to build that PCH and/or module. This makes it easier to understand where files come from in the include-tree, and eliminates certain issues where the scanner module may include more files than are necessary, or include them with different paths in the case of VFS/symlinks. It also avoids needing to re-stat these files if they are not used during scanning the importing TU. As part of this, we teach IncludeTreeFilesystem to handle ./ stripping in paths to match what ASTWriter does. (cherry picked from commit 4c3f73ec729e478d2ea2c097699576b50cb29279) (cherry picked from commit fcf503d9e583722751bdf666b11f94e470416040) --- clang/include/clang/CAS/IncludeTree.h | 22 ++++++ clang/lib/CAS/IncludeTree.cpp | 24 +++++-- .../IncludeTreeActionController.cpp | 67 +++++++++++-------- .../test/ClangScanDeps/modules-include-tree.c | 2 +- 4 files changed, 81 insertions(+), 34 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 977ff57bbcb5a..07d4123da7058 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -483,4 +483,26 @@ createIncludeTreeFileSystem(IncludeTreeRoot &Root); } // namespace cas } // namespace clang +namespace llvm { +template <> struct DenseMapInfo { + using FileEntry = clang::cas::IncludeTree::FileList::FileEntry; + + static FileEntry getEmptyKey() { + return {cas::ObjectRef::getDenseMapEmptyKey(), 0}; + } + + static FileEntry getTombstoneKey() { + return {cas::ObjectRef::getDenseMapTombstoneKey(), 0}; + } + + static unsigned getHashValue(FileEntry F) { + return F.FileRef.getDenseMapHash(); + } + + static bool isEqual(FileEntry LHS, FileEntry RHS) { + return LHS.FileRef == RHS.FileRef && LHS.Size == RHS.Size; + } +}; +} // namespace llvm + #endif diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index ae6deffe2052d..bae4ae5fcb041 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -460,9 +460,8 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { Directories{Alloc}; llvm::ErrorOr status(const Twine &Path) override { - SmallString<128> FilenameBuffer; - StringRef Filename = Path.toStringRef(FilenameBuffer); - + SmallString<128> Filename; + getPath(Path, Filename); auto FileEntry = Files.find(Filename); if (FileEntry != Files.end()) { return makeStatus(Filename, FileEntry->second.Size, @@ -482,8 +481,8 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { llvm::ErrorOr> openFileForRead(const Twine &Path) override { - SmallString<128> FilenameBuffer; - StringRef Filename = Path.toStringRef(FilenameBuffer); + SmallString<128> Filename; + getPath(Path, Filename); auto MaterializedFile = materialize(Filename); if (!MaterializedFile) return llvm::errorToErrorCode(MaterializedFile.takeError()); @@ -506,6 +505,15 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { return MaterializedFile{ContentsBlob->getData(), Entry->second}; } + /// Produce a filename compatible with our StringMaps. See comment in + /// \c createIncludeTreeFileSystem. + void getPath(const Twine &Path, SmallVectorImpl &Out) { + Path.toVector(Out); + // Strip dots, but do not eliminate a path consisting only of '.' + if (Out.size() != 1) + llvm::sys::path::remove_dots(Out); + } + static llvm::vfs::Status makeStatus(StringRef Filename, uint64_t Size, llvm::sys::fs::UniqueID UniqueID, llvm::sys::fs::file_type Type) { @@ -551,7 +559,11 @@ cas::createIncludeTreeFileSystem(IncludeTreeRoot &Root) { auto FilenameBlob = File.getFilename(); if (!FilenameBlob) return FilenameBlob.takeError(); - StringRef Filename = FilenameBlob->getData(); + SmallString<128> Filename(FilenameBlob->getData()); + // Strip './' in the filename to match the behaviour of ASTWriter; we + // also strip './' in IncludeTreeFileSystem::getPath. + assert(Filename != "."); + llvm::sys::path::remove_dots(Filename); StringRef DirName = llvm::sys::path::parent_path(Filename); if (DirName.empty()) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index ced5e93d70cec..7c6e352979e6f 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -99,6 +99,7 @@ class IncludeTreeBuilder { llvm::SmallBitVector HasIncludeChecks; }; + Error addModuleInputs(ASTReader &Reader); Expected getObjectForFile(Preprocessor &PP, FileID FID); Expected getObjectForFileNonCached(FileManager &FM, const SrcMgr::FileInfo &FI); @@ -128,7 +129,7 @@ class IncludeTreeBuilder { // are recorded in the PCH, ordered by \p FileEntry::UID index. SmallVector PreIncludedFileNames; llvm::BitVector SeenIncludeFiles; - SmallVector IncludedFiles; + llvm::SetVector IncludedFiles; Optional PredefinesBufferRef; Optional ModuleIncludesBufferRef; Optional ModuleMapFileRef; @@ -494,30 +495,8 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return Error::success(); // no need for additional work. // Go through all the recorded input files. - SmallVector NotSeenIncludes; - for (serialization::ModuleFile &MF : Reader->getModuleManager()) { - if (hasErrorOccurred()) - break; - Reader->visitInputFiles( - MF, /*IncludeSystem=*/true, /*Complain=*/false, - [&](const serialization::InputFile &IF, bool isSystem) { - Optional FE = IF.getFile(); - assert(FE); - if (FE->getUID() >= SeenIncludeFiles.size() || - !SeenIncludeFiles[FE->getUID()]) - NotSeenIncludes.push_back(*FE); - }); - } - // Sort so we can visit the files in deterministic order. - llvm::sort(NotSeenIncludes, [](const FileEntry *LHS, const FileEntry *RHS) { - return LHS->getUID() < RHS->getUID(); - }); - - for (const FileEntry *FE : NotSeenIncludes) { - auto FileNode = addToFileList(FM, FE); - if (!FileNode) - return FileNode.takeError(); - } + if (Error E = addModuleInputs(*Reader)) + return E; PreprocessorOptions &PPOpts = NewInvocation.getPreprocessorOpts(); if (PPOpts.ImplicitPCHInclude.empty()) @@ -543,7 +522,8 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, getCASTreeForFileIncludes(IncludeStack.pop_back_val()); if (!MainIncludeTree) return MainIncludeTree.takeError(); - auto FileList = cas::IncludeTree::FileList::create(DB, IncludedFiles); + auto FileList = + cas::IncludeTree::FileList::create(DB, IncludedFiles.getArrayRef()); if (!FileList) return FileList.takeError(); @@ -552,6 +532,39 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, ModuleMapFileRef); } +Error IncludeTreeBuilder::addModuleInputs(ASTReader &Reader) { + for (serialization::ModuleFile &MF : Reader.getModuleManager()) { + // Only add direct imports to avoid duplication. Each include tree is a + // superset of its imported modules' include trees. + if (!MF.isDirectlyImported()) + continue; + + assert(!MF.IncludeTreeID.empty() && "missing include-tree for import"); + + Optional ID; + if (Error E = DB.parseID(MF.IncludeTreeID).moveInto(ID)) + return E; + Optional Ref = DB.getReference(*ID); + if (!Ref) + return DB.createUnknownObjectError(*ID); + Optional Root; + if (Error E = cas::IncludeTreeRoot::get(DB, *Ref).moveInto(Root)) + return E; + Optional Files; + if (Error E = Root->getFileList().moveInto(Files)) + return E; + + Error E = Files->forEachFile([&](auto IF, auto Size) -> Error { + IncludedFiles.insert({IF.getRef(), Size}); + return Error::success(); + }); + if (E) + return E; + } + + return Error::success(); +} + Expected IncludeTreeBuilder::getObjectForFile(Preprocessor &PP, FileID FID) { SourceManager &SM = PP.getSourceManager(); @@ -627,7 +640,7 @@ IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { auto FileNode = createIncludeFile(Filename, **CASContents); if (!FileNode) return FileNode.takeError(); - IncludedFiles.push_back( + IncludedFiles.insert( {FileNode->getRef(), static_cast(FE->getSize())}); return FileNode->getRef(); diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index 8ab36ff66ab02..e09c564231db5 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -77,8 +77,8 @@ // fmodule-file, and fmodule-file-cache-key options. // CHECK: Files: -// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} From 1fa976b305dcddc31da6083129cbf089ee945e01 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 22 Mar 2023 16:59:04 -0700 Subject: [PATCH 21/88] [clang][cas] Create explicit IncludeTree::ModuleMap Instead of putting the modulemap file in the include-tree filesystem and parsing it at build time, create a data structure that represents just the parts of the modulemap we need when building the module: * Flags (explicit, system, framework, etc.) * Exports (export *, export Foo) * LinkLibraries (implicit framework autolink, link Foo) Additionally, we add modular headers lazily by inserting known headers when we encounter an include-tree header that is part of a submodule (this is needed for missing #include diagnostics). This removes the possibility of mismatches between header paths seen during modulemap parsing from the paths we embed due to #includes, for exmaple due to VFS virtual vs external paths. (cherry picked from commit a237e688354853ad782594d6158c2d28e26f8d01) Conflicts: clang/include/clang/Basic/Module.h clang/lib/Serialization/ASTWriter.cpp (cherry picked from commit c6b9298dfaa4a9a39ac63346c75e7d6123f3082f) --- .../clang/Basic/DiagnosticFrontendKinds.td | 2 + clang/include/clang/Basic/Module.h | 8 +- clang/include/clang/CAS/IncludeTree.h | 262 ++++++++++++- .../include/clang/Frontend/FrontendOptions.h | 4 + clang/lib/AST/Decl.cpp | 1 + clang/lib/CAS/IncludeTree.cpp | 343 ++++++++++++++++-- clang/lib/Frontend/CompilerInvocation.cpp | 51 +-- clang/lib/Frontend/FrontendAction.cpp | 148 +++++++- clang/lib/Frontend/FrontendActions.cpp | 2 + clang/lib/Frontend/IncludeTreePPActions.cpp | 6 + clang/lib/Sema/SemaModule.cpp | 1 + clang/lib/Serialization/ASTReaderDecl.cpp | 3 +- .../IncludeTreeActionController.cpp | 109 +++++- .../DependencyScanning/ScanAndUpdateArgs.cpp | 1 + .../modules-include-tree-implementation.c | 5 +- .../modules-include-tree-with-pch.c | 2 +- .../test/ClangScanDeps/modules-include-tree.c | 30 +- 17 files changed, 876 insertions(+), 102 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 6f76ef2b42a90..81877ea252a7b 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -229,6 +229,8 @@ def err_missing_module_name : Error< DefaultFatal; def err_missing_module : Error< "no module named '%0' declared in module map file '%1'">, DefaultFatal; +def err_missing_module_include_tree : Error< + "no module named '%0' declared in include-tree module map '%1'">, DefaultFatal; def err_no_submodule : Error<"no submodule named %0 in module '%1'">; def err_no_submodule_suggest : Error< "no submodule named %0 in module '%1'; did you mean '%2'?">; diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index e216117194974..99f4f71a7b449 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -108,6 +108,10 @@ class alignas(8) Module { /// of header files. ModuleMapModule, + /// This is a module that was defined by a module map and built out + /// of header files as part of an \c IncludeTree. + IncludeTreeModuleMap, + /// This is a C++20 module interface unit. ModuleInterfaceUnit, @@ -175,7 +179,9 @@ class alignas(8) Module { bool isPrivateModule() const { return Kind == PrivateModuleFragment; } - bool isModuleMapModule() const { return Kind == ModuleMapModule; } + bool isModuleMapModule() const { + return Kind == ModuleMapModule || Kind == IncludeTreeModuleMap; + } private: /// The submodules of this module, indexed by name. diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 07d4123da7058..9c9723fd3bc90 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -54,7 +54,9 @@ class IncludeTree : public IncludeTreeBase { class File; class FileList; class Node; + class Module; class ModuleImport; + class ModuleMap; Expected getBaseFile(); @@ -381,6 +383,262 @@ class IncludeTree::Node { NodeKind K; }; +/// Module or submodule declaration as part of the \c IncludeTreeRoot module +/// map structure. +class IncludeTree::Module : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "Modu"; } + + class ExportList; + class LinkLibraryList; + + struct ModuleFlags { + bool IsFramework : 1; + bool IsExplicit : 1; + bool IsExternC : 1; + bool IsSystem : 1; + ModuleFlags() + : IsFramework(false), IsExplicit(false), IsExternC(false), + IsSystem(false) {} + }; + + ModuleFlags getFlags() const; + + /// The name of the current (sub)module. + StringRef getName() const { return dataAfterFlags(); } + + size_t getNumSubmodules() const; + + Expected getSubmodule(size_t I) { + assert(I < getNumSubmodules()); + auto Node = getCAS().getProxy(getReference(I)); + if (!Node) + return Node.takeError(); + return Module(*Node); + } + + llvm::Error forEachSubmodule(llvm::function_ref CB); + + std::optional getExportsRef() const { + if (std::optional Index = getExportsIndex()) + return getReference(*Index); + return std::nullopt; + } + std::optional getLinkLibrariesRef() const { + if (std::optional Index = getLinkLibrariesIndex()) + return getReference(*Index); + return std::nullopt; + } + + /// The list of modules that this submodule re-exports. + Expected> getExports(); + + /// The list of modules that this submodule re-exports. + Expected> getLinkLibraries(); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, StringRef ModuleName, + ModuleFlags Flags, + ArrayRef Submodules, + std::optional ExportList, + std::optional LinkLibraries); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getData().size() > 1; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + char rawFlags() const { return getData()[0]; } + StringRef dataAfterFlags() const { return getData().drop_front(); } + bool hasExports() const; + bool hasLinkLibraries() const; + std::optional getExportsIndex() const; + std::optional getLinkLibrariesIndex() const; + + explicit Module(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class IncludeTreeRoot; + friend class ModuleMap; +}; + +/// The set of modules re-exported by another module. +class IncludeTree::Module::ExportList : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ExpL"; } + + /// An explicit export. + struct Export { + StringRef ModuleName; + bool Wildcard; + }; + + /// Whether this module exports all imported modules (`export *`). + bool hasGlobalWildcard() const; + + size_t getNumExplicitExports() const { return getNumReferences(); } + + /// Whether the explicit export at \p I has a wildcard (`export MyModule.*`). + bool exportHasWildcard(size_t I) const; + + Expected getExplicitExport(size_t I); + + /// Calls \p CB for each explicit export declaration. + llvm::Error forEachExplicitExport(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, ArrayRef Exports, + bool GlobalWildcard); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + size_t ExpectedBitSize = Base.getNumReferences() + 1; + size_t ExpectedSize = + ExpectedBitSize / 8 + (ExpectedBitSize % 8 == 0 ? 0 : 1); + return Base.getData().size() == ExpectedSize; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit ExportList(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class Module; +}; + +/// The set of libraries to link against when using a module. +class IncludeTree::Module::LinkLibraryList + : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "LnkL"; } + + struct LinkLibrary { + StringRef Library; + bool IsFramework; + }; + + size_t getNumLibraries() const { return getNumReferences(); } + + /// Whether the library at \p I is a framework. + bool isFramework(size_t I) const; + + ObjectRef getLibraryNameRef(size_t I) const { return getReference(I); } + + Expected getLinkLibrary(size_t I) { + auto Name = getCAS().getProxy(getLibraryNameRef(I)); + if (!Name) + return Name.takeError(); + return LinkLibrary{Name->getData(), isFramework(I)}; + } + + /// Calls \p CB for each link libary. + llvm::Error + forEachLinkLibrary(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, + ArrayRef Exports); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + size_t ExpectedBitSize = Base.getNumReferences(); + size_t ExpectedSize = + ExpectedBitSize / 8 + (ExpectedBitSize % 8 == 0 ? 0 : 1); + return Base.getData().size() == ExpectedSize; + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit LinkLibraryList(ObjectProxy Node) + : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class Module; +}; + +class IncludeTree::ModuleMap : public IncludeTreeBase { +public: + static constexpr StringRef getNodeKind() { return "ModM"; } + + size_t getNumModules() const { return getNumReferences(); } + + Expected getModule(size_t I) { + auto Node = getCAS().getProxy(getReference(I)); + if (!Node) + return Node.takeError(); + return Module(*Node); + } + + /// Calls \p CB for each module declaration. + llvm::Error forEachModule(llvm::function_ref CB); + + llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); + + static Expected create(ObjectStore &DB, + ArrayRef Modules); + + static bool isValid(const ObjectProxy &Node) { + if (!IncludeTreeBase::isValid(Node)) + return false; + IncludeTreeBase Base(Node); + return Base.getData().empty(); + } + static bool isValid(ObjectStore &DB, ObjectRef Ref) { + auto Node = DB.getProxy(Ref); + if (!Node) { + llvm::consumeError(Node.takeError()); + return false; + } + return isValid(*Node); + } + +private: + explicit ModuleMap(ObjectProxy Node) : IncludeTreeBase(std::move(Node)) { + assert(isValid(*this)); + } + + friend class IncludeTreeBase; + friend class IncludeTreeRoot; +}; + /// Represents the include-tree result for a translation unit. class IncludeTreeRoot : public IncludeTreeBase { public: @@ -426,12 +684,12 @@ class IncludeTreeRoot : public IncludeTreeBase { return None; } - Expected> getModuleMapFile() { + Expected> getModuleMap() { if (Optional Ref = getModuleMapRef()) { auto Node = getCAS().getProxy(*Ref); if (!Node) return Node.takeError(); - return IncludeTree::File(*Node); + return IncludeTree::ModuleMap(*Node); } return std::nullopt; } diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h index 5554c2992ba42..7ab469a797e18 100644 --- a/clang/include/clang/Frontend/FrontendOptions.h +++ b/clang/include/clang/Frontend/FrontendOptions.h @@ -248,6 +248,10 @@ class FrontendInputFile { bool IsSystem = false) : File(File.str()), IncludeTree(std::move(Tree)), Kind(Kind), IsSystem(IsSystem) {} + FrontendInputFile(cas::ObjectRef Tree, llvm::MemoryBufferRef Buffer, + InputKind Kind, bool IsSystem = false) + : Buffer(Buffer), IncludeTree(std::move(Tree)), Kind(Kind), + IsSystem(IsSystem) {} InputKind getKind() const { return Kind; } bool isSystem() const { return IsSystem; } diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 151826f0d9c2b..4584111df0256 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -1565,6 +1565,7 @@ Module *Decl::getOwningModuleForLinkage(bool IgnoreLinkage) const { switch (M->Kind) { case Module::ModuleMapModule: + case Module::IncludeTreeModuleMap: // Module map modules have no special linkage semantics. return nullptr; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index bae4ae5fcb041..dae8c92e52901 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -77,6 +77,33 @@ llvm::Error IncludeTree::forEachInclude( }); } +/// Write the bitset \p Bits to \p Writer, filling the final byte with zeros for +/// any unused values. Note: this does not store the size of the bitset. +static void writeBitSet(llvm::support::endian::Writer &Writer, + const llvm::SmallBitVector &Bits) { + uintptr_t Store; + ArrayRef BitWords = Bits.getData(Store); + size_t RemainingBitsCount = Bits.size(); + while (RemainingBitsCount > 0) { + if (BitWords.size() > 1) { + Writer.write(BitWords.front()); + BitWords = BitWords.drop_front(); + RemainingBitsCount -= sizeof(uintptr_t) * CHAR_BIT; + continue; + } + assert(RemainingBitsCount <= sizeof(uintptr_t) * CHAR_BIT); + uintptr_t LastWord = BitWords.front(); + unsigned BytesNum = RemainingBitsCount / CHAR_BIT; + if (RemainingBitsCount % CHAR_BIT != 0) + ++BytesNum; + while (BytesNum--) { + Writer.write(static_cast(LastWord & 0xFF)); + LastWord >>= CHAR_BIT; + } + break; + } +} + Expected IncludeTree::create( ObjectStore &DB, SrcMgr::CharacteristicKind FileCharacteristic, ObjectRef BaseFile, ArrayRef Includes, @@ -118,27 +145,7 @@ Expected IncludeTree::create( if (SubmoduleName) Refs.push_back(*SubmoduleName); - uintptr_t Store; - ArrayRef BitWords = Checks.getData(Store); - size_t RemainingBitsCount = Checks.size(); - while (RemainingBitsCount > 0) { - if (BitWords.size() > 1) { - Writer.write(BitWords.front()); - BitWords = BitWords.drop_front(); - RemainingBitsCount -= sizeof(uintptr_t) * CHAR_BIT; - continue; - } - assert(RemainingBitsCount <= sizeof(uintptr_t) * CHAR_BIT); - uintptr_t LastWord = BitWords.front(); - unsigned BytesNum = RemainingBitsCount / CHAR_BIT; - if (RemainingBitsCount % CHAR_BIT != 0) - ++BytesNum; - while (BytesNum--) { - Buffer.push_back(LastWord & 0xFF); - LastWord >>= CHAR_BIT; - } - break; - } + writeBitSet(Writer, Checks); return IncludeTreeBase::create(DB, Refs, Buffer); } @@ -268,6 +275,236 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { Base.getData().size() == NumFiles * sizeof(FileSizeTy); } +static constexpr char ModuleFlagFramework = 1 << 0; +static constexpr char ModuleFlagExplicit = 1 << 1; +static constexpr char ModuleFlagExternC = 1 << 2; +static constexpr char ModuleFlagSystem = 1 << 3; +static constexpr char ModuleFlagHasExports = 1 << 4; +static constexpr char ModuleFlagHasLinkLibraries = 1 << 5; + +IncludeTree::Module::ModuleFlags IncludeTree::Module::getFlags() const { + char Raw = rawFlags(); + ModuleFlags Flags; + Flags.IsFramework = Raw & ModuleFlagFramework; + Flags.IsExplicit = Raw & ModuleFlagExplicit; + Flags.IsExternC = Raw & ModuleFlagExternC; + Flags.IsSystem = Raw & ModuleFlagSystem; + return Flags; +} + +size_t IncludeTree::Module::getNumSubmodules() const { + size_t Count = getNumReferences(); + if (hasExports()) + Count -= 1; + if (hasLinkLibraries()) + Count -= 1; + return Count; +} + +llvm::Error IncludeTree::Module::forEachSubmodule( + llvm::function_ref CB) { + size_t Count = getNumSubmodules(); + return forEachReference([&](ObjectRef Ref) -> llvm::Error { + if (Count == 0) + return llvm::Error::success(); + Count -= 1; + auto Node = getCAS().getProxy(Ref); + if (!Node) + return Node.takeError(); + return CB(Module(*Node)); + }); +} + +Expected +IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, + ModuleFlags Flags, ArrayRef Submodules, + std::optional ExportList, + std::optional LinkLibraries) { + // Data: + // - 1 byte for Flags + // - ModuleName (String) + // Refs: + // - Submodules (IncludeTreeModule) + // - (optional) ExportList + // - (optional) LinkLibaryList + + char RawFlags = 0; + if (Flags.IsFramework) + RawFlags |= ModuleFlagFramework; + if (Flags.IsExplicit) + RawFlags |= ModuleFlagExplicit; + if (Flags.IsExternC) + RawFlags |= ModuleFlagExternC; + if (Flags.IsSystem) + RawFlags |= ModuleFlagSystem; + if (ExportList) + RawFlags |= ModuleFlagHasExports; + if (LinkLibraries) + RawFlags |= ModuleFlagHasLinkLibraries; + + SmallString<64> Buffer; + Buffer.push_back(RawFlags); + Buffer.append(ModuleName); + + SmallVector Refs(Submodules); + if (ExportList) + Refs.push_back(*ExportList); + if (LinkLibraries) + Refs.push_back(*LinkLibraries); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +bool IncludeTree::Module::hasExports() const { + return rawFlags() & ModuleFlagHasExports; +} +bool IncludeTree::Module::hasLinkLibraries() const { + return rawFlags() & ModuleFlagHasLinkLibraries; +} + +std::optional IncludeTree::Module::getExportsIndex() const { + if (hasExports()) + return getNumReferences() - (hasLinkLibraries() ? 2 : 1); + return std::nullopt; +} +std::optional IncludeTree::Module::getLinkLibrariesIndex() const { + if (hasLinkLibraries()) + return getNumReferences() - 1; + return std::nullopt; +} + +Expected> +IncludeTree::Module::getExports() { + if (auto Ref = getExportsRef()) { + auto N = getCAS().getProxy(*Ref); + if (!N) + return N.takeError(); + return ExportList(std::move(*N)); + } + return std::nullopt; +} + +/// The list of modules that this submodule re-exports. +Expected> +IncludeTree::Module::getLinkLibraries() { + if (auto Ref = getLinkLibrariesRef()) { + auto N = getCAS().getProxy(*Ref); + if (!N) + return N.takeError(); + return LinkLibraryList(std::move(*N)); + } + return std::nullopt; +} + +bool IncludeTree::Module::ExportList::hasGlobalWildcard() const { + // The bit after explicit exports is global. + return exportHasWildcard(getNumExplicitExports()); +} +bool IncludeTree::Module::ExportList::exportHasWildcard(size_t I) const { + assert(I < getNumExplicitExports() + 1); + unsigned ByteIndex = I / CHAR_BIT; + size_t RemainingIndex = I % CHAR_BIT; + uint8_t Bits = getData()[ByteIndex]; + return Bits & (1 << RemainingIndex); +} +Expected +IncludeTree::Module::ExportList::getExplicitExport(size_t I) { + Expected Name = getCAS().getProxy(getReference(I)); + if (!Name) + return Name.takeError(); + return Export{Name->getData(), exportHasWildcard(I)}; +} +llvm::Error IncludeTree::Module::ExportList::forEachExplicitExport( + llvm::function_ref CB) { + size_t ExportI = 0; + return forEachReference([&](ObjectRef Ref) { + Expected Name = getCAS().getProxy(Ref); + if (!Name) + return Name.takeError(); + return CB(Export{Name->getData(), exportHasWildcard(ExportI)}); + }); +} +Expected +IncludeTree::Module::ExportList::create(ObjectStore &DB, + ArrayRef Exports, + bool GlobalWildcard) { + // Data: + // - 1 bit per explicit export for wildcard + // - 1 bit for global wildcard + // Refs: export names + SmallString<64> Buffer; + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + SmallVector Refs; + llvm::SmallBitVector WildcardBits; + for (Export E : Exports) { + auto Ref = DB.storeFromString({}, E.ModuleName); + if (!Ref) + return Ref.takeError(); + Refs.push_back(*Ref); + WildcardBits.push_back(E.Wildcard); + } + WildcardBits.push_back(GlobalWildcard); + writeBitSet(Writer, WildcardBits); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +bool IncludeTree::Module::LinkLibraryList::isFramework(size_t I) const { + assert(I < getNumLibraries()); + unsigned ByteIndex = I / CHAR_BIT; + size_t RemainingIndex = I % CHAR_BIT; + uint8_t Bits = getData()[ByteIndex]; + return Bits & (1 << RemainingIndex); +} +llvm::Error IncludeTree::Module::LinkLibraryList::forEachLinkLibrary( + llvm::function_ref CB) { + size_t I = 0; + return forEachReference([&](ObjectRef Ref) { + auto Name = getCAS().getProxy(getLibraryNameRef(I)); + if (!Name) + return Name.takeError(); + return CB({Name->getData(), isFramework(I++)}); + }); +} +Expected +IncludeTree::Module::LinkLibraryList::create(ObjectStore &DB, + ArrayRef Libraries) { + // Data: + // - 1 bit per library for IsFramework + // Refs: library names + SmallString<64> Buffer; + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + SmallVector Refs; + llvm::SmallBitVector FrameworkBits; + for (LinkLibrary L : Libraries) { + auto Ref = DB.storeFromString({}, L.Library); + if (!Ref) + return Ref.takeError(); + Refs.push_back(*Ref); + FrameworkBits.push_back(L.IsFramework); + } + writeBitSet(Writer, FrameworkBits); + + return IncludeTreeBase::create(DB, Refs, Buffer); +} + +Expected +IncludeTree::ModuleMap::create(ObjectStore &DB, ArrayRef Modules) { + return IncludeTreeBase::create(DB, Modules, {}); +} + +llvm::Error IncludeTree::ModuleMap::forEachModule( + llvm::function_ref CB) { + return forEachReference([&](ObjectRef Ref) { + auto N = getCAS().getProxy(Ref); + if (!N) + return N.takeError(); + return CB(Module(std::move(*N))); + }); +} + static constexpr char HasPCH = 0x01; static constexpr char HasModuleMap = 0x02; @@ -277,7 +514,7 @@ IncludeTreeRoot::create(ObjectStore &DB, ObjectRef MainFileTree, Optional ModuleMapRef) { assert(IncludeTree::isValid(DB, MainFileTree)); assert(IncludeTree::FileList::isValid(DB, FileList)); - assert(!ModuleMapRef || IncludeTree::File::isValid(DB, *ModuleMapRef)); + assert(!ModuleMapRef || IncludeTree::ModuleMap::isValid(DB, *ModuleMapRef)); std::array Data = {0}; if (PCHRef) @@ -376,6 +613,62 @@ llvm::Error IncludeTree::Node::print(llvm::raw_ostream &OS, unsigned Indent) { } } +llvm::Error IncludeTree::Module::print(llvm::raw_ostream &OS, unsigned Indent) { + OS.indent(Indent) << getName(); + ModuleFlags Flags = getFlags(); + if (Flags.IsFramework) + OS << " (framework)"; + if (Flags.IsExplicit) + OS << " (explicit)"; + if (Flags.IsExternC) + OS << " (extern_c)"; + if (Flags.IsSystem) + OS << " (system)"; + OS << '\n'; + auto ExportList = getExports(); + if (!ExportList) + return ExportList.takeError(); + if (*ExportList) + if (llvm::Error E = (*ExportList)->print(OS, Indent + 2)) + return E; + auto LinkLibraries = getLinkLibraries(); + if (!LinkLibraries) + return LinkLibraries.takeError(); + if (*LinkLibraries) + if (llvm::Error E = (*LinkLibraries)->print(OS, Indent + 2)) + return E; + return forEachSubmodule( + [&](Module Sub) { return Sub.print(OS, Indent + 2); }); +} +llvm::Error IncludeTree::Module::ExportList::print(llvm::raw_ostream &OS, + unsigned Indent) { + if (hasGlobalWildcard()) + OS.indent(Indent) << "export *\n"; + return forEachExplicitExport([&](Export E) { + OS.indent(Indent) << "export " << E.ModuleName; + if (E.Wildcard) + OS << ".*"; + OS << '\n'; + return llvm::Error::success(); + }); +} + +llvm::Error IncludeTree::Module::LinkLibraryList::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachLinkLibrary([&](LinkLibrary E) { + OS.indent(Indent) << "link " << E.Library; + if (E.IsFramework) + OS << " (framework)"; + OS << '\n'; + return llvm::Error::success(); + }); +} + +llvm::Error IncludeTree::ModuleMap::print(llvm::raw_ostream &OS, + unsigned Indent) { + return forEachModule([&](Module M) { return M.print(OS, Indent); }); +} + llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { if (Optional PCHRef = getPCHRef()) { OS.indent(Indent) << "(PCH) "; @@ -387,11 +680,11 @@ llvm::Error IncludeTreeRoot::print(llvm::raw_ostream &OS, unsigned Indent) { return E; if (llvm::Error E = MainTree->print(OS.indent(Indent), Indent)) return E; - Optional ModuleMap; - if (llvm::Error E = getModuleMapFile().moveInto(ModuleMap)) + Optional ModuleMap; + if (llvm::Error E = getModuleMap().moveInto(ModuleMap)) return E; if (ModuleMap) { - OS.indent(Indent) << "Module Map: "; + OS.indent(Indent) << "Module Map:\n"; if (llvm::Error E = ModuleMap->print(OS, Indent)) return E; } diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index ecf1848f86738..6ff91921007fa 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -3019,7 +3019,8 @@ static void GenerateFrontendArgs(const FrontendOptions &Opts, static void determineInputFromIncludeTree( StringRef IncludeTreeID, CASOptions &CASOpts, DiagnosticsEngine &Diags, - Optional &IncludeTree, StringRef &InputFilename) { + Optional &IncludeTree, + Optional &Buffer, StringRef &InputFilename) { assert(!IncludeTreeID.empty()); auto reportError = [&](llvm::Error &&E) { Diags.Report(diag::err_fe_unable_to_load_include_tree) @@ -3037,28 +3038,27 @@ static void determineInputFromIncludeTree( auto Root = cas::IncludeTreeRoot::get(*CAS, *Object); if (!Root) return reportError(Root.takeError()); - - Optional BaseFile; - - auto MaybeModuleMap = Root->getModuleMapFile(); - if (!MaybeModuleMap) - return reportError(MaybeModuleMap.takeError()); - if (*MaybeModuleMap) { - // Building a module from a modulemap, the modulemap is the primary input. - BaseFile = *MaybeModuleMap; - } else { - auto MainTree = Root->getMainFileTree(); - if (!MainTree) - return reportError(MainTree.takeError()); - if (llvm::Error E = MainTree->getBaseFile().moveInto(BaseFile)) - return reportError(std::move(E)); - } - + auto MainTree = Root->getMainFileTree(); + if (!MainTree) + return reportError(MainTree.takeError()); + auto BaseFile = MainTree->getBaseFile(); + if (!BaseFile) + return reportError(BaseFile.takeError()); auto FilenameBlob = BaseFile->getFilename(); if (!FilenameBlob) return reportError(FilenameBlob.takeError()); + InputFilename = FilenameBlob->getData(); IncludeTree = *Root; + + if (InputFilename != Module::getModuleInputBufferName()) + return; + + // Handle buffer + auto Contents = BaseFile->getContents(); + if (!Contents) + return reportError(Contents.takeError()); + Buffer = llvm::MemoryBufferRef(Contents->getData(), InputFilename); } static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, @@ -3286,13 +3286,14 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, Opts.Inputs.clear(); Optional Tree; + Optional TreeInputBuffer; if (!Opts.CASIncludeTreeID.empty()) { if (!Inputs.empty()) { Diags.Report(diag::err_drv_inputs_and_include_tree); } StringRef InputFilename; determineInputFromIncludeTree(Opts.CASIncludeTreeID, CASOpts, Diags, Tree, - InputFilename); + TreeInputBuffer, InputFilename); if (!InputFilename.empty()) Inputs.push_back(InputFilename.str()); } @@ -3331,8 +3332,16 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, if (Tree) { FrontendInputFile &InputFile = Opts.Inputs.back(); - InputFile = FrontendInputFile(Tree->getRef(), InputFile.getFile(), - InputFile.getKind(), InputFile.isSystem()); + if (TreeInputBuffer) { + // This is automatically set to modulemap when building a module; revert + // to a source file for the module includes buffer. + auto Kind = InputFile.getKind().withFormat(InputKind::Source); + InputFile = FrontendInputFile(Tree->getRef(), *TreeInputBuffer, Kind, + InputFile.isSystem()); + } else { + InputFile = FrontendInputFile(Tree->getRef(), InputFile.getFile(), + InputFile.getKind(), InputFile.isSystem()); + } } Opts.DashX = DashX; diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index b0e8a724310bb..f0259f591aa5b 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -546,6 +546,116 @@ static Module *prepareToBuildModule(CompilerInstance &CI, return M; } +static Expected makeIncludeTreeModule(CompilerInstance &CI, + cas::IncludeTree::Module Mod, + Module *Parent) { + ModuleMap &MMap = CI.getPreprocessor().getHeaderSearchInfo().getModuleMap(); + auto Flags = Mod.getFlags(); + Module *M = nullptr; + bool NewModule = false; + std::tie(M, NewModule) = MMap.findOrCreateModule( + Mod.getName(), Parent, Flags.IsFramework, Flags.IsExplicit); + assert(NewModule); + M->Kind = Module::IncludeTreeModuleMap; + M->IsExternC = Flags.IsExternC; + M->IsSystem = Flags.IsSystem; + + auto ExportList = Mod.getExports(); + if (!ExportList) + return ExportList.takeError(); + if (*ExportList) { + if ((*ExportList)->hasGlobalWildcard()) + M->Exports.push_back(Module::ExportDecl(nullptr, true)); + + llvm::Error Err = (*ExportList)->forEachExplicitExport([&](auto Export) { + Module::UnresolvedExportDecl UED; + SmallVector ModuleComponents; + Export.ModuleName.split(ModuleComponents, '.'); + for (StringRef Name : ModuleComponents) + UED.Id.push_back({std::string(Name), SourceLocation()}); + UED.Wildcard = Export.Wildcard; + M->UnresolvedExports.push_back(std::move(UED)); + return llvm::Error::success(); + }); + if (Err) + return std::move(Err); + } + + auto LinkLibs = Mod.getLinkLibraries(); + if (!LinkLibs) + return LinkLibs.takeError(); + if (*LinkLibs) { + llvm::Error Err = (*LinkLibs)->forEachLinkLibrary([&](auto LL) { + M->LinkLibraries.emplace_back(std::string(LL.Library), LL.IsFramework); + return llvm::Error::success(); + }); + if (Err) + return std::move(Err); + } + + llvm::Error Err = Mod.forEachSubmodule([&](cas::IncludeTree::Module Sub) { + return makeIncludeTreeModule(CI, Sub, M).takeError(); + }); + if (Err) + return std::move(Err); + return M; +} + +/// Loads the include tree modulemap \p MM. +/// \returns true if there was an error. +static bool loadIncludeTreeModuleMap(CompilerInstance &CI, + cas::IncludeTree::ModuleMap MM) { + llvm::Error Err = MM.forEachModule([&](auto M) -> llvm::Error { + return makeIncludeTreeModule(CI, M, /*Parent=*/nullptr).takeError(); + }); + if (Err) { + CI.getDiagnostics().Report(diag::err_fe_unable_to_load_include_tree) + << CI.getFrontendOpts().CASIncludeTreeID << std::move(Err); + return true; + } + return false; +} + +static Module *prepareToBuildModule(CompilerInstance &CI, + cas::IncludeTree::ModuleMap MM) { + if (CI.getLangOpts().CurrentModule.empty()) { + CI.getDiagnostics().Report(diag::err_missing_module_name); + + // FIXME: Eventually, we could consider asking whether there was just + // a single module described in the module map, and use that as a + // default. Then it would be fairly trivial to just "compile" a module + // map with a single module (the common case). + return nullptr; + } + + if (loadIncludeTreeModuleMap(CI, MM)) + return nullptr; + + // Dig out the module definition. + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + Module *M = HS.lookupModule(CI.getLangOpts().CurrentModule, SourceLocation(), + /*AllowSearch=*/false); + if (!M) { + CI.getDiagnostics().Report(diag::err_missing_module_include_tree) + << CI.getLangOpts().CurrentModule + << CI.getFrontendOpts().CASIncludeTreeID; + + return nullptr; + } + + if (auto CacheKey = CI.getCompileJobCacheKey()) + M->setModuleCacheKey(CacheKey->toString()); + + // If we're being run from the command-line, the module build stack will not + // have been filled in yet, so complete it now in order to allow us to detect + // module cycles. + SourceManager &SourceMgr = CI.getSourceManager(); + if (SourceMgr.getModuleBuildStack().empty()) + SourceMgr.pushModuleBuildStack(CI.getLangOpts().CurrentModule, + FullSourceLoc(SourceLocation(), SourceMgr)); + return M; +} + /// Compute the input buffer that should be used to build the specified module. static std::unique_ptr getInputBufferForModule(CompilerInstance &CI, Module *M) { @@ -882,6 +992,21 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, if (llvm::Error E = IncludeTreeRoot->getPCHBuffer().moveInto(IncludeTreePCHBuffer)) return reportError(std::move(E)); + + auto ModMap = IncludeTreeRoot->getModuleMap(); + if (!ModMap) + return reportError(ModMap.takeError()); + if (*ModMap) { + if (CI.getFrontendOpts().ProgramAction == frontend::GenerateModule) { + auto *CurrentModule = prepareToBuildModule(CI, **ModMap); + if (!CurrentModule) + return false; + CI.getLangOpts().setCompilingModule(LangOptions::CMK_ModuleMap); + } else { + if (loadIncludeTreeModuleMap(CI, **ModMap)) + return false; + } + } } if (!CI.InitializeSourceManager(Input)) @@ -924,25 +1049,10 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // If the module contents are in the same file, skip to them. CI.getPreprocessor().setSkipMainFilePreamble(OffsetToContents, true); else { - std::unique_ptr Buffer; - if (Input.isIncludeTree()) { - // Get the existing module include buffer from the include-tree. - assert(IncludeTreeRoot); - auto MainTree = IncludeTreeRoot->getMainFileTree(); - if (!MainTree) - return reportError(MainTree.takeError()); - auto BaseFile = MainTree->getBaseFile(); - if (!BaseFile) - return reportError(BaseFile.takeError()); - if (auto E = BaseFile->getMemoryBuffer().moveInto(Buffer)) - return reportError(std::move(E)); - assert(Buffer); - } else { - // Otherwise, convert the module description to a suitable input buffer. - Buffer = getInputBufferForModule(CI, CurrentModule); - if (!Buffer) - return false; - } + // Otherwise, convert the module description to a suitable input buffer. + auto Buffer = getInputBufferForModule(CI, CurrentModule); + if (!Buffer) + return false; // Reinitialize the main file entry to refer to the new input. auto Kind = CurrentModule->IsSystem ? SrcMgr::C_System : SrcMgr::C_User; diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index e86c683c204b4..aa1c048e94ee6 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -831,6 +831,8 @@ static StringRef ModuleKindName(Module::ModuleKind MK) { switch (MK) { case Module::ModuleMapModule: return "Module Map Module"; + case Module::IncludeTreeModuleMap: + return "Include Tree Module"; case Module::ModuleInterfaceUnit: return "Interface Unit"; case Module::ModulePartitionInterface: diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index 3531f94c6d737..2d94a858c961a 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -167,6 +167,12 @@ class IncludeTreePPActions final : public PPCachedActions { return reportErrorTwine( llvm::Twine("failed to find or infer submodule '") + Sub + "'"); } + + // Add to known headers for the module. + ModuleMap &MMap = PP.getHeaderSearchInfo().getModuleMap(); + Module::Header H; + H.Entry = *FE; + MMap.addHeader(M, std::move(H), ModuleMap::NormalHeader); } return IncludeFile{FID, M}; diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index 4b01f109fc881..6284ab346ee64 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -366,6 +366,7 @@ Sema::ActOnPrivateModuleFragmentDecl(SourceLocation ModuleLoc, switch (ModuleScopes.empty() ? Module::GlobalModuleFragment : ModuleScopes.back().Module->Kind) { case Module::ModuleMapModule: + case Module::IncludeTreeModuleMap: case Module::GlobalModuleFragment: case Module::ModulePartitionImplementation: case Module::ModulePartitionInterface: diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index f6334988071e5..f45c49220bcca 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -2980,8 +2980,7 @@ static bool isConsumerInterestedIn(ASTContext &Ctx, Decl *D, bool HasBody) { // emitted when we import the relevant module. if (isPartOfPerModuleInitializer(D)) { auto *M = D->getImportedOwningModule(); - if (M && M->Kind == Module::ModuleMapModule && - Ctx.DeclMustBeEmitted(D)) + if (M && M->isModuleMapModule() && Ctx.DeclMustBeEmitted(D)) return false; } diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 7c6e352979e6f..caea6662392ac 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -132,7 +132,7 @@ class IncludeTreeBuilder { llvm::SetVector IncludedFiles; Optional PredefinesBufferRef; Optional ModuleIncludesBufferRef; - Optional ModuleMapFileRef; + Optional ModuleMapRef; /// When the builder is created from an existing tree, the main include tree. Optional MainIncludeTreeRef; SmallVector IncludeStack; @@ -426,6 +426,59 @@ void IncludeTreeBuilder::exitedSubmodule(Preprocessor &PP, Module *M, // Submodule exit is handled automatically when leaving a modular file. } +static Expected +getIncludeTreeModule(cas::ObjectStore &DB, Module *M) { + using ITModule = cas::IncludeTree::Module; + SmallVector Submodules; + for (Module *Sub : M->submodules()) { + Expected SubTree = getIncludeTreeModule(DB, Sub); + if (!SubTree) + return SubTree.takeError(); + Submodules.push_back(SubTree->getRef()); + } + + ITModule::ModuleFlags Flags; + Flags.IsFramework = M->IsFramework; + Flags.IsExplicit = M->IsExplicit; + Flags.IsExternC = M->IsExternC; + Flags.IsSystem = M->IsSystem; + + bool GlobalWildcardExport = false; + SmallVector Exports; + llvm::BumpPtrAllocator Alloc; + llvm::StringSaver Saver(Alloc); + for (Module::ExportDecl &Export : M->Exports) { + if (Export.getPointer() == nullptr && Export.getInt()) { + GlobalWildcardExport = true; + } else if (Export.getPointer()) { + StringRef Name = Saver.save(Export.getPointer()->getFullModuleName()); + Exports.push_back({Name, Export.getInt()}); + } + } + std::optional ExportList; + if (GlobalWildcardExport || !Exports.empty()) { + auto EL = ITModule::ExportList::create(DB, Exports, GlobalWildcardExport); + if (!EL) + return EL.takeError(); + ExportList = EL->getRef(); + } + + SmallVector Libraries; + for (Module::LinkLibrary &LL : M->LinkLibraries) { + Libraries.push_back({LL.Library, LL.IsFramework}); + } + std::optional LinkLibraries; + if (!Libraries.empty()) { + auto LL = ITModule::LinkLibraryList::create(DB, Libraries); + if (!LL) + return LL.takeError(); + LinkLibraries = LL->getRef(); + } + + return ITModule::create(DB, M->Name, Flags, Submodules, ExportList, + LinkLibraries); +} + Expected IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { @@ -474,21 +527,6 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, return std::move(E); } - auto &FrontendOpts = NewInvocation.getFrontendOpts(); - if (!FrontendOpts.Inputs.empty() && - FrontendOpts.Inputs[0].getKind().getFormat() == InputKind::ModuleMap) { - // FIXME: handle inferred module maps - Expected FE = FM.getFileRef(FrontendOpts.Inputs[0].getFile()); - if (!FE) - return FE.takeError(); - if (Error E = addToFileList(FM, *FE).moveInto(ModuleMapFileRef)) - return std::move(E); - } - - for (StringRef ModuleMap : FrontendOpts.ModuleMapFiles) - if (Error E = addFile(ModuleMap)) - return std::move(E); - auto FinishIncludeTree = [&]() -> Error { IntrusiveRefCntPtr Reader = ScanInstance.getASTReader(); if (!Reader) @@ -522,14 +560,49 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, getCASTreeForFileIncludes(IncludeStack.pop_back_val()); if (!MainIncludeTree) return MainIncludeTree.takeError(); + + if (!ScanInstance.getLangOpts().CurrentModule.empty()) { + SmallVector Modules; + auto AddModule = [&](Module *M) -> llvm::Error { + Expected Mod = getIncludeTreeModule(DB, M); + if (!Mod) + return Mod.takeError(); + Modules.push_back(Mod->getRef()); + return Error::success(); + }; + if (Module *M = ScanInstance.getPreprocessor().getCurrentModule()) { + if (Error E = AddModule(M)) + return std::move(E); + } else { + // When building a TU or PCH, we can have headers files that are part of + // both the public and private modules that are included textually. In + // that case we need both of those modules. + ModuleMap &MMap = + ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); + M = MMap.findModule(ScanInstance.getLangOpts().CurrentModule); + assert(M && "missing current module?"); + if (Error E = AddModule(M)) + return std::move(E); + Module *PM = + MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private"); + if (PM) + if (Error E = AddModule(M)) + return std::move(E); + } + + auto ModMap = cas::IncludeTree::ModuleMap::create(DB, Modules); + if (!ModMap) + return ModMap.takeError(); + ModuleMapRef = ModMap->getRef(); + } + auto FileList = cas::IncludeTree::FileList::create(DB, IncludedFiles.getArrayRef()); if (!FileList) return FileList.takeError(); return cas::IncludeTreeRoot::create(DB, MainIncludeTree->getRef(), - FileList->getRef(), PCHRef, - ModuleMapFileRef); + FileList->getRef(), PCHRef, ModuleMapRef); } Error IncludeTreeBuilder::addModuleInputs(ASTReader &Reader) { diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index af851c979f89c..7b3bd52de3f58 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -35,6 +35,7 @@ void tooling::dependencies::configureInvocationForCaching( if (ProduceIncludeTree) { FrontendOpts.CASIncludeTreeID = std::move(RootID); FrontendOpts.Inputs.clear(); + FrontendOpts.ModuleMapFiles.clear(); HeaderSearchOptions &HSOpts = CI.getHeaderSearchOpts(); HeaderSearchOptions OriginalHSOpts; std::swap(HSOpts, OriginalHSOpts); diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c index 8d879d21236ed..40b1d51281acb 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -31,10 +31,9 @@ // CHECK: Files: // CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap // CHECK: [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} - -// Despite not importing the module, we need its modulemap for submodule info. -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap //--- cdb.json.template [{ diff --git a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c index 97168db219320..60b2ba3daf851 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c +++ b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c @@ -84,7 +84,7 @@ // CHECK-NEXT: "-cc1" // CHECK: "-fcas-path" // CHECK-NEXT: "[[PREFIX]]/cas" -// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK-NOT: -fmodule-map-file= // CHECK: "-disable-free" // CHECK: "-fcas-include-tree" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index e09c564231db5..cb42ded1fc728 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -39,20 +39,26 @@ // CHECK: llvmcas://{{[[:xdigit:]]+}} // CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} -// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Top +// CHECK: export * // CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap // CHECK-LABEL: MODULE Left // CHECK: llvmcas://{{[[:xdigit:]]+}} // CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 (Module) Top -// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Left +// CHECK: export * // CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK-LABEL: MODULE Right @@ -60,10 +66,13 @@ // CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} // CHECK: 2:1 (Module) Top -// CHECK: Module Map: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Right +// CHECK: export * // CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK-LABEL: TRANSLATION UNIT @@ -76,8 +85,9 @@ // recorded in the include-tree; it's handled entirely by fmodule-map-file, // fmodule-file, and fmodule-file-cache-key options. +// CHECK-NOT: Module Map // CHECK: Files: -// CHECK: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: [[PREFIX]]/module.modulemap llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Left.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Top.h llvmcas://{{[[:xdigit:]]+}} // CHECK: [[PREFIX]]/Right.h llvmcas://{{[[:xdigit:]]+}} @@ -96,7 +106,7 @@ // CHECK-NEXT: "-cc1" // CHECK: "-fcas-path" // CHECK-NEXT: "[[PREFIX]]/cas" -// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK-NOT: -fmodule-map-file // CHECK: "-o" // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Left-{{.*}}.pcm" // CHECK: "-disable-free" @@ -131,7 +141,7 @@ // CHECK-NEXT: "-cc1" // CHECK: "-fcas-path" // CHECK-NEXT: "[[PREFIX]]/cas" -// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK-NOT: -fmodule-map-file // CHECK: "-o" // CHECK-NEXT: "[[PREFIX]]/outputs/{{.*}}/Right-{{.*}}.pcm" // CHECK: "-disable-free" @@ -256,7 +266,7 @@ // CHECK-NEXT: "-cc1" // CHECK: "-fcas-path" // CHECK-NEXT: "[[PREFIX]]/cas" -// CHECK: "-fmodule-map-file=[[PREFIX]]/module.modulemap" +// CHECK-NOT: -fmodule-map-file // CHECK: "-disable-free" // CHECK: "-fcas-include-tree" // CHECK-NEXT: "[[TU_TREE]]" From c9b7511e0898bfb898255994dbf60fedc83974f1 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 14:16:35 -0700 Subject: [PATCH 22/88] [clang][cas] Add tests for include-tree module map * Private modules with fmodule-name * Inferred framework modules * Framework autolink * VFS (cherry picked from commit 95659504cfb37ad5b9525933a505e2bc260a21aa) (cherry picked from commit 871b0e71a8cad1585b05d3a685999553de91d302) --- .../IncludeTreeActionController.cpp | 2 +- .../modules-include-tree-exports.c | 128 ++++++++++++++++++ ...ules-include-tree-implementation-private.c | 65 +++++++++ .../modules-include-tree-inferred.m | 48 +++++++ .../modules-include-tree-vfsoverlay.c | 99 ++++++++++++++ 5 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-exports.c create mode 100644 clang/test/ClangScanDeps/modules-include-tree-implementation-private.c create mode 100644 clang/test/ClangScanDeps/modules-include-tree-inferred.m create mode 100644 clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index caea6662392ac..93d8be92eece3 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -586,7 +586,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, Module *PM = MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private"); if (PM) - if (Error E = AddModule(M)) + if (Error E = AddModule(PM)) return std::move(E); } diff --git a/clang/test/ClangScanDeps/modules-include-tree-exports.c b/clang/test/ClangScanDeps/modules-include-tree-exports.c new file mode 100644 index 0000000000000..0884e2852320d --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-exports.c @@ -0,0 +1,128 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name TwoSubs > %t/TwoSubs.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportExplicit > %t/ExportExplicit.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportWildcard > %t/ExportWildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name ExportGlobalWildcard > %t/ExportGlobalWildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name NoExports > %t/NoExports.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu_export_explicit.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_export_wildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 2 > %t/tu_export_global_wildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 3 > %t/tu_export_none.rsp + +// Build +// RUN: %clang @%t/TwoSubs.rsp +// RUN: %clang @%t/ExportExplicit.rsp +// RUN: %clang @%t/ExportWildcard.rsp +// RUN: %clang @%t/ExportGlobalWildcard.rsp +// RUN: %clang @%t/NoExports.rsp +// RUN: not %clang @%t/tu_export_explicit.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_explicit +// RUN: %clang @%t/tu_export_wildcard.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_wildcard -allow-empty +// RUN: %clang @%t/tu_export_global_wildcard.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_global_wildcard -allow-empty +// RUN: not %clang @%t/tu_export_none.rsp 2>&1 | FileCheck %s -check-prefix=tu_export_none + +//--- cdb.json.template +[ +{ + "file": "DIR/tu_export_explicit.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_explicit.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_wildcard.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_wildcard.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_global_wildcard.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_global_wildcard.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +{ + "file": "DIR/tu_export_none.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_export_none.c -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}, +] + +//--- module.modulemap +module TwoSubs { + module Sub1 { header "Sub1.h" } + module Sub2 { header "Sub2.h" } +} + +module ExportExplicit { + header "Import.h" + export TwoSubs.Sub2 +} + +module ExportWildcard { + header "Import.h" + export TwoSubs.* +} + +module ExportGlobalWildcard { + header "Import.h" + export * +} + +module NoExports { + header "Import.h" +} + +//--- Sub1.h +void sub1(void); + +//--- Sub2.h +void sub2(void); + +//--- Import.h +#include "Sub1.h" +#include "Sub2.h" + +//--- tu_export_explicit.c +#pragma clang module import ExportExplicit +void tu1(void) { + sub2(); + // tu_export_explicit-NOT: error + sub1(); + // tu_export_explicit: error: call to undeclared function 'sub1' + // tu_export_explicit: error: missing '#include "Sub1.h"' +} + +//--- tu_export_wildcard.c +#pragma clang module import ExportWildcard +void tu1(void) { + sub1(); + sub2(); + // tu_export_wildcard-NOT: error +} + +//--- tu_export_global_wildcard.c +#pragma clang module import ExportGlobalWildcard +void tu1(void) { + sub1(); + sub2(); + // tu_export_global_wildcard-NOT: error +} + +//--- tu_export_none.c +#pragma clang module import NoExports +void tu1(void) { + sub1(); + // tu_export_none: error: call to undeclared function 'sub1' + // tu_export_none: error: missing '#include "Sub1.h"' + sub2(); + // tu_export_none: error: call to undeclared function 'sub2' + // tu_export_none: error: missing '#include "Sub2.h"' +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c new file mode 100644 index 0000000000000..9d21b042a1beb --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c @@ -0,0 +1,65 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: FileCheck %s -input-file %t/deps.json -check-prefix=NO_MODULES +// NO_MODULES: "modules": [] + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/tu.rsp + +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK: Submodule: Mod +// CHECK: 3:1 [[PREFIX]]/Mod.framework/PrivateHeaders/Priv.h llvmcas:// +// CHECK: Submodule: Mod_Private +// CHECK: Module Map: +// CHECK: Mod (framework) +// CHECK: link Mod (framework) +// CHECK: Mod_Private (framework) +// CHECK: link Mod_Private (framework) + +// CHECK: Files: +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/PrivateHeaders/Priv.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -F DIR -fmodule-name=Mod -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- Mod.framework/Modules/module.modulemap +framework module Mod { header "Mod.h" } + +//--- Mod.framework/Modules/module.private.modulemap +framework module Mod_Private { header "Priv.h" } + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- Mod.framework/PrivateHeaders/Priv.h +void priv(void); + +//--- tu.m +#import +#import +void tu(void) { + pub(); + priv(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-inferred.m b/clang/test/ClangScanDeps/modules-include-tree-inferred.m new file mode 100644 index 0000000000000..df0668d54e8bd --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-inferred.m @@ -0,0 +1,48 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name Mod > %t/Mod.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: cat %t/Mod.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Mod.casid +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Mod.casid | FileCheck %s -DPREFIX=%/t +// RUN: %clang @%t/Mod.rsp +// RUN: %clang @%t/tu.rsp + +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK: Submodule: Mod +// CHECK: Module Map: +// CHECK: Mod (framework) +// CHECK: link Mod (framework) +// CHECK: Files: +// CHECK-NOT: [[PREFIX]]/module.modulemap +// CHECK: [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// +// CHECK-NOT: [[PREFIX]]/module.modulemap + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +framework module * {} + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- tu.m +#import +void tu(void) { + pub(); +} diff --git a/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c new file mode 100644 index 0000000000000..1f5e102916775 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c @@ -0,0 +1,99 @@ +// Check include-tree-based caching works with vfsoverlay files. + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed -e "s|DIR|%/t|g" %t/vfs.yaml.template > %t/vfs.yaml + +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: -cas-path %t/cas > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name=A > %t/A.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/A.rsp +// RUN: %clang @%t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/A.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/A.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE A" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/A.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE A +// CHECK: llvmcas:// +// CHECK: 2:1 [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK: Submodule: A +// CHECK: Module Map: +// CHECK: A (framework) +// CHECK: link A (framework) +// CHECK: Files: +// CHECK-NOT: modulemap +// CHECK: [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK-NOT: modulemap + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: Files: +// CHECK-NOT: .modulemap +// CHECK-NOT: .yaml +// CHECK: [[PREFIX]]/elsewhere2/A.h llvmcas:// +// CHECK-NOT: .modulemap +// CHECK-NOT: .yaml + +//--- cdb.json.template +[{ + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -ivfsoverlay DIR/vfs.yaml -F DIR", + "file": "DIR/tu.c" +}] + +//--- vfs.yaml.template +{ + "version": 0, + "case-sensitive": "false", + "roots": [ + { + "name": "DIR/A.framework", + "type": "directory" + "contents": [ + { + "name": "Modules", + "type": "directory" + "contents": [ + { + "external-contents": "DIR/elsewhere1/A.modulemap", + "name": "module.modulemap", + "type": "file" + } + ] + }, + { + "name": "Headers", + "type": "directory" + "contents": [ + { + "external-contents": "DIR/elsewhere2/A.h", + "name": "A.h", + "type": "file" + } + ] + } + ] + } + ] +} + +//--- elsewhere1/A.modulemap +framework module A { header "A.h" } + +//--- elsewhere2/A.h +typedef int A_t; + +//--- tu.c +#include "A/A.h" +A_t a = 0; From 4b6a64eda5b88a0fed85a53e72cc12dfd36da7c5 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 15:07:12 -0700 Subject: [PATCH 23/88] [clang][cas] Correctly split out modules that are only imported for visibility Modules imported for visibility only can happen when -fmodule-name matches a modular header that is imported more than once so that we trigger the correct re-exports but without trying to import a module. (cherry picked from commit d844c16c5b19e28db5736ae3da13e9ca05f3d866) (cherry picked from commit 02b64b7278e7c6165cba5f84f2d2f0b74f515793) --- .../include/clang/Basic/DiagnosticLexKinds.td | 3 ++ clang/include/clang/CAS/IncludeTree.h | 9 +++-- clang/include/clang/Lex/PPCachedActions.h | 2 + clang/lib/CAS/IncludeTree.cpp | 15 ++++++-- clang/lib/Frontend/IncludeTreePPActions.cpp | 2 +- clang/lib/Lex/PPDirectives.cpp | 37 ++++++++++++++----- .../DependencyScanningWorker.cpp | 4 ++ .../IncludeTreeActionController.cpp | 5 ++- ...ules-include-tree-implementation-private.c | 4 ++ .../modules-include-tree-implementation.c | 2 +- 10 files changed, 63 insertions(+), 20 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td index 12aed1386cdd8..3cba656af179d 100644 --- a/clang/include/clang/Basic/DiagnosticLexKinds.td +++ b/clang/include/clang/Basic/DiagnosticLexKinds.td @@ -918,6 +918,9 @@ def warn_defined_in_function_type_macro : Extension< "macro expansion producing 'defined' has undefined behavior">, InGroup; +def err_pp_missing_module_include_tree : Error< + "no module named '%0' declared in include-tree module map">, DefaultFatal; + let CategoryName = "Nullability Issue" in { def err_pp_assume_nonnull_syntax : Error<"expected 'begin' or 'end'">; diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 9c9723fd3bc90..77cbd3fcc47b6 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -331,9 +331,12 @@ class IncludeTree::ModuleImport : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "ModI"; } - static Expected create(ObjectStore &DB, StringRef ModuleName); + static Expected create(ObjectStore &DB, StringRef ModuleName, + bool VisibilityOnly); - StringRef getModuleName() { return getData(); } + StringRef getModuleName() const { return getData().drop_front(); } + /// Whether this module should only be "marked visible" rather than imported. + bool visibilityOnly() const { return (bool)getData()[0]; } llvm::Error print(llvm::raw_ostream &OS, unsigned Indent = 0); @@ -341,7 +344,7 @@ class IncludeTree::ModuleImport : public IncludeTreeBase { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - return Base.getNumReferences() == 0 && !Base.getData().empty(); + return Base.getNumReferences() == 0 && Base.getData().size() > 1; } static bool isValid(ObjectStore &DB, ObjectRef Ref) { auto Node = DB.getProxy(Ref); diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index dde2c6f30ed93..c741fc022a5c4 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -39,6 +39,8 @@ class PPCachedActions { /// The module that is imported by an \c #include directive or \c @import. struct IncludeModule { SmallVector, 2> ImportPath; + // Whether this module should only be "marked visible" rather than imported. + bool VisibilityOnly; }; virtual ~PPCachedActions() = default; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index dae8c92e52901..68bae3b5bf1a5 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -212,9 +212,12 @@ bool IncludeTree::isValid(const ObjectProxy &Node) { } Expected -IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName) { - return IncludeTreeBase::create(DB, {}, - llvm::arrayRefFromStringRef(ModuleName)); +IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName, + bool VisibilityOnly) { + SmallString<64> Buffer; + Buffer.push_back((char)VisibilityOnly); + Buffer.append(ModuleName); + return IncludeTreeBase::create(DB, {}, Buffer); } IncludeTree::FileList::FileSizeTy @@ -600,7 +603,11 @@ llvm::Error IncludeTree::FileList::print(llvm::raw_ostream &OS, llvm::Error IncludeTree::ModuleImport::print(llvm::raw_ostream &OS, unsigned Indent) { - OS << "(Module) " << getModuleName() << '\n'; + if (visibilityOnly()) + OS << "(Module for visibility only) "; + else + OS << "(Module) "; + OS << getModuleName() << '\n'; return llvm::Error::success(); } diff --git a/clang/lib/Frontend/IncludeTreePPActions.cpp b/clang/lib/Frontend/IncludeTreePPActions.cpp index 2d94a858c961a..87db33b85d52a 100644 --- a/clang/lib/Frontend/IncludeTreePPActions.cpp +++ b/clang/lib/Frontend/IncludeTreePPActions.cpp @@ -123,7 +123,7 @@ class IncludeTreePPActions final : public PPCachedActions { Import.getModuleName().split(ModuleComponents, '.'); for (StringRef Component : ModuleComponents) Path.emplace_back(PP.getIdentifierInfo(Component), IncludeLoc); - return IncludeModule{std::move(Path)}; + return IncludeModule{std::move(Path), Import.visibilityOnly()}; } assert(Node->getKind() == cas::IncludeTree::NodeKind::Tree); diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 92b68ddf10a9c..c2278ba0b5380 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -2011,17 +2011,36 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, return; } if (auto *Import = std::get_if(&Include)) { - ModuleLoadResult Imported = TheModuleLoader.loadModule( - IncludeTok.getLocation(), Import->ImportPath, Module::Hidden, - /*IsIncludeDirective=*/true); - if (!Imported) { - assert(hadModuleLoaderFatalFailure() && "unexpected failure kind"); - if (hadModuleLoaderFatalFailure()) { - IncludeTok.setKind(tok::eof); - CurLexer->cutOffLexing(); + ModuleLoadResult Imported; + if (Import->VisibilityOnly) { + ModuleMap &MMap = getHeaderSearchInfo().getModuleMap(); + Module *M = nullptr; + for (auto &NameLoc : Import->ImportPath) { + M = MMap.lookupModuleQualified(NameLoc.first->getName(), M); + if (!M) + break; + } + if (!M) { + Diags->Report(diag::err_pp_missing_module_include_tree) + << getLangOpts().CurrentModule; + + return; + } + Imported = M; + } else { + Imported = TheModuleLoader.loadModule( + IncludeTok.getLocation(), Import->ImportPath, Module::Hidden, + /*IsIncludeDirective=*/true); + if (!Imported) { + assert(hadModuleLoaderFatalFailure() && "unexpected failure kind"); + if (hadModuleLoaderFatalFailure()) { + IncludeTok.setKind(tok::eof); + CurLexer->cutOffLexing(); + } + return; } - return; } + makeModuleVisible(Imported, EndLoc); if (IncludeTok.getIdentifierInfo()->getPPKeywordID() != tok::pp___include_macros) diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 5d5cd6c8326f5..fcefb3e7d4ffc 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -458,6 +458,10 @@ class DependencyScanningAction : public tooling::ToolAction { else Action = std::make_unique(); + // Normally this would be handled by GeneratePCHAction + if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH) + ScanInstance.getLangOpts().CompilingPCH = true; + if (Error E = Controller.initialize(ScanInstance, OriginalInvocation)) return reportError(std::move(E)); diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 93d8be92eece3..9ea4bfa1763e2 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -397,8 +397,9 @@ void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc) { - auto Import = - check(cas::IncludeTree::ModuleImport::create(DB, M->getFullModuleName())); + bool VisibilityOnly = M->isForBuilding(PP.getLangOpts()); + auto Import = check(cas::IncludeTree::ModuleImport::create( + DB, M->getFullModuleName(), VisibilityOnly)); if (!Import) return; diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c index 9d21b042a1beb..912a72da3a174 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c @@ -23,6 +23,8 @@ // CHECK: Submodule: Mod // CHECK: 3:1 [[PREFIX]]/Mod.framework/PrivateHeaders/Priv.h llvmcas:// // CHECK: Submodule: Mod_Private +// CHECK: 4:1 (Module for visibility only) Mod +// CHECK: 5:1 (Module for visibility only) Mod_Private // CHECK: Module Map: // CHECK: Mod (framework) // CHECK: link Mod (framework) @@ -59,6 +61,8 @@ void priv(void); //--- tu.m #import #import +#import +#import void tu(void) { pub(); priv(); diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c index 40b1d51281acb..b0ccbec601856 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -27,7 +27,7 @@ // CHECK: 2:1 [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} // CHECK: Submodule: Mod -// CHECK: 3:1 (Module) Mod +// CHECK: 3:1 (Module for visibility only) Mod // CHECK: Files: // CHECK: [[PREFIX]]/tu.c llvmcas://{{[[:xdigit:]]+}} From 50087abf881115e2963c1ec1a1484ac493cd822c Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 23 Mar 2023 16:42:04 -0700 Subject: [PATCH 24/88] [clang][cas] Fix include-tree with pch importing public/private modules We were dropping the modular import of a private module if its public module was imported via PCH, because we skipped parsing the private modulemap during scanning. (cherry picked from commit 6034ccd1ea9d2fb5c2fea793c66d95dceedba8ac) (cherry picked from commit ab658ceefcece986379e7fbf6a0fcf0cab3a0808) --- .../IncludeTreeActionController.cpp | 56 ++++++++++++++++ .../modules-include-tree-pch-with-private.c | 65 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 9ea4bfa1763e2..60a2eb3a7b48f 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -153,6 +153,19 @@ struct PPCallbacksDependencyCollector : public DependencyCollector { PP.addPPCallbacks(std::move(CB)); } }; +/// A utility for adding \c ASTReaderListener to a compiler instance at the +/// appropriate time. +struct ASTReaderListenerDependencyCollector : public DependencyCollector { + using MakeL = + llvm::unique_function(ASTReader &R)>; + MakeL Create; + ASTReaderListenerDependencyCollector(MakeL Create) : Create(std::move(Create)) {} + void attachToASTReader(ASTReader &R) final { + std::unique_ptr L = Create(R); + assert(L); + R.addListener(std::move(L)); + } +}; struct IncludeTreePPCallbacks : public PPCallbacks { IncludeTreeBuilder &Builder; @@ -217,6 +230,40 @@ struct IncludeTreePPCallbacks : public PPCallbacks { Builder.exitedSubmodule(PP, M, ImportLoc, ForPragma); } }; + +/// Utility to trigger module lookup in header search for modules loaded via +/// PCH. This causes dependency scanning via PCH to parse modulemap files at +/// roughly the same point they would with modulemap files embedded in the pcms, +/// which is disabled with include-tree modules. Without this, we can fail to +/// find modules that are in the same directory as a named import, since +/// it may be skipped during search (see \c loadFrameworkModule). +/// +/// The specific lookup we do matches what happens in ASTReader for the +/// MODULE_DIRECTORY record, and ignores the result. +class LookupPCHModulesListener : public ASTReaderListener { +public: + LookupPCHModulesListener(Preprocessor &PP) : PP(PP) {} + +private: + void ReadModuleName(StringRef ModuleName) final { + if (PCHFinished) + return; + // Match MODULE_DIRECTORY: allow full search and ignore failure to find + // the module. + (void)PP.getHeaderSearchInfo().lookupModule( + ModuleName, SourceLocation(), /*AllowSearch=*/true, + /*AllowExtraModuleMapSearch=*/true); + } + void visitModuleFile(StringRef Filename, + serialization::ModuleKind Kind) final { + if (Kind == serialization::MK_PCH) + PCHFinished = true; + } + +private: + Preprocessor &PP; + bool PCHFinished = false; +}; } // namespace /// The PCH recorded file paths with canonical paths, create a VFS that @@ -274,6 +321,15 @@ Error IncludeTreeActionController::initialize( }); ScanInstance.addDependencyCollector(std::move(DC)); + // Attach callback for modules loaded via PCH. + if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) { + auto DC = std::make_shared( + [&](ASTReader &R) { + return std::make_unique(R.getPreprocessor()); + }); + ScanInstance.addDependencyCollector(std::move(DC)); + } + // Enable caching in the resulting commands. ScanInstance.getFrontendOpts().CacheCompileJob = true; CASOpts = ScanInstance.getCASOpts(); diff --git a/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c new file mode 100644 index 0000000000000..99910c5bb62f0 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c @@ -0,0 +1,65 @@ +// Test importing a private module whose public module was previously imported +// via a PCH. + +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Mod > %t/Mod.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Mod.rsp +// RUN: %clang @%t/pch.rsp + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --module-name Mod_Private > %t/Mod_Private.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Mod_Private.rsp +// RUN: %clang @%t/tu.rsp + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -include prefix.h -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x objective-c-header DIR/prefix.h -o DIR/prefix.h.pch -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +}] + +//--- Mod.framework/Modules/module.modulemap +framework module Mod { header "Mod.h" } + +//--- Mod.framework/Modules/module.private.modulemap +framework module Mod_Private { header "Priv.h" } + +//--- Mod.framework/Headers/Mod.h +void pub(void); + +//--- Mod.framework/PrivateHeaders/Priv.h +void priv(void); + +//--- prefix.h +#import + +//--- tu.m +#import +void tu(void) { + pub(); + priv(); +} From 5a0e470f149c2fa4484c667892f595a9a3f34f3d Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Fri, 24 Mar 2023 15:48:09 -0700 Subject: [PATCH 25/88] [clang][cas] Make missing include-tree id an error This was review feedback on a previous PR that somehow went missing. Bring it back. (cherry picked from commit f1ccf9d17cd58d5c013e05fde249d008dd72d4dd) (cherry picked from commit 0b727cd37892e11441a9beac4b8ac8f4c415fd36) --- .../IncludeTreeActionController.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 60a2eb3a7b48f..29b72c4221efd 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -392,11 +392,14 @@ Error IncludeTreeActionController::finalizeModuleBuild( Error IncludeTreeActionController::finalizeModuleInvocation( CompilerInvocation &CI, const ModuleDeps &MD) { - if (auto ID = MD.IncludeTreeID) { - configureInvocationForCaching(CI, CASOpts, std::move(*ID), - /*CASFSWorkingDir=*/"", - /*ProduceIncludeTree=*/true); - } + if (!MD.IncludeTreeID) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "missing include-tree for module '%s'", + MD.ID.ModuleName.c_str()); + + configureInvocationForCaching(CI, CASOpts, *MD.IncludeTreeID, + /*CASFSWorkingDir=*/"", + /*ProduceIncludeTree=*/true); DepscanPrefixMapping::remapInvocationPaths(CI, PrefixMapper); return Error::success(); From 29f49354444e6ff5a1f9a200dcceefcc570e2337 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Sun, 26 Mar 2023 14:37:29 -0700 Subject: [PATCH 26/88] [clang][cas] Allow non-existent fmodule-name Missing the module named by fmodule-name is not an error for a TU or PCH. (cherry picked from commit fcd06ba0187401237b71018f8d9e509c0ea87e28) (cherry picked from commit 3e11b20d2ed0b175a215ca07e2561f65df9e1840) --- .../IncludeTreeActionController.cpp | 12 +++++------- .../modules-include-tree-implementation.c | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 29b72c4221efd..e218ae9c734dd 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -639,13 +639,11 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, // that case we need both of those modules. ModuleMap &MMap = ScanInstance.getPreprocessor().getHeaderSearchInfo().getModuleMap(); - M = MMap.findModule(ScanInstance.getLangOpts().CurrentModule); - assert(M && "missing current module?"); - if (Error E = AddModule(M)) - return std::move(E); - Module *PM = - MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private"); - if (PM) + if (Module *M = MMap.findModule(ScanInstance.getLangOpts().CurrentModule)) + if (Error E = AddModule(M)) + return std::move(E); + if (Module *PM = + MMap.findModule(ScanInstance.getLangOpts().ModuleName + "_Private")) if (Error E = AddModule(PM)) return std::move(E); } diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation.c b/clang/test/ClangScanDeps/modules-include-tree-implementation.c index b0ccbec601856..282742ad803f1 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation.c @@ -4,7 +4,7 @@ // RUN: split-file %s %t // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: clang-scan-deps -compilation-database %t/cdb.json -j 1 \ // RUN: -cas-path %t/cas -module-files-dir %t/outputs \ // RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ // RUN: > %t/deps.json @@ -35,12 +35,22 @@ // CHECK: [[PREFIX]]/Mod.h llvmcas://{{[[:xdigit:]]+}} // CHECK-NOT: [[PREFIX]]/module.modulemap +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_missing_module.rsp +// RUN: %clang @%t/tu_missing_module.rsp + //--- cdb.json.template -[{ +[ +{ "file": "DIR/tu.c", "directory": "DIR", "command": "clang -fsyntax-only DIR/tu.c -I DIR -fmodule-name=Mod -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" -}] +}, +{ + "file": "DIR/tu_missing_module.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu_missing_module.c -I DIR -fmodule-name=NonExistent -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache" +} +] //--- module.modulemap module Mod { header "Mod.h" } @@ -55,3 +65,6 @@ void top(void); void tu(void) { top(); } + +//--- tu_missing_module.c + From 5d9a93405f540e821d85f4f09a82ec7c46cfb22d Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 27 Mar 2023 10:28:47 -0700 Subject: [PATCH 27/88] [clang][cas] Switch to chained representation of IncludeTree::FileList Instead of flattening file lists when importing PCH/modules, just create a chained reference to the list and flatten it only in FileList::forEachFile. This reduces the storage cost and performance overhead of merging file lists. (cherry picked from commit ee7ddda1e8e2fc16648ca18b12be168858751d4a) (cherry picked from commit 2fbfa490d0dab86eb042e9008969024ce1633863) --- clang/include/clang/CAS/IncludeTree.h | 44 ++------- clang/lib/CAS/IncludeTree.cpp | 78 +++++++++++++--- .../IncludeTreeActionController.cpp | 33 ++++--- clang/unittests/CAS/IncludeTreeTest.cpp | 92 ++++++++++++++----- 4 files changed, 163 insertions(+), 84 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 77cbd3fcc47b6..b478a0614bb56 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -264,25 +264,15 @@ class IncludeTree::File : public IncludeTreeBase { } }; -/// A flat list of \c File entries. This is used along with a simple -/// implementation of a \p vfs::FileSystem produced via -/// \p createIncludeTreeFileSystem(). +/// A list of \c File entries. Multiple \c FileList can be combined without +/// copying their contents. This is used along with a simple implementation of a +/// \p vfs::FileSystem produced via \p createIncludeTreeFileSystem(). class IncludeTree::FileList : public IncludeTreeBase { public: static constexpr StringRef getNodeKind() { return "List"; } using FileSizeTy = uint32_t; - size_t getNumFiles() const { return getNumReferences(); } - - ObjectRef getFileRef(size_t I) const { - assert(I < getNumFiles()); - return getReference(I); - } - - Expected getFile(size_t I) { return getFile(getFileRef(I)); } - FileSizeTy getFileSize(size_t I) const; - /// \returns each \c File entry along with its file size. llvm::Error forEachFile(llvm::function_ref Callback); @@ -294,7 +284,8 @@ class IncludeTree::FileList : public IncludeTreeBase { ObjectRef FileRef; FileSizeTy Size; }; - static Expected create(ObjectStore &DB, ArrayRef Files); + static Expected create(ObjectStore &DB, ArrayRef Files, + ArrayRef FileLists); static Expected get(ObjectStore &CAS, ObjectRef Ref); @@ -308,6 +299,9 @@ class IncludeTree::FileList : public IncludeTreeBase { assert(isValid(*this)); } + size_t getNumFilesCurrentList() const; + FileSizeTy getFileSize(size_t I) const; + Expected getFile(ObjectRef Ref) { auto Node = getCAS().getProxy(Ref); if (!Node) @@ -744,26 +738,4 @@ createIncludeTreeFileSystem(IncludeTreeRoot &Root); } // namespace cas } // namespace clang -namespace llvm { -template <> struct DenseMapInfo { - using FileEntry = clang::cas::IncludeTree::FileList::FileEntry; - - static FileEntry getEmptyKey() { - return {cas::ObjectRef::getDenseMapEmptyKey(), 0}; - } - - static FileEntry getTombstoneKey() { - return {cas::ObjectRef::getDenseMapTombstoneKey(), 0}; - } - - static unsigned getHashValue(FileEntry F) { - return F.FileRef.getDenseMapHash(); - } - - static bool isEqual(FileEntry LHS, FileEntry RHS) { - return LHS.FileRef == RHS.FileRef && LHS.Size == RHS.Size; - } -}; -} // namespace llvm - #endif diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 68bae3b5bf1a5..af850b68b0233 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -7,8 +7,11 @@ //===----------------------------------------------------------------------===// #include "clang/CAS/IncludeTree.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/Support/EndianStream.h" +#include "llvm/Support/Error.h" +#include using namespace clang; using namespace clang::cas; @@ -220,10 +223,15 @@ IncludeTree::ModuleImport::create(ObjectStore &DB, StringRef ModuleName, return IncludeTreeBase::create(DB, {}, Buffer); } +size_t IncludeTree::FileList::getNumFilesCurrentList() const { + return llvm::support::endian::read( + getData().data()); +} + IncludeTree::FileList::FileSizeTy IncludeTree::FileList::getFileSize(size_t I) const { - assert(I < getNumFiles()); - StringRef Data = getData(); + assert(I < getNumFilesCurrentList()); + StringRef Data = getData().drop_front(sizeof(uint32_t)); assert(Data.size() >= (I + 1) * sizeof(FileSizeTy)); return llvm::support::endian::read( Data.data() + I * sizeof(FileSizeTy)); @@ -232,29 +240,44 @@ IncludeTree::FileList::getFileSize(size_t I) const { llvm::Error IncludeTree::FileList::forEachFile( llvm::function_ref Callback) { size_t I = 0; + size_t FileCount = getNumFilesCurrentList(); return forEachReference([&](ObjectRef Ref) -> llvm::Error { - auto Include = getFile(Ref); - if (!Include) - return Include.takeError(); - return Callback(std::move(*Include), getFileSize(I++)); + if (I < FileCount) { + auto Include = getFile(Ref); + if (!Include) + return Include.takeError(); + return Callback(std::move(*Include), getFileSize(I++)); + } + // Otherwise, it's a chained FileList. + ++I; + auto Proxy = getCAS().getProxy(Ref); + if (!Proxy) + return Proxy.takeError(); + FileList FL(std::move(*Proxy)); + return FL.forEachFile(Callback); }); } Expected -IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files) { +IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files, + ArrayRef FileLists) { SmallVector Refs; - Refs.reserve(Files.size()); + Refs.reserve(Files.size() + FileLists.size()); SmallString<256> Buffer; - Buffer.reserve(Files.size() * sizeof(FileSizeTy)); + Buffer.reserve(sizeof(uint32_t) + Files.size() * sizeof(FileSizeTy)); llvm::raw_svector_ostream BufOS(Buffer); llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + Writer.write(static_cast(Files.size())); for (const FileEntry &Entry : Files) { assert(File::isValid(DB, Entry.FileRef)); Refs.push_back(Entry.FileRef); Writer.write(Entry.Size); } + + Refs.append(FileLists.begin(), FileLists.end()); + return IncludeTreeBase::create(DB, Refs, Buffer); } @@ -273,9 +296,13 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - unsigned NumFiles = Base.getNumReferences(); - return NumFiles != 0 && - Base.getData().size() == NumFiles * sizeof(FileSizeTy); + StringRef Data = Base.getData(); + if (Data.size() < sizeof(uint32_t)) + return false; + unsigned NumFiles = + llvm::support::endian::read(Data.data()); + return NumFiles != 0 && NumFiles <= Base.getNumReferences() && + Data.size() == sizeof(uint32_t) + NumFiles * sizeof(FileSizeTy); } static constexpr char ModuleFlagFramework = 1 << 0; @@ -845,20 +872,47 @@ class IncludeTreeFileSystem : public llvm::vfs::FileSystem { }; } // namespace +static llvm::Error diagnoseFileChange(IncludeTree::File F, ObjectRef Content) { + auto FilenameBlob = F.getFilename(); + if (!FilenameBlob) + return FilenameBlob.takeError(); + cas::ObjectStore &DB = F.getCAS(); + std::string Filename(FilenameBlob->getData()); + std::string OldID = DB.getID(Content).toString(); + std::string NewID = DB.getID(F.getContentsRef()).toString(); + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "file '%s' changed during build; include-tree " + "contents changed from %s to %s", + Filename.c_str(), OldID.c_str(), + NewID.c_str()); +} + Expected> cas::createIncludeTreeFileSystem(IncludeTreeRoot &Root) { auto FileList = Root.getFileList(); if (!FileList) return FileList.takeError(); + // Map from FilenameRef to ContentsRef. + llvm::DenseMap SeenContents; + IntrusiveRefCntPtr IncludeTreeFS = new IncludeTreeFileSystem(Root.getCAS()); llvm::Error E = FileList->forEachFile( [&](IncludeTree::File File, IncludeTree::FileList::FileSizeTy Size) -> llvm::Error { + auto InsertPair = SeenContents.insert( + std::make_pair(File.getFilenameRef(), File.getContentsRef())); + if (!InsertPair.second) { + if (InsertPair.first->second != File.getContentsRef()) + return diagnoseFileChange(File, InsertPair.first->second); + return llvm::Error::success(); + } + auto FilenameBlob = File.getFilename(); if (!FilenameBlob) return FilenameBlob.takeError(); + SmallString<128> Filename(FilenameBlob->getData()); // Strip './' in the filename to match the behaviour of ASTWriter; we // also strip './' in IncludeTreeFileSystem::getPath. diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index e218ae9c734dd..10df1417740d4 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -129,7 +129,8 @@ class IncludeTreeBuilder { // are recorded in the PCH, ordered by \p FileEntry::UID index. SmallVector PreIncludedFileNames; llvm::BitVector SeenIncludeFiles; - llvm::SetVector IncludedFiles; + SmallVector IncludedFiles; + SmallVector IncludedFileLists; Optional PredefinesBufferRef; Optional ModuleIncludesBufferRef; Optional ModuleMapRef; @@ -454,9 +455,23 @@ void IncludeTreeBuilder::handleHasIncludeCheck(Preprocessor &PP, bool Result) { IncludeStack.back().HasIncludeChecks.push_back(Result); } +// FIXME: duplicates code in PPDirectives +static bool isForModuleBuilding(const Module *M, StringRef CurrentModule, + StringRef ModuleName) { + StringRef TopLevelName = M->getTopLevelModuleName(); + + // When building framework Foo, we wanna make sure that Foo *and* Foo_Private + // are textually included and no modules are built for both. + if (M->getTopLevelModule()->IsFramework && CurrentModule == ModuleName && + !CurrentModule.endswith("_Private") && TopLevelName.endswith("_Private")) + TopLevelName = TopLevelName.drop_back(8); + + return TopLevelName == CurrentModule; +} + void IncludeTreeBuilder::moduleImport(Preprocessor &PP, const Module *M, SourceLocation EndLoc) { - bool VisibilityOnly = M->isForBuilding(PP.getLangOpts()); + bool VisibilityOnly = isForModuleBuilding(M, PP.getLangOpts().CurrentModule, PP.getLangOpts().ModuleName); auto Import = check(cas::IncludeTree::ModuleImport::create( DB, M->getFullModuleName(), VisibilityOnly)); if (!Import) @@ -655,7 +670,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, } auto FileList = - cas::IncludeTree::FileList::create(DB, IncludedFiles.getArrayRef()); + cas::IncludeTree::FileList::create(DB, IncludedFiles, IncludedFileLists); if (!FileList) return FileList.takeError(); @@ -681,16 +696,8 @@ Error IncludeTreeBuilder::addModuleInputs(ASTReader &Reader) { Optional Root; if (Error E = cas::IncludeTreeRoot::get(DB, *Ref).moveInto(Root)) return E; - Optional Files; - if (Error E = Root->getFileList().moveInto(Files)) - return E; - Error E = Files->forEachFile([&](auto IF, auto Size) -> Error { - IncludedFiles.insert({IF.getRef(), Size}); - return Error::success(); - }); - if (E) - return E; + IncludedFileLists.push_back(Root->getFileListRef()); } return Error::success(); @@ -771,7 +778,7 @@ IncludeTreeBuilder::addToFileList(FileManager &FM, const FileEntry *FE) { auto FileNode = createIncludeFile(Filename, **CASContents); if (!FileNode) return FileNode.takeError(); - IncludedFiles.insert( + IncludedFiles.push_back( {FileNode->getRef(), static_cast(FE->getSize())}); return FileNode->getRef(); diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index c7739611bfce3..c85b4f767d625 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -4,6 +4,7 @@ #include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/ObjectStore.h" +#include "llvm/Support/Error.h" #include "llvm/Testing/Support/Error.h" #include "gtest/gtest.h" @@ -135,29 +136,74 @@ TEST(IncludeTree, IncludeTreeScan) { Optional FileList; ASSERT_THAT_ERROR(Root->getFileList().moveInto(FileList), llvm::Succeeded()); - ASSERT_EQ(FileList->getNumFiles(), size_t(4)); - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(0).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), MainFile->getRef()); - EXPECT_EQ(FileList->getFileSize(0), MainContents.size()); - } - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(1).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), A1File->getRef()); - EXPECT_EQ(FileList->getFileSize(1), A1Contents.size()); - } - { - Optional File; - ASSERT_THAT_ERROR(FileList->getFile(2).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), B1File->getRef()); - EXPECT_EQ(FileList->getFileSize(2), IncludeTree::FileList::FileSizeTy(0)); - } - { + + SmallVector> + Files; + ASSERT_THAT_ERROR(FileList->forEachFile([&](auto F, auto S) -> llvm::Error { + Files.push_back({F, S}); + return llvm::Error::success(); + }), + llvm::Succeeded()); + + ASSERT_EQ(Files.size(), size_t(4)); + EXPECT_EQ(Files[0].first.getRef(), MainFile->getRef()); + EXPECT_EQ(Files[0].second, MainContents.size()); + EXPECT_EQ(Files[1].first.getRef(), A1File->getRef()); + EXPECT_EQ(Files[1].second, A1Contents.size()); + EXPECT_EQ(Files[2].first.getRef(), B1File->getRef()); + EXPECT_EQ(Files[2].second, IncludeTree::FileList::FileSizeTy(0)); + EXPECT_EQ(Files[3].first.getRef(), SysFile->getRef()); + EXPECT_EQ(Files[3].second, IncludeTree::FileList::FileSizeTy(0)); +} + +TEST(IncludeTree, IncludeTreeFileList) { + std::shared_ptr DB = llvm::cas::createInMemoryCAS(); + SmallVector Files; + for (unsigned I = 0; I < 10; ++I) { Optional File; - ASSERT_THAT_ERROR(FileList->getFile(3).moveInto(File), llvm::Succeeded()); - EXPECT_EQ(File->getRef(), SysFile->getRef()); - EXPECT_EQ(FileList->getFileSize(3), IncludeTree::FileList::FileSizeTy(0)); + std::string Path = "/file" + std::to_string(I); + static constexpr StringRef Bytes = "123456789"; + Optional Content; + ASSERT_THAT_ERROR( + DB->storeFromString({}, Bytes.substr(0, I)).moveInto(Content), + llvm::Succeeded()); + ASSERT_THAT_ERROR( + IncludeTree::File::create(*DB, Path, *Content).moveInto(File), + llvm::Succeeded()); + Files.push_back(std::move(*File)); } + + auto MakeFileList = [&](unsigned Begin, unsigned End, + ArrayRef Lists) { + SmallVector Entries; + for (; Begin != End; ++Begin) + Entries.push_back({Files[Begin].getRef(), Begin}); + return IncludeTree::FileList::create(*DB, Entries, Lists); + }; + + Optional L89, L7, L29, L; + ASSERT_THAT_ERROR(MakeFileList(8, 10, {}).moveInto(L89), llvm::Succeeded()); + EXPECT_EQ(L89->getNumReferences(), 2u); + ASSERT_THAT_ERROR(MakeFileList(7, 8, {}).moveInto(L7), llvm::Succeeded()); + EXPECT_EQ(L7->getNumReferences(), 1u); + ASSERT_THAT_ERROR( + MakeFileList(2, 7, {L7->getRef(), L89->getRef()}).moveInto(L29), + llvm::Succeeded()); + EXPECT_EQ(L29->getNumReferences(), 7u); // 2,3,4,5,6, {7}, {8, 9} + ASSERT_THAT_ERROR(MakeFileList(0, 2, {L29->getRef()}).moveInto(L), + llvm::Succeeded()); + EXPECT_EQ(L->getNumReferences(), 3u); // 0, 1, {2, ...} + + size_t I = 0; + ASSERT_THAT_ERROR( + L->forEachFile([&](IncludeTree::File F, auto Size) -> llvm::Error { + EXPECT_EQ(F.getFilenameRef(), Files[I].getFilenameRef()) + << "filename mismatch at " << I; + EXPECT_EQ(F.getContentsRef(), Files[I].getContentsRef()) + << "contents mismatch at " << I; + EXPECT_EQ(Size, I) << "size mismatch at " << I; + I += 1; + return llvm::Error::success(); + }), + llvm::Succeeded()); } From e3288dff6a089ada39a7853685e07106c869906e Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Sun, 26 Mar 2023 14:37:25 -0700 Subject: [PATCH 28/88] [clang][cas] Add simple prefix-mapping test for include-tree modules (cherry picked from commit f796c8fd6ed20acd97fe1b686a2bd07d09e96507) (cherry picked from commit c2817eacc92fce41ce6427ccf93709affa2b5d72) --- .../modules-include-tree-prefix-map.c | 305 ++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-prefix-map.c diff --git a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c new file mode 100644 index 0000000000000..d96f6b21233b8 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c @@ -0,0 +1,305 @@ +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t/dir1 +// RUN: cp -r %t/dir1 %t/dir2 +// RUN: sed "s|DIR|%/t/dir1|g" %t/dir1/cdb.json.template > %t/cdb1.json +// RUN: sed "s|DIR|%/t/dir2|g" %t/dir1/cdb.json.template > %t/cdb2.json + +// RUN: clang-scan-deps -compilation-database %t/cdb1.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/dir1/outputs \ +// RUN: -prefix-map=%t/dir1/outputs=/^modules -prefix-map=%t/dir1=/^src -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// Extract the include-tree commands +// RUN: %deps-to-rsp %t/deps.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Top.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Top.casid +// RUN: cat %t/Left.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Left.casid +// RUN: cat %t/Right.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Right.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Top" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Top.casid >> %t/result.txt +// RUN: echo "MODULE Left" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Left.casid >> %t/result.txt +// RUN: echo "MODULE Right" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Right.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t -check-prefix=NO_PATHS +// NO_PATHS-NOT: [[PREFIX]] + +// RUN: cat %t/deps.json >> %t/result.txt + +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Top +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: Module Map: +// CHECK: Top +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap + +// CHECK-LABEL: MODULE Left +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: +// CHECK: Left +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: MODULE Right +// CHECK: llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 /^src/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Top +// CHECK: Module Map: +// CHECK: Right +// CHECK: export * +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Right.h llvmcas://{{[[:xdigit:]]+}} +// CHECK-NOT: module.modulemap +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: /^src/tu.m llvmcas://{{[[:xdigit:]]+}} +// CHECK: 1:1 llvmcas://{{[[:xdigit:]]+}} +// CHECK: 2:1 (Module) Left +// CHECK: 3:1 (Module) Right + +// Note: the modules with explicit imports are imported via parser and are not +// recorded in the include-tree; it's handled entirely by fmodule-map-file, +// fmodule-file, and fmodule-file-cache-key options. + +// CHECK-NOT: Module Map +// CHECK: Files: +// CHECK-NOT: module.modulemap +// CHECK: /^src/Left.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: /^src/Top.h llvmcas://{{[[:xdigit:]]+}} +// CHECK: /^src/Right.h llvmcas://{{[[:xdigit:]]+}} + +// CHECK: { +// CHECK-NEXT "modules": [ +// CHECK-NEXT { +// CHECK: "cas-include-tree-id": "[[LEFT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[LEFT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Left" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Left.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK: ] +// CHECK: "name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Top" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file-cache-key +// CHECK-NEXT: "/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Top=/^modules/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Right.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TOP_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "[[PREFIX]]/dir1/module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK: "-o" +// CHECK-NEXT: "[[PREFIX]]/dir1/outputs/{{.*}}/Top-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TOP_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Top" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/Top.h" +// CHECK-NEXT: "[[PREFIX]]/dir1/module.modulemap" +// CHECK-NEXT: ] +// CHECK: "name": "Top" +// CHECK: } +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "cas-include-tree-id": "[[TU_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Left" +// CHECK: } +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "[[PREFIX]]/cas" +// CHECK-NOT: -fmodule-map-file +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[TU_TREE]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-fsyntax-only" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Left-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fmodule-file=Left=/^modules/{{.*}}/Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=/^modules/{{.*}}/Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/dir1/tu.m" +// CHECK-NEXT: ] +// CHECK: "input-file": "[[PREFIX]]/dir1/tu.m" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +// Build the include-tree commands +// RUN: %clang @%t/Top.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/Left.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/Right.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS +// RUN: %clang @%t/tu.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_MISS + +// Scan in a different directory +// RUN: clang-scan-deps -compilation-database %t/cdb2.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/dir2/outputs \ +// RUN: -prefix-map=%t/dir2/outputs=/^modules -prefix-map=%t/dir2=/^src -prefix-map-sdk=/^sdk -prefix-map-toolchain=/^tc \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps2.json + +// RUN: %deps-to-rsp %t/deps2.json --module-name Top > %t/Top2.rsp +// RUN: %deps-to-rsp %t/deps2.json --module-name Left > %t/Left2.rsp +// RUN: %deps-to-rsp %t/deps2.json --module-name Right > %t/Right2.rsp +// RUN: %deps-to-rsp %t/deps2.json --tu-index 0 > %t/tu2.rsp + +// Check cache hits +// RUN: %clang @%t/Top2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Left2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/Right2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT +// RUN: %clang @%t/tu2.rsp 2>&1 | FileCheck %s -check-prefix=CACHE_HIT + +// CACHE_MISS: compile job cache miss +// CACHE_HIT: compile job cache hit + +//--- cdb.json.template +[{ + "file": "DIR/tu.m", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -I DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { + int x; +}; +void top(void); + +//--- Left.h +#include "Top.h" +void left(void); + +//--- Right.h +#include "Top.h" +void right(void); + +//--- tu.m +#import "Left.h" +#import + +void tu(void) { + top(); + left(); + right(); +} From d5226b1eb00e62425c53b5b3a51420b06f3895b4 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 27 Mar 2023 14:12:20 -0700 Subject: [PATCH 29/88] [clang][cas] Skip duplicates in forEachFile If we see the same File or FileList more than once, skip visiting it again. This improves performance if the same module is reached through multiple dependency chains, or the same file is in multiple modules/PCH/TU file lists explicitly. (cherry picked from commit 591fa525181ff2c17033f56f160a43f977386da1) (cherry picked from commit 653e01be976e31a5c804a6e8e1b01a04434fc2b8) --- clang/include/clang/CAS/IncludeTree.h | 4 ++ clang/lib/CAS/IncludeTree.cpp | 23 ++++++++--- clang/unittests/CAS/IncludeTreeTest.cpp | 53 +++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index b478a0614bb56..826bb85397f6b 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -309,6 +309,10 @@ class IncludeTree::FileList : public IncludeTreeBase { return File(std::move(*Node)); } + llvm::Error + forEachFileImpl(llvm::DenseSet &Seen, + llvm::function_ref Callback); + static bool isValid(const ObjectProxy &Node); static bool isValid(ObjectStore &CAS, ObjectRef Ref) { auto Node = CAS.getProxy(Ref); diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index af850b68b0233..54fa397340ecc 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -237,27 +237,38 @@ IncludeTree::FileList::getFileSize(size_t I) const { Data.data() + I * sizeof(FileSizeTy)); } -llvm::Error IncludeTree::FileList::forEachFile( +llvm::Error IncludeTree::FileList::forEachFileImpl( + llvm::DenseSet &Seen, llvm::function_ref Callback) { - size_t I = 0; + size_t Next = 0; size_t FileCount = getNumFilesCurrentList(); return forEachReference([&](ObjectRef Ref) -> llvm::Error { - if (I < FileCount) { + size_t Index = Next++; + if (!Seen.insert(Ref).second) + return llvm::Error::success(); + + if (Index < FileCount) { auto Include = getFile(Ref); if (!Include) return Include.takeError(); - return Callback(std::move(*Include), getFileSize(I++)); + return Callback(std::move(*Include), getFileSize(Index)); } + // Otherwise, it's a chained FileList. - ++I; auto Proxy = getCAS().getProxy(Ref); if (!Proxy) return Proxy.takeError(); FileList FL(std::move(*Proxy)); - return FL.forEachFile(Callback); + return FL.forEachFileImpl(Seen, Callback); }); } +llvm::Error IncludeTree::FileList::forEachFile( + llvm::function_ref Callback) { + llvm::DenseSet Seen; + return forEachFileImpl(Seen, Callback); +} + Expected IncludeTree::FileList::create(ObjectStore &DB, ArrayRef Files, ArrayRef FileLists) { diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index c85b4f767d625..1514cba3f89c5 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -197,6 +197,8 @@ TEST(IncludeTree, IncludeTreeFileList) { size_t I = 0; ASSERT_THAT_ERROR( L->forEachFile([&](IncludeTree::File F, auto Size) -> llvm::Error { + if (I >= Files.size()) + return llvm::Error::success(); // diagnosed later. EXPECT_EQ(F.getFilenameRef(), Files[I].getFilenameRef()) << "filename mismatch at " << I; EXPECT_EQ(F.getContentsRef(), Files[I].getContentsRef()) @@ -206,4 +208,55 @@ TEST(IncludeTree, IncludeTreeFileList) { return llvm::Error::success(); }), llvm::Succeeded()); + EXPECT_EQ(I, Files.size()); +} + +TEST(IncludeTree, IncludeTreeFileListDuplicates) { + std::shared_ptr DB = llvm::cas::createInMemoryCAS(); + SmallVector Files; + for (unsigned I = 0; I < 10; ++I) { + std::optional File; + std::string Path = "/file" + std::to_string(I); + static constexpr StringRef Bytes = "123456789"; + std::optional Content; + ASSERT_THAT_ERROR( + DB->storeFromString({}, Bytes.substr(0, I)).moveInto(Content), + llvm::Succeeded()); + ASSERT_THAT_ERROR( + IncludeTree::File::create(*DB, Path, *Content).moveInto(File), + llvm::Succeeded()); + Files.push_back(std::move(*File)); + } + + auto MakeFileList = [&](unsigned Begin, unsigned End, + ArrayRef Lists) { + SmallVector Entries; + for (; Begin != End; ++Begin) + Entries.push_back({Files[Begin].getRef(), Begin}); + return IncludeTree::FileList::create(*DB, Entries, Lists); + }; + + std::optional L89, L; + ASSERT_THAT_ERROR(MakeFileList(8, 10, {}).moveInto(L89), llvm::Succeeded()); + EXPECT_EQ(L89->getNumReferences(), 2u); + ASSERT_THAT_ERROR( + MakeFileList(0, 9, {L89->getRef(), L89->getRef()}).moveInto(L), + llvm::Succeeded()); + EXPECT_EQ(L->getNumReferences(), 11u); // 0, 1, ..., 8, {8, 9}, {8, 9} + + size_t I = 0; + ASSERT_THAT_ERROR( + L->forEachFile([&](IncludeTree::File F, auto Size) -> llvm::Error { + if (I >= Files.size()) + return llvm::Error::success(); // diagnosed later. + EXPECT_EQ(F.getFilenameRef(), Files[I].getFilenameRef()) + << "filename mismatch at " << I; + EXPECT_EQ(F.getContentsRef(), Files[I].getContentsRef()) + << "contents mismatch at " << I; + EXPECT_EQ(Size, I) << "size mismatch at " << I; + I += 1; + return llvm::Error::success(); + }), + llvm::Succeeded()); + EXPECT_EQ(I, Files.size()); } From 2ed7779f391ad026060f7c36d282ff7a2cc772eb Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 27 Mar 2023 21:23:42 -0700 Subject: [PATCH 30/88] [clang][cas] Update tests for stable/20221013 branch * Framework autolink is not automatic unless the binary exists * Inputs are sorted by name in dep scan --- clang/test/ClangScanDeps/modules-include-tree-exports.c | 6 +++--- .../modules-include-tree-implementation-private.c | 2 -- clang/test/ClangScanDeps/modules-include-tree-inferred.m | 1 - clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c | 1 - 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/clang/test/ClangScanDeps/modules-include-tree-exports.c b/clang/test/ClangScanDeps/modules-include-tree-exports.c index 0884e2852320d..381854d4b94f3 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-exports.c +++ b/clang/test/ClangScanDeps/modules-include-tree-exports.c @@ -16,9 +16,9 @@ // RUN: %deps-to-rsp %t/deps.json --module-name ExportGlobalWildcard > %t/ExportGlobalWildcard.rsp // RUN: %deps-to-rsp %t/deps.json --module-name NoExports > %t/NoExports.rsp // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu_export_explicit.rsp -// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_export_wildcard.rsp -// RUN: %deps-to-rsp %t/deps.json --tu-index 2 > %t/tu_export_global_wildcard.rsp -// RUN: %deps-to-rsp %t/deps.json --tu-index 3 > %t/tu_export_none.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 1 > %t/tu_export_global_wildcard.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 2 > %t/tu_export_none.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 3 > %t/tu_export_wildcard.rsp // Build // RUN: %clang @%t/TwoSubs.rsp diff --git a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c index 912a72da3a174..279a88fcf3d06 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c +++ b/clang/test/ClangScanDeps/modules-include-tree-implementation-private.c @@ -27,9 +27,7 @@ // CHECK: 5:1 (Module for visibility only) Mod_Private // CHECK: Module Map: // CHECK: Mod (framework) -// CHECK: link Mod (framework) // CHECK: Mod_Private (framework) -// CHECK: link Mod_Private (framework) // CHECK: Files: // CHECK: [[PREFIX]]/tu.m llvmcas:// diff --git a/clang/test/ClangScanDeps/modules-include-tree-inferred.m b/clang/test/ClangScanDeps/modules-include-tree-inferred.m index df0668d54e8bd..f42847ef69004 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-inferred.m +++ b/clang/test/ClangScanDeps/modules-include-tree-inferred.m @@ -22,7 +22,6 @@ // CHECK: Submodule: Mod // CHECK: Module Map: // CHECK: Mod (framework) -// CHECK: link Mod (framework) // CHECK: Files: // CHECK-NOT: [[PREFIX]]/module.modulemap // CHECK: [[PREFIX]]/Mod.framework/Headers/Mod.h llvmcas:// diff --git a/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c index 1f5e102916775..59589ed535a58 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c +++ b/clang/test/ClangScanDeps/modules-include-tree-vfsoverlay.c @@ -31,7 +31,6 @@ // CHECK: Submodule: A // CHECK: Module Map: // CHECK: A (framework) -// CHECK: link A (framework) // CHECK: Files: // CHECK-NOT: modulemap // CHECK: [[PREFIX]]/elsewhere2/A.h llvmcas:// From 06e30202675456e6701ca6cb58b11568f979a16d Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Mon, 27 Mar 2023 22:29:54 -0700 Subject: [PATCH 31/88] [dsymutil] Initialize the debug map before loading the main binary Fix a crash when a warning is emitted while loading the symbols from the main binary. The warning helper assumes that the resulting debug map is initialized, but this happened after loading the main binary. Since there's no dependency between the two the initialization can be moved up. rdar://107298776 (cherry picked from commit 568be31c9e50a7d7263417841ee1b12334529903) --- llvm/tools/dsymutil/MachODebugMapParser.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/tools/dsymutil/MachODebugMapParser.cpp b/llvm/tools/dsymutil/MachODebugMapParser.cpp index 955272b72f566..1435bf2914387 100644 --- a/llvm/tools/dsymutil/MachODebugMapParser.cpp +++ b/llvm/tools/dsymutil/MachODebugMapParser.cpp @@ -113,6 +113,8 @@ class MachODebugMapParser { StringRef BinaryPath); void Warning(const Twine &Msg, StringRef File = StringRef()) { + assert(Result && + "The debug map must be initialized before calling this function"); WithColor::warning() << "(" << MachOUtils::getArchName( Result->getTriple().getArchName()) @@ -199,10 +201,9 @@ static std::string getArchName(const object::MachOObjectFile &Obj) { std::unique_ptr MachODebugMapParser::parseOneBinary(const MachOObjectFile &MainBinary, StringRef BinaryPath) { + Result = std::make_unique(MainBinary.getArchTriple(), BinaryPath, + MainBinary.getUuid()); loadMainBinarySymbols(MainBinary); - ArrayRef UUID = MainBinary.getUuid(); - Result = - std::make_unique(MainBinary.getArchTriple(), BinaryPath, UUID); MainBinaryStrings = MainBinary.getStringTableData(); for (const SymbolRef &Symbol : MainBinary.symbols()) { const DataRefImpl &DRI = Symbol.getRawDataRefImpl(); From 9ba3d7a89be7b5c2907da188718c0a8b02ea50f7 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 23 Mar 2023 21:43:32 -0700 Subject: [PATCH 32/88] [lldb] Add ability to hide the root name of a value When printing a value, allow the root value's name to be elided, without omiting the names of child values. At the API level, this adds `SetHideRootName()`, which joins the existing `SetHideName()` function. This functionality is used by `dwim-print` and `expression`. Fixes an issue identified by @jgorbe in https://reviews.llvm.org/D145609. Differential Revision: https://reviews.llvm.org/D146783 (cherry picked from commit 23349d83a98f23e67ff0321dad7c378b117ce6aa) --- .../DataFormatters/DumpValueObjectOptions.h | 3 +++ .../lldb/DataFormatters/ValueObjectPrinter.h | 2 ++ .../Commands/CommandObjectDWIMPrint.cpp | 2 +- .../Commands/CommandObjectExpression.cpp | 2 +- .../DataFormatters/DumpValueObjectOptions.cpp | 14 +++++++---- .../DataFormatters/ValueObjectPrinter.cpp | 24 ++++++++++++++----- .../API/commands/dwim-print/TestDWIMPrint.py | 8 +++++++ lldb/test/API/commands/dwim-print/main.c | 7 ++++++ 8 files changed, 50 insertions(+), 12 deletions(-) diff --git a/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h b/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h index c40ccd1cdb439..d92d7fff2a00f 100644 --- a/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h +++ b/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h @@ -105,6 +105,8 @@ class DumpValueObjectOptions { DumpValueObjectOptions &SetHideRootType(bool hide_root_type = false); + DumpValueObjectOptions &SetHideRootName(bool hide_root_name); + DumpValueObjectOptions &SetHideName(bool hide_name = false); DumpValueObjectOptions &SetHideValue(bool hide_value = false); @@ -146,6 +148,7 @@ class DumpValueObjectOptions { bool m_show_location : 1; bool m_use_objc : 1; bool m_hide_root_type : 1; + bool m_hide_root_name : 1; bool m_hide_name : 1; bool m_hide_value : 1; bool m_run_validator : 1; diff --git a/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h b/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h index 90e54021a71f7..e1a4b8e12e59c 100644 --- a/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h +++ b/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h @@ -120,6 +120,8 @@ class ValueObjectPrinter { bool HasReachedMaximumDepth(); private: + bool ShouldShowName() const; + ValueObject *m_orig_valobj; ValueObject *m_valobj; Stream *m_stream; diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp index e9d8ee874a2d4..1b733c55ff259 100644 --- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp +++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp @@ -84,7 +84,7 @@ bool CommandObjectDWIMPrint::DoExecute(StringRef command, DumpValueObjectOptions dump_options = m_varobj_options.GetAsDumpOptions( m_expr_options.m_verbosity, m_format_options.GetFormat()); - dump_options.SetHideName(eval_options.GetSuppressPersistentResult()); + dump_options.SetHideRootName(eval_options.GetSuppressPersistentResult()); StackFrame *frame = m_exe_ctx.GetFramePtr(); diff --git a/lldb/source/Commands/CommandObjectExpression.cpp b/lldb/source/Commands/CommandObjectExpression.cpp index b49ca3ce8a419..a513b3fdfc897 100644 --- a/lldb/source/Commands/CommandObjectExpression.cpp +++ b/lldb/source/Commands/CommandObjectExpression.cpp @@ -482,7 +482,7 @@ bool CommandObjectExpression::EvaluateExpression(llvm::StringRef expr, DumpValueObjectOptions options(m_varobj_options.GetAsDumpOptions( m_command_options.m_verbosity, format)); - options.SetHideName(eval_options.GetSuppressPersistentResult()); + options.SetHideRootName(eval_options.GetSuppressPersistentResult()); options.SetVariableFormatDisplayLanguage( result_valobj_sp->GetPreferredDisplayLanguage()); diff --git a/lldb/source/DataFormatters/DumpValueObjectOptions.cpp b/lldb/source/DataFormatters/DumpValueObjectOptions.cpp index 38de4428bb235..3fbff86a14172 100644 --- a/lldb/source/DataFormatters/DumpValueObjectOptions.cpp +++ b/lldb/source/DataFormatters/DumpValueObjectOptions.cpp @@ -19,10 +19,10 @@ DumpValueObjectOptions::DumpValueObjectOptions() m_decl_printing_helper(), m_pointer_as_array(), m_use_synthetic(true), m_scope_already_checked(false), m_flat_output(false), m_ignore_cap(false), m_show_types(false), m_show_location(false), m_use_objc(false), - m_hide_root_type(false), m_hide_name(false), m_hide_value(false), - m_run_validator(false), m_use_type_display_name(true), - m_allow_oneliner_mode(true), m_hide_pointer_value(false), - m_reveal_empty_aggregates(true) {} + m_hide_root_type(false), m_hide_root_name(false), m_hide_name(false), + m_hide_value(false), m_run_validator(false), + m_use_type_display_name(true), m_allow_oneliner_mode(true), + m_hide_pointer_value(false), m_reveal_empty_aggregates(true) {} DumpValueObjectOptions::DumpValueObjectOptions(ValueObject &valobj) : DumpValueObjectOptions() { @@ -143,6 +143,12 @@ DumpValueObjectOptions::SetHideRootType(bool hide_root_type) { return *this; } +DumpValueObjectOptions & +DumpValueObjectOptions::SetHideRootName(bool hide_root_name) { + m_hide_root_name = hide_root_name; + return *this; +} + DumpValueObjectOptions &DumpValueObjectOptions::SetHideName(bool hide_name) { m_hide_name = hide_name; return *this; diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp index 5f0336ebd890e..5d4a2b1a12e48 100644 --- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp +++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp @@ -275,7 +275,7 @@ void ValueObjectPrinter::PrintDecl() { StreamString varName; - if (!m_options.m_hide_name) { + if (ShouldShowName()) { if (m_options.m_flat_output) m_valobj->GetExpressionPath(varName); else @@ -314,7 +314,7 @@ void ValueObjectPrinter::PrintDecl() { m_stream->Printf("(%s) ", typeName.GetData()); if (!varName.Empty()) m_stream->Printf("%s =", varName.GetData()); - else if (!m_options.m_hide_name) + else if (ShouldShowName()) m_stream->Printf(" ="); } } @@ -437,7 +437,7 @@ bool ValueObjectPrinter::PrintValueAndSummaryIfNeeded(bool &value_printed, if (m_options.m_hide_pointer_value && IsPointerValue(m_valobj->GetCompilerType())) { } else { - if (!m_options.m_hide_name) + if (ShouldShowName()) m_stream->PutChar(' '); m_stream->PutCString(m_value); value_printed = true; @@ -459,7 +459,7 @@ bool ValueObjectPrinter::PrintObjectDescriptionIfNeeded(bool value_printed, // let's avoid the overly verbose no description error for a nil thing if (m_options.m_use_objc && !IsNil() && !IsUninitialized() && (!m_options.m_pointer_as_array)) { - if (!m_options.m_hide_value || !m_options.m_hide_name) + if (!m_options.m_hide_value || ShouldShowName()) m_stream->Printf(" "); const char *object_desc = nullptr; if (value_printed || summary_printed) @@ -587,8 +587,14 @@ void ValueObjectPrinter::PrintChildrenPreamble() { if (ShouldPrintValueObject()) m_stream->EOL(); } else { - if (ShouldPrintValueObject()) - m_stream->PutCString(IsRef() ? ": {\n" : " {\n"); + if (ShouldPrintValueObject()) { + if (IsRef()) { + m_stream->PutCString(": "); + } else if (ShouldShowName()) { + m_stream->PutChar(' '); + } + m_stream->PutCString("{\n"); + } m_stream->IndentMore(); } } @@ -841,3 +847,9 @@ void ValueObjectPrinter::PrintChildrenIfNeeded(bool value_printed, bool ValueObjectPrinter::HasReachedMaximumDepth() { return m_curr_depth >= m_options.m_max_depth; } + +bool ValueObjectPrinter::ShouldShowName() const { + if (m_curr_depth == 0) + return !m_options.m_hide_root_name && !m_options.m_hide_name; + return !m_options.m_hide_name; +} diff --git a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py index 705e2ef79ddeb..a5766a46532cd 100644 --- a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py +++ b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py @@ -112,3 +112,11 @@ def test_expression_language(self): lldbutil.run_to_name_breakpoint(self, "main") self._expect_cmd(f"dwim-print -l c++ -- argc", "frame variable") self._expect_cmd(f"dwim-print -l c++ -- argc + 1", "expression") + + def test_nested_values(self): + """Test dwim-print with nested values (structs, etc).""" + self.build() + lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.c")) + self.runCmd("settings set auto-one-line-summaries false") + self._expect_cmd(f"dwim-print s", "frame variable") + self._expect_cmd(f"dwim-print (struct Structure)s", "expression") diff --git a/lldb/test/API/commands/dwim-print/main.c b/lldb/test/API/commands/dwim-print/main.c index 5c2fa9bb6a78e..e2eccf3a88b4b 100644 --- a/lldb/test/API/commands/dwim-print/main.c +++ b/lldb/test/API/commands/dwim-print/main.c @@ -1,3 +1,10 @@ +struct Structure { + int number; +}; + int main(int argc, char **argv) { + struct Structure s; + s.number = 30; + // break here return 0; } From 670230167346693f2039ed83d243dffd39c502df Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Tue, 28 Mar 2023 14:13:49 -0700 Subject: [PATCH 33/88] [lldb] Support Universal Mach-O binaries with a fat64 header Support universal Mach-O binaries with a fat64 header. After 4d683f7fa7d4, dsymutil can now generate such binaries when the offsets would otherwise overflow the 32-bit offsets in the regular fat header. rdar://107289570 Differential revision: https://reviews.llvm.org/D147012 (cherry picked from commit fda53ad9374b60fc83f1a6065ae3e7985182a5d2) --- .../Python/lldbsuite/test/make/Makefile.rules | 2 +- .../ObjectContainerUniversalMachO.cpp | 63 ++++++++++++------- .../ObjectContainerUniversalMachO.h | 39 +++++++++++- lldb/test/API/macosx/universal64/Makefile | 24 +++++++ .../API/macosx/universal64/TestUniversal64.py | 39 ++++++++++++ lldb/test/API/macosx/universal64/main.c | 5 ++ 6 files changed, 145 insertions(+), 27 deletions(-) create mode 100644 lldb/test/API/macosx/universal64/Makefile create mode 100644 lldb/test/API/macosx/universal64/TestUniversal64.py create mode 100644 lldb/test/API/macosx/universal64/main.c diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules index afcfc7417bd52..ac6c26285299b 100644 --- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules +++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules @@ -183,7 +183,7 @@ ARCHFLAG ?= -arch #---------------------------------------------------------------------- ifeq "$(OS)" "Darwin" DS := $(DSYMUTIL) - DSFLAGS = + DSFLAGS := $(DSFLAGS_EXTRAS) DSYM = $(EXE).dSYM AR := $(CROSS_COMPILE)libtool ARFLAGS := -static -o diff --git a/lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.cpp b/lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.cpp index 78e0a4b2f499f..9af9c0120dd7d 100644 --- a/lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.cpp +++ b/lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.cpp @@ -57,7 +57,8 @@ ObjectContainer *ObjectContainerUniversalMachO::CreateInstance( bool ObjectContainerUniversalMachO::MagicBytesMatch(const DataExtractor &data) { lldb::offset_t offset = 0; uint32_t magic = data.GetU32(&offset); - return magic == FAT_MAGIC || magic == FAT_CIGAM; + return magic == FAT_MAGIC || magic == FAT_CIGAM || magic == FAT_MAGIC_64 || + magic == FAT_CIGAM_64; } ObjectContainerUniversalMachO::ObjectContainerUniversalMachO( @@ -82,38 +83,51 @@ bool ObjectContainerUniversalMachO::ParseHeader() { bool ObjectContainerUniversalMachO::ParseHeader( lldb_private::DataExtractor &data, llvm::MachO::fat_header &header, - std::vector &fat_archs) { - bool success = false; + std::vector &fat_archs) { // Store the file offset for this universal file as we could have a universal // .o file in a BSD archive, or be contained in another kind of object. - // Universal mach-o files always have their headers in big endian. lldb::offset_t offset = 0; data.SetByteOrder(eByteOrderBig); header.magic = data.GetU32(&offset); fat_archs.clear(); - if (header.magic == FAT_MAGIC) { - - data.SetAddressByteSize(4); + // Universal mach-o files always have their headers in big endian. + if (header.magic == FAT_MAGIC || header.magic == FAT_MAGIC_64) { + const bool is_fat64 = header.magic == FAT_MAGIC_64; + data.SetAddressByteSize(is_fat64 ? 8 : 4); header.nfat_arch = data.GetU32(&offset); // Now we should have enough data for all of the fat headers, so lets index // them so we know how many architectures that this universal binary // contains. - uint32_t arch_idx = 0; - for (arch_idx = 0; arch_idx < header.nfat_arch; ++arch_idx) { + for (uint32_t arch_idx = 0; arch_idx < header.nfat_arch; ++arch_idx) { if (data.ValidOffsetForDataOfSize(offset, sizeof(fat_arch))) { - fat_arch arch; - if (data.GetU32(&offset, &arch, sizeof(fat_arch) / sizeof(uint32_t))) - fat_archs.push_back(arch); + if (is_fat64) { + fat_arch_64 arch; + arch.cputype = data.GetU32(&offset); + arch.cpusubtype = data.GetU32(&offset); + arch.offset = data.GetU64(&offset); + arch.size = data.GetU64(&offset); + arch.align = data.GetU32(&offset); + arch.reserved = data.GetU32(&offset); + fat_archs.emplace_back(arch); + } else { + fat_arch arch; + arch.cputype = data.GetU32(&offset); + arch.cpusubtype = data.GetU32(&offset); + arch.offset = data.GetU32(&offset); + arch.size = data.GetU32(&offset); + arch.align = data.GetU32(&offset); + fat_archs.emplace_back(arch); + } } } - success = true; - } else { - memset(&header, 0, sizeof(header)); + return true; } - return success; + + memset(&header, 0, sizeof(header)); + return true; } size_t ObjectContainerUniversalMachO::GetNumArchitectures() const { @@ -123,8 +137,8 @@ size_t ObjectContainerUniversalMachO::GetNumArchitectures() const { bool ObjectContainerUniversalMachO::GetArchitectureAtIndex( uint32_t idx, ArchSpec &arch) const { if (idx < m_header.nfat_arch) { - arch.SetArchitecture(eArchTypeMachO, m_fat_archs[idx].cputype, - m_fat_archs[idx].cpusubtype); + arch.SetArchitecture(eArchTypeMachO, m_fat_archs[idx].GetCPUType(), + m_fat_archs[idx].GetCPUSubType()); return true; } return false; @@ -166,8 +180,8 @@ ObjectContainerUniversalMachO::GetObjectFile(const FileSpec *file) { DataBufferSP data_sp; lldb::offset_t data_offset = 0; return ObjectFile::FindPlugin( - module_sp, file, m_offset + m_fat_archs[arch_idx].offset, - m_fat_archs[arch_idx].size, data_sp, data_offset); + module_sp, file, m_offset + m_fat_archs[arch_idx].GetOffset(), + m_fat_archs[arch_idx].GetSize(), data_sp, data_offset); } } return ObjectFileSP(); @@ -184,11 +198,12 @@ size_t ObjectContainerUniversalMachO::GetModuleSpecifications( if (ObjectContainerUniversalMachO::MagicBytesMatch(data)) { llvm::MachO::fat_header header; - std::vector fat_archs; + std::vector fat_archs; if (ParseHeader(data, header, fat_archs)) { - for (const llvm::MachO::fat_arch &fat_arch : fat_archs) { - const lldb::offset_t slice_file_offset = fat_arch.offset + file_offset; - if (fat_arch.offset < file_size && file_size > slice_file_offset) { + for (const FatArch &fat_arch : fat_archs) { + const lldb::offset_t slice_file_offset = + fat_arch.GetOffset() + file_offset; + if (fat_arch.GetOffset() < file_size && file_size > slice_file_offset) { ObjectFile::GetModuleSpecifications( file, slice_file_offset, file_size - slice_file_offset, specs); } diff --git a/lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.h b/lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.h index 4fbea936ac85c..20f1f051e07a0 100644 --- a/lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.h +++ b/lldb/source/Plugins/ObjectContainer/Universal-Mach-O/ObjectContainerUniversalMachO.h @@ -63,11 +63,46 @@ class ObjectContainerUniversalMachO : public lldb_private::ObjectContainer { protected: llvm::MachO::fat_header m_header; - std::vector m_fat_archs; + + struct FatArch { + FatArch(llvm::MachO::fat_arch arch) : m_arch(arch), m_is_fat64(false) {} + FatArch(llvm::MachO::fat_arch_64 arch) : m_arch(arch), m_is_fat64(true) {} + + uint32_t GetCPUType() const { + return m_is_fat64 ? m_arch.fat_arch_64.cputype : m_arch.fat_arch.cputype; + } + + uint32_t GetCPUSubType() const { + return m_is_fat64 ? m_arch.fat_arch_64.cpusubtype + : m_arch.fat_arch.cpusubtype; + } + + uint64_t GetOffset() const { + return m_is_fat64 ? m_arch.fat_arch_64.offset : m_arch.fat_arch.offset; + } + + uint64_t GetSize() const { + return m_is_fat64 ? m_arch.fat_arch_64.size : m_arch.fat_arch.size; + } + + uint32_t GetAlign() const { + return m_is_fat64 ? m_arch.fat_arch_64.align : m_arch.fat_arch.align; + } + + private: + const union Arch { + Arch(llvm::MachO::fat_arch arch) : fat_arch(arch) {} + Arch(llvm::MachO::fat_arch_64 arch) : fat_arch_64(arch) {} + llvm::MachO::fat_arch fat_arch; + llvm::MachO::fat_arch_64 fat_arch_64; + } m_arch; + const bool m_is_fat64; + }; + std::vector m_fat_archs; static bool ParseHeader(lldb_private::DataExtractor &data, llvm::MachO::fat_header &header, - std::vector &fat_archs); + std::vector &fat_archs); }; #endif // LLDB_SOURCE_PLUGINS_OBJECTCONTAINER_UNIVERSAL_MACH_O_OBJECTCONTAINERUNIVERSALMACHO_H diff --git a/lldb/test/API/macosx/universal64/Makefile b/lldb/test/API/macosx/universal64/Makefile new file mode 100644 index 0000000000000..f763f3ae2f6c9 --- /dev/null +++ b/lldb/test/API/macosx/universal64/Makefile @@ -0,0 +1,24 @@ +EXE := fat.out + +ifdef FAT64_DSYM + DSFLAGS_EXTRAS=-fat64 +endif + +include Makefile.rules + +all: fat.out + +fat.out: fat.arm64.out fat.x86_64.out + lipo -fat64 -create -o $@ $^ + +fat.x86_64.out: fat.x86_64.o + $(CC) -isysroot $(SDKROOT) -target x86_64-apple-macosx10.9 -o $@ $< + +fat.arm64.out: fat.arm64.o + $(CC) -isysroot $(SDKROOT) -target arm64-apple-macosx10.9 -o $@ $< + +fat.x86_64.o: main.c + $(CC) -isysroot $(SDKROOT) -g -O0 -target x86_64-apple-macosx11 -c -o $@ $< + +fat.arm64.o: main.c + $(CC) -isysroot $(SDKROOT) -g -O0 -target arm64-apple-macosx11 -c -o $@ $< diff --git a/lldb/test/API/macosx/universal64/TestUniversal64.py b/lldb/test/API/macosx/universal64/TestUniversal64.py new file mode 100644 index 0000000000000..0c4226fac5380 --- /dev/null +++ b/lldb/test/API/macosx/universal64/TestUniversal64.py @@ -0,0 +1,39 @@ +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class Universal64TestCase(TestBase): + NO_DEBUG_INFO_TESTCASE = True + + def do_test(self): + # Get the executable. + exe = self.getBuildArtifact("fat.out") + + # Create a target. + self.target = self.dbg.CreateTarget(exe) + + # Create a breakpoint on main. + main_bp = self.target.BreakpointCreateByName("main") + self.assertTrue(main_bp, VALID_BREAKPOINT) + + # Make sure the binary and the dSYM are in the image list. + self.expect("image list ", patterns=['fat.out', 'fat.out.dSYM']) + + # The dynamic loader doesn't support fat64 executables so we can't + # actually launch them here. + + @skipUnlessDarwin + @skipIfDarwinEmbedded + def test_universal64_executable(self): + """Test fat64 universal executable""" + self.build(debug_info="dsym") + self.do_test() + + @skipUnlessDarwin + @skipIfDarwinEmbedded + @skipIf(compiler="clang", compiler_version=['<', '7.0']) + def test_universal64_dsym(self): + """Test fat64 universal dSYM""" + self.build(debug_info="dsym", dictionary={'FAT64_DSYM': '1'}) + self.do_test() diff --git a/lldb/test/API/macosx/universal64/main.c b/lldb/test/API/macosx/universal64/main.c new file mode 100644 index 0000000000000..5124e0afbfc19 --- /dev/null +++ b/lldb/test/API/macosx/universal64/main.c @@ -0,0 +1,5 @@ +#include + +int foo() { return 0; } + +int main(int argc, char **argv) { return foo(); } From 372efe77b87bd0b3342b05c15d9d09f2a4c9a569 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Tue, 21 Mar 2023 16:33:28 -0700 Subject: [PATCH 34/88] [libclang] Add APIs to report dependency scanning info as opaque objects This is useful so we can evolve the reported scanning information without breaking ABI. (cherry picked from commit 55779305c69335faba3e020faea98a56a1e05b62) --- clang/include/clang-c/CXString.h | 9 + clang/include/clang-c/Dependencies.h | 245 +++++++++++++++++++++++++ clang/tools/c-index-test/core_main.cpp | 135 ++++++++------ clang/tools/libclang/CDependencies.cpp | 235 +++++++++++++++++++++++- clang/tools/libclang/libclang.map | 22 +++ 5 files changed, 589 insertions(+), 57 deletions(-) diff --git a/clang/include/clang-c/CXString.h b/clang/include/clang-c/CXString.h index f117010c71a46..f132a2191a79e 100644 --- a/clang/include/clang-c/CXString.h +++ b/clang/include/clang-c/CXString.h @@ -16,6 +16,7 @@ #include "clang-c/ExternC.h" #include "clang-c/Platform.h" +#include LLVM_CLANG_C_EXTERN_C_BEGIN @@ -44,6 +45,14 @@ typedef struct { unsigned Count; } CXStringSet; +/** + * An array of C strings. + */ +typedef struct { + const char **Strings; + size_t Count; +} CXCStringArray; + /** * Retrieve the character data associated with the given string. */ diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 2daf9f38b0fc5..ec04090365eec 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -365,6 +365,251 @@ clang_experimental_DependencyScannerWorker_getFileDependencies_v5( CXModuleLookupOutputCallback *MLO, unsigned Options, CXFileDependenciesList **Out, CXDiagnosticSet *OutDiags); +/** + * Output of \c clang_experimental_DependencyScannerWorker_getDepGraph. + */ +typedef struct CXOpaqueDepGraph *CXDepGraph; + +/** + * An individual module dependency that is part of an overall compilation + * \c CXDepGraph. + */ +typedef struct CXOpaqueDepGraphModule *CXDepGraphModule; + +/** + * An individual command-line invocation that is part of an overall compilation + * \c CXDepGraph. + */ +typedef struct CXOpaqueDepGraphTUCommand *CXDepGraphTUCommand; + +/** + * Settings to use for the + * \c clang_experimental_DependencyScannerWorker_getDepGraph action. + */ +typedef struct CXOpaqueDependencyScannerWorkerScanSettings + *CXDependencyScannerWorkerScanSettings; + +/** + * Creates a set of settings for + * \c clang_experimental_DependencyScannerWorker_getDepGraph action. + * Must be disposed with + * \c clang_experimental_DependencyScannerWorkerScanSettings_dispose. + * Memory for settings is not copied. Any provided pointers must be valid until + * the call to \c clang_experimental_DependencyScannerWorker_getDepGraph. + * + * \param argc the number of compiler invocation arguments (including argv[0]). + * \param argv the compiler driver invocation arguments (including argv[0]). + * \param ModuleName If non-null, the dependencies of the named module are + * returned. Otherwise, the dependencies of the whole + * translation unit are returned. + * \param WorkingDirectory the directory in which the invocation runs. + * \param MLOContext the context that will be passed to \c MLO each time it is + * called. + * \param MLO a callback that is called to determine the paths of output files + * for each module dependency. This may receive the same module on + * different workers. This should be NULL if + * \c clang_experimental_DependencyScannerService_create_v1 was + * called with \c CXDependencyMode_Flat. This callback will be called + * on the same thread that called \c + * clang_experimental_DependencyScannerWorker_getDepGraph. + */ +CINDEX_LINKAGE CXDependencyScannerWorkerScanSettings +clang_experimental_DependencyScannerWorkerScanSettings_create( + int argc, const char *const *argv, const char *ModuleName, + const char *WorkingDirectory, void *MLOContext, + CXModuleLookupOutputCallback *MLO); + +/** + * Dispose of a \c CXDependencyScannerWorkerScanSettings object. + */ +CINDEX_LINKAGE void + clang_experimental_DependencyScannerWorkerScanSettings_dispose( + CXDependencyScannerWorkerScanSettings); + +/** + * Produces the dependency graph for a particular compiler invocation. + * + * \param Settings object created via + * \c clang_experimental_DependencyScannerWorkerScanSettings_create. + * \param [out] Out A non-NULL pointer to store the resulting dependencies. The + * output must be freed by calling + * \c clang_experimental_DepGraph_dispose. + * + * \returns \c CXError_Success on success; otherwise a non-zero \c CXErrorCode + * indicating the kind of error. When returning \c CXError_Failure there will + * be a \c CXDepGraph object on \p Out that can be used to get diagnostics via + * \c clang_experimental_DepGraph_getDiagnostics. + */ +CINDEX_LINKAGE enum CXErrorCode +clang_experimental_DependencyScannerWorker_getDepGraph( + CXDependencyScannerWorker, CXDependencyScannerWorkerScanSettings Settings, + CXDepGraph *Out); + +/** + * Dispose of a \c CXDepGraph object. + */ +CINDEX_LINKAGE void clang_experimental_DepGraph_dispose(CXDepGraph); + +/** + * \returns the number of \c CXDepGraphModule objects in the graph. + */ +CINDEX_LINKAGE size_t clang_experimental_DepGraph_getNumModules(CXDepGraph); + +/** + * \returns the \c CXDepGraphModule object at the given \p Index. + * + * The \c CXDepGraphModule object is only valid to use while \c CXDepGraph is + * valid. Must be disposed with \c clang_experimental_DepGraphModule_dispose. + */ +CINDEX_LINKAGE CXDepGraphModule +clang_experimental_DepGraph_getModule(CXDepGraph, size_t Index); + +CINDEX_LINKAGE void clang_experimental_DepGraphModule_dispose(CXDepGraphModule); + +/** + * \returns the name of the module. This may include `:` for C++20 module + * partitions, or a header-name for C++20 header units. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getName(CXDepGraphModule); + +/** + * \returns the context hash of a module represents the set of compiler options + * that may make one version of a module incompatible from another. This + * includes things like language mode, predefined macros, header search paths, + * etc... + * + * Modules with the same name but a different \c ContextHash should be treated + * as separate modules for the purpose of a build. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getContextHash(CXDepGraphModule); + +/** + * \returns the path to the modulemap file which defines this module. If there's + * no modulemap (e.g. for a C++ module) returns \c NULL. + * + * This can be used to explicitly build this module. This file will + * additionally appear in \c FileDeps as a dependency. + * + * The string is only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphModule_getModuleMapPath(CXDepGraphModule); + +/** + * \returns the list of files which this module directly depends on. + * + * If any of these change then the module needs to be rebuilt. + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getFileDeps(CXDepGraphModule); + +/** + * \returns the list of modules which this module direct depends on. + * + * This does include the context hash. The format is + * `:` + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getModuleDeps(CXDepGraphModule); + +/** + * \returns the canonical command line to build this module. + * + * The strings are only valid to use while the \c CXDepGraphModule object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule); + +/** + * \returns the number \c CXDepGraphTUCommand objects in the graph. + */ +CINDEX_LINKAGE size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph); + +/** + * \returns the \c CXDepGraphTUCommand object at the given \p Index. + * + * The \c CXDepGraphTUCommand object is only valid to use while \c CXDepGraph is + * valid. Must be disposed with \c clang_experimental_DepGraphTUCommand_dispose. + */ +CINDEX_LINKAGE CXDepGraphTUCommand +clang_experimental_DepGraph_getTUCommand(CXDepGraph, size_t Index); + +/** + * Dispose of a \c CXDepGraphTUCommand object. + */ +CINDEX_LINKAGE void + clang_experimental_DepGraphTUCommand_dispose(CXDepGraphTUCommand); + +/** + * \returns the executable name for the command. + * + * The string is only valid to use while the \c CXDepGraphTUCommand object is + * valid. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphTUCommand_getExecutable(CXDepGraphTUCommand); + +/** + * \returns the canonical command line to build this translation unit. + * + * The strings are only valid to use while the \c CXDepGraphTUCommand object is + * valid. + */ +CINDEX_LINKAGE CXCStringArray + clang_experimental_DepGraphTUCommand_getBuildArguments(CXDepGraphTUCommand); + +/** + * \returns the list of files which this translation unit directly depends on. + * + * The strings are only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +CXCStringArray clang_experimental_DepGraph_getTUFileDeps(CXDepGraph); + +/** + * \returns the list of modules which this translation unit direct depends on. + * + * This does include the context hash. The format is + * `:` + * + * The strings are only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +CXCStringArray clang_experimental_DepGraph_getTUModuleDeps(CXDepGraph); + +/** + * \returns the context hash of the C++20 module this translation unit exports. + * + * If the translation unit is not a module then this is empty. + * + * The string is only valid to use while the \c CXDepGraph object is valid. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph); + +/** + * \returns The diagnostics emitted during scanning. These must be always freed + * by calling \c clang_disposeDiagnosticSet. + */ +CINDEX_LINKAGE +CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph); + /** * @} */ diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 2fa66355c8daf..e5512c9d33678 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -719,35 +719,6 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DependencyScannerService_dispose_v0(Service); }); - auto Callback = [&](CXModuleDependencySet *MDS) { - llvm::outs() << "modules:\n"; - for (const auto &M : llvm::makeArrayRef(MDS->Modules, MDS->Count)) { - llvm::outs() << " module:\n" - << " name: " << clang_getCString(M.Name) << "\n" - << " context-hash: " << clang_getCString(M.ContextHash) - << "\n" - << " module-map-path: " - << clang_getCString(M.ModuleMapPath) << "\n" - << " module-deps:\n"; - for (const auto &ModuleName : - llvm::makeArrayRef(M.ModuleDeps->Strings, M.ModuleDeps->Count)) - llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; - llvm::outs() << " file-deps:\n"; - for (const auto &FileName : - llvm::makeArrayRef(M.FileDeps->Strings, M.FileDeps->Count)) - llvm::outs() << " " << clang_getCString(FileName) << "\n"; - llvm::outs() << " build-args:"; - for (const auto &Arg : llvm::makeArrayRef(M.BuildArguments->Strings, - M.BuildArguments->Count)) - llvm::outs() << " " << clang_getCString(Arg); - llvm::outs() << "\n"; - } - clang_experimental_ModuleDependencySet_dispose(MDS); - }; - - auto CB = - functionObjectToCCallbackRef(Callback); - auto LookupOutput = [&](const char *ModuleName, const char *ContextHash, CXOutputKind Kind, char *Output, size_t MaxLen) { std::string Out = OutputPath + "/" + ModuleName + "_" + ContextHash; @@ -781,46 +752,98 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, char *Output, size_t MaxLen)>(LookupOutput); unsigned CommandIndex = 0; - auto HandleCommand = [&](CXString ContextHash, CXStringSet *ModuleDeps, - CXStringSet *FileDeps, CXStringSet *Args) { + auto HandleCommand = [&](const char *ContextHash, CXCStringArray ModuleDeps, + CXCStringArray FileDeps, CXCStringArray Args) { llvm::outs() << " command " << CommandIndex++ << ":\n"; - llvm::outs() << " context-hash: " << clang_getCString(ContextHash) - << "\n" + llvm::outs() << " context-hash: " << ContextHash << "\n" << " module-deps:\n"; for (const auto &ModuleName : - llvm::makeArrayRef(ModuleDeps->Strings, ModuleDeps->Count)) - llvm::outs() << " " << clang_getCString(ModuleName) << "\n"; + llvm::makeArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) + llvm::outs() << " " << ModuleName << "\n"; llvm::outs() << " file-deps:\n"; - for (const auto &FileName : - llvm::makeArrayRef(FileDeps->Strings, FileDeps->Count)) - llvm::outs() << " " << clang_getCString(FileName) << "\n"; + for (const auto &FileName : ArrayRef(FileDeps.Strings, FileDeps.Count)) + llvm::outs() << " " << FileName << "\n"; llvm::outs() << " build-args:"; - for (const auto &Arg : llvm::makeArrayRef(Args->Strings, Args->Count)) - llvm::outs() << " " << clang_getCString(Arg); + for (const auto &Arg : ArrayRef(Args.Strings, Args.Count)) + llvm::outs() << " " << Arg; llvm::outs() << "\n"; }; - CXFileDependenciesList *Result = nullptr; - CXDiagnosticSet Diags; - auto DisposeDiagnosticSet = - llvm::make_scope_exit([&]() { clang_disposeDiagnosticSet(Diags); }); - CXErrorCode Err = - clang_experimental_DependencyScannerWorker_getFileDependencies_v5( - Worker, Args.size(), Args.data(), - ModuleName ? ModuleName->c_str() : nullptr, WorkingDirectory.c_str(), - CB.Context, CB.Callback, LookupOutputCB.Context, - LookupOutputCB.Callback, - /*Options=*/0, &Result, &Diags); + CXDependencyScannerWorkerScanSettings ScanSettings = + clang_experimental_DependencyScannerWorkerScanSettings_create( + Args.size(), Args.data(), ModuleName ? ModuleName->c_str() : nullptr, + WorkingDirectory.c_str(), LookupOutputCB.Context, + LookupOutputCB.Callback); + auto DisposeScanSettings = llvm::make_scope_exit([&]() { + clang_experimental_DependencyScannerWorkerScanSettings_dispose( + ScanSettings); + }); + CXDepGraph Graph = nullptr; + auto DisposeDepGraph = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraph_dispose(Graph); }); + CXErrorCode Err = clang_experimental_DependencyScannerWorker_getDepGraph( + Worker, ScanSettings, &Graph); + if (Err == CXError_Success) { + llvm::outs() << "modules:\n"; + for (size_t I = 0, E = clang_experimental_DepGraph_getNumModules(Graph); + I < E; ++I) { + CXDepGraphModule Mod = clang_experimental_DepGraph_getModule(Graph, I); + const char *Name = clang_experimental_DepGraphModule_getName(Mod); + const char *ContextHash = + clang_experimental_DepGraphModule_getContextHash(Mod); + const char *ModuleMapPath = + clang_experimental_DepGraphModule_getModuleMapPath(Mod); + CXCStringArray ModuleDeps = + clang_experimental_DepGraphModule_getModuleDeps(Mod); + CXCStringArray FileDeps = + clang_experimental_DepGraphModule_getFileDeps(Mod); + CXCStringArray BuildArguments = + clang_experimental_DepGraphModule_getBuildArguments(Mod); + auto Dispose = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraphModule_dispose(Mod); }); + llvm::outs() << " module:\n" + << " name: " << Name << "\n" + << " context-hash: " << ContextHash << "\n" + << " module-map-path: " + << (ModuleMapPath ? ModuleMapPath : "") << "\n" + << " module-deps:\n"; + for (const auto &ModuleName : + ArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) + llvm::outs() << " " << ModuleName << "\n"; + llvm::outs() << " file-deps:\n"; + for (const auto &FileName : ArrayRef(FileDeps.Strings, FileDeps.Count)) + llvm::outs() << " " << FileName << "\n"; + llvm::outs() << " build-args:"; + for (const auto &Arg : + ArrayRef(BuildArguments.Strings, BuildArguments.Count)) + llvm::outs() << " " << Arg; + llvm::outs() << "\n"; + } + llvm::outs() << "dependencies:\n"; - for (size_t I = 0; I < Result->NumCommands; ++I) - HandleCommand( - Result->Commands[I].ContextHash, Result->Commands[I].ModuleDeps, - Result->Commands[I].FileDeps, Result->Commands[I].BuildArguments); - clang_experimental_FileDependenciesList_dispose(Result); + const char *TUContextHash = + clang_experimental_DepGraph_getTUContextHash(Graph); + CXCStringArray TUModuleDeps = + clang_experimental_DepGraph_getTUModuleDeps(Graph); + CXCStringArray TUFileDeps = + clang_experimental_DepGraph_getTUFileDeps(Graph); + for (size_t I = 0, E = clang_experimental_DepGraph_getNumTUCommands(Graph); + I < E; ++I) { + CXDepGraphTUCommand Cmd = + clang_experimental_DepGraph_getTUCommand(Graph, I); + CXCStringArray Args = + clang_experimental_DepGraphTUCommand_getBuildArguments(Cmd); + auto Dispose = llvm::make_scope_exit( + [&]() { clang_experimental_DepGraphTUCommand_dispose(Cmd); }); + HandleCommand(TUContextHash, TUModuleDeps, TUFileDeps, Args); + } return 0; } llvm::errs() << "error: failed to get dependencies\n"; + CXDiagnosticSet Diags = clang_experimental_DepGraph_getDiagnostics(Graph); + auto DisposeDiagnosticSet = + llvm::make_scope_exit([&]() { clang_disposeDiagnosticSet(Diags); }); for (unsigned I = 0, N = clang_getNumDiagnosticsInSet(Diags); I < N; ++I) { CXDiagnostic Diag = clang_getDiagnosticInSet(Diags, I); CXString Spelling = clang_getDiagnosticSpelling(Diag); diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 4e202d2153282..f8c052dab6688 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -239,7 +239,7 @@ static CXErrorCode getFullDependencies(DependencyScanningWorker *Worker, TranslationUnitDeps TU = DepConsumer.takeTranslationUnitDeps(); - if (!TU.ModuleGraph.empty()) { + if (MDC && !TU.ModuleGraph.empty()) { CXModuleDependencySet *MDS = new CXModuleDependencySet; MDS->Count = TU.ModuleGraph.size(); MDS->Modules = new CXModuleDependency[MDS->Count]; @@ -385,6 +385,239 @@ CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v5( return Result; } +namespace { + +struct DependencyScannerWorkerScanSettings { + int argc; + const char *const *argv; + const char *ModuleName; + const char *WorkingDirectory; + void *MLOContext; + CXModuleLookupOutputCallback *MLO; +}; + +struct CStringsManager { + SmallVector>> OwnedCStr; + SmallVector>> OwnedStdStr; + + /// Doesn't own the string contents. + CXCStringArray createCStringsRef(ArrayRef Strings) { + OwnedCStr.push_back(std::make_unique>()); + std::vector &CStrings = *OwnedCStr.back(); + CStrings.reserve(Strings.size()); + for (const auto &String : Strings) + CStrings.push_back(String.c_str()); + return {CStrings.data(), CStrings.size()}; + } + + /// Doesn't own the string contents. + CXCStringArray createCStringsRef(const llvm::StringSet<> &StringsUnordered) { + std::vector Strings; + + for (auto SI = StringsUnordered.begin(), SE = StringsUnordered.end(); + SI != SE; ++SI) + Strings.push_back(SI->getKey()); + + llvm::sort(Strings); + + OwnedCStr.push_back(std::make_unique>()); + std::vector &CStrings = *OwnedCStr.back(); + CStrings.reserve(Strings.size()); + for (const auto &String : Strings) + CStrings.push_back(String.data()); + return {CStrings.data(), CStrings.size()}; + } + + /// Gets ownership of string contents. + CXCStringArray createCStringsOwned(std::vector &&Strings) { + OwnedStdStr.push_back( + std::make_unique>(std::move(Strings))); + return createCStringsRef(*OwnedStdStr.back()); + } +}; + +struct DependencyGraph { + TranslationUnitDeps TUDeps; + CXDiagnosticSetDiagnosticConsumer DiagConsumer; + CStringsManager StrMgr{}; +}; + +struct DependencyGraphModule { + ModuleDeps *ModDeps; + CStringsManager StrMgr{}; +}; + +struct DependencyGraphTUCommand { + Command *TUCmd; + CStringsManager StrMgr{}; +}; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyScannerWorkerScanSettings, + CXDependencyScannerWorkerScanSettings) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraph, CXDepGraph) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraphModule, CXDepGraphModule) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DependencyGraphTUCommand, + CXDepGraphTUCommand) + +} // end anonymous namespace + +CXDependencyScannerWorkerScanSettings +clang_experimental_DependencyScannerWorkerScanSettings_create( + int argc, const char *const *argv, const char *ModuleName, + const char *WorkingDirectory, void *MLOContext, + CXModuleLookupOutputCallback *MLO) { + return wrap(new DependencyScannerWorkerScanSettings{ + argc, argv, ModuleName, WorkingDirectory, MLOContext, MLO}); +} + +void clang_experimental_DependencyScannerWorkerScanSettings_dispose( + CXDependencyScannerWorkerScanSettings Settings) { + delete unwrap(Settings); +} + +enum CXErrorCode clang_experimental_DependencyScannerWorker_getDepGraph( + CXDependencyScannerWorker W, + CXDependencyScannerWorkerScanSettings CXSettings, CXDepGraph *Out) { + DependencyScannerWorkerScanSettings &Settings = *unwrap(CXSettings); + int argc = Settings.argc; + const char *const *argv = Settings.argv; + const char *ModuleName = Settings.ModuleName; + const char *WorkingDirectory = Settings.WorkingDirectory; + void *MLOContext = Settings.MLOContext; + CXModuleLookupOutputCallback *MLO = Settings.MLO; + + OutputLookup OL(MLOContext, MLO); + auto LookupOutputs = [&](const ModuleID &ID, ModuleOutputKind MOK) { + return OL.lookupModuleOutput(ID, MOK); + }; + + if (!Out) + return CXError_InvalidArguments; + *Out = nullptr; + + DependencyGraph *DepGraph = new DependencyGraph(); + + CXErrorCode Result = getFileDependencies( + W, argc, argv, WorkingDirectory, /*MDC=*/nullptr, /*MDCContext=*/nullptr, + /*Error=*/nullptr, &DepGraph->DiagConsumer, LookupOutputs, + ModuleName ? Optional(ModuleName) : std::nullopt, + [&](TranslationUnitDeps TU) { DepGraph->TUDeps = std::move(TU); }); + + *Out = wrap(DepGraph); + return Result; +} + +void clang_experimental_DepGraph_dispose(CXDepGraph Graph) { + delete unwrap(Graph); +} + +size_t clang_experimental_DepGraph_getNumModules(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.ModuleGraph.size(); +} + +CXDepGraphModule clang_experimental_DepGraph_getModule(CXDepGraph Graph, + size_t Index) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return wrap(new DependencyGraphModule{&TUDeps.ModuleGraph[Index]}); +} + +void clang_experimental_DepGraphModule_dispose(CXDepGraphModule CXDepMod) { + delete unwrap(CXDepMod); +} + +const char * +clang_experimental_DepGraphModule_getName(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return ModDeps.ID.ModuleName.c_str(); +} + +const char * +clang_experimental_DepGraphModule_getContextHash(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return ModDeps.ID.ContextHash.c_str(); +} + +const char * +clang_experimental_DepGraphModule_getModuleMapPath(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + if (ModDeps.ClangModuleMapFile.empty()) + return nullptr; + return ModDeps.ClangModuleMapFile.c_str(); +} + +CXCStringArray +clang_experimental_DepGraphModule_getFileDeps(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.FileDeps); +} + +CXCStringArray +clang_experimental_DepGraphModule_getModuleDeps(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + std::vector Modules; + Modules.reserve(ModDeps.ClangModuleDeps.size()); + for (const ModuleID &MID : ModDeps.ClangModuleDeps) + Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); + return unwrap(CXDepMod)->StrMgr.createCStringsOwned(std::move(Modules)); +} + +CXCStringArray +clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.BuildArguments); +} + +size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.Commands.size(); +} + +CXDepGraphTUCommand clang_experimental_DepGraph_getTUCommand(CXDepGraph Graph, + size_t Index) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return wrap(new DependencyGraphTUCommand{&TUDeps.Commands[Index]}); +} + +void clang_experimental_DepGraphTUCommand_dispose(CXDepGraphTUCommand CXCmd) { + delete unwrap(CXCmd); +} + +const char * +clang_experimental_DepGraphTUCommand_getExecutable(CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + return TUCmd.Executable.c_str(); +} + +CXCStringArray clang_experimental_DepGraphTUCommand_getBuildArguments( + CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + return unwrap(CXCmd)->StrMgr.createCStringsRef(TUCmd.Arguments); +} + +CXCStringArray clang_experimental_DepGraph_getTUFileDeps(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return unwrap(Graph)->StrMgr.createCStringsRef(TUDeps.FileDeps); +} + +CXCStringArray clang_experimental_DepGraph_getTUModuleDeps(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + std::vector Modules; + Modules.reserve(TUDeps.ClangModuleDeps.size()); + for (const ModuleID &MID : TUDeps.ClangModuleDeps) + Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); + return unwrap(Graph)->StrMgr.createCStringsOwned(std::move(Modules)); +} + +const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph Graph) { + TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; + return TUDeps.ID.ContextHash.c_str(); +} + +CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) { + return unwrap(Graph)->DiagConsumer.getDiagnosticSet(); +} + static std::string lookupModuleOutput(const ModuleID &ID, ModuleOutputKind MOK, void *MLOContext, CXModuleLookupOutputCallback *MLO) { diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 6ba53c57c1f15..e9b5ffbc610e4 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -489,7 +489,29 @@ LLVM_16 { clang_experimental_DependencyScannerServiceOptions_setCASDatabases; clang_experimental_DependencyScannerServiceOptions_setDependencyMode; clang_experimental_DependencyScannerServiceOptions_setObjectStore; + clang_experimental_DependencyScannerWorker_getDepGraph; clang_experimental_DependencyScannerWorker_getFileDependencies_v5; + clang_experimental_DependencyScannerWorkerScanSettings_create; + clang_experimental_DependencyScannerWorkerScanSettings_dispose; + clang_experimental_DepGraph_dispose; + clang_experimental_DepGraph_getDiagnostics; + clang_experimental_DepGraph_getModule; + clang_experimental_DepGraph_getNumModules; + clang_experimental_DepGraph_getNumTUCommands; + clang_experimental_DepGraph_getTUCommand; + clang_experimental_DepGraph_getTUContextHash; + clang_experimental_DepGraph_getTUFileDeps; + clang_experimental_DepGraph_getTUModuleDeps; + clang_experimental_DepGraphModule_dispose; + clang_experimental_DepGraphModule_getBuildArguments; + clang_experimental_DepGraphModule_getContextHash; + clang_experimental_DepGraphModule_getFileDeps; + clang_experimental_DepGraphModule_getModuleDeps; + clang_experimental_DepGraphModule_getModuleMapPath; + clang_experimental_DepGraphModule_getName; + clang_experimental_DepGraphTUCommand_dispose; + clang_experimental_DepGraphTUCommand_getBuildArguments; + clang_experimental_DepGraphTUCommand_getExecutable; clang_getUnqualifiedType; clang_getNonReferenceType; clang_CXXMethod_isDeleted; From e8395d11e1d2778a98a7f937090cd3493c38e5b2 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 28 Mar 2023 13:47:39 -0700 Subject: [PATCH 35/88] [clang][cas] Fall back to textual include for missing inferred submodules When an inferred submodule is missing, because the umbrella header does not actually include it, we need to fall back to textual include. This was not working when included via PCH, because we were not serializing the inference flag(s) in the include-tree. rdar://107281193 (cherry picked from commit ecf79468fe5bff8acaa1435c722a838ef005652c) --- clang/include/clang/CAS/IncludeTree.h | 12 ++- clang/lib/CAS/IncludeTree.cpp | 49 ++++++++-- clang/lib/Frontend/FrontendAction.cpp | 3 + .../IncludeTreeActionController.cpp | 3 + .../modules-include-tree-missing-submodule.c | 92 +++++++++++++++++++ 5 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c diff --git a/clang/include/clang/CAS/IncludeTree.h b/clang/include/clang/CAS/IncludeTree.h index 826bb85397f6b..cf36c8c90277b 100644 --- a/clang/include/clang/CAS/IncludeTree.h +++ b/clang/include/clang/CAS/IncludeTree.h @@ -398,9 +398,13 @@ class IncludeTree::Module : public IncludeTreeBase { bool IsExplicit : 1; bool IsExternC : 1; bool IsSystem : 1; + bool InferSubmodules : 1; + bool InferExplicitSubmodules : 1; + bool InferExportWildcard : 1; ModuleFlags() : IsFramework(false), IsExplicit(false), IsExternC(false), - IsSystem(false) {} + IsSystem(false), InferSubmodules(false), + InferExplicitSubmodules(false), InferExportWildcard(false) {} }; ModuleFlags getFlags() const; @@ -449,7 +453,7 @@ class IncludeTree::Module : public IncludeTreeBase { if (!IncludeTreeBase::isValid(Node)) return false; IncludeTreeBase Base(Node); - return Base.getData().size() > 1; + return Base.getData().size() > 2; } static bool isValid(ObjectStore &DB, ObjectRef Ref) { auto Node = DB.getProxy(Ref); @@ -461,8 +465,8 @@ class IncludeTree::Module : public IncludeTreeBase { } private: - char rawFlags() const { return getData()[0]; } - StringRef dataAfterFlags() const { return getData().drop_front(); } + uint16_t rawFlags() const; + StringRef dataAfterFlags() const { return getData().drop_front(2); } bool hasExports() const; bool hasLinkLibraries() const; std::optional getExportsIndex() const; diff --git a/clang/lib/CAS/IncludeTree.cpp b/clang/lib/CAS/IncludeTree.cpp index 54fa397340ecc..0cd6869339db1 100644 --- a/clang/lib/CAS/IncludeTree.cpp +++ b/clang/lib/CAS/IncludeTree.cpp @@ -316,20 +316,26 @@ bool IncludeTree::FileList::isValid(const ObjectProxy &Node) { Data.size() == sizeof(uint32_t) + NumFiles * sizeof(FileSizeTy); } -static constexpr char ModuleFlagFramework = 1 << 0; -static constexpr char ModuleFlagExplicit = 1 << 1; -static constexpr char ModuleFlagExternC = 1 << 2; -static constexpr char ModuleFlagSystem = 1 << 3; -static constexpr char ModuleFlagHasExports = 1 << 4; -static constexpr char ModuleFlagHasLinkLibraries = 1 << 5; +static constexpr uint16_t ModuleFlagFramework = 1 << 0; +static constexpr uint16_t ModuleFlagExplicit = 1 << 1; +static constexpr uint16_t ModuleFlagExternC = 1 << 2; +static constexpr uint16_t ModuleFlagSystem = 1 << 3; +static constexpr uint16_t ModuleFlagInferSubmodules = 1 << 4; +static constexpr uint16_t ModuleFlagInferExplicitSubmodules = 1 << 5; +static constexpr uint16_t ModuleFlagInferInferExportWildcard = 1 << 6; +static constexpr uint16_t ModuleFlagHasExports = 1 << 7; +static constexpr uint16_t ModuleFlagHasLinkLibraries = 1 << 8; IncludeTree::Module::ModuleFlags IncludeTree::Module::getFlags() const { - char Raw = rawFlags(); + uint16_t Raw = rawFlags(); ModuleFlags Flags; Flags.IsFramework = Raw & ModuleFlagFramework; Flags.IsExplicit = Raw & ModuleFlagExplicit; Flags.IsExternC = Raw & ModuleFlagExternC; Flags.IsSystem = Raw & ModuleFlagSystem; + Flags.InferSubmodules = Raw & ModuleFlagInferSubmodules; + Flags.InferExplicitSubmodules = Raw & ModuleFlagInferExplicitSubmodules; + Flags.InferExportWildcard = Raw & ModuleFlagInferInferExportWildcard; return Flags; } @@ -362,14 +368,14 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, std::optional ExportList, std::optional LinkLibraries) { // Data: - // - 1 byte for Flags + // - 2 bytes for Flags // - ModuleName (String) // Refs: // - Submodules (IncludeTreeModule) // - (optional) ExportList // - (optional) LinkLibaryList - char RawFlags = 0; + uint16_t RawFlags = 0; if (Flags.IsFramework) RawFlags |= ModuleFlagFramework; if (Flags.IsExplicit) @@ -378,13 +384,22 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, RawFlags |= ModuleFlagExternC; if (Flags.IsSystem) RawFlags |= ModuleFlagSystem; + if (Flags.InferSubmodules) + RawFlags |= ModuleFlagInferSubmodules; + if (Flags.InferExplicitSubmodules) + RawFlags |= ModuleFlagInferExplicitSubmodules; + if (Flags.InferExportWildcard) + RawFlags |= ModuleFlagInferInferExportWildcard; if (ExportList) RawFlags |= ModuleFlagHasExports; if (LinkLibraries) RawFlags |= ModuleFlagHasLinkLibraries; SmallString<64> Buffer; - Buffer.push_back(RawFlags); + llvm::raw_svector_ostream BufOS(Buffer); + llvm::support::endian::Writer Writer(BufOS, llvm::support::little); + Writer.write(RawFlags); + Buffer.append(ModuleName); SmallVector Refs(Submodules); @@ -396,6 +411,11 @@ IncludeTree::Module::create(ObjectStore &DB, StringRef ModuleName, return IncludeTreeBase::create(DB, Refs, Buffer); } +uint16_t IncludeTree::Module::rawFlags() const { + return llvm::support::endian::read( + getData().data()); +} + bool IncludeTree::Module::hasExports() const { return rawFlags() & ModuleFlagHasExports; } @@ -670,6 +690,15 @@ llvm::Error IncludeTree::Module::print(llvm::raw_ostream &OS, unsigned Indent) { if (Flags.IsSystem) OS << " (system)"; OS << '\n'; + if (Flags.InferSubmodules) { + if (Flags.InferExplicitSubmodules) + OS << " explicit module *"; + else + OS << " module *"; + if (Flags.InferExportWildcard) + OS << " { export * }"; + OS << '\n'; + } auto ExportList = getExports(); if (!ExportList) return ExportList.takeError(); diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index f0259f591aa5b..f91fc30e78242 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -559,6 +559,9 @@ static Expected makeIncludeTreeModule(CompilerInstance &CI, M->Kind = Module::IncludeTreeModuleMap; M->IsExternC = Flags.IsExternC; M->IsSystem = Flags.IsSystem; + M->InferSubmodules = Flags.InferSubmodules; + M->InferExplicitSubmodules = Flags.InferExplicitSubmodules; + M->InferExportWildcard = Flags.InferExportWildcard; auto ExportList = Mod.getExports(); if (!ExportList) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 10df1417740d4..95cb66b067f5e 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -517,6 +517,9 @@ getIncludeTreeModule(cas::ObjectStore &DB, Module *M) { Flags.IsExplicit = M->IsExplicit; Flags.IsExternC = M->IsExternC; Flags.IsSystem = M->IsSystem; + Flags.InferSubmodules = M->InferSubmodules; + Flags.InferExplicitSubmodules = M->InferExplicitSubmodules; + Flags.InferExportWildcard = M->InferExportWildcard; bool GlobalWildcardExport = false; SmallVector Exports; diff --git a/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c b/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c new file mode 100644 index 0000000000000..f599b6bb94687 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-missing-submodule.c @@ -0,0 +1,92 @@ +// Ensure we fallback to textual inclusion for headers in incomplete umbrellas. + +// REQUIRES: ondisk_cas + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json + +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Foo > %t/Foo.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Foo.rsp +// RUN: %clang @%t/pch.rsp + +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp + +// Extract include-tree casids +// RUN: cat %t/Foo.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Foo.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Foo" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Foo.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt +// RUN: FileCheck %s -input-file %t/result.txt -DPREFIX=%/t + +// CHECK-LABEL: MODULE Foo +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Foo.framework/Headers/Foo.h llvmcas:// +// CHECK: Submodule: Foo +// CHECK-NOT: Bar +// CHECK: Module Map: +// CHECK: Foo (framework) +// CHECK-NOT: Bar +// CHECK: module * +// CHECK-NOT: Bar + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: (PCH) llvmcas:// +// CHECK: [[PREFIX]]/tu.c llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/Foo.framework/Headers/Bar.h llvmcas:// + +// RUN: %clang @%t/tu.rsp + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.c -include prefix.h -F DIR -fmodules -fimplicit-modules -fimplicit-module-maps -fmodules-cache-path=DIR/module-cache -Rcompile-job-cache" +}] + +//--- Foo.framework/Modules/module.modulemap +framework module Foo { + umbrella header "Foo.h" + module * { export * } +} + +//--- Foo.framework/Headers/Foo.h +// Do not import Bar.h +void foo(void); + +//--- Foo.framework/Headers/Bar.h +void bar(void); + +//--- prefix.h +#include + +//--- tu.c +#include +// FIXME: -Wincomplete-umbrella warning +void tu(void) { + bar(); +} From 182443fd194106c4cd273a3e856bea5362210972 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Mon, 27 Mar 2023 14:58:10 -0700 Subject: [PATCH 36/88] [lldb] Fix value printing for a specific case Fixes printing of spaces in cases where the following are true: 1. Persistent results are disabled 2. The type has a summary string As reported by @jgorbe in D146783, two spaces were being printed before the summary string, and no spaces were printed after. Differential Revision: https://reviews.llvm.org/D147006 (cherry picked from commit 51dd8a20c1c32c60d03719e0e40bbc8e6c125477) --- lldb/include/lldb/DataFormatters/ValueObjectPrinter.h | 2 +- lldb/source/DataFormatters/ValueObjectPrinter.cpp | 11 +++++++---- lldb/test/API/commands/dwim-print/TestDWIMPrint.py | 9 +++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h b/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h index e1a4b8e12e59c..a4946a20591a5 100644 --- a/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h +++ b/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h @@ -98,7 +98,7 @@ class ValueObjectPrinter { ValueObject *GetValueObjectForChildrenGeneration(); - void PrintChildrenPreamble(); + void PrintChildrenPreamble(bool value_printed, bool summary_printed); void PrintChildrenPostamble(bool print_dotdotdot); diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp index 5d4a2b1a12e48..39213c6878935 100644 --- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp +++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp @@ -445,7 +445,9 @@ bool ValueObjectPrinter::PrintValueAndSummaryIfNeeded(bool &value_printed, } if (m_summary.size()) { - m_stream->Printf(" %s", m_summary.c_str()); + if (ShouldShowName() || value_printed) + m_stream->PutChar(' '); + m_stream->PutCString(m_summary); summary_printed = true; } } @@ -582,7 +584,8 @@ ValueObject *ValueObjectPrinter::GetValueObjectForChildrenGeneration() { return m_valobj; } -void ValueObjectPrinter::PrintChildrenPreamble() { +void ValueObjectPrinter::PrintChildrenPreamble(bool value_printed, + bool summary_printed) { if (m_options.m_flat_output) { if (ShouldPrintValueObject()) m_stream->EOL(); @@ -590,7 +593,7 @@ void ValueObjectPrinter::PrintChildrenPreamble() { if (ShouldPrintValueObject()) { if (IsRef()) { m_stream->PutCString(": "); - } else if (ShouldShowName()) { + } else if (value_printed || summary_printed || ShouldShowName()) { m_stream->PutChar(' '); } m_stream->PutCString("{\n"); @@ -716,7 +719,7 @@ void ValueObjectPrinter::PrintChildren( for (size_t idx = 0; idx < num_children; ++idx) { if (ValueObjectSP child_sp = GenerateChild(synth_m_valobj, idx)) { if (!any_children_printed) { - PrintChildrenPreamble(); + PrintChildrenPreamble(value_printed, summary_printed); any_children_printed = true; } PrintChild(child_sp, curr_ptr_depth); diff --git a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py index a5766a46532cd..624284d36ba34 100644 --- a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py +++ b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py @@ -120,3 +120,12 @@ def test_nested_values(self): self.runCmd("settings set auto-one-line-summaries false") self._expect_cmd(f"dwim-print s", "frame variable") self._expect_cmd(f"dwim-print (struct Structure)s", "expression") + + def test_summary_strings(self): + """Test dwim-print with nested values (structs, etc).""" + self.build() + lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.c")) + self.runCmd("settings set auto-one-line-summaries false") + self.runCmd("type summary add -e -s 'stub summary' Structure") + self._expect_cmd(f"dwim-print s", "frame variable") + self._expect_cmd(f"dwim-print (struct Structure)s", "expression") From f388e6758ab6aa0fa906b382073504cd26e92f38 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 9 Mar 2023 11:10:32 -0800 Subject: [PATCH 37/88] [lldb] Fix dwim-print error message for missing expr (cherry picked from commit 4d18d97b594ccaa3cbd79beb4afef45e4156dc8d) --- lldb/source/Commands/CommandObjectDWIMPrint.cpp | 10 ++++++---- lldb/test/API/commands/dwim-print/TestDWIMPrint.py | 7 +++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp index 1b733c55ff259..437082ca80aa8 100644 --- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp +++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp @@ -63,14 +63,16 @@ bool CommandObjectDWIMPrint::DoExecute(StringRef command, OptionsWithRaw args{command}; StringRef expr = args.GetRawPart(); + if (expr.empty()) { + result.AppendErrorWithFormatv("'{0}' takes a variable or expression", + m_cmd_name); + return false; + } + if (args.HasArgs()) { if (!ParseOptionsAndNotify(args.GetArgs(), result, m_option_group, m_exe_ctx)) return false; - } else if (command.empty()) { - result.AppendErrorWithFormatv("'{0}' takes a variable or expression", - m_cmd_name); - return false; } auto verbosity = GetDebugger().GetDWIMPrintVerbosity(); diff --git a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py index 624284d36ba34..61aa7f2284e8c 100644 --- a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py +++ b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py @@ -113,6 +113,13 @@ def test_expression_language(self): self._expect_cmd(f"dwim-print -l c++ -- argc", "frame variable") self._expect_cmd(f"dwim-print -l c++ -- argc + 1", "expression") + def test_empty_expression(self): + self.build() + lldbutil.run_to_name_breakpoint(self, "main") + error_msg = "error: 'dwim-print' takes a variable or expression" + self.expect(f"dwim-print", error=True, startstr=error_msg) + self.expect(f"dwim-print -- ", error=True, startstr=error_msg) + def test_nested_values(self): """Test dwim-print with nested values (structs, etc).""" self.build() From e2c55df55676107199e88ff50455d9b4e61b73eb Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 30 Mar 2023 09:59:19 -0700 Subject: [PATCH 38/88] Revert "[lldb] Disable po tests in TestSwiftPOSysTypes" This reverts commit 2ac7c4ca69f44520d9c25d141dd09e7defba06da. --- lldb/test/API/lang/swift/po/sys_types/main.swift | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lldb/test/API/lang/swift/po/sys_types/main.swift b/lldb/test/API/lang/swift/po/sys_types/main.swift index eb51c43bc5cad..84ca55dda4ff1 100644 --- a/lldb/test/API/lang/swift/po/sys_types/main.swift +++ b/lldb/test/API/lang/swift/po/sys_types/main.swift @@ -29,10 +29,8 @@ func main() { #endif //% self.expect("po nsarr", substrs = ['1','2','3','4']) var nsobject = NSObject() //% self.expect("po clr", substrs = ['1 0 0 1']) # may change depending on OS/platform - // Disabled: rdar://106152599 - // self.expect("po nsobject", substrs = [' Date: Tue, 14 Feb 2023 11:02:12 -0800 Subject: [PATCH 39/88] X86: force enable the mandatory tailcall on Win64 Enable the guaranteed tailcall on Windows x86_64 when we encounter a Swift function call. This function will not conform to the Windows prologue requirements which will implicitly break the unwinder and the debugger, but is required to enable the proper Concurrency behaviour for non-trivial programs. This requires associated changes in the Swift repository to update the test expectations. --- clang/lib/Basic/Targets/X86.h | 3 +-- llvm/lib/Target/X86/X86FrameLowering.cpp | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h index c1e1da22978bf..f5efd290e63b7 100644 --- a/clang/lib/Basic/Targets/X86.h +++ b/clang/lib/Basic/Targets/X86.h @@ -819,11 +819,10 @@ class LLVM_LIBRARY_VISIBILITY WindowsX86_64TargetInfo case CC_PreserveAll: case CC_X86_64SysV: case CC_Swift: + case CC_SwiftAsync: case CC_X86RegCall: case CC_OpenCLKernel: return CCCR_OK; - case CC_SwiftAsync: - return CCCR_Error; default: return CCCR_Warning; } diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index b6fa3d5d4e543..11987d265efd0 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -1510,7 +1510,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF, // Space reserved for stack-based arguments when making a (ABI-guaranteed) // tail call. unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta(); - if (TailCallArgReserveSize && IsWin64Prologue) + if (TailCallArgReserveSize && IsWin64Prologue && + !MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftAsync)) report_fatal_error("Can't handle guaranteed tail call under win64 yet"); const bool EmitStackProbeCall = From 894d63a03ea7d107c6facf6326b9a0bc90b66d35 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Wed, 29 Mar 2023 21:51:45 -0700 Subject: [PATCH 40/88] [DWARFLinker] Honor verbose flag for input verification Fix an inconsistency between input and output verification in dsymutil. Previously, output verification would be controlled by the verbose flag, while input verification would unconditionally dump to stdout. Make input and output verification behave the same by printing verification error to stderr in verbose mode only. (cherry picked from commit 9f8678b38cd52b2e6a23cfbbd73d6cdd3b2d4c03) --- llvm/lib/DWARFLinker/DWARFLinker.cpp | 3 ++- llvm/test/tools/dsymutil/X86/verify.test | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp index 4908b09553825..022f25772811c 100644 --- a/llvm/lib/DWARFLinker/DWARFLinker.cpp +++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp @@ -2837,8 +2837,9 @@ Error DWARFLinker::cloneModuleUnit(LinkContext &Context, RefModuleUnit &Unit, bool DWARFLinker::verify(const DWARFFile &File) { assert(File.Dwarf); + raw_ostream &os = Options.Verbose ? errs() : nulls(); DIDumpOptions DumpOpts; - if (!File.Dwarf->verify(llvm::outs(), DumpOpts.noImplicitRecursion())) { + if (!File.Dwarf->verify(os, DumpOpts.noImplicitRecursion())) { reportWarning("input verification failed", File); return false; } diff --git a/llvm/test/tools/dsymutil/X86/verify.test b/llvm/test/tools/dsymutil/X86/verify.test index eb8aa045c0f7b..2a7b1938e843f 100644 --- a/llvm/test/tools/dsymutil/X86/verify.test +++ b/llvm/test/tools/dsymutil/X86/verify.test @@ -16,7 +16,8 @@ # RUN: dsymutil -verify-dwarf=input -verbose -oso-prepend-path=%p/../Inputs -y %s -o %t 2>&1 | FileCheck %s --check-prefix=QUIET-INPUT-FAIL # RUN: dsymutil -verify-dwarf=none -verbose -oso-prepend-path=%p/../Inputs -y %s -o %t 2>&1 | FileCheck %s --check-prefixes=QUIET-SUCCESS # RUN: not dsymutil -verify-dwarf=bogus -verbose -oso-prepend-path=%p/../Inputs -y %s -o %t 2>&1 | FileCheck %s --check-prefixes=BOGUS -# RUN: not dsymutil -verify-dwarf=all -oso-prepend-path=%p/../Inputs -y %s -o %t 2>&1 | FileCheck %s --check-prefixes=QUIET-OUTPUT-FAIL,QUIET-INPUT-FAIL,VERBOSE-INPUT-FAIL +# RUN: not dsymutil -verify-dwarf=all -oso-prepend-path=%p/../Inputs -y %s -o %t 2>&1 | FileCheck %s --check-prefixes=QUIET-OUTPUT-FAIL,QUIET-INPUT-FAIL +# RUN: not dsymutil -verify-dwarf=all -verbose -oso-prepend-path=%p/../Inputs -y %s -o %t 2>&1 | FileCheck %s --check-prefixes=VERBOSE-INPUT-FAIL # VERBOSE-INPUT-FAIL-DAG: error: Abbreviation declaration contains multiple DW_AT_language attributes. # QUIET-INPUT-FAIL-DAG: warning: input verification failed From 7895f3191ea36f967a61c84caa334eaec03408e6 Mon Sep 17 00:00:00 2001 From: Daniel Grumberg Date: Thu, 30 Mar 2023 14:51:45 +0100 Subject: [PATCH 41/88] [clang][ExtractAPI] Reland ExtractAPI for libclang improvements This relands the changes that were originally introduced by: - https://reviews.llvm.org/D146656 - https://reviews.llvm.org/D147138 This also fixes the leak that led to these changes being reverted Differential Revision: https://reviews.llvm.org/D147234 --- .../clang/ExtractAPI/ExtractAPIVisitor.h | 604 +++++++++++++++++- .../TypedefUnderlyingTypeResolver.h | 0 clang/lib/ExtractAPI/CMakeLists.txt | 1 - clang/lib/ExtractAPI/DeclarationFragments.cpp | 2 +- clang/lib/ExtractAPI/ExtractAPIConsumer.cpp | 30 +- clang/lib/ExtractAPI/ExtractAPIVisitor.cpp | 560 ---------------- .../Serialization/SymbolGraphSerializer.cpp | 4 - .../TypedefUnderlyingTypeResolver.cpp | 2 +- clang/test/Index/extract-api-cursor.m | 177 ++--- clang/test/Index/extract-api-usr.m | 18 +- clang/tools/c-index-test/c-index-test.c | 21 + clang/tools/libclang/CXExtractAPI.cpp | 132 +++- 12 files changed, 842 insertions(+), 709 deletions(-) rename clang/{lib => include/clang}/ExtractAPI/TypedefUnderlyingTypeResolver.h (100%) delete mode 100644 clang/lib/ExtractAPI/ExtractAPIVisitor.cpp diff --git a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h index f6546fb4776a6..a31648b80195a 100644 --- a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h +++ b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h @@ -14,24 +14,27 @@ #ifndef LLVM_CLANG_EXTRACTAPI_EXTRACT_API_VISITOR_H #define LLVM_CLANG_EXTRACTAPI_EXTRACT_API_VISITOR_H +#include "llvm/ADT/FunctionExtras.h" + +#include "clang/AST/ASTContext.h" +#include "clang/AST/ParentMapContext.h" #include "clang/AST/RecursiveASTVisitor.h" #include "clang/Basic/SourceManager.h" #include "clang/ExtractAPI/API.h" -#include "llvm/ADT/FunctionExtras.h" +#include "clang/ExtractAPI/TypedefUnderlyingTypeResolver.h" +#include namespace clang { namespace extractapi { +namespace impl { -/// The RecursiveASTVisitor to traverse symbol declarations and collect API -/// information. -class ExtractAPIVisitor : public RecursiveASTVisitor { -public: - ExtractAPIVisitor(ASTContext &Context, - llvm::unique_function LocationChecker, - APISet &API) - : Context(Context), API(API), - LocationChecker(std::move(LocationChecker)) {} +template +class ExtractAPIVisitorBase : public RecursiveASTVisitor { +protected: + ExtractAPIVisitorBase(ASTContext &Context, APISet &API) + : Context(Context), API(API) {} +public: const APISet &getAPI() const { return API; } bool VisitVarDecl(const VarDecl *Decl); @@ -50,7 +53,11 @@ class ExtractAPIVisitor : public RecursiveASTVisitor { bool VisitObjCCategoryDecl(const ObjCCategoryDecl *Decl); -private: + bool shouldDeclBeIncluded(const Decl *Decl) const; + + const RawComment *fetchRawCommentForDecl(const Decl *Decl) const; + +protected: /// Collect API information for the enum constants and associate with the /// parent enum. void recordEnumConstants(EnumRecord *EnumRecord, @@ -77,9 +84,582 @@ class ExtractAPIVisitor : public RecursiveASTVisitor { void recordObjCProtocols(ObjCContainerRecord *Container, ObjCInterfaceDecl::protocol_range Protocols); + ASTContext &Context; APISet &API; - llvm::unique_function LocationChecker; + + StringRef getTypedefName(const TagDecl *Decl) { + if (const auto *TypedefDecl = Decl->getTypedefNameForAnonDecl()) + return TypedefDecl->getName(); + + return {}; + } + + bool isInSystemHeader(const Decl *D) { + return Context.getSourceManager().isInSystemHeader(D->getLocation()); + } + +private: + Derived &getDerivedExtractAPIVisitor() { + return *static_cast(this); + } +}; + +template +bool ExtractAPIVisitorBase::VisitVarDecl(const VarDecl *Decl) { + // skip function parameters. + if (isa(Decl)) + return true; + + // Skip non-global variables in records (struct/union/class). + if (Decl->getDeclContext()->isRecord()) + return true; + + // Skip local variables inside function or method. + if (!Decl->isDefinedOutsideFunctionOrMethod()) + return true; + + // If this is a template but not specialization or instantiation, skip. + if (Decl->getASTContext().getTemplateOrSpecializationInfo(Decl) && + Decl->getTemplateSpecializationKind() == TSK_Undeclared) + return true; + + if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl)) + return true; + + // Collect symbol information. + StringRef Name = Decl->getName(); + StringRef USR = API.recordUSR(Decl); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + LinkageInfo Linkage = Decl->getLinkageAndVisibility(); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the variable. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForVar(Decl); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Decl); + + // Add the global variable record to the API set. + API.addGlobalVar(Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, + Declaration, SubHeading, isInSystemHeader(Decl)); + return true; +} + +template +bool ExtractAPIVisitorBase::VisitFunctionDecl( + const FunctionDecl *Decl) { + if (const auto *Method = dyn_cast(Decl)) { + // Skip member function in class templates. + if (Method->getParent()->getDescribedClassTemplate() != nullptr) + return true; + + // Skip methods in records. + for (auto P : Context.getParents(*Method)) { + if (P.template get()) + return true; + } + + // Skip ConstructorDecl and DestructorDecl. + if (isa(Method) || isa(Method)) + return true; + } + + // Skip templated functions. + switch (Decl->getTemplatedKind()) { + case FunctionDecl::TK_NonTemplate: + case FunctionDecl::TK_DependentNonTemplate: + break; + case FunctionDecl::TK_MemberSpecialization: + case FunctionDecl::TK_FunctionTemplateSpecialization: + if (auto *TemplateInfo = Decl->getTemplateSpecializationInfo()) { + if (!TemplateInfo->isExplicitInstantiationOrSpecialization()) + return true; + } + break; + case FunctionDecl::TK_FunctionTemplate: + case FunctionDecl::TK_DependentFunctionTemplateSpecialization: + return true; + } + + if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl)) + return true; + + // Collect symbol information. + StringRef Name = Decl->getName(); + StringRef USR = API.recordUSR(Decl); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + LinkageInfo Linkage = Decl->getLinkageAndVisibility(); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments, sub-heading, and signature of the function. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForFunction(Decl); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Decl); + FunctionSignature Signature = + DeclarationFragmentsBuilder::getFunctionSignature(Decl); + + // Add the function record to the API set. + API.addGlobalFunction(Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, + Declaration, SubHeading, Signature, + isInSystemHeader(Decl)); + return true; +} + +template +bool ExtractAPIVisitorBase::VisitEnumDecl(const EnumDecl *Decl) { + if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl)) + return true; + + SmallString<128> QualifiedNameBuffer; + // Collect symbol information. + StringRef Name = Decl->getName(); + if (Name.empty()) + Name = getTypedefName(Decl); + if (Name.empty()) { + llvm::raw_svector_ostream OS(QualifiedNameBuffer); + Decl->printQualifiedName(OS); + Name = QualifiedNameBuffer.str(); + } + + StringRef USR = API.recordUSR(Decl); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the enum. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForEnum(Decl); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Decl); + + EnumRecord *EnumRecord = + API.addEnum(API.copyString(Name), USR, Loc, AvailabilitySet(Decl), + Comment, Declaration, SubHeading, isInSystemHeader(Decl)); + + // Now collect information about the enumerators in this enum. + getDerivedExtractAPIVisitor().recordEnumConstants(EnumRecord, + Decl->enumerators()); + + return true; +} + +template +bool ExtractAPIVisitorBase::VisitRecordDecl(const RecordDecl *Decl) { + // Skip C++ structs/classes/unions + // TODO: support C++ records + if (isa(Decl)) + return true; + + if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl)) + return true; + + // Collect symbol information. + StringRef Name = Decl->getName(); + if (Name.empty()) + Name = getTypedefName(Decl); + if (Name.empty()) + return true; + + StringRef USR = API.recordUSR(Decl); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the struct. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForStruct(Decl); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Decl); + + StructRecord *StructRecord = + API.addStruct(Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration, + SubHeading, isInSystemHeader(Decl)); + + // Now collect information about the fields in this struct. + getDerivedExtractAPIVisitor().recordStructFields(StructRecord, + Decl->fields()); + + return true; +} + +template +bool ExtractAPIVisitorBase::VisitObjCInterfaceDecl( + const ObjCInterfaceDecl *Decl) { + if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl)) + return true; + + // Collect symbol information. + StringRef Name = Decl->getName(); + StringRef USR = API.recordUSR(Decl); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + LinkageInfo Linkage = Decl->getLinkageAndVisibility(); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the interface. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForObjCInterface(Decl); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Decl); + + // Collect super class information. + SymbolReference SuperClass; + if (const auto *SuperClassDecl = Decl->getSuperClass()) { + SuperClass.Name = SuperClassDecl->getObjCRuntimeNameAsString(); + SuperClass.USR = API.recordUSR(SuperClassDecl); + } + + ObjCInterfaceRecord *ObjCInterfaceRecord = API.addObjCInterface( + Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, Declaration, + SubHeading, SuperClass, isInSystemHeader(Decl)); + + // Record all methods (selectors). This doesn't include automatically + // synthesized property methods. + getDerivedExtractAPIVisitor().recordObjCMethods(ObjCInterfaceRecord, + Decl->methods()); + getDerivedExtractAPIVisitor().recordObjCProperties(ObjCInterfaceRecord, + Decl->properties()); + getDerivedExtractAPIVisitor().recordObjCInstanceVariables(ObjCInterfaceRecord, + Decl->ivars()); + getDerivedExtractAPIVisitor().recordObjCProtocols(ObjCInterfaceRecord, + Decl->protocols()); + + return true; +} + +template +bool ExtractAPIVisitorBase::VisitObjCProtocolDecl( + const ObjCProtocolDecl *Decl) { + if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl)) + return true; + + // Collect symbol information. + StringRef Name = Decl->getName(); + StringRef USR = API.recordUSR(Decl); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the protocol. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForObjCProtocol(Decl); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Decl); + + ObjCProtocolRecord *ObjCProtocolRecord = + API.addObjCProtocol(Name, USR, Loc, AvailabilitySet(Decl), Comment, + Declaration, SubHeading, isInSystemHeader(Decl)); + + getDerivedExtractAPIVisitor().recordObjCMethods(ObjCProtocolRecord, + Decl->methods()); + getDerivedExtractAPIVisitor().recordObjCProperties(ObjCProtocolRecord, + Decl->properties()); + getDerivedExtractAPIVisitor().recordObjCProtocols(ObjCProtocolRecord, + Decl->protocols()); + + return true; +} + +template +bool ExtractAPIVisitorBase::VisitTypedefNameDecl( + const TypedefNameDecl *Decl) { + // Skip ObjC Type Parameter for now. + if (isa(Decl)) + return true; + + if (!Decl->isDefinedOutsideFunctionOrMethod()) + return true; + + if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl)) + return true; + + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + StringRef Name = Decl->getName(); + StringRef USR = API.recordUSR(Decl); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + QualType Type = Decl->getUnderlyingType(); + SymbolReference SymRef = + TypedefUnderlyingTypeResolver(Context).getSymbolReferenceForType(Type, + API); + + API.addTypedef(Name, USR, Loc, AvailabilitySet(Decl), Comment, + DeclarationFragmentsBuilder::getFragmentsForTypedef(Decl), + DeclarationFragmentsBuilder::getSubHeading(Decl), SymRef, + isInSystemHeader(Decl)); + + return true; +} + +template +bool ExtractAPIVisitorBase::VisitObjCCategoryDecl( + const ObjCCategoryDecl *Decl) { + if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl)) + return true; + + StringRef Name = Decl->getName(); + StringRef USR = API.recordUSR(Decl); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Decl)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + // Build declaration fragments and sub-heading for the category. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForObjCCategory(Decl); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Decl); + + const ObjCInterfaceDecl *InterfaceDecl = Decl->getClassInterface(); + SymbolReference Interface(InterfaceDecl->getName(), + API.recordUSR(InterfaceDecl)); + + ObjCCategoryRecord *ObjCCategoryRecord = API.addObjCCategory( + Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration, SubHeading, + Interface, isInSystemHeader(Decl)); + + getDerivedExtractAPIVisitor().recordObjCMethods(ObjCCategoryRecord, + Decl->methods()); + getDerivedExtractAPIVisitor().recordObjCProperties(ObjCCategoryRecord, + Decl->properties()); + getDerivedExtractAPIVisitor().recordObjCInstanceVariables(ObjCCategoryRecord, + Decl->ivars()); + getDerivedExtractAPIVisitor().recordObjCProtocols(ObjCCategoryRecord, + Decl->protocols()); + + return true; +} + +/// Collect API information for the enum constants and associate with the +/// parent enum. +template +void ExtractAPIVisitorBase::recordEnumConstants( + EnumRecord *EnumRecord, const EnumDecl::enumerator_range Constants) { + for (const auto *Constant : Constants) { + // Collect symbol information. + StringRef Name = Constant->getName(); + StringRef USR = API.recordUSR(Constant); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Constant->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Constant)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the enum constant. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForEnumConstant(Constant); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Constant); + + API.addEnumConstant(EnumRecord, Name, USR, Loc, AvailabilitySet(Constant), + Comment, Declaration, SubHeading, + isInSystemHeader(Constant)); + } +} + +/// Collect API information for the struct fields and associate with the +/// parent struct. +template +void ExtractAPIVisitorBase::recordStructFields( + StructRecord *StructRecord, const RecordDecl::field_range Fields) { + for (const auto *Field : Fields) { + // Collect symbol information. + StringRef Name = Field->getName(); + StringRef USR = API.recordUSR(Field); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Field->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Field)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the struct field. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForField(Field); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Field); + + API.addStructField(StructRecord, Name, USR, Loc, AvailabilitySet(Field), + Comment, Declaration, SubHeading, + isInSystemHeader(Field)); + } +} + +/// Collect API information for the Objective-C methods and associate with the +/// parent container. +template +void ExtractAPIVisitorBase::recordObjCMethods( + ObjCContainerRecord *Container, + const ObjCContainerDecl::method_range Methods) { + for (const auto *Method : Methods) { + // Don't record selectors for properties. + if (Method->isPropertyAccessor()) + continue; + + StringRef Name = API.copyString(Method->getSelector().getAsString()); + StringRef USR = API.recordUSR(Method); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Method->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Method)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments, sub-heading, and signature for the method. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForObjCMethod(Method); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Method); + FunctionSignature Signature = + DeclarationFragmentsBuilder::getFunctionSignature(Method); + + API.addObjCMethod(Container, Name, USR, Loc, AvailabilitySet(Method), + Comment, Declaration, SubHeading, Signature, + Method->isInstanceMethod(), isInSystemHeader(Method)); + } +} + +template +void ExtractAPIVisitorBase::recordObjCProperties( + ObjCContainerRecord *Container, + const ObjCContainerDecl::prop_range Properties) { + for (const auto *Property : Properties) { + StringRef Name = Property->getName(); + StringRef USR = API.recordUSR(Property); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Property->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Property)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the property. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForObjCProperty(Property); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Property); + + StringRef GetterName = + API.copyString(Property->getGetterName().getAsString()); + StringRef SetterName = + API.copyString(Property->getSetterName().getAsString()); + + // Get the attributes for property. + unsigned Attributes = ObjCPropertyRecord::NoAttr; + if (Property->getPropertyAttributes() & + ObjCPropertyAttribute::kind_readonly) + Attributes |= ObjCPropertyRecord::ReadOnly; + + API.addObjCProperty( + Container, Name, USR, Loc, AvailabilitySet(Property), Comment, + Declaration, SubHeading, + static_cast(Attributes), GetterName, + SetterName, Property->isOptional(), + !(Property->getPropertyAttributes() & + ObjCPropertyAttribute::kind_class), + isInSystemHeader(Property)); + } +} + +template +void ExtractAPIVisitorBase::recordObjCInstanceVariables( + ObjCContainerRecord *Container, + const llvm::iterator_range< + DeclContext::specific_decl_iterator> + Ivars) { + for (const auto *Ivar : Ivars) { + StringRef Name = Ivar->getName(); + StringRef USR = API.recordUSR(Ivar); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Ivar->getLocation()); + DocComment Comment; + if (auto *RawComment = + getDerivedExtractAPIVisitor().fetchRawCommentForDecl(Ivar)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading for the instance variable. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForField(Ivar); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Ivar); + + ObjCInstanceVariableRecord::AccessControl Access = + Ivar->getCanonicalAccessControl(); + + API.addObjCInstanceVariable(Container, Name, USR, Loc, + AvailabilitySet(Ivar), Comment, Declaration, + SubHeading, Access, isInSystemHeader(Ivar)); + } +} + +template +void ExtractAPIVisitorBase::recordObjCProtocols( + ObjCContainerRecord *Container, + ObjCInterfaceDecl::protocol_range Protocols) { + for (const auto *Protocol : Protocols) + Container->Protocols.emplace_back(Protocol->getName(), + API.recordUSR(Protocol)); +} + +} // namespace impl + +/// The RecursiveASTVisitor to traverse symbol declarations and collect API +/// information. +template +class ExtractAPIVisitor + : public impl::ExtractAPIVisitorBase, ExtractAPIVisitor<>, Derived>> { + using Base = impl::ExtractAPIVisitorBase, ExtractAPIVisitor<>, Derived>>; + +public: + ExtractAPIVisitor(ASTContext &Context, APISet &API) : Base(Context, API) {} + + bool shouldDeclBeIncluded(const Decl *D) const { return true; } + const RawComment *fetchRawCommentForDecl(const Decl *D) const { + return this->Context.getRawCommentForDeclNoCache(D); + } }; } // namespace extractapi diff --git a/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.h b/clang/include/clang/ExtractAPI/TypedefUnderlyingTypeResolver.h similarity index 100% rename from clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.h rename to clang/include/clang/ExtractAPI/TypedefUnderlyingTypeResolver.h diff --git a/clang/lib/ExtractAPI/CMakeLists.txt b/clang/lib/ExtractAPI/CMakeLists.txt index d73cfdec98feb..e7de57d2984cc 100644 --- a/clang/lib/ExtractAPI/CMakeLists.txt +++ b/clang/lib/ExtractAPI/CMakeLists.txt @@ -7,7 +7,6 @@ add_clang_library(clangExtractAPI APIIgnoresList.cpp AvailabilityInfo.cpp ExtractAPIConsumer.cpp - ExtractAPIVisitor.cpp DeclarationFragments.cpp Serialization/SerializerBase.cpp Serialization/SymbolGraphSerializer.cpp diff --git a/clang/lib/ExtractAPI/DeclarationFragments.cpp b/clang/lib/ExtractAPI/DeclarationFragments.cpp index b8de1270b5f02..312ff5fd33e4f 100644 --- a/clang/lib/ExtractAPI/DeclarationFragments.cpp +++ b/clang/lib/ExtractAPI/DeclarationFragments.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "clang/ExtractAPI/DeclarationFragments.h" -#include "TypedefUnderlyingTypeResolver.h" +#include "clang/ExtractAPI/TypedefUnderlyingTypeResolver.h" #include "clang/Index/USRGeneration.h" #include "llvm/ADT/StringSwitch.h" diff --git a/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp b/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp index ccc8765138390..5a490080f38c6 100644 --- a/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp +++ b/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp @@ -12,8 +12,10 @@ /// //===----------------------------------------------------------------------===// +#include "clang/AST/ASTConcept.h" #include "clang/AST/ASTConsumer.h" #include "clang/AST/ASTContext.h" +#include "clang/AST/DeclObjC.h" #include "clang/Basic/DiagnosticFrontend.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManager.h" @@ -33,6 +35,7 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" @@ -218,11 +221,34 @@ struct LocationFileChecker { llvm::DenseSet ExternalFileEntries; }; +struct BatchExtractAPIVisitor : ExtractAPIVisitor { + bool shouldDeclBeIncluded(const Decl *D) const { + bool ShouldBeIncluded = true; + // Check that we have the definition for redeclarable types. + if (auto *TD = llvm::dyn_cast(D)) + ShouldBeIncluded = TD->isThisDeclarationADefinition(); + else if (auto *Interface = llvm::dyn_cast(D)) + ShouldBeIncluded = Interface->isThisDeclarationADefinition(); + else if (auto *Protocol = llvm::dyn_cast(D)) + ShouldBeIncluded = Protocol->isThisDeclarationADefinition(); + + ShouldBeIncluded = ShouldBeIncluded && LCF(D->getLocation()); + return ShouldBeIncluded; + } + + BatchExtractAPIVisitor(LocationFileChecker &LCF, ASTContext &Context, + APISet &API) + : ExtractAPIVisitor(Context, API), LCF(LCF) {} + +private: + LocationFileChecker &LCF; +}; + class ExtractAPIConsumer : public ASTConsumer { public: ExtractAPIConsumer(ASTContext &Context, std::unique_ptr LCF, APISet &API) - : Visitor(Context, *LCF, API), LCF(std::move(LCF)) {} + : Visitor(*LCF, Context, API), LCF(std::move(LCF)) {} void HandleTranslationUnit(ASTContext &Context) override { // Use ExtractAPIVisitor to traverse symbol declarations in the context. @@ -230,7 +256,7 @@ class ExtractAPIConsumer : public ASTConsumer { } private: - ExtractAPIVisitor Visitor; + BatchExtractAPIVisitor Visitor; std::unique_ptr LCF; }; diff --git a/clang/lib/ExtractAPI/ExtractAPIVisitor.cpp b/clang/lib/ExtractAPI/ExtractAPIVisitor.cpp deleted file mode 100644 index 24260cf89383d..0000000000000 --- a/clang/lib/ExtractAPI/ExtractAPIVisitor.cpp +++ /dev/null @@ -1,560 +0,0 @@ -//===- ExtractAPI/ExtractAPIVisitor.cpp -------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// This file implements the ExtractAPIVisitor an ASTVisitor to collect API -/// information. -/// -//===----------------------------------------------------------------------===// - -#include "clang/ExtractAPI/ExtractAPIVisitor.h" - -#include "TypedefUnderlyingTypeResolver.h" -#include "clang/AST/ASTConsumer.h" -#include "clang/AST/ASTContext.h" -#include "clang/AST/Decl.h" -#include "clang/AST/DeclCXX.h" -#include "clang/AST/ParentMapContext.h" -#include "clang/AST/RawCommentList.h" -#include "clang/Basic/SourceLocation.h" -#include "clang/Basic/SourceManager.h" -#include "clang/Basic/TargetInfo.h" -#include "clang/ExtractAPI/API.h" -#include "clang/ExtractAPI/AvailabilityInfo.h" -#include "clang/ExtractAPI/DeclarationFragments.h" -#include "clang/Frontend/ASTConsumers.h" -#include "clang/Frontend/FrontendOptions.h" -#include "llvm/Support/raw_ostream.h" - -using namespace clang; -using namespace extractapi; - -namespace { - -StringRef getTypedefName(const TagDecl *Decl) { - if (const auto *TypedefDecl = Decl->getTypedefNameForAnonDecl()) - return TypedefDecl->getName(); - - return {}; -} - -template -bool isInSystemHeader(const ASTContext &Context, const DeclTy *D) { - return Context.getSourceManager().isInSystemHeader(D->getLocation()); -} - -} // namespace - -bool ExtractAPIVisitor::VisitVarDecl(const VarDecl *Decl) { - // skip function parameters. - if (isa(Decl)) - return true; - - // Skip non-global variables in records (struct/union/class). - if (Decl->getDeclContext()->isRecord()) - return true; - - // Skip local variables inside function or method. - if (!Decl->isDefinedOutsideFunctionOrMethod()) - return true; - - // If this is a template but not specialization or instantiation, skip. - if (Decl->getASTContext().getTemplateOrSpecializationInfo(Decl) && - Decl->getTemplateSpecializationKind() == TSK_Undeclared) - return true; - - if (!LocationChecker(Decl->getLocation())) - return true; - - // Collect symbol information. - StringRef Name = Decl->getName(); - StringRef USR = API.recordUSR(Decl); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Decl->getLocation()); - LinkageInfo Linkage = Decl->getLinkageAndVisibility(); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the variable. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForVar(Decl); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Decl); - - // Add the global variable record to the API set. - API.addGlobalVar(Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, - Declaration, SubHeading, isInSystemHeader(Context, Decl)); - return true; -} - -bool ExtractAPIVisitor::VisitFunctionDecl(const FunctionDecl *Decl) { - if (const auto *Method = dyn_cast(Decl)) { - // Skip member function in class templates. - if (Method->getParent()->getDescribedClassTemplate() != nullptr) - return true; - - // Skip methods in records. - for (auto P : Context.getParents(*Method)) { - if (P.get()) - return true; - } - - // Skip ConstructorDecl and DestructorDecl. - if (isa(Method) || isa(Method)) - return true; - } - - // Skip templated functions. - switch (Decl->getTemplatedKind()) { - case FunctionDecl::TK_NonTemplate: - case FunctionDecl::TK_DependentNonTemplate: - break; - case FunctionDecl::TK_MemberSpecialization: - case FunctionDecl::TK_FunctionTemplateSpecialization: - if (auto *TemplateInfo = Decl->getTemplateSpecializationInfo()) { - if (!TemplateInfo->isExplicitInstantiationOrSpecialization()) - return true; - } - break; - case FunctionDecl::TK_FunctionTemplate: - case FunctionDecl::TK_DependentFunctionTemplateSpecialization: - return true; - } - - if (!LocationChecker(Decl->getLocation())) - return true; - - // Collect symbol information. - StringRef Name = Decl->getName(); - StringRef USR = API.recordUSR(Decl); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Decl->getLocation()); - LinkageInfo Linkage = Decl->getLinkageAndVisibility(); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments, sub-heading, and signature of the function. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForFunction(Decl); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Decl); - FunctionSignature Signature = - DeclarationFragmentsBuilder::getFunctionSignature(Decl); - - // Add the function record to the API set. - API.addGlobalFunction(Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, - Declaration, SubHeading, Signature, - isInSystemHeader(Context, Decl)); - return true; -} - -bool ExtractAPIVisitor::VisitEnumDecl(const EnumDecl *Decl) { - if (!Decl->isComplete()) - return true; - - // Skip forward declaration. - if (!Decl->isThisDeclarationADefinition()) - return true; - - if (!LocationChecker(Decl->getLocation())) - return true; - - SmallString<128> QualifiedNameBuffer; - // Collect symbol information. - StringRef Name = Decl->getName(); - if (Name.empty()) - Name = getTypedefName(Decl); - if (Name.empty()) { - llvm::raw_svector_ostream OS(QualifiedNameBuffer); - Decl->printQualifiedName(OS); - Name = QualifiedNameBuffer.str(); - } - - StringRef USR = API.recordUSR(Decl); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Decl->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the enum. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForEnum(Decl); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Decl); - - EnumRecord *EnumRecord = API.addEnum( - API.copyString(Name), USR, Loc, AvailabilitySet(Decl), Comment, - Declaration, SubHeading, isInSystemHeader(Context, Decl)); - - // Now collect information about the enumerators in this enum. - recordEnumConstants(EnumRecord, Decl->enumerators()); - - return true; -} - -bool ExtractAPIVisitor::VisitRecordDecl(const RecordDecl *Decl) { - if (!Decl->isCompleteDefinition()) - return true; - - // Skip C++ structs/classes/unions - // TODO: support C++ records - if (isa(Decl)) - return true; - - if (!LocationChecker(Decl->getLocation())) - return true; - - // Collect symbol information. - StringRef Name = Decl->getName(); - if (Name.empty()) - Name = getTypedefName(Decl); - if (Name.empty()) - return true; - - StringRef USR = API.recordUSR(Decl); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Decl->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the struct. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForStruct(Decl); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Decl); - - StructRecord *StructRecord = - API.addStruct(Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration, - SubHeading, isInSystemHeader(Context, Decl)); - - // Now collect information about the fields in this struct. - recordStructFields(StructRecord, Decl->fields()); - - return true; -} - -bool ExtractAPIVisitor::VisitObjCInterfaceDecl(const ObjCInterfaceDecl *Decl) { - // Skip forward declaration for classes (@class) - if (!Decl->isThisDeclarationADefinition()) - return true; - - if (!LocationChecker(Decl->getLocation())) - return true; - - // Collect symbol information. - StringRef Name = Decl->getName(); - StringRef USR = API.recordUSR(Decl); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Decl->getLocation()); - LinkageInfo Linkage = Decl->getLinkageAndVisibility(); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the interface. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForObjCInterface(Decl); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Decl); - - // Collect super class information. - SymbolReference SuperClass; - if (const auto *SuperClassDecl = Decl->getSuperClass()) { - SuperClass.Name = SuperClassDecl->getObjCRuntimeNameAsString(); - SuperClass.USR = API.recordUSR(SuperClassDecl); - } - - ObjCInterfaceRecord *ObjCInterfaceRecord = API.addObjCInterface( - Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, Declaration, - SubHeading, SuperClass, isInSystemHeader(Context, Decl)); - - // Record all methods (selectors). This doesn't include automatically - // synthesized property methods. - recordObjCMethods(ObjCInterfaceRecord, Decl->methods()); - recordObjCProperties(ObjCInterfaceRecord, Decl->properties()); - recordObjCInstanceVariables(ObjCInterfaceRecord, Decl->ivars()); - recordObjCProtocols(ObjCInterfaceRecord, Decl->protocols()); - - return true; -} - -bool ExtractAPIVisitor::VisitObjCProtocolDecl(const ObjCProtocolDecl *Decl) { - // Skip forward declaration for protocols (@protocol). - if (!Decl->isThisDeclarationADefinition()) - return true; - - if (!LocationChecker(Decl->getLocation())) - return true; - - // Collect symbol information. - StringRef Name = Decl->getName(); - StringRef USR = API.recordUSR(Decl); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Decl->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the protocol. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForObjCProtocol(Decl); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Decl); - - ObjCProtocolRecord *ObjCProtocolRecord = API.addObjCProtocol( - Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration, SubHeading, - isInSystemHeader(Context, Decl)); - - recordObjCMethods(ObjCProtocolRecord, Decl->methods()); - recordObjCProperties(ObjCProtocolRecord, Decl->properties()); - recordObjCProtocols(ObjCProtocolRecord, Decl->protocols()); - - return true; -} - -bool ExtractAPIVisitor::VisitTypedefNameDecl(const TypedefNameDecl *Decl) { - // Skip ObjC Type Parameter for now. - if (isa(Decl)) - return true; - - if (!Decl->isDefinedOutsideFunctionOrMethod()) - return true; - - if (!LocationChecker(Decl->getLocation())) - return true; - - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Decl->getLocation()); - StringRef Name = Decl->getName(); - StringRef USR = API.recordUSR(Decl); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - QualType Type = Decl->getUnderlyingType(); - SymbolReference SymRef = - TypedefUnderlyingTypeResolver(Context).getSymbolReferenceForType(Type, - API); - - API.addTypedef(Name, USR, Loc, AvailabilitySet(Decl), Comment, - DeclarationFragmentsBuilder::getFragmentsForTypedef(Decl), - DeclarationFragmentsBuilder::getSubHeading(Decl), SymRef, - isInSystemHeader(Context, Decl)); - - return true; -} - -bool ExtractAPIVisitor::VisitObjCCategoryDecl(const ObjCCategoryDecl *Decl) { - // Collect symbol information. - StringRef Name = Decl->getName(); - StringRef USR = API.recordUSR(Decl); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Decl->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Decl)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - // Build declaration fragments and sub-heading for the category. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForObjCCategory(Decl); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Decl); - - const ObjCInterfaceDecl *InterfaceDecl = Decl->getClassInterface(); - SymbolReference Interface(InterfaceDecl->getName(), - API.recordUSR(InterfaceDecl)); - - ObjCCategoryRecord *ObjCCategoryRecord = API.addObjCCategory( - Name, USR, Loc, AvailabilitySet(Decl), Comment, Declaration, SubHeading, - Interface, isInSystemHeader(Context, Decl)); - - recordObjCMethods(ObjCCategoryRecord, Decl->methods()); - recordObjCProperties(ObjCCategoryRecord, Decl->properties()); - recordObjCInstanceVariables(ObjCCategoryRecord, Decl->ivars()); - recordObjCProtocols(ObjCCategoryRecord, Decl->protocols()); - - return true; -} - -/// Collect API information for the enum constants and associate with the -/// parent enum. -void ExtractAPIVisitor::recordEnumConstants( - EnumRecord *EnumRecord, const EnumDecl::enumerator_range Constants) { - for (const auto *Constant : Constants) { - // Collect symbol information. - StringRef Name = Constant->getName(); - StringRef USR = API.recordUSR(Constant); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Constant->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Constant)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the enum constant. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForEnumConstant(Constant); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Constant); - - API.addEnumConstant(EnumRecord, Name, USR, Loc, AvailabilitySet(Constant), - Comment, Declaration, SubHeading, - isInSystemHeader(Context, Constant)); - } -} - -/// Collect API information for the struct fields and associate with the -/// parent struct. -void ExtractAPIVisitor::recordStructFields( - StructRecord *StructRecord, const RecordDecl::field_range Fields) { - for (const auto *Field : Fields) { - // Collect symbol information. - StringRef Name = Field->getName(); - StringRef USR = API.recordUSR(Field); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Field->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Field)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the struct field. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForField(Field); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Field); - - API.addStructField(StructRecord, Name, USR, Loc, AvailabilitySet(Field), - Comment, Declaration, SubHeading, - isInSystemHeader(Context, Field)); - } -} - -/// Collect API information for the Objective-C methods and associate with the -/// parent container. -void ExtractAPIVisitor::recordObjCMethods( - ObjCContainerRecord *Container, - const ObjCContainerDecl::method_range Methods) { - for (const auto *Method : Methods) { - // Don't record selectors for properties. - if (Method->isPropertyAccessor()) - continue; - - StringRef Name = API.copyString(Method->getSelector().getAsString()); - StringRef USR = API.recordUSR(Method); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Method->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Method)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments, sub-heading, and signature for the method. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForObjCMethod(Method); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Method); - FunctionSignature Signature = - DeclarationFragmentsBuilder::getFunctionSignature(Method); - - API.addObjCMethod(Container, Name, USR, Loc, AvailabilitySet(Method), - Comment, Declaration, SubHeading, Signature, - Method->isInstanceMethod(), - isInSystemHeader(Context, Method)); - } -} - -void ExtractAPIVisitor::recordObjCProperties( - ObjCContainerRecord *Container, - const ObjCContainerDecl::prop_range Properties) { - for (const auto *Property : Properties) { - StringRef Name = Property->getName(); - StringRef USR = API.recordUSR(Property); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Property->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Property)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the property. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForObjCProperty(Property); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Property); - - StringRef GetterName = - API.copyString(Property->getGetterName().getAsString()); - StringRef SetterName = - API.copyString(Property->getSetterName().getAsString()); - - // Get the attributes for property. - unsigned Attributes = ObjCPropertyRecord::NoAttr; - if (Property->getPropertyAttributes() & - ObjCPropertyAttribute::kind_readonly) - Attributes |= ObjCPropertyRecord::ReadOnly; - - API.addObjCProperty( - Container, Name, USR, Loc, AvailabilitySet(Property), Comment, - Declaration, SubHeading, - static_cast(Attributes), GetterName, - SetterName, Property->isOptional(), - !(Property->getPropertyAttributes() & - ObjCPropertyAttribute::kind_class), - isInSystemHeader(Context, Property)); - } -} - -void ExtractAPIVisitor::recordObjCInstanceVariables( - ObjCContainerRecord *Container, - const llvm::iterator_range< - DeclContext::specific_decl_iterator> - Ivars) { - for (const auto *Ivar : Ivars) { - StringRef Name = Ivar->getName(); - StringRef USR = API.recordUSR(Ivar); - PresumedLoc Loc = - Context.getSourceManager().getPresumedLoc(Ivar->getLocation()); - DocComment Comment; - if (auto *RawComment = Context.getRawCommentForDeclNoCache(Ivar)) - Comment = RawComment->getFormattedLines(Context.getSourceManager(), - Context.getDiagnostics()); - - // Build declaration fragments and sub-heading for the instance variable. - DeclarationFragments Declaration = - DeclarationFragmentsBuilder::getFragmentsForField(Ivar); - DeclarationFragments SubHeading = - DeclarationFragmentsBuilder::getSubHeading(Ivar); - - ObjCInstanceVariableRecord::AccessControl Access = - Ivar->getCanonicalAccessControl(); - - API.addObjCInstanceVariable( - Container, Name, USR, Loc, AvailabilitySet(Ivar), Comment, Declaration, - SubHeading, Access, isInSystemHeader(Context, Ivar)); - } -} - -void ExtractAPIVisitor::recordObjCProtocols( - ObjCContainerRecord *Container, - ObjCInterfaceDecl::protocol_range Protocols) { - for (const auto *Protocol : Protocols) - Container->Protocols.emplace_back(Protocol->getName(), - API.recordUSR(Protocol)); -} diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp index a9c69e058f63c..26f65325ec20b 100644 --- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp +++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp @@ -544,10 +544,6 @@ Array generateParentContexts(const RecordTy &Record, const APISet &API, serializeParentContext(PC, Lang)); }); - // The last component would be the record itself so let's remove it. - if (!ParentContexts.empty()) - ParentContexts.pop_back(); - return ParentContexts; } diff --git a/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.cpp b/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.cpp index 3da2424ea7263..3a5f62c9b2e6c 100644 --- a/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.cpp +++ b/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.cpp @@ -11,7 +11,7 @@ /// //===----------------------------------------------------------------------===// -#include "TypedefUnderlyingTypeResolver.h" +#include "clang/ExtractAPI/TypedefUnderlyingTypeResolver.h" #include "clang/Index/USRGeneration.h" using namespace clang; diff --git a/clang/test/Index/extract-api-cursor.m b/clang/test/Index/extract-api-cursor.m index 078f2f52e215c..1b27b6f61437b 100644 --- a/clang/test/Index/extract-api-cursor.m +++ b/clang/test/Index/extract-api-cursor.m @@ -1,3 +1,5 @@ +// Test is line- and column-sensitive. Run lines are below + /// Foo docs struct Foo { /// Bar docs @@ -25,91 +27,94 @@ @interface Derived: Base - (void)derivedMethodWithValue:(id)value; @end -/// This won't show up in docs because we can't serialize it -@interface Derived () -/// Derived method in category docs, won't show up either. -- (void)derivedMethodInCategory; +@implementation Derived +- (void)derivedMethodWithValue:(id)value { + int a = 5; +} @end -// RUN: c-index-test -single-symbol-sgfs local %s | FileCheck %s - -// Checking for Foo -// CHECK: "parentContexts":[] -// CHECK-SAME: "relatedSymbols":[] -// CHECK-SAME: "relationships":[] -// CHECK-SAME: "text":"Foo docs" -// CHECK-SAME: "kind":{"displayName":"Structure","identifier":"objective-c.struct"} -// CHECK-SAME: "title":"Foo" - -// Checking for bar -// CHECK-NEXT: "parentContexts":[{"kind":"objective-c.struct","name":"Foo","usr":"c:@S@Foo"}] -// CHECK-SAME: "relatedSymbols":[] -// CHECK-SAME: "relationships":[{"kind":"memberOf","source":"c:@S@Foo@FI@bar","target":"c:@S@Foo" -// CHECK-SAME: "text":"Bar docs" -// CHECK-SAME: "kind":{"displayName":"Instance Property","identifier":"objective-c.property"} -// CHECK-SAME: "title":"bar" - -// Checking for Base -// CHECK-NEXT: "parentContexts":[] -// CHECK-SAME: "relatedSymbols":[] -// CHECK-SAME: "relationships":[] -// CHECK-SAME: "text":"Base docs" -// CHECK-SAME: "kind":{"displayName":"Class","identifier":"objective-c.class"} -// CHECK-SAME: "title":"Base" - -// Checking for baseProperty -// CHECK-NEXT: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"}] -// CHECK-SAME: "relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" -// CHECK-SAME: "isSystem":false -// CHECK-SAME: "usr":"c:@S@Foo"}] -// CHECK-SAME: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Base(py)baseProperty","target":"c:objc(cs)Base" -// CHECK-SAME: "text":"Base property docs" -// CHECK-SAME: "kind":{"displayName":"Instance Property","identifier":"objective-c.property"} -// CHECK-SAME: "title":"baseProperty" - -// Checking for baseMethodWithArg -// CHECK-NEXT: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"}] -// CHECK-SAME: "relatedSymbols":[] -// CHECK-SAME: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Base(im)baseMethodWithArg:","target":"c:objc(cs)Base" -// CHECK-SAME: "text":"Base method docs" -// CHECK-SAME: "kind":{"displayName":"Instance Method","identifier":"objective-c.method"} -// CHECK-SAME: "title":"baseMethodWithArg:" - -// Checking for Protocol -// CHECK-NEXT: "parentContexts":[] -// CHECK-SAME: "relatedSymbols":[] -// CHECK-SAME: "relationships":[] -// CHECK-SAME: "text":"Protocol docs" -// CHECK-SAME: "kind":{"displayName":"Protocol","identifier":"objective-c.protocol"} -// CHECK-SAME: "title":"Protocol" - -// Checking for protocolProperty -// CHECK-NEXT: "parentContexts":[{"kind":"objective-c.protocol","name":"Protocol","usr":"c:objc(pl)Protocol"}] -// CHECK-SAME: "relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" -// CHECK-SAME: "isSystem":false -// CHECK-SAME: "usr":"c:@S@Foo"}] -// CHECK-SAME: "relationships":[{"kind":"memberOf","source":"c:objc(pl)Protocol(py)protocolProperty","target":"c:objc(pl)Protocol" -// CHECK-SAME: "text":"Protocol property docs" -// CHECK-SAME: "kind":{"displayName":"Instance Property","identifier":"objective-c.property"} -// CHECK-SAME: "title":"protocolProperty" - -// Checking for Derived -// CHECK-NEXT: "parentContexts":[] -// CHECK-SAME: "relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" -// CHECK-SAME: "isSystem":false -// CHECK-SAME: "usr":"c:objc(cs)Base"}] -// CHECK-SAME: "relationships":[{"kind":"inheritsFrom","source":"c:objc(cs)Derived","target":"c:objc(cs)Base" -// CHECK-SAME: "text":"Derived docs" -// CHECK-SAME: "kind":{"displayName":"Class","identifier":"objective-c.class"} -// CHECK-SAME: "title":"Derived" - -// Checking for derivedMethodWithValue -// CHECK-NEXT: "parentContexts":[{"kind":"objective-c.class","name":"Derived","usr":"c:objc(cs)Derived"}] -// CHECK-SAME: "relatedSymbols":[] -// CHECK-SAME: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Derived(im)derivedMethodWithValue:","target":"c:objc(cs)Derived" -// CHECK-SAME: "text":"Derived method docs" -// CHECK-SAME: "kind":{"displayName":"Instance Method","identifier":"objective-c.method"} -// CHECK-SAME: "title":"derivedMethodWithValue:" - -// CHECK-NOT: This won't show up in docs because we can't serialize it -// CHECK-NOT: Derived method in category docs, won't show up either. +// RUN: c-index-test -single-symbol-sgf-at=%s:4:9 local %s | FileCheck -check-prefix=CHECK-FOO %s +// CHECK-FOO: "parentContexts":[{"kind":"objective-c.struct","name":"Foo","usr":"c:@S@Foo"}] +// CHECK-FOO: "relatedSymbols":[] +// CHECK-FOO: "relationships":[] +// CHECK-FOO: "text":"Foo docs" +// CHECK-FOO: "kind":{"displayName":"Structure","identifier":"objective-c.struct"} +// CHECK-FOO: "title":"Foo" + +// RUN: c-index-test -single-symbol-sgf-at=%s:6:9 local %s | FileCheck -check-prefix=CHECK-BAR %s +// CHECK-BAR: "parentContexts":[{"kind":"objective-c.struct","name":"Foo","usr":"c:@S@Foo"},{"kind":"objective-c.property","name":"bar","usr":"c:@S@Foo@FI@bar"}] +// CHECK-BAR: "relatedSymbols":[] +// CHECK-BAR: "relationships":[{"kind":"memberOf","source":"c:@S@Foo@FI@bar","target":"c:@S@Foo" +// CHECK-BAR: "text":"Bar docs" +// CHECK-BAR: "kind":{"displayName":"Instance Property","identifier":"objective-c.property"} +// CHECK-BAR: "title":"bar" + +// RUN: c-index-test -single-symbol-sgf-at=%s:10:11 local %s | FileCheck -check-prefix=CHECK-BASE %s +// CHECK-BASE: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"}] +// CHECK-BASE: "relatedSymbols":[] +// CHECK-BASE: "relationships":[] +// CHECK-BASE: "text":"Base docs" +// CHECK-BASE: "kind":{"displayName":"Class","identifier":"objective-c.class"} +// CHECK-BASE: "title":"Base" + +// RUN: c-index-test -single-symbol-sgf-at=%s:12:25 local %s | FileCheck -check-prefix=CHECK-BASE-PROP %s +// CHECK-BASE-PROP: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"},{"kind":"objective-c.property","name":"baseProperty","usr":"c:objc(cs)Base(py)baseProperty"}] +// CHECK-BASE-PROP: "relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" +// CHECK-BASE-PROP: "isSystem":false +// CHECK-BASE-PROP: "usr":"c:@S@Foo"}] +// CHECK-BASE-PROP: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Base(py)baseProperty","target":"c:objc(cs)Base" +// CHECK-BASE-PROP: "text":"Base property docs" +// CHECK-BASE-PROP: "kind":{"displayName":"Instance Property","identifier":"objective-c.property"} +// CHECK-BASE-PROP: "title":"baseProperty" + +// RUN: c-index-test -single-symbol-sgf-at=%s:15:9 local %s | FileCheck -check-prefix=CHECK-BASE-METHOD %s +// CHECK-BASE-METHOD: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"},{"kind":"objective-c.method","name":"baseMethodWithArg:","usr":"c:objc(cs)Base(im)baseMethodWithArg:"}] +// CHECK-BASE-METHOD: "relatedSymbols":[] +// CHECK-BASE-METHOD: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Base(im)baseMethodWithArg:","target":"c:objc(cs)Base" +// CHECK-BASE-METHOD: "text":"Base method docs" +// CHECK-BASE-METHOD: "kind":{"displayName":"Instance Method","identifier":"objective-c.method"} +// CHECK-BASE-METHOD: "title":"baseMethodWithArg:" + +// RUN: c-index-test -single-symbol-sgf-at=%s:19:11 local %s | FileCheck -check-prefix=CHECK-PROTOCOL %s +// CHECK-PROTOCOL: "parentContexts":[{"kind":"objective-c.protocol","name":"Protocol","usr":"c:objc(pl)Protocol"}] +// CHECK-PROTOCOL: "relatedSymbols":[] +// CHECK-PROTOCOL: "relationships":[] +// CHECK-PROTOCOL: "text":"Protocol docs" +// CHECK-PROTOCOL: "kind":{"displayName":"Protocol","identifier":"objective-c.protocol"} +// CHECK-PROTOCOL: "title":"Protocol" + +// RUN: c-index-test -single-symbol-sgf-at=%s:21:27 local %s | FileCheck -check-prefix=CHECK-PROTOCOL-PROP %s +// CHECK-PROTOCOL-PROP: "parentContexts":[{"kind":"objective-c.protocol","name":"Protocol","usr":"c:objc(pl)Protocol"},{"kind":"objective-c.property","name":"protocolProperty","usr":"c:objc(pl)Protocol(py)protocolProperty"}] +// CHECK-PROTOCOL-PROP: "relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" +// CHECK-PROTOCOL-PROP: "isSystem":false +// CHECK-PROTOCOL-PROP: "usr":"c:@S@Foo"}] +// CHECK-PROTOCOL-PROP: "relationships":[{"kind":"memberOf","source":"c:objc(pl)Protocol(py)protocolProperty","target":"c:objc(pl)Protocol" +// CHECK-PROTOCOL-PROP: "text":"Protocol property docs" +// CHECK-PROTOCOL-PROP: "kind":{"displayName":"Instance Property","identifier":"objective-c.property"} +// CHECK-PROTOCOL-PROP: "title":"protocolProperty" + +// RUN: c-index-test -single-symbol-sgf-at=%s:25:15 local %s | FileCheck -check-prefix=CHECK-DERIVED %s +// CHECK-DERIVED: "parentContexts":[{"kind":"objective-c.class","name":"Derived","usr":"c:objc(cs)Derived"}] +// CHECK-DERIVED: "relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" +// CHECK-DERIVED: "isSystem":false +// CHECK-DERIVED: "usr":"c:objc(cs)Base"}] +// CHECK-DERIVED: "relationships":[{"kind":"inheritsFrom","source":"c:objc(cs)Derived","target":"c:objc(cs)Base" +// CHECK-DERIVED: "text":"Derived docs" +// CHECK-DERIVED: "kind":{"displayName":"Class","identifier":"objective-c.class"} +// CHECK-DERIVED: "title":"Derived" + +// RUN: c-index-test -single-symbol-sgf-at=%s:27:11 local %s | FileCheck -check-prefix=CHECK-DERIVED-METHOD %s +// CHECK-DERIVED-METHOD: "parentContexts":[{"kind":"objective-c.class","name":"Derived","usr":"c:objc(cs)Derived"},{"kind":"objective-c.method","name":"derivedMethodWithValue:","usr":"c:objc(cs)Derived(im)derivedMethodWithValue:"}] +// CHECK-DERIVED-METHOD: "relatedSymbols":[] +// CHECK-DERIVED-METHOD: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Derived(im)derivedMethodWithValue:","target":"c:objc(cs)Derived" +// CHECK-DERIVED-METHOD: "text":"Derived method docs" +// CHECK-DERIVED-METHOD: "kind":{"displayName":"Instance Method","identifier":"objective-c.method"} +// CHECK-DERIVED-METHOD: "title":"derivedMethodWithValue:" + +// RUN: c-index-test -single-symbol-sgf-at=%s:31:11 local %s | FileCheck -check-prefix=CHECK-DERIVED-METHOD-IMPL %s +// CHECK-DERIVED-METHOD-IMPL: "parentContexts":[{"kind":"objective-c.class","name":"Derived","usr":"c:objc(cs)Derived"},{"kind":"objective-c.method","name":"derivedMethodWithValue:","usr":"c:objc(cs)Derived(im)derivedMethodWithValue:"}] +// CHECK-DERIVED-METHOD-IMPL: "relatedSymbols":[] +// CHECK-DERIVED-METHOD-IMPL: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Derived(im)derivedMethodWithValue:","target":"c:objc(cs)Derived" +// CHECK-DERIVED-METHOD-IMPL: "text":"Derived method docs" +// CHECK-DERIVED-METHOD-IMPL: "kind":{"displayName":"Instance Method","identifier":"objective-c.method"} +// CHECK-DERIVED-METHOD-IMPL: "title":"derivedMethodWithValue:" diff --git a/clang/test/Index/extract-api-usr.m b/clang/test/Index/extract-api-usr.m index 12bfb0a676e66..0b468ee916708 100644 --- a/clang/test/Index/extract-api-usr.m +++ b/clang/test/Index/extract-api-usr.m @@ -28,7 +28,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for Foo // RUN: c-index-test "-single-symbol-sgf-for=c:@S@Foo" %s | FileCheck -check-prefix=CHECK-FOO %s -// CHECK-FOO: "parentContexts":[] +// CHECK-FOO: "parentContexts":[{"kind":"objective-c.struct","name":"Foo","usr":"c:@S@Foo"}] // CHECK-FOO-SAME: "relatedSymbols":[] // CHECK-FOO-SAME: "relationships":[] // CHECK-FOO-SAME: "text":"Foo docs" @@ -38,7 +38,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for bar // RUN: c-index-test "-single-symbol-sgf-for=c:@S@Foo@FI@bar" %s | FileCheck -check-prefix=CHECK-BAR %s -// CHECK-BAR: "parentContexts":[{"kind":"objective-c.struct","name":"Foo","usr":"c:@S@Foo"}] +// CHECK-BAR: "parentContexts":[{"kind":"objective-c.struct","name":"Foo","usr":"c:@S@Foo"},{"kind":"objective-c.property","name":"bar","usr":"c:@S@Foo@FI@bar"}] // CHECK-BAR-SAME: "relatedSymbols":[] // CHECK-BAR-SAME: "relationships":[{"kind":"memberOf","source":"c:@S@Foo@FI@bar","target":"c:@S@Foo" // CHECK-BAR-SAME: "text":"Bar docs" @@ -47,7 +47,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for Base // RUN: c-index-test "-single-symbol-sgf-for=c:objc(cs)Base" %s | FileCheck -check-prefix=CHECK-BASE %s -// CHECK-BASE: "parentContexts":[] +// CHECK-BASE: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"}] // CHECK-BASE-SAME: "relatedSymbols":[] // CHECK-BASE-SAME: "relationships":[] // CHECK-BASE-SAME: "text":"Base docs" @@ -56,7 +56,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for baseProperty // RUN: c-index-test "-single-symbol-sgf-for=c:objc(cs)Base(py)baseProperty" %s | FileCheck -check-prefix=CHECK-BASEPROP %s -// CHECK-BASEPROP: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"}] +// CHECK-BASEPROP: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"},{"kind":"objective-c.property","name":"baseProperty","usr":"c:objc(cs)Base(py)baseProperty"}] // CHECK-BASEPROP-SAME:"relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" // CHECK-BASEPROP-SAME: "isSystem":false // CHECK-BASEPROP-SAME: "usr":"c:@S@Foo"}] @@ -67,7 +67,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for baseMethodWithArg // RUN: c-index-test "-single-symbol-sgf-for=c:objc(cs)Base(im)baseMethodWithArg:" %s | FileCheck -check-prefix=CHECK-BASEMETHOD %s -// CHECK-BASEMETHOD: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"}] +// CHECK-BASEMETHOD: "parentContexts":[{"kind":"objective-c.class","name":"Base","usr":"c:objc(cs)Base"},{"kind":"objective-c.method","name":"baseMethodWithArg:","usr":"c:objc(cs)Base(im)baseMethodWithArg:"}] // CHECK-BASEMETHOD-SAME:"relatedSymbols":[] // CHECK-BASEMETHOD-SAME: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Base(im)baseMethodWithArg:","target":"c:objc(cs)Base" // CHECK-BASEMETHOD-SAME: "text":"Base method docs" @@ -76,7 +76,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for Protocol // RUN: c-index-test "-single-symbol-sgf-for=c:objc(pl)Protocol" %s | FileCheck -check-prefix=CHECK-PROT %s -// CHECK-PROT: "parentContexts":[] +// CHECK-PROT: "parentContexts":[{"kind":"objective-c.protocol","name":"Protocol","usr":"c:objc(pl)Protocol"}] // CHECK-PROT-SAME: "relatedSymbols":[] // CHECK-PROT-SAME: "relationships":[] // CHECK-PROT-SAME: "text":"Protocol docs" @@ -85,7 +85,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for protocolProperty // RUN: c-index-test "-single-symbol-sgf-for=c:objc(pl)Protocol(py)protocolProperty" %s | FileCheck -check-prefix=CHECK-PROTPROP %s -// CHECK-PROTPROP: "parentContexts":[{"kind":"objective-c.protocol","name":"Protocol","usr":"c:objc(pl)Protocol"}] +// CHECK-PROTPROP: "parentContexts":[{"kind":"objective-c.protocol","name":"Protocol","usr":"c:objc(pl)Protocol"},{"kind":"objective-c.property","name":"protocolProperty","usr":"c:objc(pl)Protocol(py)protocolProperty"}] // CHECK-PROTPROP-SAME:"relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" // CHECK-PROTPROP-SAME: "isSystem":false // CHECK-PROTPROP-SAME: "usr":"c:@S@Foo"}] @@ -96,7 +96,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for Derived // RUN: c-index-test "-single-symbol-sgf-for=c:objc(cs)Derived" %s | FileCheck -check-prefix=CHECK-DERIVED %s -// CHECK-DERIVED: "parentContexts":[] +// CHECK-DERIVED: "parentContexts":[{"kind":"objective-c.class","name":"Derived","usr":"c:objc(cs)Derived"}] // CHECK-DERIVED-SAME:"relatedSymbols":[{"accessLevel":"public","declarationLanguage":"objective-c" // CHECK-DERIVED-SAME: "isSystem":false // CHECK-DERIVED-SAME: "usr":"c:objc(cs)Base"}] @@ -107,7 +107,7 @@ - (void)derivedMethodWithValue:(id)value; // Checking for derivedMethodWithValue // RUN: c-index-test "-single-symbol-sgf-for=c:objc(cs)Derived(im)derivedMethodWithValue:" %s | FileCheck -check-prefix=CHECK-DERIVEDMETHOD %s -// CHECK-DERIVEDMETHOD: "parentContexts":[{"kind":"objective-c.class","name":"Derived","usr":"c:objc(cs)Derived"}] +// CHECK-DERIVEDMETHOD: "parentContexts":[{"kind":"objective-c.class","name":"Derived","usr":"c:objc(cs)Derived"},{"kind":"objective-c.method","name":"derivedMethodWithValue:","usr":"c:objc(cs)Derived(im)derivedMethodWithValue:"}] // CHECK-DERIVEDMETHOD-SAME:"relatedSymbols":[] // CHECK-DERIVEDMETHOD-SAME: "relationships":[{"kind":"memberOf","source":"c:objc(cs)Derived(im)derivedMethodWithValue:","target":"c:objc(cs)Derived" // CHECK-DERIVEDMETHOD-SAME: "text":"Derived method docs" diff --git a/clang/tools/c-index-test/c-index-test.c b/clang/tools/c-index-test/c-index-test.c index 052c39a15d33e..cd0034d938aa8 100644 --- a/clang/tools/c-index-test/c-index-test.c +++ b/clang/tools/c-index-test/c-index-test.c @@ -3,6 +3,7 @@ #include "clang-c/BuildSystem.h" #include "clang-c/CXCompilationDatabase.h" #include "clang-c/CXErrorCode.h" +#include "clang-c/CXSourceLocation.h" #include "clang-c/CXString.h" #include "clang-c/Documentation.h" #include "clang-c/Index.h" @@ -4887,6 +4888,22 @@ static int perform_test_single_symbol_sgf(const char *input, int argc, return result; } +static void inspect_single_symbol_sgf_cursor(CXCursor Cursor) { + CXSourceLocation CursorLoc; + CXString SGFData; + const char *SGF; + unsigned line, column; + CursorLoc = clang_getCursorLocation(Cursor); + clang_getSpellingLocation(CursorLoc, 0, &line, &column, 0); + + SGFData = clang_getSymbolGraphForCursor(Cursor); + SGF = clang_getCString(SGFData); + if (SGF) + printf("%d:%d: %s\n", line, column, SGF); + + clang_disposeString(SGFData); +} + /******************************************************************************/ /* Command line processing. */ /******************************************************************************/ @@ -4946,6 +4963,7 @@ static void print_usage(void) { " c-index-test -print-usr-file \n"); fprintf(stderr, " c-index-test -single-symbol-sgfs {*}\n" + " c-index-test -single-symbol-sgf-at= {*}\n" " c-index-test -single-symbol-sgf-for= {}*\n"); fprintf(stderr, " c-index-test -write-pch \n" @@ -5082,6 +5100,9 @@ int cindextest_main(int argc, const char **argv) { else if (argc > 3 && strcmp(argv[1], "-single-symbol-sgfs") == 0) return perform_test_load_source(argc - 3, argv + 3, argv[2], PrintSingleSymbolSGFs, NULL); + else if (argc > 2 && strstr(argv[1], "-single-symbol-sgf-at=") == argv[1]) + return inspect_cursor_at( + argc, argv, "-single-symbol-sgf-at=", inspect_single_symbol_sgf_cursor); else if (argc > 2 && strstr(argv[1], "-single-symbol-sgf-for=") == argv[1]) return perform_test_single_symbol_sgf(argv[1], argc - 2, argv + 2); diff --git a/clang/tools/libclang/CXExtractAPI.cpp b/clang/tools/libclang/CXExtractAPI.cpp index 787334ab1bbb2..9128e891538a8 100644 --- a/clang/tools/libclang/CXExtractAPI.cpp +++ b/clang/tools/libclang/CXExtractAPI.cpp @@ -18,6 +18,7 @@ #include "clang-c/Index.h" #include "clang-c/Platform.h" #include "clang/AST/Decl.h" +#include "clang/AST/DeclObjC.h" #include "clang/Basic/TargetInfo.h" #include "clang/ExtractAPI/API.h" #include "clang/ExtractAPI/ExtractAPIVisitor.h" @@ -34,13 +35,73 @@ using namespace clang; using namespace clang::extractapi; +namespace { +struct LibClangExtractAPIVisitor + : ExtractAPIVisitor { + using Base = ExtractAPIVisitor; + + LibClangExtractAPIVisitor(ASTContext &Context, APISet &API) + : ExtractAPIVisitor(Context, API) {} + + const RawComment *fetchRawCommentForDecl(const Decl *D) const { + return Context.getRawCommentForAnyRedecl(D); + } + + // We need to visit implementations as well to ensure that when a user clicks + // on a method defined only within the implementation that we can still + // provide a symbol graph for it. + bool VisitObjCImplementationDecl(const ObjCImplementationDecl *Decl) { + if (!shouldDeclBeIncluded(Decl)) + return true; + + const ObjCInterfaceDecl *Interface = Decl->getClassInterface(); + StringRef Name = Interface->getName(); + StringRef USR = API.recordUSR(Decl); + PresumedLoc Loc = + Context.getSourceManager().getPresumedLoc(Decl->getLocation()); + LinkageInfo Linkage = Decl->getLinkageAndVisibility(); + DocComment Comment; + if (auto *RawComment = fetchRawCommentForDecl(Interface)) + Comment = RawComment->getFormattedLines(Context.getSourceManager(), + Context.getDiagnostics()); + + // Build declaration fragments and sub-heading by generating them for the + // interface. + DeclarationFragments Declaration = + DeclarationFragmentsBuilder::getFragmentsForObjCInterface(Interface); + DeclarationFragments SubHeading = + DeclarationFragmentsBuilder::getSubHeading(Decl); + + // Collect super class information. + SymbolReference SuperClass; + if (const auto *SuperClassDecl = Decl->getSuperClass()) { + SuperClass.Name = SuperClassDecl->getObjCRuntimeNameAsString(); + SuperClass.USR = API.recordUSR(SuperClassDecl); + } + + ObjCInterfaceRecord *ObjCInterfaceRecord = API.addObjCInterface( + Name, USR, Loc, AvailabilitySet(Decl), Linkage, Comment, Declaration, + SubHeading, SuperClass, isInSystemHeader(Decl)); + + // Record all methods (selectors). This doesn't include automatically + // synthesized property methods. + recordObjCMethods(ObjCInterfaceRecord, Decl->methods()); + recordObjCProperties(ObjCInterfaceRecord, Decl->properties()); + recordObjCInstanceVariables(ObjCInterfaceRecord, Decl->ivars()); + + return true; + } +}; +} // namespace + DEFINE_SIMPLE_CONVERSION_FUNCTIONS(APISet, CXAPISet) -static void WalkupFromMostDerivedType(ExtractAPIVisitor &Visitor, Decl *D); +static void WalkupFromMostDerivedType(LibClangExtractAPIVisitor &Visitor, + Decl *D); template static bool WalkupParentContext(DeclContext *Parent, - ExtractAPIVisitor &Visitor) { + LibClangExtractAPIVisitor &Visitor) { if (auto *D = dyn_cast(Parent)) { WalkupFromMostDerivedType(Visitor, D); return true; @@ -48,7 +109,8 @@ static bool WalkupParentContext(DeclContext *Parent, return false; } -static void WalkupFromMostDerivedType(ExtractAPIVisitor &Visitor, Decl *D) { +static void WalkupFromMostDerivedType(LibClangExtractAPIVisitor &Visitor, + Decl *D) { switch (D->getKind()) { #define ABSTRACT_DECL(DECL) #define DECL(CLASS, BASE) \ @@ -84,8 +146,7 @@ enum CXErrorCode clang_createAPISet(CXTranslationUnit tu, CXAPISet *out_api) { auto Lang = Unit->getInputKind().getLanguage(); APISet *API = new APISet(Ctx.getTargetInfo().getTriple(), Lang, Unit->getMainFileName().str()); - ExtractAPIVisitor Visitor( - Ctx, [](SourceLocation Loc) { return true; }, *API); + LibClangExtractAPIVisitor Visitor(Ctx, *API); for (auto It = Unit->top_level_begin(); It != Unit->top_level_end(); ++It) { Visitor.TraverseDecl(*It); @@ -107,45 +168,50 @@ CXString clang_getSymbolGraphForUSR(const char *usr, CXAPISet api) { } CXString clang_getSymbolGraphForCursor(CXCursor cursor) { + cursor = clang_getCursorReferenced(cursor); CXCursorKind Kind = clang_getCursorKind(cursor); - if (clang_isDeclaration(Kind)) { - const Decl *D = cxcursor::getCursorDecl(cursor); + if (!clang_isDeclaration(Kind)) + return cxstring::createNull(); - if (!D) - return cxstring::createNull(); + const Decl *D = cxcursor::getCursorDecl(cursor); - CXTranslationUnit TU = cxcursor::getCursorTU(cursor); - if (!TU) - return cxstring::createNull(); + if (!D) + return cxstring::createNull(); - ASTUnit *Unit = cxtu::getASTUnit(TU); + CXTranslationUnit TU = cxcursor::getCursorTU(cursor); + if (!TU) + return cxstring::createNull(); - auto &Ctx = Unit->getASTContext(); - auto Lang = Unit->getInputKind().getLanguage(); - APISet API(Ctx.getTargetInfo().getTriple(), Lang, - Unit->getMainFileName().str()); - ExtractAPIVisitor Visitor( - Ctx, [](SourceLocation Loc) { return true; }, API); + ASTUnit *Unit = cxtu::getASTUnit(TU); - SmallString<128> USR; - if (index::generateUSRForDecl(D, USR)) - return cxstring::createNull(); + auto &Ctx = Unit->getASTContext(); - WalkupFromMostDerivedType(Visitor, const_cast(D)); - auto *Record = API.findRecordForUSR(USR); + auto Lang = Unit->getInputKind().getLanguage(); + APISet API(Ctx.getTargetInfo().getTriple(), Lang, + Unit->getMainFileName().str()); + LibClangExtractAPIVisitor Visitor(Ctx, API); - if (!Record) - return cxstring::createNull(); + const Decl *CanonicalDecl = D->getCanonicalDecl(); + CanonicalDecl = CanonicalDecl ? CanonicalDecl : D; - for (const auto &Fragment : Record->Declaration.getFragments()) { - if (Fragment.Declaration) - WalkupFromMostDerivedType(Visitor, - const_cast(Fragment.Declaration)); - } + SmallString<128> USR; + if (index::generateUSRForDecl(CanonicalDecl, USR)) + return cxstring::createNull(); - if (auto SGF = SymbolGraphSerializer::serializeSingleSymbolSGF(USR, API)) - return GenerateCXStringFromSymbolGraphData(std::move(*SGF)); + WalkupFromMostDerivedType(Visitor, const_cast(CanonicalDecl)); + auto *Record = API.findRecordForUSR(USR); + + if (!Record) + return cxstring::createNull(); + + for (const auto &Fragment : Record->Declaration.getFragments()) { + if (Fragment.Declaration) + WalkupFromMostDerivedType(Visitor, + const_cast(Fragment.Declaration)); } + if (auto SGF = SymbolGraphSerializer::serializeSingleSymbolSGF(USR, API)) + return GenerateCXStringFromSymbolGraphData(std::move(*SGF)); + return cxstring::createNull(); } From 4faf613c811caa16d894d9be121ea85625c1e90e Mon Sep 17 00:00:00 2001 From: Hamish Knight Date: Fri, 31 Mar 2023 17:42:17 +0100 Subject: [PATCH 42/88] [lldb] Update for full enablement of experimental string processing rdar://107419385 rdar://101765556 --- .../ExpressionParser/Swift/SwiftExpressionParser.cpp | 2 -- lldb/test/API/lang/swift/regex/TestSwiftRegex.py | 5 +++++ lldb/test/Shell/SwiftREPL/Regex.test | 8 +++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lldb/source/Plugins/ExpressionParser/Swift/SwiftExpressionParser.cpp b/lldb/source/Plugins/ExpressionParser/Swift/SwiftExpressionParser.cpp index 95154dbca10f0..5d6b114be9bb7 100644 --- a/lldb/source/Plugins/ExpressionParser/Swift/SwiftExpressionParser.cpp +++ b/lldb/source/Plugins/ExpressionParser/Swift/SwiftExpressionParser.cpp @@ -1343,8 +1343,6 @@ static llvm::Expected ParseAndImport( sc.target_sp->GetSwiftEnableBareSlashRegex(); invocation.getLangOptions().EnableBareSlashRegexLiterals = enable_bare_slash_regex_literals; - invocation.getLangOptions().EnableExperimentalStringProcessing = - enable_bare_slash_regex_literals; invocation.getLangOptions().Features.insert(swift::Feature::VariadicGenerics); auto should_use_prestable_abi = [&]() { diff --git a/lldb/test/API/lang/swift/regex/TestSwiftRegex.py b/lldb/test/API/lang/swift/regex/TestSwiftRegex.py index 63f73fb0bb23b..9538b77e8d406 100644 --- a/lldb/test/API/lang/swift/regex/TestSwiftRegex.py +++ b/lldb/test/API/lang/swift/regex/TestSwiftRegex.py @@ -50,6 +50,11 @@ def test_swift_regex_in_exp(self): self.build() lldbutil.run_to_source_breakpoint( self, 'Set breakpoint here', self.main_source_spec) + + # Make sure we can use the extended syntax without enabling anything. + self.expect('e -- #/Order from <(.*)>, type: (.*), count in dozen: ([0-9]+)/#', + substrs=['_StringProcessing.Regex<(Substring, Substring, Substring, Substring)>']) + self.runCmd( "settings set target.experimental.swift-enable-bare-slash-regex true") self.expect('e -- /Order from <(.*)>, type: (.*), count in dozen: ([0-9]+)/', diff --git a/lldb/test/Shell/SwiftREPL/Regex.test b/lldb/test/Shell/SwiftREPL/Regex.test index a9e72e91ba3a5..698eb2fbfb4c2 100644 --- a/lldb/test/Shell/SwiftREPL/Regex.test +++ b/lldb/test/Shell/SwiftREPL/Regex.test @@ -3,8 +3,13 @@ // RUN: %lldb --repl < %s | FileCheck %s +// Make sure we can use an extended literal without enabling anything. +let regex1 = #/Order from <(.*)>, type: (.*), count in dozen: ([0-9]+)/# +// CHECK: _StringProcessing.Regex<(Substring, Substring, Substring, Substring)> = + :settings set target.experimental.swift-enable-bare-slash-regex true -let regex = /Order from <(.*)>, type: (.*), count in dozen: ([0-9]+)/ + +let regex2 = /Order from <(.*)>, type: (.*), count in dozen: ([0-9]+)/ // CHECK: _StringProcessing.Regex<(Substring, Substring, Substring, Substring)> = import RegexBuilder @@ -17,5 +22,6 @@ if #unavailable(macOS 13, iOS 16, watchOS 9, tvOS 16) { // Work around the fact that the REPL doesn't print results for // variables declared inside of a conditional. print("_StringProcessing.Regex<(Substring, Substring, Substring, Substring)> =") + print("_StringProcessing.Regex<(Substring, Substring, Substring, Substring)> =") print("_StringProcessing.Regex = ") } From 9f1a8d73fbfc51b0d0d35b30c7c57a089484f047 Mon Sep 17 00:00:00 2001 From: Augusto Noronha Date: Thu, 30 Mar 2023 14:01:36 -0700 Subject: [PATCH 43/88] [clang] Add "transparent_stepping" attribute The `transparent_stepping` attribute is intended as a hint for debuggers that this function itself is not interesting, but it calls a function that might be. So, when stepping in arrives at a function with this attribute, debuggers should transparently step-in through it into the functions called by the annotated function (but not by subsequent calls made by those functions), stopping at the first one its normal rules for whether to stop says to stop at - or stepping out again if none qualify. Also, when stepping out arrives at a function with this attribute, the debugger should continue stepping out to its caller. Differential Revision: https://reviews.llvm.org/D146595 (cherry picked from commit 115648dd3a25362f02fc14942b5edcffbff2b117) --- clang/include/clang/Basic/Attr.td | 6 ++ clang/include/clang/Basic/AttrDocs.td | 60 +++++++++++++++++++ clang/lib/CodeGen/CGDebugInfo.cpp | 17 ++++++ clang/lib/Sema/SemaDeclAttr.cpp | 9 +++ .../attr-transparent-stepping-method.cpp | 16 +++++ .../test/CodeGen/attr-transparent-stepping.c | 10 ++++ ...a-attribute-supported-attributes-list.test | 1 + clang/test/Sema/attr-transparent-stepping.c | 7 +++ .../attr-transparent-stepping-method.cpp | 11 ++++ llvm/include/llvm/IR/DebugInfoFlags.def | 3 +- llvm/include/llvm/IR/DebugInfoMetadata.h | 3 + llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp | 3 + .../disubprogram-transparent-stepping.ll | 39 ++++++++++++ .../disubprogram-transparent-stepping.ll | 42 +++++++++++++ llvm/unittests/IR/MetadataTest.cpp | 18 ++++-- 15 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 clang/test/CodeGen/attr-transparent-stepping-method.cpp create mode 100644 clang/test/CodeGen/attr-transparent-stepping.c create mode 100644 clang/test/Sema/attr-transparent-stepping.c create mode 100644 clang/test/SemaCXX/attr-transparent-stepping-method.cpp create mode 100644 llvm/test/Assembler/disubprogram-transparent-stepping.ll create mode 100644 llvm/test/DebugInfo/AArch64/disubprogram-transparent-stepping.ll diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 3c84585e1ae3b..1ae927d63f9d4 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -771,6 +771,12 @@ def Artificial : InheritableAttr { let SimpleHandler = 1; } +def TransparentStepping: InheritableAttr { + let Spellings = [Clang<"transparent_stepping">]; + let Subjects = SubjectList<[Function]>; + let Documentation = [TransparentSteppingDocs]; +} + def XRayInstrument : InheritableAttr { let Spellings = [Clang<"xray_always_instrument">, Clang<"xray_never_instrument">]; diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index f19af47749d74..d1f3bd0af3087 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -6924,6 +6924,66 @@ As such, this function attribute is currently only supported on X86 targets. }]; } +def TransparentSteppingDocs : Documentation { + let Category = DocCatFunction; + let Content = [{ +The ``transparent_stepping`` attribute is intended as a hint for debuggers that this +function itself is not interesting, but it calls a function that might be. So, when +stepping in arrives at a function with this attribute, debuggers should transparently +step-in through it into the functions called by the annotated function (but not by +subsequent calls made by those functions), stopping at the first one its normal rules +for whether to stop says to stop at - or stepping out again if none qualify. Also, when +stepping out arrives at a function with this attribute, the debugger should continue +stepping out to its caller. + +For example: + +.. code-block:: c + + int bar(void) { + return 42; + } + + __attribute__((transparent_stepping)) + int foo(void) { + return bar(); + } + + int caller(void) { + return foo(); + } + +Stepping into ``foo`` should step directly into ``bar`` instead, and stepping out of ``bar`` +should stop in ``caller``. + +Functions with the ``transparent_stepping`` attribute can be chained together: + +.. code-block:: c + + int baz(void) { + return 42; + } + + __attribute__((transparent_stepping)) + int bar(void) { + return baz(); + } + + __attribute__((transparent_stepping)) + int foo(void) { + return bar(); + } + + int caller(void) { + return foo(); + } + +In this example, stepping into ``foo`` should step directly into ``baz``, and stepping out of +``baz`` should stop in ``caller``. + }]; +} + + def ReadOnlyPlacementDocs : Documentation { let Category = DocCatType; let Content = [{This attribute is attached to a structure, class or union declaration. diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 0a1f72f9f07d2..7d6f22d6cf51e 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -67,6 +67,12 @@ static uint32_t getDeclAlignIfRequired(const Decl *D, const ASTContext &Ctx) { return D->hasAttr() ? D->getMaxAlignment() : 0; } +static bool getIsTransparentStepping(const Decl *D) { + if (!D) + return false; + return D->hasAttr(); +} + CGDebugInfo::CGDebugInfo(CodeGenModule &CGM) : CGM(CGM), DebugKind(CGM.getCodeGenOpts().getDebugInfo()), DebugTypeExtRefs(CGM.getCodeGenOpts().DebugTypeExtRefs), @@ -1882,6 +1888,8 @@ llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction( SPFlags |= llvm::DISubprogram::SPFlagLocalToUnit; if (CGM.getLangOpts().Optimize) SPFlags |= llvm::DISubprogram::SPFlagOptimized; + if (getIsTransparentStepping(Method)) + SPFlags |= llvm::DISubprogram::SPFlagIsTransparentStepping; // In this debug mode, emit type info for a class when its constructor type // info is emitted. @@ -3809,6 +3817,8 @@ llvm::DISubprogram *CGDebugInfo::getFunctionFwdDeclOrStub(GlobalDecl GD, if (Stub) { Flags |= getCallSiteRelatedAttrs(); SPFlags |= llvm::DISubprogram::SPFlagDefinition; + if (getIsTransparentStepping(FD)) + SPFlags |= llvm::DISubprogram::SPFlagIsTransparentStepping; return DBuilder.createFunction( DContext, Name, LinkageName, Unit, Line, getOrCreateFunctionType(GD.getDecl(), FnType, Unit), 0, Flags, SPFlags, @@ -3958,6 +3968,8 @@ llvm::DISubprogram *CGDebugInfo::getObjCMethodDeclaration( if (It == TypeCache.end()) return nullptr; auto *InterfaceType = cast(It->second); + if (getIsTransparentStepping(D)) + SPFlags |= llvm::DISubprogram::SPFlagIsTransparentStepping; llvm::DISubprogram *FD = DBuilder.createFunction( InterfaceType, getObjCMethodName(OMD), StringRef(), InterfaceType->getFile(), LineNo, FnType, LineNo, Flags, SPFlags); @@ -4124,6 +4136,8 @@ void CGDebugInfo::emitFunctionStart(GlobalDecl GD, SourceLocation Loc, SPFlags |= llvm::DISubprogram::SPFlagLocalToUnit; if (CGM.getLangOpts().Optimize) SPFlags |= llvm::DISubprogram::SPFlagOptimized; + if (getIsTransparentStepping(D)) + SPFlags |= llvm::DISubprogram::SPFlagIsTransparentStepping; llvm::DINode::DIFlags FlagsForDef = Flags | getCallSiteRelatedAttrs(); llvm::DISubprogram::DISPFlags SPFlagsForDef = @@ -4210,6 +4224,9 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc, llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(D); llvm::DISubroutineType *STy = getOrCreateFunctionType(D, FnType, Unit); + if (getIsTransparentStepping(D)) + SPFlags |= llvm::DISubprogram::SPFlagIsTransparentStepping; + llvm::DISubprogram *SP = DBuilder.createFunction( FDContext, Name, LinkageName, Unit, LineNo, STy, ScopeLine, Flags, SPFlags, TParamsArray.get(), nullptr, nullptr, Annotations); diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 39218c9c287ae..b0b1e86fbe4b0 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -6687,6 +6687,12 @@ static void handleSwiftAsyncName(Sema &S, Decl *D, const ParsedAttr &AL) { D->addAttr(::new (S.Context) SwiftAsyncNameAttr(S.Context, AL, Name)); } +static void handleTransparentStepping(Sema &S, Decl *D, + const ParsedAttr &AL) { + D->addAttr(::new (S.Context) + TransparentSteppingAttr(S.Context, AL)); +} + static void handleSwiftNewType(Sema &S, Decl *D, const ParsedAttr &AL) { // Make sure that there is an identifier as the annotation's single argument. if (!AL.checkExactlyNumArgs(S, 1)) @@ -8948,6 +8954,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, case ParsedAttr::AT_NoDebug: handleNoDebugAttr(S, D, AL); break; + case ParsedAttr::AT_TransparentStepping: + handleTransparentStepping(S, D, AL); + break; case ParsedAttr::AT_CmseNSEntry: handleCmseNSEntryAttr(S, D, AL); break; diff --git a/clang/test/CodeGen/attr-transparent-stepping-method.cpp b/clang/test/CodeGen/attr-transparent-stepping-method.cpp new file mode 100644 index 0000000000000..5eb91b90bd65b --- /dev/null +++ b/clang/test/CodeGen/attr-transparent-stepping-method.cpp @@ -0,0 +1,16 @@ +// RUN: %clang_cc1 -debug-info-kind=limited -emit-llvm -o - %s | FileCheck %s + +void bar(void) {} + +struct A { +[[clang::transparent_stepping()]] +void foo(void) { + bar(); +} +}; + +int main() { + A().foo(); +} + +// CHECK: DISubprogram(name: "foo"{{.*}} DISPFlagIsTransparentStepping diff --git a/clang/test/CodeGen/attr-transparent-stepping.c b/clang/test/CodeGen/attr-transparent-stepping.c new file mode 100644 index 0000000000000..6ae42dce78123 --- /dev/null +++ b/clang/test/CodeGen/attr-transparent-stepping.c @@ -0,0 +1,10 @@ +// RUN: %clang_cc1 -debug-info-kind=limited -emit-llvm -o - %s | FileCheck %s + +void bar(void) {} + +__attribute__((transparent_stepping)) +void foo(void) { + bar(); +} + +// CHECK: DISubprogram(name: "foo"{{.*}} DISPFlagIsTransparentStepping diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test index f7c5ce9468841..d55874611cabe 100644 --- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test +++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test @@ -185,6 +185,7 @@ // CHECK-NEXT: Target (SubjectMatchRule_function) // CHECK-NEXT: TargetClones (SubjectMatchRule_function) // CHECK-NEXT: TestTypestate (SubjectMatchRule_function_is_member) +// CHECK-NEXT: TransparentStepping (SubjectMatchRule_function) // CHECK-NEXT: TrivialABI (SubjectMatchRule_record) // CHECK-NEXT: Uninitialized (SubjectMatchRule_variable_is_local) // CHECK-NEXT: UseHandle (SubjectMatchRule_variable_is_parameter) diff --git a/clang/test/Sema/attr-transparent-stepping.c b/clang/test/Sema/attr-transparent-stepping.c new file mode 100644 index 0000000000000..8bcaa823f9542 --- /dev/null +++ b/clang/test/Sema/attr-transparent-stepping.c @@ -0,0 +1,7 @@ +// RUN: %clang_cc1 %s -verify -fsyntax-only + +__attribute__((transparent_stepping)) +void correct(void) {} + +__attribute__((transparent_stepping(1))) // expected-error {{'transparent_stepping' attribute takes no arguments}} +void wrong_arg(void) {} diff --git a/clang/test/SemaCXX/attr-transparent-stepping-method.cpp b/clang/test/SemaCXX/attr-transparent-stepping-method.cpp new file mode 100644 index 0000000000000..7f2be98a7c584 --- /dev/null +++ b/clang/test/SemaCXX/attr-transparent-stepping-method.cpp @@ -0,0 +1,11 @@ +// RUN: %clang_cc1 %s -verify -fsyntax-only + + +struct S { +[[clang::transparent_stepping]] +void correct(void) {} + +[[clang::transparent_stepping(1)]] // expected-error {{'transparent_stepping' attribute takes no arguments}} +void one_arg(void) {} +}; + diff --git a/llvm/include/llvm/IR/DebugInfoFlags.def b/llvm/include/llvm/IR/DebugInfoFlags.def index df375b6c68e81..6e9be38e7272e 100644 --- a/llvm/include/llvm/IR/DebugInfoFlags.def +++ b/llvm/include/llvm/IR/DebugInfoFlags.def @@ -91,11 +91,12 @@ HANDLE_DISP_FLAG((1u << 8), MainSubprogram) // for defaulted functions HANDLE_DISP_FLAG((1u << 9), Deleted) HANDLE_DISP_FLAG((1u << 11), ObjCDirect) +HANDLE_DISP_FLAG((1u << 12), IsTransparentStepping) #ifdef DISP_FLAG_LARGEST_NEEDED // Intended to be used with ADT/BitmaskEnum.h. // NOTE: Always must be equal to largest flag, check this when adding new flags. -HANDLE_DISP_FLAG((1 << 11), Largest) +HANDLE_DISP_FLAG((1 << 12), Largest) #undef DISP_FLAG_LARGEST_NEEDED #endif diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h index b8f10fa31c7b0..e4392bc286697 100644 --- a/llvm/include/llvm/IR/DebugInfoMetadata.h +++ b/llvm/include/llvm/IR/DebugInfoMetadata.h @@ -2025,6 +2025,9 @@ class DISubprogram : public DILocalScope { bool isElemental() const { return getSPFlags() & SPFlagElemental; } bool isRecursive() const { return getSPFlags() & SPFlagRecursive; } bool isObjCDirect() const { return getSPFlags() & SPFlagObjCDirect; } + bool getIsTransparentStepping() const { + return getSPFlags() & SPFlagIsTransparentStepping; + } /// Check if this is deleted member function. /// diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp index 3c9736d382951..4ee848e1713f8 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp @@ -1362,6 +1362,9 @@ void DwarfUnit::applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie, if (!SP->getTargetFuncName().empty()) addString(SPDie, dwarf::DW_AT_trampoline, SP->getTargetFuncName()); + if (SP->getIsTransparentStepping()) + addFlag(SPDie, dwarf::DW_AT_trampoline); + if (DD->getDwarfVersion() >= 5 && SP->isDeleted()) addFlag(SPDie, dwarf::DW_AT_deleted); } diff --git a/llvm/test/Assembler/disubprogram-transparent-stepping.ll b/llvm/test/Assembler/disubprogram-transparent-stepping.ll new file mode 100644 index 0000000000000..51550c5eb1bc2 --- /dev/null +++ b/llvm/test/Assembler/disubprogram-transparent-stepping.ll @@ -0,0 +1,39 @@ +; This test verifies that the DISPFlagIsTransparentStepping attribute in a DISubprogram +; is assembled/disassembled correctly. +; +; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s +; +; CHECK: !DISubprogram(name: "baz",{{.*}} DISPFlagIsTransparentStepping +; +; ModuleID = 't.c' +source_filename = "t.c" +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx13.0.0" + +; Function Attrs: noinline nounwind optnone ssp uwtable(sync) +define void @baz() #0 !dbg !10 { +entry: + ret void, !dbg !14 +} + +attributes #0 = { noinline nounwind optnone ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+zcm,+zcz" } + +!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6} +!llvm.dbg.cu = !{!7} +!llvm.ident = !{!9} + +!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 13, i32 2]} +!1 = !{i32 7, !"Dwarf Version", i32 4} +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 1, !"wchar_size", i32 4} +!4 = !{i32 8, !"PIC Level", i32 2} +!5 = !{i32 7, !"uwtable", i32 1} +!6 = !{i32 7, !"frame-pointer", i32 1} +!7 = distinct !DICompileUnit(language: DW_LANG_C11, file: !8, producer: "clang version 17.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!8 = !DIFile(filename: "t.c", directory: "/") +!9 = !{!"clang version 17.0.0"} +!10 = distinct !DISubprogram(name: "baz", scope: !8, file: !8, line: 3, type: !11, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagIsTransparentStepping, unit: !7, retainedNodes: !13) +!11 = !DISubroutineType(types: !12) +!12 = !{null} +!13 = !{} +!14 = !DILocation(line: 4, column: 1, scope: !10) diff --git a/llvm/test/DebugInfo/AArch64/disubprogram-transparent-stepping.ll b/llvm/test/DebugInfo/AArch64/disubprogram-transparent-stepping.ll new file mode 100644 index 0000000000000..0e93af5b4c728 --- /dev/null +++ b/llvm/test/DebugInfo/AArch64/disubprogram-transparent-stepping.ll @@ -0,0 +1,42 @@ +; This test verifies that the proper DWARF debug info is emitted +; for a trampoline function with no target. +; +; RUN: llc -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s +; +; CHECK: DW_TAG_subprogram +; CHECK: DW_AT_name ("baz") +; CHECK: DW_AT_trampoline (true) +; +; ModuleID = 't.c' +source_filename = "t.c" +target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" +target triple = "arm64-apple-macosx13.0.0" + +; Function Attrs: noinline nounwind optnone ssp uwtable(sync) +define void @baz() #0 !dbg !10 { +entry: + ret void, !dbg !14 +} + +attributes #0 = { noinline nounwind optnone ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+zcm,+zcz" } + +!llvm.module.flags = !{!0, !1, !2, !3, !4, !5, !6} +!llvm.dbg.cu = !{!7} +!llvm.ident = !{!9} + +!0 = !{i32 2, !"SDK Version", [2 x i32] [i32 13, i32 2]} +!1 = !{i32 7, !"Dwarf Version", i32 4} +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 1, !"wchar_size", i32 4} +!4 = !{i32 8, !"PIC Level", i32 2} +!5 = !{i32 7, !"uwtable", i32 1} +!6 = !{i32 7, !"frame-pointer", i32 1} +!7 = distinct !DICompileUnit(language: DW_LANG_C11, file: !8, producer: "clang version 17.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!8 = !DIFile(filename: "t.c", directory: "/") +!9 = !{!"clang version 17.0.0"} +!10 = distinct !DISubprogram(name: "baz", scope: !8, file: !8, line: 3, type: !11, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagIsTransparentStepping, unit: !7, retainedNodes: !13) +!11 = !DISubroutineType(types: !12) +!12 = !{null} +!13 = !{} +!14 = !DILocation(line: 4, column: 1, scope: !10) + diff --git a/llvm/unittests/IR/MetadataTest.cpp b/llvm/unittests/IR/MetadataTest.cpp index 23d7db2394119..20b9cb3dda008 100644 --- a/llvm/unittests/IR/MetadataTest.cpp +++ b/llvm/unittests/IR/MetadataTest.cpp @@ -2319,6 +2319,7 @@ TEST_F(DISubprogramTest, get) { assert(!IsLocalToUnit && IsDefinition && !IsOptimized && "bools and SPFlags have to match"); SPFlags |= DISubprogram::SPFlagDefinition; + SPFlags |= DISubprogram::SPFlagIsTransparentStepping; auto *N = DISubprogram::get( Context, Scope, Name, LinkageName, File, Line, Type, ScopeLine, @@ -2402,12 +2403,17 @@ TEST_F(DISubprogramTest, get) { Flags, SPFlags ^ DISubprogram::SPFlagDefinition, Unit, TemplateParams, Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName)); - EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line, - Type, ScopeLine + 1, ContainingType, - VirtualIndex, ThisAdjustment, Flags, SPFlags, - Unit, TemplateParams, Declaration, - RetainedNodes, ThrownTypes, Annotations, - TargetFuncName)); + EXPECT_NE(N, DISubprogram::get( + Context, Scope, Name, LinkageName, File, Line, Type, + ScopeLine, ContainingType, VirtualIndex, ThisAdjustment, + Flags, SPFlags ^ DISubprogram::SPFlagIsTransparentStepping, + Unit, TemplateParams, Declaration, RetainedNodes, + ThrownTypes, Annotations, TargetFuncName)); + EXPECT_NE(N, DISubprogram::get( + Context, Scope, Name, LinkageName, File, Line, Type, + ScopeLine + 1, ContainingType, VirtualIndex, ThisAdjustment, + Flags, SPFlags, Unit, TemplateParams, Declaration, + RetainedNodes, ThrownTypes, Annotations, TargetFuncName)); EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line, Type, ScopeLine, getCompositeType(), VirtualIndex, ThisAdjustment, Flags, SPFlags, From 23ea83a9e6451f2e12fbaf3fd6675ebd7aea2e34 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 14 Dec 2022 13:53:15 -0800 Subject: [PATCH 44/88] [lldb] Report clang module build remarks Update the Clang diagnostic consumer (in ClangModulesDeclVendor) to report progress on Clang module builds, as both progress events and expression logs. Module build remarks are enabled by with clang's `-Rmodule-build` flag. With this change, command line users of lldb will see progress events showing which modules are being built, and - by how long they stay on screen - how much time it takes to build them. IDEs that show progress events can show these updates if desired. This does not show module-import remarks, although that may be added as a future change. Differential Revision: https://reviews.llvm.org/D140056 (cherry picked from commit 9c5b97570502c5c6648730f75d097910ae2faa22) --- .../Clang/ClangModulesDeclVendor.cpp | 73 +++++++++++++++++-- .../progress_reporting/clang_modules/Makefile | 4 + .../clang_modules/MyModule.h | 1 + .../TestClangModuleBuildProgress.py | 46 ++++++++++++ .../progress_reporting/clang_modules/main.m | 1 + .../clang_modules/module.modulemap | 4 + 6 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 lldb/test/API/functionalities/progress_reporting/clang_modules/Makefile create mode 100644 lldb/test/API/functionalities/progress_reporting/clang_modules/MyModule.h create mode 100644 lldb/test/API/functionalities/progress_reporting/clang_modules/TestClangModuleBuildProgress.py create mode 100644 lldb/test/API/functionalities/progress_reporting/clang_modules/main.m create mode 100644 lldb/test/API/functionalities/progress_reporting/clang_modules/module.modulemap diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp index ec4e9f12fc974..ed64f112ed742 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp @@ -6,6 +6,9 @@ // //===----------------------------------------------------------------------===// +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticFrontend.h" +#include "clang/Basic/DiagnosticSerialization.h" #include "clang/Basic/TargetInfo.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendActions.h" @@ -15,7 +18,9 @@ #include "clang/Parse/Parser.h" #include "clang/Sema/Lookup.h" #include "clang/Serialization/ASTReader.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Path.h" #include "llvm/Support/Threading.h" @@ -25,6 +30,7 @@ #include "Plugins/TypeSystem/Clang/TypeSystemClang.h" #include "lldb/Core/ModuleList.h" +#include "lldb/Core/Progress.h" #include "lldb/Host/Host.h" #include "lldb/Host/HostInfo.h" #include "lldb/Symbol/CompileUnit.h" @@ -61,6 +67,9 @@ class StoringDiagnosticConsumer : public clang::DiagnosticConsumer { void EndSourceFile() override; private: + bool HandleModuleRemark(const clang::Diagnostic &info); + void SetCurrentModuleProgress(llvm::StringRef module_name); + typedef std::pair IDAndDiagnostic; std::vector m_diagnostics; @@ -72,6 +81,9 @@ class StoringDiagnosticConsumer : public clang::DiagnosticConsumer { /// Output string filled by m_os. Will be reused for different diagnostics. std::string m_output; Log *m_log; + /// A Progress with explicitly managed lifetime. + std::unique_ptr m_current_progress_up; + std::vector m_module_build_stack; }; /// The private implementation of our ClangModulesDeclVendor. Contains all the @@ -140,6 +152,9 @@ StoringDiagnosticConsumer::StoringDiagnosticConsumer() { void StoringDiagnosticConsumer::HandleDiagnostic( clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) { + if (HandleModuleRemark(info)) + return; + // Print the diagnostic to m_output. m_output.clear(); m_diag_printer->HandleDiagnostic(DiagLevel, info); @@ -170,9 +185,54 @@ void StoringDiagnosticConsumer::BeginSourceFile( } void StoringDiagnosticConsumer::EndSourceFile() { + m_current_progress_up = nullptr; m_diag_printer->EndSourceFile(); } +bool StoringDiagnosticConsumer::HandleModuleRemark( + const clang::Diagnostic &info) { + Log *log = GetLog(LLDBLog::Expressions); + switch (info.getID()) { + case clang::diag::remark_module_build: { + const auto &module_name = info.getArgStdStr(0); + SetCurrentModuleProgress(module_name); + m_module_build_stack.push_back(module_name); + + const auto &module_path = info.getArgStdStr(1); + LLDB_LOG(log, "Building Clang module {0} as {1}", module_name, module_path); + return true; + } + case clang::diag::remark_module_build_done: { + // The current module is done. + m_module_build_stack.pop_back(); + if (m_module_build_stack.empty()) { + m_current_progress_up = nullptr; + } else { + // Update the progress to re-show the module that was currently being + // built from the time the now completed module was originally began. + const auto &resumed_module_name = m_module_build_stack.back(); + SetCurrentModuleProgress(resumed_module_name); + } + + const auto &module_name = info.getArgStdStr(0); + LLDB_LOG(log, "Finished building Clang module {0}", module_name); + return true; + } + default: + return false; + } +} + +void StoringDiagnosticConsumer::SetCurrentModuleProgress( + llvm::StringRef module_name) { + // Ensure the ordering of: + // 1. Completing the existing progress event. + // 2. Beginining a new progress event. + m_current_progress_up = nullptr; + m_current_progress_up = std::make_unique( + llvm::formatv("Currently building module {0}", module_name)); +} + ClangModulesDeclVendor::ClangModulesDeclVendor() : ClangDeclVendor(eClangModuleDeclVendor) {} @@ -609,7 +669,8 @@ ClangModulesDeclVendor::Create(Target &target) { "-target", arch.GetTriple().str(), "-fmodules-validate-system-headers", - "-Werror=non-modular-include-in-framework-module"}; + "-Werror=non-modular-include-in-framework-module", + "-Rmodule-build"}; target.GetPlatform()->AddClangModuleCompilationOptions( &target, compiler_invocation_arguments); @@ -647,16 +708,18 @@ ClangModulesDeclVendor::Create(Target &target) { } } - llvm::IntrusiveRefCntPtr diagnostics_engine = - clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions, - new StoringDiagnosticConsumer); - std::vector compiler_invocation_argument_cstrs; compiler_invocation_argument_cstrs.reserve( compiler_invocation_arguments.size()); for (const std::string &arg : compiler_invocation_arguments) compiler_invocation_argument_cstrs.push_back(arg.c_str()); + auto diag_options_up = + clang::CreateAndPopulateDiagOpts(compiler_invocation_argument_cstrs); + llvm::IntrusiveRefCntPtr diagnostics_engine = + clang::CompilerInstance::createDiagnostics(diag_options_up.release(), + new StoringDiagnosticConsumer); + Log *log = GetLog(LLDBLog::Expressions); LLDB_LOG(log, "ClangModulesDeclVendor's compiler flags {0:$[ ]}", llvm::make_range(compiler_invocation_arguments.begin(), diff --git a/lldb/test/API/functionalities/progress_reporting/clang_modules/Makefile b/lldb/test/API/functionalities/progress_reporting/clang_modules/Makefile new file mode 100644 index 0000000000000..4ad4c54783a46 --- /dev/null +++ b/lldb/test/API/functionalities/progress_reporting/clang_modules/Makefile @@ -0,0 +1,4 @@ +OBJC_SOURCES := main.m +CFLAGS_EXTRAS = -fmodules -I$(BUILDDIR) + +include Makefile.rules diff --git a/lldb/test/API/functionalities/progress_reporting/clang_modules/MyModule.h b/lldb/test/API/functionalities/progress_reporting/clang_modules/MyModule.h new file mode 100644 index 0000000000000..ab96a103ec4ec --- /dev/null +++ b/lldb/test/API/functionalities/progress_reporting/clang_modules/MyModule.h @@ -0,0 +1 @@ +extern int doesNotActuallyExist; diff --git a/lldb/test/API/functionalities/progress_reporting/clang_modules/TestClangModuleBuildProgress.py b/lldb/test/API/functionalities/progress_reporting/clang_modules/TestClangModuleBuildProgress.py new file mode 100644 index 0000000000000..af0b59cd555f1 --- /dev/null +++ b/lldb/test/API/functionalities/progress_reporting/clang_modules/TestClangModuleBuildProgress.py @@ -0,0 +1,46 @@ +""" +Test clang module build progress events. +""" +import os +import shutil + +import lldb +import lldbsuite.test.lldbutil as lldbutil +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * + + +class TestCase(TestBase): + @skipUnlessDarwin + def test_clang_module_build_progress_report(self): + """Test receipt of progress events for clang module builds""" + self.build() + + # Ensure an empty module cache. + mod_cache = self.getBuildArtifact("new-modules") + if os.path.isdir(mod_cache): + shutil.rmtree(mod_cache) + self.runCmd(f"settings set symbols.clang-modules-cache-path '{mod_cache}'") + + # TODO: The need for this seems like a bug. + self.runCmd( + f"settings set target.clang-module-search-paths '{self.getSourceDir()}'" + ) + + lldbutil.run_to_name_breakpoint(self, "main") + + # Just before triggering module builds, start listening for progress + # events. Listening any earlier would result in a queue filled with + # other unrelated progress events. + broadcaster = self.dbg.GetBroadcaster() + listener = lldbutil.start_listening_from( + broadcaster, lldb.SBDebugger.eBroadcastBitProgress + ) + + # Trigger module builds. + self.expect("expression @import MyModule") + + event = lldbutil.fetch_next_event(self, listener, broadcaster) + payload = lldb.SBDebugger.GetProgressFromEvent(event) + message = payload[0] + self.assertEqual(message, "Currently building module MyModule") diff --git a/lldb/test/API/functionalities/progress_reporting/clang_modules/main.m b/lldb/test/API/functionalities/progress_reporting/clang_modules/main.m new file mode 100644 index 0000000000000..237c8ce181774 --- /dev/null +++ b/lldb/test/API/functionalities/progress_reporting/clang_modules/main.m @@ -0,0 +1 @@ +int main() {} diff --git a/lldb/test/API/functionalities/progress_reporting/clang_modules/module.modulemap b/lldb/test/API/functionalities/progress_reporting/clang_modules/module.modulemap new file mode 100644 index 0000000000000..88caf2cb678ff --- /dev/null +++ b/lldb/test/API/functionalities/progress_reporting/clang_modules/module.modulemap @@ -0,0 +1,4 @@ +module MyModule { + header "MyModule.h" + export * +} From 24b40874ec38efa55934036fa0896f4fb240dde4 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Fri, 10 Feb 2023 11:55:02 -0800 Subject: [PATCH 45/88] [lldb] Add an SB API to get progress events as SBStructuredData This is a preparatory patch to add an SB API to get the progress data as SBStructuredData. The advantage of using SBStructuredData is that the dictionary can grow over time with more fields. This approach is identical to the way this is implemented for diagnostic events. Differential revision: https://reviews.llvm.org/D143687 (cherry picked from commit 0ac8dfd0587a1a95e8ed464bc59741837aae9c1f) --- lldb/bindings/interface/SBDebugger.i | 2 ++ .../python/static-binding/LLDBWrapPython.cpp | 31 +++++++++++++++++++ lldb/bindings/python/static-binding/lldb.py | 5 +++ lldb/include/lldb/API/SBDebugger.h | 3 ++ lldb/include/lldb/Core/DebuggerEvents.h | 4 +++ lldb/source/API/SBDebugger.cpp | 15 +++++++++ lldb/source/Core/DebuggerEvents.cpp | 19 ++++++++++++ .../TestProgressReporting.py | 11 +++++++ 8 files changed, 90 insertions(+) diff --git a/lldb/bindings/interface/SBDebugger.i b/lldb/bindings/interface/SBDebugger.i index 07f807b739499..28094081e555e 100644 --- a/lldb/bindings/interface/SBDebugger.i +++ b/lldb/bindings/interface/SBDebugger.i @@ -131,6 +131,8 @@ public: uint64_t &OUTPUT, bool &OUTPUT); + static lldb::SBStructuredData GetProgressDataFromEvent(const lldb::SBEvent &event); + static lldb::SBStructuredData GetDiagnosticFromEvent(const lldb::SBEvent &event); SBBroadcaster GetBroadcaster(); diff --git a/lldb/bindings/python/static-binding/LLDBWrapPython.cpp b/lldb/bindings/python/static-binding/LLDBWrapPython.cpp index b67527430e79c..fc418ca3518ee 100644 --- a/lldb/bindings/python/static-binding/LLDBWrapPython.cpp +++ b/lldb/bindings/python/static-binding/LLDBWrapPython.cpp @@ -21677,6 +21677,36 @@ SWIGINTERN PyObject *_wrap_SBDebugger_GetProgressFromEvent(PyObject *self, PyObj } +SWIGINTERN PyObject *_wrap_SBDebugger_GetProgressDataFromEvent(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + lldb::SBEvent *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + lldb::SBStructuredData result; + + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1, SWIGTYPE_p_lldb__SBEvent, 0 | 0); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SBDebugger_GetProgressDataFromEvent" "', argument " "1"" of type '" "lldb::SBEvent const &""'"); + } + if (!argp1) { + SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "SBDebugger_GetProgressDataFromEvent" "', argument " "1"" of type '" "lldb::SBEvent const &""'"); + } + arg1 = reinterpret_cast< lldb::SBEvent * >(argp1); + { + SWIG_PYTHON_THREAD_BEGIN_ALLOW; + result = lldb::SBDebugger::GetProgressDataFromEvent((lldb::SBEvent const &)*arg1); + SWIG_PYTHON_THREAD_END_ALLOW; + } + resultobj = SWIG_NewPointerObj((new lldb::SBStructuredData(result)), SWIGTYPE_p_lldb__SBStructuredData, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_SBDebugger_GetDiagnosticFromEvent(PyObject *self, PyObject *args) { PyObject *resultobj = 0; lldb::SBEvent *arg1 = 0 ; @@ -84702,6 +84732,7 @@ static PyMethodDef SwigMethods[] = { { "SBData_swigregister", SBData_swigregister, METH_O, NULL}, { "SBData_swiginit", SBData_swiginit, METH_VARARGS, NULL}, { "SBDebugger_GetProgressFromEvent", _wrap_SBDebugger_GetProgressFromEvent, METH_O, "SBDebugger_GetProgressFromEvent(SBEvent event) -> char const *"}, + { "SBDebugger_GetProgressDataFromEvent", _wrap_SBDebugger_GetProgressDataFromEvent, METH_O, "SBDebugger_GetProgressDataFromEvent(SBEvent event) -> SBStructuredData"}, { "SBDebugger_GetDiagnosticFromEvent", _wrap_SBDebugger_GetDiagnosticFromEvent, METH_O, "SBDebugger_GetDiagnosticFromEvent(SBEvent event) -> SBStructuredData"}, { "SBDebugger_GetBroadcaster", _wrap_SBDebugger_GetBroadcaster, METH_O, "SBDebugger_GetBroadcaster(SBDebugger self) -> SBBroadcaster"}, { "SBDebugger_Initialize", _wrap_SBDebugger_Initialize, METH_NOARGS, "SBDebugger_Initialize()"}, diff --git a/lldb/bindings/python/static-binding/lldb.py b/lldb/bindings/python/static-binding/lldb.py index 2c489165fc293..077c8b4cf31df 100644 --- a/lldb/bindings/python/static-binding/lldb.py +++ b/lldb/bindings/python/static-binding/lldb.py @@ -4191,6 +4191,11 @@ def GetProgressFromEvent(event): r"""GetProgressFromEvent(SBEvent event) -> char const *""" return _lldb.SBDebugger_GetProgressFromEvent(event) + @staticmethod + def GetProgressDataFromEvent(event): + r"""GetProgressDataFromEvent(SBEvent event) -> SBStructuredData""" + return _lldb.SBDebugger_GetProgressDataFromEvent(event) + @staticmethod def GetDiagnosticFromEvent(event): r"""GetDiagnosticFromEvent(SBEvent event) -> SBStructuredData""" diff --git a/lldb/include/lldb/API/SBDebugger.h b/lldb/include/lldb/API/SBDebugger.h index a246057626869..35970b011023a 100644 --- a/lldb/include/lldb/API/SBDebugger.h +++ b/lldb/include/lldb/API/SBDebugger.h @@ -83,6 +83,9 @@ class LLDB_API SBDebugger { uint64_t &completed, uint64_t &total, bool &is_debugger_specific); + static lldb::SBStructuredData + GetProgressDataFromEvent(const lldb::SBEvent &event); + static lldb::SBStructuredData GetDiagnosticFromEvent(const lldb::SBEvent &event); diff --git a/lldb/include/lldb/Core/DebuggerEvents.h b/lldb/include/lldb/Core/DebuggerEvents.h index e1203dcf38c8c..8be0b3fc435d4 100644 --- a/lldb/include/lldb/Core/DebuggerEvents.h +++ b/lldb/include/lldb/Core/DebuggerEvents.h @@ -32,6 +32,10 @@ class ProgressEventData : public EventData { void Dump(Stream *s) const override; static const ProgressEventData *GetEventDataFromEvent(const Event *event_ptr); + + static StructuredData::DictionarySP + GetAsStructuredData(const Event *event_ptr); + uint64_t GetID() const { return m_id; } bool IsFinite() const { return m_total != UINT64_MAX; } uint64_t GetCompleted() const { return m_completed; } diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp index cc186862f24ad..58115ea73a828 100644 --- a/lldb/source/API/SBDebugger.cpp +++ b/lldb/source/API/SBDebugger.cpp @@ -167,6 +167,21 @@ const char *SBDebugger::GetProgressFromEvent(const lldb::SBEvent &event, return progress_data->GetMessage().c_str(); } +lldb::SBStructuredData +SBDebugger::GetProgressDataFromEvent(const lldb::SBEvent &event) { + LLDB_INSTRUMENT_VA(event); + + StructuredData::DictionarySP dictionary_sp = + ProgressEventData::GetAsStructuredData(event.get()); + + if (!dictionary_sp) + return {}; + + SBStructuredData data; + data.m_impl_up->SetObjectSP(std::move(dictionary_sp)); + return data; +} + lldb::SBStructuredData SBDebugger::GetDiagnosticFromEvent(const lldb::SBEvent &event) { LLDB_INSTRUMENT_VA(event); diff --git a/lldb/source/Core/DebuggerEvents.cpp b/lldb/source/Core/DebuggerEvents.cpp index 0cc9ee9ad96f5..a52a4b26dc40f 100644 --- a/lldb/source/Core/DebuggerEvents.cpp +++ b/lldb/source/Core/DebuggerEvents.cpp @@ -49,6 +49,25 @@ ProgressEventData::GetEventDataFromEvent(const Event *event_ptr) { return GetEventDataFromEventImpl(event_ptr); } +StructuredData::DictionarySP +ProgressEventData::GetAsStructuredData(const Event *event_ptr) { + const ProgressEventData *progress_data = + ProgressEventData::GetEventDataFromEvent(event_ptr); + + if (!progress_data) + return {}; + + auto dictionary_sp = std::make_shared(); + dictionary_sp->AddStringItem("message", progress_data->GetMessage()); + dictionary_sp->AddIntegerItem("progress_id", progress_data->GetID()); + dictionary_sp->AddIntegerItem("completed", progress_data->GetCompleted()); + dictionary_sp->AddIntegerItem("total", progress_data->GetTotal()); + dictionary_sp->AddBooleanItem("debugger_specific", + progress_data->IsDebuggerSpecific()); + + return dictionary_sp; +} + llvm::StringRef DiagnosticEventData::GetPrefix() const { switch (m_type) { case Type::Warning: diff --git a/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py b/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py index 164ccbf72bf8e..8d1821e4e92d3 100644 --- a/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py +++ b/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py @@ -28,3 +28,14 @@ def test_dwarf_symbol_loading_progress_report(self): message = ret_args[0] self.assertGreater(len(message), 0) + def test_dwarf_symbol_loading_progress_report_structured_data(self): + """Test that we are able to fetch dwarf symbol loading progress events + using the structured data API""" + self.build() + + lldbutil.run_to_source_breakpoint(self, 'break here', lldb.SBFileSpec('main.c')) + + event = lldbutil.fetch_next_event(self, self.listener, self.broadcaster) + progress_data = lldb.SBDebugger.GetProgressDataFromEvent(event) + message = progress_data.GetValueForKey("message").GetStringValue(100) + self.assertGreater(len(message), 0) From 11a2593e13b79df3d339dfc64f5be9ce96c7779a Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Sun, 12 Feb 2023 10:55:39 -0800 Subject: [PATCH 46/88] [lldb] Add the ability to provide a message to a progress event update Consider the following example as motivation. Say you have to load symbols for 3 dynamic libraries: `libFoo`, `libBar` and `libBaz`. Currently, there are two ways to report process for this operation: 1. As 3 separate progress instances. In this case you create a progress instance with the message "Loading symbols: libFoo", "Loading symbols: libBar", and "Loading symbols: libBaz" respectively. Each progress event gets a unique ID and therefore cannot be correlated by the consumer. 2. As 1 progress instance with 3 units of work. The title would be "Loading symbols" and you call Progress::Increment for each of the libraries. The 3 progress events share the same ID and can easily be correlated, however, in the current design, there's no way to include the name of the libraries. The second approach is preferred when the amount of work is known in advance, because determinate progress can be reported (i.e. x out of y operations completed). An additional benefit is that the progress consumer can decide to ignore certain progress updates by their ID if they are deemed to noisy, which isn't trivial for the first approach due to the use of different progress IDs. This patch adds the ability to add a message (detail) to a progress event update. For the example described above, progress can now be displayed as shown: [1/3] Loading symbols: libFoo [2/3] Loading symbols: libBar [3/3] Loading symbols: libBaz Differential revision: https://reviews.llvm.org/D143690 (cherry picked from commit 129eb5bcab91a12ed3c4712279f201834ae2d8e1) --- lldb/include/lldb/Core/Debugger.h | 5 +++-- lldb/include/lldb/Core/DebuggerEvents.h | 29 ++++++++++++++++++++----- lldb/include/lldb/Core/Progress.h | 6 +++-- lldb/source/API/SBDebugger.cpp | 4 +++- lldb/source/Core/Debugger.cpp | 21 ++++++++++-------- lldb/source/Core/DebuggerEvents.cpp | 6 ++++- lldb/source/Core/Progress.cpp | 10 ++++----- 7 files changed, 56 insertions(+), 25 deletions(-) diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h index 05bd2445e2fbf..42b9f27014f3d 100644 --- a/lldb/include/lldb/Core/Debugger.h +++ b/lldb/include/lldb/Core/Debugger.h @@ -514,8 +514,9 @@ class Debugger : public std::enable_shared_from_this, /// debugger identifier that this progress should be delivered to. If this /// optional parameter does not have a value, the progress will be /// delivered to all debuggers. - static void ReportProgress(uint64_t progress_id, const std::string &message, - uint64_t completed, uint64_t total, + static void ReportProgress(uint64_t progress_id, std::string title, + std::string details, uint64_t completed, + uint64_t total, llvm::Optional debugger_id); static void ReportDiagnosticImpl(DiagnosticEventData::Type type, diff --git a/lldb/include/lldb/Core/DebuggerEvents.h b/lldb/include/lldb/Core/DebuggerEvents.h index 8be0b3fc435d4..5240d1b0c1dfb 100644 --- a/lldb/include/lldb/Core/DebuggerEvents.h +++ b/lldb/include/lldb/Core/DebuggerEvents.h @@ -20,10 +20,11 @@ class Stream; class ProgressEventData : public EventData { public: - ProgressEventData(uint64_t progress_id, const std::string &message, + ProgressEventData(uint64_t progress_id, std::string title, std::string update, uint64_t completed, uint64_t total, bool debugger_specific) - : m_message(message), m_id(progress_id), m_completed(completed), - m_total(total), m_debugger_specific(debugger_specific) {} + : m_title(std::move(title)), m_details(std::move(update)), + m_id(progress_id), m_completed(completed), m_total(total), + m_debugger_specific(debugger_specific) {} static ConstString GetFlavorString(); @@ -40,12 +41,30 @@ class ProgressEventData : public EventData { bool IsFinite() const { return m_total != UINT64_MAX; } uint64_t GetCompleted() const { return m_completed; } uint64_t GetTotal() const { return m_total; } - const std::string &GetMessage() const { return m_message; } + std::string GetMessage() const { + std::string message = m_title; + if (!m_details.empty()) { + message.append(": "); + message.append(m_details); + } + return message; + } + const std::string &GetTitle() const { return m_title; } + const std::string &GetDetails() const { return m_details; } bool IsDebuggerSpecific() const { return m_debugger_specific; } private: - std::string m_message; + /// The title of this progress event. The value is expected to remain stable + /// for a given progress ID. + std::string m_title; + + /// Details associated with this progress event update. The value is expected + /// to change between progress events. + std::string m_details; + + /// Unique ID used to associate progress events. const uint64_t m_id; + uint64_t m_completed; const uint64_t m_total; const bool m_debugger_specific; diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h index f625d014f2684..3cce73564c489 100644 --- a/lldb/include/lldb/Core/Progress.h +++ b/lldb/include/lldb/Core/Progress.h @@ -86,10 +86,12 @@ class Progress { /// anything nor send any progress updates. /// /// @param [in] amount The amount to increment m_completed by. - void Increment(uint64_t amount = 1); + /// + /// @param [in] an optional message associated with this update. + void Increment(uint64_t amount = 1, std::string update = {}); private: - void ReportProgress(); + void ReportProgress(std::string update = {}); static std::atomic g_id; /// The title of the progress activity. std::string m_title; diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp index 58115ea73a828..a7080e081432d 100644 --- a/lldb/source/API/SBDebugger.cpp +++ b/lldb/source/API/SBDebugger.cpp @@ -156,6 +156,7 @@ const char *SBDebugger::GetProgressFromEvent(const lldb::SBEvent &event, uint64_t &total, bool &is_debugger_specific) { LLDB_INSTRUMENT_VA(event); + const ProgressEventData *progress_data = ProgressEventData::GetEventDataFromEvent(event.get()); if (progress_data == nullptr) @@ -164,7 +165,8 @@ const char *SBDebugger::GetProgressFromEvent(const lldb::SBEvent &event, completed = progress_data->GetCompleted(); total = progress_data->GetTotal(); is_debugger_specific = progress_data->IsDebuggerSpecific(); - return progress_data->GetMessage().c_str(); + ConstString message(progress_data->GetMessage()); + return message.AsCString(); } lldb::SBStructuredData diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp index 6def870d9c75f..e8be96171515d 100644 --- a/lldb/source/Core/Debugger.cpp +++ b/lldb/source/Core/Debugger.cpp @@ -1347,7 +1347,7 @@ void Debugger::SetLoggingCallback(lldb::LogOutputCallback log_callback, } static void PrivateReportProgress(Debugger &debugger, uint64_t progress_id, - const std::string &message, + std::string title, std::string details, uint64_t completed, uint64_t total, bool is_debugger_specific) { // Only deliver progress events if we have any progress listeners. @@ -1355,13 +1355,15 @@ static void PrivateReportProgress(Debugger &debugger, uint64_t progress_id, if (!debugger.GetBroadcaster().EventTypeHasListeners(event_type)) return; EventSP event_sp(new Event( - event_type, new ProgressEventData(progress_id, message, completed, total, - is_debugger_specific))); + event_type, + new ProgressEventData(progress_id, std::move(title), std::move(details), + completed, total, is_debugger_specific))); debugger.GetBroadcaster().BroadcastEvent(event_sp); } -void Debugger::ReportProgress(uint64_t progress_id, const std::string &message, - uint64_t completed, uint64_t total, +void Debugger::ReportProgress(uint64_t progress_id, std::string title, + std::string details, uint64_t completed, + uint64_t total, llvm::Optional debugger_id) { // Check if this progress is for a specific debugger. if (debugger_id) { @@ -1369,8 +1371,9 @@ void Debugger::ReportProgress(uint64_t progress_id, const std::string &message, // still exists. DebuggerSP debugger_sp = FindDebuggerWithID(*debugger_id); if (debugger_sp) - PrivateReportProgress(*debugger_sp, progress_id, message, completed, - total, /*is_debugger_specific*/ true); + PrivateReportProgress(*debugger_sp, progress_id, std::move(title), + std::move(details), completed, total, + /*is_debugger_specific*/ true); return; } // The progress event is not debugger specific, iterate over all debuggers @@ -1379,8 +1382,8 @@ void Debugger::ReportProgress(uint64_t progress_id, const std::string &message, std::lock_guard guard(*g_debugger_list_mutex_ptr); DebuggerList::iterator pos, end = g_debugger_list_ptr->end(); for (pos = g_debugger_list_ptr->begin(); pos != end; ++pos) - PrivateReportProgress(*(*pos), progress_id, message, completed, total, - /*is_debugger_specific*/ false); + PrivateReportProgress(*(*pos), progress_id, title, details, completed, + total, /*is_debugger_specific*/ false); } } diff --git a/lldb/source/Core/DebuggerEvents.cpp b/lldb/source/Core/DebuggerEvents.cpp index a52a4b26dc40f..3af2e31b83851 100644 --- a/lldb/source/Core/DebuggerEvents.cpp +++ b/lldb/source/Core/DebuggerEvents.cpp @@ -33,7 +33,9 @@ ConstString ProgressEventData::GetFlavor() const { } void ProgressEventData::Dump(Stream *s) const { - s->Printf(" id = %" PRIu64 ", message = \"%s\"", m_id, m_message.c_str()); + s->Printf(" id = %" PRIu64 ", title = \"%s\"", m_id, m_title.c_str()); + if (!m_details.empty()) + s->Printf(", details = \"%s\"", m_details.c_str()); if (m_completed == 0 || m_completed == m_total) s->Printf(", type = %s", m_completed == 0 ? "start" : "end"); else @@ -58,6 +60,8 @@ ProgressEventData::GetAsStructuredData(const Event *event_ptr) { return {}; auto dictionary_sp = std::make_shared(); + dictionary_sp->AddStringItem("title", progress_data->GetTitle()); + dictionary_sp->AddStringItem("details", progress_data->GetDetails()); dictionary_sp->AddStringItem("message", progress_data->GetMessage()); dictionary_sp->AddIntegerItem("progress_id", progress_data->GetID()); dictionary_sp->AddIntegerItem("completed", progress_data->GetCompleted()); diff --git a/lldb/source/Core/Progress.cpp b/lldb/source/Core/Progress.cpp index c54e7774adf3a..08be73f1470f3 100644 --- a/lldb/source/Core/Progress.cpp +++ b/lldb/source/Core/Progress.cpp @@ -36,7 +36,7 @@ Progress::~Progress() { } } -void Progress::Increment(uint64_t amount) { +void Progress::Increment(uint64_t amount, std::string update) { if (amount > 0) { std::lock_guard guard(m_mutex); // Watch out for unsigned overflow and make sure we don't increment too @@ -45,16 +45,16 @@ void Progress::Increment(uint64_t amount) { m_completed = m_total; else m_completed += amount; - ReportProgress(); + ReportProgress(update); } } -void Progress::ReportProgress() { +void Progress::ReportProgress(std::string update) { if (!m_complete) { // Make sure we only send one notification that indicates the progress is // complete. m_complete = m_completed == m_total; - Debugger::ReportProgress(m_id, m_title, m_completed, m_total, - m_debugger_id); + Debugger::ReportProgress(m_id, m_title, std::move(update), m_completed, + m_total, m_debugger_id); } } From 95525bab4a96f244a446ee83481546cc1f51198b Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 22 Mar 2023 18:38:07 -0700 Subject: [PATCH 47/88] [lldb] Use one Progress event per root module build Following the work done by @jdevlieghere in D143690, this changes how Clang module build events are emitted. Instead of one Progress event per module being built, a single Progress event is used to encompass all modules, and each module build is sent as an `Increment` update. Differential Revision: https://reviews.llvm.org/D147248 (cherry picked from commit 7edff3c1b298f696c632625fa863acbc7d68d446) --- .../Clang/ClangModulesDeclVendor.cpp | 20 +++++++++---------- .../TestClangModuleBuildProgress.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp index ed64f112ed742..d5317a4408c53 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp @@ -68,7 +68,7 @@ class StoringDiagnosticConsumer : public clang::DiagnosticConsumer { private: bool HandleModuleRemark(const clang::Diagnostic &info); - void SetCurrentModuleProgress(llvm::StringRef module_name); + void SetCurrentModuleProgress(std::string module_name); typedef std::pair IDAndDiagnostic; @@ -208,8 +208,9 @@ bool StoringDiagnosticConsumer::HandleModuleRemark( if (m_module_build_stack.empty()) { m_current_progress_up = nullptr; } else { - // Update the progress to re-show the module that was currently being - // built from the time the now completed module was originally began. + // When the just completed module began building, a module that depends on + // it ("module A") was effectively paused. Update the progress to re-show + // "module A" as continuing to be built. const auto &resumed_module_name = m_module_build_stack.back(); SetCurrentModuleProgress(resumed_module_name); } @@ -224,13 +225,12 @@ bool StoringDiagnosticConsumer::HandleModuleRemark( } void StoringDiagnosticConsumer::SetCurrentModuleProgress( - llvm::StringRef module_name) { - // Ensure the ordering of: - // 1. Completing the existing progress event. - // 2. Beginining a new progress event. - m_current_progress_up = nullptr; - m_current_progress_up = std::make_unique( - llvm::formatv("Currently building module {0}", module_name)); + std::string module_name) { + if (!m_current_progress_up) + m_current_progress_up = + std::make_unique("Building Clang modules"); + + m_current_progress_up->Increment(1, std::move(module_name)); } ClangModulesDeclVendor::ClangModulesDeclVendor() diff --git a/lldb/test/API/functionalities/progress_reporting/clang_modules/TestClangModuleBuildProgress.py b/lldb/test/API/functionalities/progress_reporting/clang_modules/TestClangModuleBuildProgress.py index af0b59cd555f1..228f676aedf6a 100644 --- a/lldb/test/API/functionalities/progress_reporting/clang_modules/TestClangModuleBuildProgress.py +++ b/lldb/test/API/functionalities/progress_reporting/clang_modules/TestClangModuleBuildProgress.py @@ -43,4 +43,4 @@ def test_clang_module_build_progress_report(self): event = lldbutil.fetch_next_event(self, listener, broadcaster) payload = lldb.SBDebugger.GetProgressFromEvent(event) message = payload[0] - self.assertEqual(message, "Currently building module MyModule") + self.assertEqual(message, "Building Clang modules") From a5cde6afe234227122f0ea5a0acc4a69690e0f63 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Thu, 30 Mar 2023 17:54:57 -0700 Subject: [PATCH 48/88] [libclang/depscan] Fix use-after-free issue when using diagnostics of `clang_experimental_DepGraph_getDiagnostics` The `StoredDiagnostic`s captured from the `getFileDependencies()` call reference a `SourceManager` object that gets destroyed and is invalid to use for getting diagnostic location info. To address this, capture diagnostics as a serialized diagnostics buffer and "materialize" it for the `clang_experimental_DepGraph_getDiagnostics` call, by re-using existing libclang machinery for loading serialized diagnostics. Also delete `clang_experimental_DependencyScannerWorker_getFileDependencies_v5` since it's unsafe to use to get diagnostics and its functionality is superseeded by `clang_experimental_DependencyScannerWorker_getDepGraph`. rdar://105978877 (cherry picked from commit 397a30d9d20ed76388700605076da468d02dbf3b) --- clang/include/clang-c/Dependencies.h | 21 ++---- .../Frontend/SerializedDiagnosticReader.h | 3 + .../Frontend/SerializedDiagnosticPrinter.cpp | 3 + .../Frontend/SerializedDiagnosticReader.cpp | 7 +- clang/test/Index/Core/scan-deps-with-diags.m | 6 ++ clang/tools/c-index-test/core_main.cpp | 3 +- clang/tools/libclang/CDependencies.cpp | 70 +++++++------------ clang/tools/libclang/CXLoadedDiagnostic.cpp | 52 ++++++++++---- clang/tools/libclang/CXLoadedDiagnostic.h | 13 +++- clang/tools/libclang/libclang.map | 1 - 10 files changed, 99 insertions(+), 80 deletions(-) create mode 100644 clang/test/Index/Core/scan-deps-with-diags.m diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index ec04090365eec..955217fa3301a 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -310,18 +310,8 @@ typedef size_t CXModuleLookupOutputCallback(void *Context, char *Output, size_t MaxLen); /** - * See \c clang_experimental_DependencyScannerWorker_getFileDependencies_v5. - * Returns diagnostics in an unstructured CXString instead of CXDiagnosticSet. - */ -CINDEX_LINKAGE enum CXErrorCode -clang_experimental_DependencyScannerWorker_getFileDependencies_v4( - CXDependencyScannerWorker Worker, int argc, const char *const *argv, - const char *ModuleName, const char *WorkingDirectory, void *MDCContext, - CXModuleDiscoveredCallback *MDC, void *MLOContext, - CXModuleLookupOutputCallback *MLO, unsigned Options, - CXFileDependenciesList **Out, CXString *error); - -/** + * Deprecated, use \c clang_experimental_DependencyScannerWorker_getDepGraph. + * * Calculates the list of file dependencies for a particular compiler * invocation. * @@ -351,19 +341,18 @@ clang_experimental_DependencyScannerWorker_getFileDependencies_v4( * \param [out] Out A non-NULL pointer to store the resulting dependencies. The * output must be freed by calling * \c clang_experimental_FileDependenciesList_dispose. - * \param [out] OutDiags The diagnostics emitted during scanning. These must be - * always freed by calling \c clang_disposeDiagnosticSet. + * \param [out] error the error string to pass back to client (if any). * * \returns \c CXError_Success on success; otherwise a non-zero \c CXErrorCode * indicating the kind of error. */ CINDEX_LINKAGE enum CXErrorCode -clang_experimental_DependencyScannerWorker_getFileDependencies_v5( +clang_experimental_DependencyScannerWorker_getFileDependencies_v4( CXDependencyScannerWorker Worker, int argc, const char *const *argv, const char *ModuleName, const char *WorkingDirectory, void *MDCContext, CXModuleDiscoveredCallback *MDC, void *MLOContext, CXModuleLookupOutputCallback *MLO, unsigned Options, - CXFileDependenciesList **Out, CXDiagnosticSet *OutDiags); + CXFileDependenciesList **Out, CXString *error); /** * Output of \c clang_experimental_DependencyScannerWorker_getDepGraph. diff --git a/clang/include/clang/Frontend/SerializedDiagnosticReader.h b/clang/include/clang/Frontend/SerializedDiagnosticReader.h index be7d3a10a1705..1d95fd23c774d 100644 --- a/clang/include/clang/Frontend/SerializedDiagnosticReader.h +++ b/clang/include/clang/Frontend/SerializedDiagnosticReader.h @@ -65,6 +65,9 @@ class SerializedDiagnosticReader { /// Read the diagnostics in \c File std::error_code readDiagnostics(StringRef File); + /// Read the diagnostics in \c Buffer. + std::error_code readDiagnostics(llvm::MemoryBufferRef Buffer); + private: enum class Cursor; diff --git a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp index 8d5039935aa7f..a75ad2bbe066d 100644 --- a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp +++ b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp @@ -608,6 +608,9 @@ void SDiagsWriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel, return; } + // Call base class to update diagnostic counts. + DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info); + // Enter the block for a non-note diagnostic immediately, rather than waiting // for beginDiagnostic, in case associated notes are emitted before we get // there. diff --git a/clang/lib/Frontend/SerializedDiagnosticReader.cpp b/clang/lib/Frontend/SerializedDiagnosticReader.cpp index aca3f77c2de3e..984d039fe7820 100644 --- a/clang/lib/Frontend/SerializedDiagnosticReader.cpp +++ b/clang/lib/Frontend/SerializedDiagnosticReader.cpp @@ -34,7 +34,12 @@ std::error_code SerializedDiagnosticReader::readDiagnostics(StringRef File) { if (!Buffer) return SDError::CouldNotLoad; - llvm::BitstreamCursor Stream(**Buffer); + return readDiagnostics(**Buffer); +} + +std::error_code +SerializedDiagnosticReader::readDiagnostics(llvm::MemoryBufferRef Buffer) { + llvm::BitstreamCursor Stream(Buffer); Optional BlockInfo; if (Stream.AtEndOfStream()) diff --git a/clang/test/Index/Core/scan-deps-with-diags.m b/clang/test/Index/Core/scan-deps-with-diags.m new file mode 100644 index 0000000000000..a47d3646385ff --- /dev/null +++ b/clang/test/Index/Core/scan-deps-with-diags.m @@ -0,0 +1,6 @@ +// RUN: not c-index-test core --scan-deps %S -output-dir=%t -- \ +// RUN: %clang -c %s -o %t/t.o 2> %t.err.txt +// RUN: FileCheck -input-file=%t.err.txt %s + +// CHECK: [[@LINE+1]]:10: fatal error: 'not-existent.h' file not found +#include "not-existent.h" diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index e5512c9d33678..a7cade1af6ec2 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -846,7 +846,8 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, llvm::make_scope_exit([&]() { clang_disposeDiagnosticSet(Diags); }); for (unsigned I = 0, N = clang_getNumDiagnosticsInSet(Diags); I < N; ++I) { CXDiagnostic Diag = clang_getDiagnosticInSet(Diags, I); - CXString Spelling = clang_getDiagnosticSpelling(Diag); + CXString Spelling = + clang_formatDiagnostic(Diag, clang_defaultDiagnosticDisplayOptions()); llvm::errs() << clang_getCString(Spelling) << "\n"; clang_disposeString(Spelling); clang_disposeDiagnostic(Diag); diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index f8c052dab6688..925901bb09ed5 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -13,11 +13,13 @@ #include "CASUtils.h" #include "CXDiagnosticSetDiagnosticConsumer.h" +#include "CXLoadedDiagnostic.h" #include "CXString.h" #include "clang-c/Dependencies.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/SerializedDiagnosticPrinter.h" #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" @@ -342,49 +344,6 @@ CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v4( return Result; } -CXErrorCode clang_experimental_DependencyScannerWorker_getFileDependencies_v5( - CXDependencyScannerWorker W, int argc, const char *const *argv, - const char *ModuleName, const char *WorkingDirectory, void *MDCContext, - CXModuleDiscoveredCallback *MDC, void *MLOContext, - CXModuleLookupOutputCallback *MLO, unsigned, CXFileDependenciesList **Out, - CXDiagnosticSet *OutDiags) { - OutputLookup OL(MLOContext, MLO); - auto LookupOutputs = [&](const ModuleID &ID, ModuleOutputKind MOK) { - return OL.lookupModuleOutput(ID, MOK); - }; - - if (!Out) - return CXError_InvalidArguments; - *Out = nullptr; - - CXDiagnosticSetDiagnosticConsumer DiagConsumer; - - CXErrorCode Result = getFileDependencies( - W, argc, argv, WorkingDirectory, MDC, MDCContext, nullptr, &DiagConsumer, - LookupOutputs, ModuleName ? Optional(ModuleName) : None, - [&](TranslationUnitDeps TU) { - assert(TU.DriverCommandLine.empty()); - std::vector Modules; - for (const ModuleID &MID : TU.ClangModuleDeps) - Modules.push_back(MID.ModuleName + ":" + MID.ContextHash); - auto *Commands = new CXTranslationUnitCommand[TU.Commands.size()]; - for (size_t I = 0, E = TU.Commands.size(); I < E; ++I) { - Commands[I].ContextHash = cxstring::createDup(TU.ID.ContextHash); - Commands[I].FileDeps = cxstring::createSet(TU.FileDeps); - Commands[I].ModuleDeps = cxstring::createSet(Modules); - Commands[I].Executable = - cxstring::createDup(TU.Commands[I].Executable); - Commands[I].BuildArguments = - cxstring::createSet(TU.Commands[I].Arguments); - } - *Out = new CXFileDependenciesList{TU.Commands.size(), Commands}; - }); - - *OutDiags = DiagConsumer.getDiagnosticSet(); - - return Result; -} - namespace { struct DependencyScannerWorkerScanSettings { @@ -438,8 +397,18 @@ struct CStringsManager { struct DependencyGraph { TranslationUnitDeps TUDeps; - CXDiagnosticSetDiagnosticConsumer DiagConsumer; + SmallString<256> SerialDiagBuf; CStringsManager StrMgr{}; + + CXDiagnosticSet getDiagnosticSet() const { + CXLoadDiag_Error Error; + CXString ErrorString; + CXDiagnosticSet DiagSet = loadCXDiagnosticsFromBuffer( + llvm::MemoryBufferRef(SerialDiagBuf, ""), &Error, &ErrorString); + assert(Error == CXLoadDiag_None); + clang_disposeString(ErrorString); + return DiagSet; + } }; struct DependencyGraphModule { @@ -497,9 +466,18 @@ enum CXErrorCode clang_experimental_DependencyScannerWorker_getDepGraph( DependencyGraph *DepGraph = new DependencyGraph(); + // We capture diagnostics as a serialized diagnostics buffer, so that we don't + // need to keep a valid SourceManager in order to access diagnostic locations. + auto DiagOpts = llvm::makeIntrusiveRefCnt(); + auto DiagOS = + std::make_unique(DepGraph->SerialDiagBuf); + std::unique_ptr SerialDiagConsumer = + serialized_diags::create("", DiagOpts.get(), + /*MergeChildRecords=*/false, std::move(DiagOS)); + CXErrorCode Result = getFileDependencies( W, argc, argv, WorkingDirectory, /*MDC=*/nullptr, /*MDCContext=*/nullptr, - /*Error=*/nullptr, &DepGraph->DiagConsumer, LookupOutputs, + /*Error=*/nullptr, SerialDiagConsumer.get(), LookupOutputs, ModuleName ? Optional(ModuleName) : std::nullopt, [&](TranslationUnitDeps TU) { DepGraph->TUDeps = std::move(TU); }); @@ -615,7 +593,7 @@ const char *clang_experimental_DepGraph_getTUContextHash(CXDepGraph Graph) { } CXDiagnosticSet clang_experimental_DepGraph_getDiagnostics(CXDepGraph Graph) { - return unwrap(Graph)->DiagConsumer.getDiagnosticSet(); + return unwrap(Graph)->getDiagnosticSet(); } static std::string lookupModuleOutput(const ModuleID &ID, ModuleOutputKind MOK, diff --git a/clang/tools/libclang/CXLoadedDiagnostic.cpp b/clang/tools/libclang/CXLoadedDiagnostic.cpp index 3d2b3047f47b5..6481109a17824 100644 --- a/clang/tools/libclang/CXLoadedDiagnostic.cpp +++ b/clang/tools/libclang/CXLoadedDiagnostic.cpp @@ -249,6 +249,9 @@ class DiagLoader : serialized_diags::SerializedDiagnosticReader { } CXDiagnosticSet load(const char *file); + CXDiagnosticSet load(llvm::MemoryBufferRef Buffer); + + CXDiagnosticSet reportError(std::error_code EC); }; } // end anonymous namespace @@ -256,24 +259,38 @@ CXDiagnosticSet DiagLoader::load(const char *file) { TopDiags = std::make_unique(); std::error_code EC = readDiagnostics(file); - if (EC) { - switch (EC.value()) { - case static_cast(serialized_diags::SDError::HandlerFailed): - // We've already reported the problem. - break; - case static_cast(serialized_diags::SDError::CouldNotLoad): - reportBad(CXLoadDiag_CannotLoad, EC.message()); - break; - default: - reportInvalidFile(EC.message()); - break; - } - return nullptr; - } + if (EC) + return reportError(EC); + + return (CXDiagnosticSet)TopDiags.release(); +} + +CXDiagnosticSet DiagLoader::load(llvm::MemoryBufferRef Buffer) { + TopDiags = std::make_unique(); + + std::error_code EC = readDiagnostics(Buffer); + if (EC) + return reportError(EC); return (CXDiagnosticSet)TopDiags.release(); } +CXDiagnosticSet DiagLoader::reportError(std::error_code EC) { + assert(EC); + switch (EC.value()) { + case static_cast(serialized_diags::SDError::HandlerFailed): + // We've already reported the problem. + break; + case static_cast(serialized_diags::SDError::CouldNotLoad): + reportBad(CXLoadDiag_CannotLoad, EC.message()); + break; + default: + reportInvalidFile(EC.message()); + break; + } + return nullptr; +} + std::error_code DiagLoader::readLocation(const serialized_diags::Location &SDLoc, CXLoadedDiagnostic::Location &LoadedLoc) { @@ -420,3 +437,10 @@ CXDiagnosticSet clang_loadDiagnostics(const char *file, DiagLoader L(error, errorString); return L.load(file); } + +CXDiagnosticSet clang::loadCXDiagnosticsFromBuffer(llvm::MemoryBufferRef buffer, + enum CXLoadDiag_Error *error, + CXString *errorString) { + DiagLoader L(error, errorString); + return L.load(buffer); +} diff --git a/clang/tools/libclang/CXLoadedDiagnostic.h b/clang/tools/libclang/CXLoadedDiagnostic.h index 93995d7bb798c..eb76dfbfe11b5 100644 --- a/clang/tools/libclang/CXLoadedDiagnostic.h +++ b/clang/tools/libclang/CXLoadedDiagnostic.h @@ -19,6 +19,10 @@ #include "clang/Basic/LLVM.h" #include +namespace llvm { +class MemoryBufferRef; +} + namespace clang { class CXLoadedDiagnostic : public CXDiagnosticImpl { public: @@ -88,6 +92,13 @@ class CXLoadedDiagnostic : public CXDiagnosticImpl { unsigned severity; unsigned category; }; -} + +/// Read a serialized diagnostics \p buffer and create a \c CXDiagnosticSet +/// object for the loaded diagnostics. +CXDiagnosticSet loadCXDiagnosticsFromBuffer(llvm::MemoryBufferRef buffer, + enum CXLoadDiag_Error *error, + CXString *errorString); + +} // namespace clang #endif diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index e9b5ffbc610e4..00ded6d77e700 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -490,7 +490,6 @@ LLVM_16 { clang_experimental_DependencyScannerServiceOptions_setDependencyMode; clang_experimental_DependencyScannerServiceOptions_setObjectStore; clang_experimental_DependencyScannerWorker_getDepGraph; - clang_experimental_DependencyScannerWorker_getFileDependencies_v5; clang_experimental_DependencyScannerWorkerScanSettings_create; clang_experimental_DependencyScannerWorkerScanSettings_dispose; clang_experimental_DepGraph_dispose; From 2af4818d8de2fd1ae2d8c274470b8c0cab2e26ca Mon Sep 17 00:00:00 2001 From: Augusto Noronha Date: Thu, 30 Mar 2023 19:04:03 -0700 Subject: [PATCH 49/88] [lldb] Add support for the DW_AT_trampoline attribute with a boolean This patch adds support for the DW_AT_trampoline attribute whose value is a boolean. Which is a "generic trampoline". Stepping into a generic trampoline by default will step through the function, checking at every branch, until we stop in a function which makes sense to stop at (a function with debug info, which isn't a trampoline, for example). Differential Revision: https://reviews.llvm.org/D147292 (cherry picked from commit 4a8e54505077cca76e7eb0776b2d0d807abfc883) --- lldb/include/lldb/Symbol/Function.h | 8 +- lldb/include/lldb/Target/Target.h | 6 + lldb/include/lldb/Target/Thread.h | 21 +++ lldb/include/lldb/Target/ThreadPlan.h | 1 + .../lldb/Target/ThreadPlanStepOverRange.h | 1 - .../include/lldb/Target/ThreadPlanStepRange.h | 1 + .../ThreadPlanStepThroughGenericTrampoline.h | 52 +++++++ lldb/source/Core/Module.cpp | 5 + .../SymbolFile/DWARF/DWARFASTParserClang.cpp | 6 +- .../SymbolFile/DWARF/DWARFASTParserSwift.cpp | 6 +- .../Plugins/SymbolFile/DWARF/DWARFDIE.cpp | 7 + .../Plugins/SymbolFile/DWARF/DWARFDIE.h | 2 + .../SymbolFile/DWARF/DWARFDebugInfoEntry.cpp | 6 + .../SymbolFile/DWARF/DWARFDebugInfoEntry.h | 2 + lldb/source/Symbol/Function.cpp | 7 +- lldb/source/Target/CMakeLists.txt | 1 + lldb/source/Target/Target.cpp | 6 + lldb/source/Target/TargetProperties.td | 3 + lldb/source/Target/Thread.cpp | 12 ++ .../Target/ThreadPlanShouldStopHere.cpp | 15 +- lldb/source/Target/ThreadPlanStepInRange.cpp | 11 ++ .../source/Target/ThreadPlanStepOverRange.cpp | 31 ---- lldb/source/Target/ThreadPlanStepRange.cpp | 32 ++++ lldb/source/Target/ThreadPlanStepThrough.cpp | 4 + ...ThreadPlanStepThroughGenericTrampoline.cpp | 137 ++++++++++++++++++ .../API/lang/c/trampoline_stepping/Makefile | 3 + .../TestTrampolineStepping.py | 104 +++++++++++++ .../API/lang/c/trampoline_stepping/main.c | 52 +++++++ 28 files changed, 499 insertions(+), 43 deletions(-) create mode 100644 lldb/include/lldb/Target/ThreadPlanStepThroughGenericTrampoline.h create mode 100644 lldb/source/Target/ThreadPlanStepThroughGenericTrampoline.cpp create mode 100644 lldb/test/API/lang/c/trampoline_stepping/Makefile create mode 100644 lldb/test/API/lang/c/trampoline_stepping/TestTrampolineStepping.py create mode 100644 lldb/test/API/lang/c/trampoline_stepping/main.c diff --git a/lldb/include/lldb/Symbol/Function.h b/lldb/include/lldb/Symbol/Function.h index fe2416aed783d..de9400e7af97c 100644 --- a/lldb/include/lldb/Symbol/Function.h +++ b/lldb/include/lldb/Symbol/Function.h @@ -442,7 +442,7 @@ class Function : public UserID, public SymbolContextScope { Function(CompileUnit *comp_unit, lldb::user_id_t func_uid, lldb::user_id_t func_type_uid, const Mangled &mangled, Type *func_type, const AddressRange &range, - bool can_throw = false); + bool can_throw = false, bool generic_trampoline = false); /// Destructor. ~Function() override; @@ -554,6 +554,10 @@ class Function : public UserID, public SymbolContextScope { /// A type object pointer. Type *GetType(); + bool IsGenericTrampoline() const { + return m_is_generic_trampoline; + } + /// Get const accessor for the type that describes the function return value /// type, and parameter types. /// @@ -659,6 +663,8 @@ class Function : public UserID, public SymbolContextScope { /// information. Mangled m_mangled; + bool m_is_generic_trampoline; + /// All lexical blocks contained in this function. Block m_block; diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h index 8936402da644f..5e1c6874eec6f 100644 --- a/lldb/include/lldb/Target/Target.h +++ b/lldb/include/lldb/Target/Target.h @@ -289,6 +289,12 @@ class TargetProperties : public Properties { bool GetDebugUtilityExpression() const; + /// Trampoline support includes stepping through trampolines directly to their + /// targets, stepping out of trampolines directly to their callers, and + /// automatically filtering out trampolines as possible breakpoint locations + /// when set by name. + bool GetEnableTrampolineSupport() const; + private: // Callbacks for m_launch_info. void Arg0ValueChangedCallback(); diff --git a/lldb/include/lldb/Target/Thread.h b/lldb/include/lldb/Target/Thread.h index 94bc413d93ff8..73daf1ebcd541 100644 --- a/lldb/include/lldb/Target/Thread.h +++ b/lldb/include/lldb/Target/Thread.h @@ -904,6 +904,27 @@ class Thread : public std::enable_shared_from_this, bool abort_other_plans, bool stop_other_threads, Status &status); + /// Gets the plan used to step through a function with a generic trampoline. A + /// generic trampoline is one without a function target, which the thread plan + /// will attempt to step through until it finds a place where it makes sense + /// to stop at. + /// \param[in] abort_other_plans + /// \b true if we discard the currently queued plans and replace them with + /// this one. + /// Otherwise this plan will go on the end of the plan stack. + /// + /// \param[in] stop_other_threads + /// \b true if we will stop other threads while we single step this one. + /// + /// \param[out] status + /// A status with an error if queuing failed. + /// + /// \return + /// A shared pointer to the newly queued thread plan, or nullptr if the + /// plan could not be queued. + virtual lldb::ThreadPlanSP QueueThreadPlanForStepThroughGenericTrampoline( + bool abort_other_plans, lldb::RunMode stop_other_threads, Status &status); + /// Gets the plan used to continue from the current PC. /// This is a simple plan, mostly useful as a backstop when you are continuing /// for some particular purpose. diff --git a/lldb/include/lldb/Target/ThreadPlan.h b/lldb/include/lldb/Target/ThreadPlan.h index 57fee942441b4..cd5d4a4ecc1f8 100644 --- a/lldb/include/lldb/Target/ThreadPlan.h +++ b/lldb/include/lldb/Target/ThreadPlan.h @@ -302,6 +302,7 @@ class ThreadPlan : public std::enable_shared_from_this, eKindStepInRange, eKindRunToAddress, eKindStepThrough, + eKindStepThroughGenericTrampoline, eKindStepUntil }; diff --git a/lldb/include/lldb/Target/ThreadPlanStepOverRange.h b/lldb/include/lldb/Target/ThreadPlanStepOverRange.h index 8585ac62f09b3..d2bddcb573a2e 100644 --- a/lldb/include/lldb/Target/ThreadPlanStepOverRange.h +++ b/lldb/include/lldb/Target/ThreadPlanStepOverRange.h @@ -30,7 +30,6 @@ class ThreadPlanStepOverRange : public ThreadPlanStepRange, bool ShouldStop(Event *event_ptr) override; protected: - bool DoPlanExplainsStop(Event *event_ptr) override; bool DoWillResume(lldb::StateType resume_state, bool current_plan) override; void SetFlagsToDefault() override { diff --git a/lldb/include/lldb/Target/ThreadPlanStepRange.h b/lldb/include/lldb/Target/ThreadPlanStepRange.h index 2fe8852771000..606135022dfc2 100644 --- a/lldb/include/lldb/Target/ThreadPlanStepRange.h +++ b/lldb/include/lldb/Target/ThreadPlanStepRange.h @@ -41,6 +41,7 @@ class ThreadPlanStepRange : public ThreadPlan { void AddRange(const AddressRange &new_range); protected: + bool DoPlanExplainsStop(Event *event_ptr) override; bool InRange(); lldb::FrameComparison CompareCurrentFrameToStartFrame(); bool InSymbol(); diff --git a/lldb/include/lldb/Target/ThreadPlanStepThroughGenericTrampoline.h b/lldb/include/lldb/Target/ThreadPlanStepThroughGenericTrampoline.h new file mode 100644 index 0000000000000..03ebb83918858 --- /dev/null +++ b/lldb/include/lldb/Target/ThreadPlanStepThroughGenericTrampoline.h @@ -0,0 +1,52 @@ +//===-- ThreadPlanStepInRange.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_TARGET_THREADPLANSTEPTHROUGHGENERICTRAMPOLINE_H +#define LLDB_TARGET_THREADPLANSTEPTHROUGHGENERICTRAMPOLINE_H + +#include "lldb/Target/Thread.h" +#include "lldb/Target/ThreadPlanShouldStopHere.h" +#include "lldb/Target/ThreadPlanStepRange.h" + +namespace lldb_private { + +class ThreadPlanStepThroughGenericTrampoline : public ThreadPlanStepRange, + public ThreadPlanShouldStopHere { +public: + ThreadPlanStepThroughGenericTrampoline(Thread &thread, + lldb::RunMode stop_others); + + ~ThreadPlanStepThroughGenericTrampoline() override; + + void GetDescription(Stream *s, lldb::DescriptionLevel level) override; + + bool ShouldStop(Event *event_ptr) override; + bool ValidatePlan(Stream *error) override; + +protected: + void SetFlagsToDefault() override { + GetFlags().Set( + ThreadPlanStepThroughGenericTrampoline::s_default_flag_values); + } + +private: + // Need an appropriate marker for the current stack so we can tell step out + // from step in. + + static uint32_t + s_default_flag_values; // These are the default flag values + // for the ThreadPlanStepThroughGenericTrampoline. + ThreadPlanStepThroughGenericTrampoline( + const ThreadPlanStepThroughGenericTrampoline &) = delete; + const ThreadPlanStepThroughGenericTrampoline & + operator=(const ThreadPlanStepThroughGenericTrampoline &) = delete; +}; + +} // namespace lldb_private + +#endif // LLDB_TARGET_THREADPLANSTEPTHROUGHGENERICTRAMPOLINE_H diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp index 2582d7bcd66fa..e885a1798265e 100644 --- a/lldb/source/Core/Module.cpp +++ b/lldb/source/Core/Module.cpp @@ -799,7 +799,12 @@ void Module::LookupInfo::Prune(SymbolContextList &sc_list, if (!sc_list.GetContextAtIndex(i, sc)) break; + bool is_trampoline = + Target::GetGlobalProperties().GetEnableTrampolineSupport() && + sc.function && sc.function->IsGenericTrampoline(); + bool keep_it = + !is_trampoline && NameMatchesLookupInfo(sc.GetFunctionName(), sc.GetLanguage()); if (keep_it) ++i; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp index 5c9c69b77443b..81d846ba2a25f 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp @@ -2395,12 +2395,16 @@ DWARFASTParserClang::ParseFunctionFromDWARF(CompileUnit &comp_unit, assert(func_type == nullptr || func_type != DIE_IS_BEING_PARSED); + bool is_generic_trampoline = die.IsGenericTrampoline(); + const user_id_t func_user_id = die.GetID(); func_sp = std::make_shared(&comp_unit, func_user_id, // UserID is the DIE offset func_user_id, func_name, func_type, - func_range); // first address range + func_range, // first address range + false, // canThrow + is_generic_trampoline); if (func_sp.get() != nullptr) { if (frame_base.IsValid()) diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserSwift.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserSwift.cpp index e2b1375863c5a..bf93346aad510 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserSwift.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserSwift.cpp @@ -274,9 +274,11 @@ Function *DWARFASTParserSwift::ParseFunctionFromDWARF( decl_column)); const user_id_t func_user_id = die.GetID(); + bool is_generic_trampoline = die.IsGenericTrampoline(); func_sp.reset(new Function(&comp_unit, func_user_id, func_user_id, - func_name, nullptr, func_range, - can_throw)); // first address range + func_name, nullptr, + func_range, // first address range + can_throw, is_generic_trampoline)); if (func_sp.get() != NULL) { if (frame_base.IsValid()) diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp index e8492079af88a..2b8032eae8032 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp @@ -203,6 +203,13 @@ const char *DWARFDIE::GetMangledName() const { return nullptr; } +bool DWARFDIE::IsGenericTrampoline() const { + if (IsValid()) + return m_die->GetIsGenericTrampoline(m_cu); + else + return false; +} + const char *DWARFDIE::GetPubname() const { if (IsValid()) return m_die->GetPubname(m_cu); diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h index 7ce9550a081e9..db353071546f4 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h @@ -28,6 +28,8 @@ class DWARFDIE : public DWARFBaseDIE { // Accessing information about a DIE const char *GetMangledName() const; + bool IsGenericTrampoline() const; + const char *GetPubname() const; const char *GetQualifiedName(std::string &storage) const; diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp index 5a584e15c397d..769ca30e458ea 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp @@ -679,6 +679,12 @@ DWARFDebugInfoEntry::GetMangledName(const DWARFUnit *cu, return name; } +bool +DWARFDebugInfoEntry::GetIsGenericTrampoline(const DWARFUnit *cu) const { + DWARFFormValue form_value; + return GetAttributeValue(cu, DW_AT_trampoline, form_value, nullptr, true) != 0; +} + // GetPubname // // Get value the name for a DIE as it should appear for a .debug_pubnames or diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h index 63d6aa79240b3..3fc9a90d7a129 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h @@ -97,6 +97,8 @@ class DWARFDebugInfoEntry { const char *GetMangledName(const DWARFUnit *cu, bool substitute_name_allowed = true) const; + bool GetIsGenericTrampoline(const DWARFUnit *cu) const; + const char *GetPubname(const DWARFUnit *cu) const; const char *GetQualifiedName(DWARFUnit *cu, std::string &storage) const; diff --git a/lldb/source/Symbol/Function.cpp b/lldb/source/Symbol/Function.cpp index b60e6634fc430..5223f0d122f0b 100644 --- a/lldb/source/Symbol/Function.cpp +++ b/lldb/source/Symbol/Function.cpp @@ -233,10 +233,11 @@ Function *IndirectCallEdge::GetCallee(ModuleList &images, // Function::Function(CompileUnit *comp_unit, lldb::user_id_t func_uid, lldb::user_id_t type_uid, const Mangled &mangled, Type *type, - const AddressRange &range, bool canThrow) + const AddressRange &range, bool canThrow, bool is_generic_trampoline) : UserID(func_uid), m_comp_unit(comp_unit), m_type_uid(type_uid), - m_type(type), m_mangled(mangled), m_block(func_uid), m_range(range), - m_frame_base(), m_flags(), m_prologue_byte_size(0) { + m_type(type), m_mangled(mangled), + m_is_generic_trampoline(is_generic_trampoline), m_block(func_uid), + m_range(range), m_frame_base(), m_flags(), m_prologue_byte_size(0) { m_block.SetParentScope(this); if (canThrow) m_flags.Set(flagsFunctionCanThrow); diff --git a/lldb/source/Target/CMakeLists.txt b/lldb/source/Target/CMakeLists.txt index 8129cf7c421d7..b921118b2e4f3 100644 --- a/lldb/source/Target/CMakeLists.txt +++ b/lldb/source/Target/CMakeLists.txt @@ -65,6 +65,7 @@ add_lldb_library(lldbTarget ThreadPlanStepOverRange.cpp ThreadPlanStepRange.cpp ThreadPlanStepThrough.cpp + ThreadPlanStepThroughGenericTrampoline.cpp ThreadPlanStepUntil.cpp ThreadPlanTracer.cpp ThreadPlanStack.cpp diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp index edc00a9c8e190..f884a9414fd0f 100644 --- a/lldb/source/Target/Target.cpp +++ b/lldb/source/Target/Target.cpp @@ -5305,6 +5305,12 @@ bool TargetProperties::GetDebugUtilityExpression() const { nullptr, idx, g_target_properties[idx].default_uint_value != 0); } +bool TargetProperties::GetEnableTrampolineSupport() const { + const uint32_t idx = ePropertyEnableTrampolineSupport; + return m_collection_sp->GetPropertyAtIndexAsBoolean( + nullptr, idx, g_target_properties[idx].default_uint_value != 0); +} + void TargetProperties::SetDebugUtilityExpression(bool debug) { const uint32_t idx = ePropertyDebugUtilityExpression; m_collection_sp->SetPropertyAtIndexAsBoolean(nullptr, idx, debug); diff --git a/lldb/source/Target/TargetProperties.td b/lldb/source/Target/TargetProperties.td index 2bdc218fd8181..de93957633f7b 100644 --- a/lldb/source/Target/TargetProperties.td +++ b/lldb/source/Target/TargetProperties.td @@ -218,6 +218,9 @@ let Definition = "target" in { def DebugUtilityExpression: Property<"debug-utility-expression", "Boolean">, DefaultFalse, Desc<"Enable debugging of LLDB-internal utility expressions.">; + def EnableTrampolineSupport: Property<"enable-trampoline-support", "Boolean">, + Global, DefaultTrue, + Desc<"Enable trampoline support in LLDB. Trampoline support includes stepping through trampolines directly to their targets, stepping out of trampolines directly to their callers, and automatically filtering out trampolines as possible breakpoint locations when set by name.">; } let Definition = "process_experimental" in { diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp index 428da24f0fcbb..6de3453f41567 100644 --- a/lldb/source/Target/Thread.cpp +++ b/lldb/source/Target/Thread.cpp @@ -41,6 +41,7 @@ #include "lldb/Target/ThreadPlanStepOverBreakpoint.h" #include "lldb/Target/ThreadPlanStepOverRange.h" #include "lldb/Target/ThreadPlanStepThrough.h" +#include "lldb/Target/ThreadPlanStepThroughGenericTrampoline.h" #include "lldb/Target/ThreadPlanStepUntil.h" #include "lldb/Target/ThreadSpec.h" #include "lldb/Target/UnwindLLDB.h" @@ -1419,6 +1420,17 @@ ThreadPlanSP Thread::QueueThreadPlanForStepThrough(StackID &return_stack_id, return thread_plan_sp; } +ThreadPlanSP Thread::QueueThreadPlanForStepThroughGenericTrampoline( + bool abort_other_plans, lldb::RunMode stop_other_threads, Status &status) { + ThreadPlanSP thread_plan_sp( + new ThreadPlanStepThroughGenericTrampoline(*this, stop_other_threads)); + + if (!thread_plan_sp || !thread_plan_sp->ValidatePlan(nullptr)) + return ThreadPlanSP(); + status = QueueThreadPlan(thread_plan_sp, abort_other_plans); + return thread_plan_sp; +} + ThreadPlanSP Thread::QueueThreadPlanForRunToAddress(bool abort_other_plans, Address &target_addr, bool stop_other_threads, diff --git a/lldb/source/Target/ThreadPlanShouldStopHere.cpp b/lldb/source/Target/ThreadPlanShouldStopHere.cpp index a470e969037bd..d59178689f9c1 100644 --- a/lldb/source/Target/ThreadPlanShouldStopHere.cpp +++ b/lldb/source/Target/ThreadPlanShouldStopHere.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "lldb/Target/ThreadPlanShouldStopHere.h" +#include "lldb/Symbol/Function.h" #include "lldb/Symbol/Symbol.h" #include "lldb/Target/LanguageRuntime.h" #include "lldb/Target/Process.h" @@ -96,10 +97,16 @@ bool ThreadPlanShouldStopHere::DefaultShouldStopHereCallback( // independently. If this ever // becomes expensive (this one isn't) we can try to have this set a state // that the StepFromHere can use. - if (frame) { - SymbolContext sc; - sc = frame->GetSymbolContext(eSymbolContextLineEntry); - if (sc.line_entry.line == 0) + SymbolContext sc; + sc = frame->GetSymbolContext(eSymbolContextLineEntry); + + if (sc.line_entry.line == 0) + should_stop_here = false; + + // If we're in a trampoline, don't stop by default. + if (Target::GetGlobalProperties().GetEnableTrampolineSupport()) { + sc = frame->GetSymbolContext(lldb::eSymbolContextFunction); + if (sc.function && sc.function->IsGenericTrampoline()) should_stop_here = false; } diff --git a/lldb/source/Target/ThreadPlanStepInRange.cpp b/lldb/source/Target/ThreadPlanStepInRange.cpp index 3e1cd4910949f..d679d9587dc53 100644 --- a/lldb/source/Target/ThreadPlanStepInRange.cpp +++ b/lldb/source/Target/ThreadPlanStepInRange.cpp @@ -221,6 +221,17 @@ bool ThreadPlanStepInRange::ShouldStop(Event *event_ptr) { // We may have set the plan up above in the FrameIsOlder section: + if (!m_sub_plan_sp) + m_sub_plan_sp = thread.QueueThreadPlanForStepThroughGenericTrampoline( + false, m_stop_others, m_status); + if (log) { + if (m_sub_plan_sp) + LLDB_LOGF(log, "Found a generic step through plan: %s", + m_sub_plan_sp->GetName()); + else + LLDB_LOGF(log, "No generic step through plan found."); + } + if (!m_sub_plan_sp) m_sub_plan_sp = thread.QueueThreadPlanForStepThrough( m_stack_id, false, stop_others, m_status); diff --git a/lldb/source/Target/ThreadPlanStepOverRange.cpp b/lldb/source/Target/ThreadPlanStepOverRange.cpp index b1cb070e0a3d0..0db2fe855854d 100644 --- a/lldb/source/Target/ThreadPlanStepOverRange.cpp +++ b/lldb/source/Target/ThreadPlanStepOverRange.cpp @@ -334,37 +334,6 @@ bool ThreadPlanStepOverRange::ShouldStop(Event *event_ptr) { return false; } -bool ThreadPlanStepOverRange::DoPlanExplainsStop(Event *event_ptr) { - // For crashes, breakpoint hits, signals, etc, let the base plan (or some - // plan above us) handle the stop. That way the user can see the stop, step - // around, and then when they are done, continue and have their step - // complete. The exception is if we've hit our "run to next branch" - // breakpoint. Note, unlike the step in range plan, we don't mark ourselves - // complete if we hit an unexplained breakpoint/crash. - - Log *log = GetLog(LLDBLog::Step); - StopInfoSP stop_info_sp = GetPrivateStopInfo(); - bool return_value; - - if (stop_info_sp) { - StopReason reason = stop_info_sp->GetStopReason(); - - if (reason == eStopReasonTrace) { - return_value = true; - } else if (reason == eStopReasonBreakpoint) { - return_value = NextRangeBreakpointExplainsStop(stop_info_sp); - } else { - if (log) - log->PutCString("ThreadPlanStepInRange got asked if it explains the " - "stop for some reason other than step."); - return_value = false; - } - } else - return_value = true; - - return return_value; -} - bool ThreadPlanStepOverRange::DoWillResume(lldb::StateType resume_state, bool current_plan) { if (resume_state != eStateSuspended && m_first_resume) { diff --git a/lldb/source/Target/ThreadPlanStepRange.cpp b/lldb/source/Target/ThreadPlanStepRange.cpp index e2323bb31577c..409590db90b1a 100644 --- a/lldb/source/Target/ThreadPlanStepRange.cpp +++ b/lldb/source/Target/ThreadPlanStepRange.cpp @@ -494,3 +494,35 @@ bool ThreadPlanStepRange::IsPlanStale() { } return false; } + + +bool ThreadPlanStepRange::DoPlanExplainsStop(Event *event_ptr) { + // For crashes, breakpoint hits, signals, etc, let the base plan (or some + // plan above us) handle the stop. That way the user can see the stop, step + // around, and then when they are done, continue and have their step + // complete. The exception is if we've hit our "run to next branch" + // breakpoint. Note, unlike the step in range plan, we don't mark ourselves + // complete if we hit an unexplained breakpoint/crash. + + Log *log = GetLog(LLDBLog::Step); + StopInfoSP stop_info_sp = GetPrivateStopInfo(); + bool return_value; + + if (stop_info_sp) { + StopReason reason = stop_info_sp->GetStopReason(); + + if (reason == eStopReasonTrace) { + return_value = true; + } else if (reason == eStopReasonBreakpoint) { + return_value = NextRangeBreakpointExplainsStop(stop_info_sp); + } else { + if (log) + log->PutCString("ThreadPlanStepRange got asked if it explains the " + "stop for some reason other than step."); + return_value = false; + } + } else + return_value = true; + + return return_value; +} diff --git a/lldb/source/Target/ThreadPlanStepThrough.cpp b/lldb/source/Target/ThreadPlanStepThrough.cpp index e195abf7e6526..0856aa506e569 100644 --- a/lldb/source/Target/ThreadPlanStepThrough.cpp +++ b/lldb/source/Target/ThreadPlanStepThrough.cpp @@ -33,6 +33,10 @@ ThreadPlanStepThrough::ThreadPlanStepThrough(Thread &thread, m_start_address(0), m_backstop_bkpt_id(LLDB_INVALID_BREAK_ID), m_backstop_addr(LLDB_INVALID_ADDRESS), m_return_stack_id(m_stack_id), m_stop_others(stop_others) { + // If trampoline support is disabled, there's nothing for us to do. + if (!Target::GetGlobalProperties().GetEnableTrampolineSupport()) + return; + LookForPlanToStepThroughFromCurrentPC(); // If we don't get a valid step through plan, don't bother to set up a diff --git a/lldb/source/Target/ThreadPlanStepThroughGenericTrampoline.cpp b/lldb/source/Target/ThreadPlanStepThroughGenericTrampoline.cpp new file mode 100644 index 0000000000000..769eee476967d --- /dev/null +++ b/lldb/source/Target/ThreadPlanStepThroughGenericTrampoline.cpp @@ -0,0 +1,137 @@ +//===-- ThreadPlanStepThroughGenericTrampoline.cpp +//-----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "lldb/Target/ThreadPlanStepThroughGenericTrampoline.h" +#include "lldb/Symbol/Function.h" +#include "lldb/Target/RegisterContext.h" +#include "lldb/Utility/LLDBLog.h" +#include "lldb/Utility/Log.h" +#include "lldb/Utility/Stream.h" + +using namespace lldb; +using namespace lldb_private; + +uint32_t ThreadPlanStepThroughGenericTrampoline::s_default_flag_values = + ThreadPlanShouldStopHere::eStepInAvoidNoDebug; + +ThreadPlanStepThroughGenericTrampoline::ThreadPlanStepThroughGenericTrampoline( + Thread &thread, lldb::RunMode stop_others) + : ThreadPlanStepRange(ThreadPlan::eKindStepThroughGenericTrampoline, + "Step through generic trampoline", thread, {}, {}, + stop_others), + ThreadPlanShouldStopHere(this) { + + SetFlagsToDefault(); + auto frame = GetThread().GetFrameWithStackID(m_stack_id); + if (!frame) + return; + SymbolContext sc = frame->GetSymbolContext(eSymbolContextFunction); + + if (!sc.function) + return; + AddRange(sc.function->GetAddressRange()); +} + +ThreadPlanStepThroughGenericTrampoline:: + ~ThreadPlanStepThroughGenericTrampoline() = default; + +void ThreadPlanStepThroughGenericTrampoline::GetDescription( + Stream *s, lldb::DescriptionLevel level) { + + auto PrintFailureIfAny = [&]() { + if (m_status.Success()) + return; + s->Printf(" failed (%s)", m_status.AsCString()); + }; + + if (level == lldb::eDescriptionLevelBrief) { + s->Printf("step through generic trampoline"); + PrintFailureIfAny(); + return; + } + + auto frame = GetThread().GetFrameWithStackID(m_stack_id); + if (!frame) { + s->Printf(""); + return; + } + + SymbolContext sc = frame->GetSymbolContext(eSymbolContextFunction); + if (!sc.function) { + s->Printf(""); + return; + } + + s->Printf("Stepping through generic trampoline %s", + sc.function->GetName().AsCString()); + + lldb::StackFrameSP curr_frame = GetThread().GetStackFrameAtIndex(0); + if (!curr_frame) + return; + + SymbolContext curr_frame_sc = + curr_frame->GetSymbolContext(eSymbolContextFunction); + if (!curr_frame_sc.function) + return; + s->Printf(", current function: %s", + curr_frame_sc.function->GetName().GetCString()); + + PrintFailureIfAny(); + + s->PutChar('.'); +} + +bool ThreadPlanStepThroughGenericTrampoline::ShouldStop(Event *event_ptr) { + Log *log = GetLog(LLDBLog::Step); + + if (log) { + StreamString s; + DumpAddress(s.AsRawOstream(), GetThread().GetRegisterContext()->GetPC(), + GetTarget().GetArchitecture().GetAddressByteSize()); + LLDB_LOGF(log, "ThreadPlanStepThroughGenericTrampoline reached %s.", + s.GetData()); + } + + if (IsPlanComplete()) + return true; + + m_no_more_plans = false; + + Thread &thread = GetThread(); + lldb::StackFrameSP curr_frame = thread.GetStackFrameAtIndex(0); + if (!curr_frame) + return false; + + SymbolContext sc = curr_frame->GetSymbolContext(eSymbolContextFunction); + + if (sc.function && sc.function->IsGenericTrampoline() && + SetNextBranchBreakpoint()) { + // While whatever frame we're in is a generic trampoline, + // continue stepping to the next branch, until we + // end up in a function which isn't a trampoline. + return false; + } + + m_no_more_plans = true; + SetPlanComplete(); + return true; +} + +bool ThreadPlanStepThroughGenericTrampoline::ValidatePlan(Stream *error) { + // If trampoline support is disabled, there's nothing for us to do. + if (!Target::GetGlobalProperties().GetEnableTrampolineSupport()) + return false; + + auto frame = GetThread().GetFrameWithStackID(m_stack_id); + if (!frame) + return false; + + SymbolContext sc = frame->GetSymbolContext(eSymbolContextFunction); + return sc.function && sc.function->IsGenericTrampoline(); +} diff --git a/lldb/test/API/lang/c/trampoline_stepping/Makefile b/lldb/test/API/lang/c/trampoline_stepping/Makefile new file mode 100644 index 0000000000000..10495940055b6 --- /dev/null +++ b/lldb/test/API/lang/c/trampoline_stepping/Makefile @@ -0,0 +1,3 @@ +C_SOURCES := main.c + +include Makefile.rules diff --git a/lldb/test/API/lang/c/trampoline_stepping/TestTrampolineStepping.py b/lldb/test/API/lang/c/trampoline_stepping/TestTrampolineStepping.py new file mode 100644 index 0000000000000..4531a727c4900 --- /dev/null +++ b/lldb/test/API/lang/c/trampoline_stepping/TestTrampolineStepping.py @@ -0,0 +1,104 @@ +"""Test that stepping in/out of trampolines works as expected. +""" + + + +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + + +class TestTrampoline(TestBase): + def setup(self, bkpt_str): + self.build() + + _, _, thread, _ = lldbutil.run_to_source_breakpoint( + self, bkpt_str, lldb.SBFileSpec('main.c')) + return thread + + def test_direct_call(self): + thread = self.setup('Break here for direct') + + # Sanity check that we start out in the correct function. + name = thread.frames[0].GetFunctionName() + self.assertIn('direct_trampoline_call', name) + + # Check that stepping in will take us directly to the trampoline target. + thread.StepInto() + name = thread.frames[0].GetFunctionName() + self.assertIn('foo', name) + + # Check that stepping out takes us back to the trampoline caller. + thread.StepOut() + name = thread.frames[0].GetFunctionName() + self.assertIn('direct_trampoline_call', name) + + # Check that stepping over the end of the trampoline target + # takes us back to the trampoline caller. + thread.StepInto() + thread.StepOver() + name = thread.frames[0].GetFunctionName() + self.assertIn('direct_trampoline_call', name) + + + def test_chained_call(self): + thread = self.setup('Break here for chained') + + # Sanity check that we start out in the correct function. + name = thread.frames[0].GetFunctionName() + self.assertIn('chained_trampoline_call', name) + + # Check that stepping in will take us directly to the trampoline target. + thread.StepInto() + name = thread.frames[0].GetFunctionName() + self.assertIn('foo', name) + + # Check that stepping out takes us back to the trampoline caller. + thread.StepOut() + name = thread.frames[0].GetFunctionName() + self.assertIn('chained_trampoline_call', name) + + # Check that stepping over the end of the trampoline target + # takes us back to the trampoline caller. + thread.StepInto() + thread.StepOver() + name = thread.frames[0].GetFunctionName() + self.assertIn('chained_trampoline_call', name) + + def test_trampoline_after_nodebug(self): + thread = self.setup('Break here for nodebug then trampoline') + + # Sanity check that we start out in the correct function. + name = thread.frames[0].GetFunctionName() + self.assertIn('trampoline_after_nodebug', name) + + # Check that stepping in will take us directly to the trampoline target. + thread.StepInto() + name = thread.frames[0].GetFunctionName() + self.assertIn('foo', name) + + # Check that stepping out takes us back to the trampoline caller. + thread.StepOut() + name = thread.frames[0].GetFunctionName() + self.assertIn('trampoline_after_nodebug', name) + + # Check that stepping over the end of the trampoline target + # takes us back to the trampoline caller. + thread.StepInto() + thread.StepOver() + name = thread.frames[0].GetFunctionName() + self.assertIn('trampoline_after_nodebug', name) + + def test_unused_target(self): + thread = self.setup('Break here for unused') + + # Sanity check that we start out in the correct function. + name = thread.frames[0].GetFunctionName() + self.assertIn('unused_target', name) + + # Check that stepping into a trampoline that doesn't call its target + # jumps back to its caller. + thread.StepInto() + name = thread.frames[0].GetFunctionName() + self.assertIn('unused_target', name) + diff --git a/lldb/test/API/lang/c/trampoline_stepping/main.c b/lldb/test/API/lang/c/trampoline_stepping/main.c new file mode 100644 index 0000000000000..cb98be00ca1f6 --- /dev/null +++ b/lldb/test/API/lang/c/trampoline_stepping/main.c @@ -0,0 +1,52 @@ +void foo(void) {} + +__attribute__((transparent_stepping)) +void bar(void) { + foo(); +} + +__attribute__((transparent_stepping)) +void baz(void) { + bar(); +} + +__attribute__((nodebug)) +void nodebug(void) {} + +__attribute__((transparent_stepping)) +void nodebug_then_trampoline(void) { + nodebug(); + baz(); +} + +__attribute__((transparent_stepping)) +void doesnt_call_trampoline(void) {} + +void direct_trampoline_call(void) { + bar(); // Break here for direct + bar(); +} + +void chained_trampoline_call(void) { + baz(); // Break here for chained + baz(); +} + +void trampoline_after_nodebug(void) { + nodebug_then_trampoline(); // Break here for nodebug then trampoline + nodebug_then_trampoline(); +} + +void unused_target(void) { + doesnt_call_trampoline(); // Break here for unused +} + + +int main(void) { + direct_trampoline_call(); + chained_trampoline_call(); + trampoline_after_nodebug(); + unused_target(); + return 0; +} + From b807ec430eb0bc31b70aad70a008175ea11b7a3b Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Mon, 20 Mar 2023 17:10:51 -0700 Subject: [PATCH 50/88] [clang/cas/dep-scan] Include the translation unit cache key in the scanning output Also add translation unit and module cache keys in the json output of `clang-scan-deps` for testing purposes. (cherry picked from commit c1784bd8f6ecc21eb41e820b77e65deafaded82a) --- .../DependencyScanningWorker.h | 3 ++ .../DependencyScanningWorker.cpp | 17 +++++++++-- clang/test/ClangScanDeps/cas-trees.c | 28 ++++++++++++++++--- clang/test/ClangScanDeps/include-tree.c | 13 ++++++++- .../modules-cas-full-by-mod-name.c | 13 +++++++-- clang/test/ClangScanDeps/modules-cas-trees.c | 10 +++++-- clang/tools/clang-scan-deps/ClangScanDeps.cpp | 4 +++ 7 files changed, 76 insertions(+), 12 deletions(-) diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index a0df3970696c7..fa634e22246a4 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -39,6 +39,9 @@ struct DepscanPrefixMapping; struct Command { std::string Executable; std::vector Arguments; + + /// The \c ActionCache key for this translation unit, if any. + std::optional TUCacheKey; }; class DependencyConsumer { diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index fcefb3e7d4ffc..6c95ff872db7e 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -15,6 +15,7 @@ #include "clang/Driver/Driver.h" #include "clang/Driver/Job.h" #include "clang/Driver/Tool.h" +#include "clang/Frontend/CompileJobCacheKey.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" @@ -484,6 +485,13 @@ class DependencyScanningAction : public tooling::ToolAction { LastCC1Arguments = OriginalInvocation.getCC1CommandLine(); + if (ScanInstance.getFrontendOpts().CacheCompileJob) { + auto &CAS = ScanInstance.getOrCreateObjectStore(); + if (auto Key = createCompileJobCacheKey( + CAS, ScanInstance.getDiagnostics(), OriginalInvocation)) + TUCacheKey = Key->toString(); + } + return true; } @@ -498,6 +506,8 @@ class DependencyScanningAction : public tooling::ToolAction { return Result; } + const std::optional &getTUCacheKey() const { return TUCacheKey; } + IntrusiveRefCntPtr getDepScanFS() { if (DepFS) { assert(!DepCASFS && "CAS DepFS should not be set"); @@ -528,6 +538,7 @@ class DependencyScanningAction : public tooling::ToolAction { Optional ScanInstanceStorage; std::shared_ptr MDC; std::vector LastCC1Arguments; + std::optional TUCacheKey; bool Scanned = false; raw_ostream *VerboseOS; }; @@ -706,7 +717,8 @@ bool DependencyScanningWorker::computeDependencies( // consumer. Consumer.handleBuildCommand( {Cmd.getExecutable(), - {Cmd.getArguments().begin(), Cmd.getArguments().end()}}); + {Cmd.getArguments().begin(), Cmd.getArguments().end()}, + /*TUCacheKey=*/std::nullopt}); return true; } @@ -727,7 +739,8 @@ bool DependencyScanningWorker::computeDependencies( return false; std::vector Args = Action.takeLastCC1Arguments(); - Consumer.handleBuildCommand({Cmd.getExecutable(), std::move(Args)}); + Consumer.handleBuildCommand( + {Cmd.getExecutable(), std::move(Args), Action.getTUCacheKey()}); return true; }); diff --git a/clang/test/ClangScanDeps/cas-trees.c b/clang/test/ClangScanDeps/cas-trees.c index c54f9a276e8de..47dbac6c47e44 100644 --- a/clang/test/ClangScanDeps/cas-trees.c +++ b/clang/test/ClangScanDeps/cas-trees.c @@ -18,7 +18,8 @@ // FULL-TREE-NEXT: "modules": [], // FULL-TREE-NEXT: "translation-units": [ // FULL-TREE-NEXT: { -// FULL-TREE: "casfs-root-id": "[[T1_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE: "cache-key": "[[T1_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE-NEXT: "casfs-root-id": "[[T1_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // FULL-TREE-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", // FULL-TREE-NEXT: "clang-module-deps": [], // FULL-TREE-NEXT: "command-line": [ @@ -36,7 +37,8 @@ // FULL-TREE-NEXT: "input-file": "[[PREFIX]]/t1.c" // FULL-TREE-NEXT: } // FULL-TREE: { -// FULL-TREE: "casfs-root-id": "[[T2_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE: "cache-key": "[[T2_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// FULL-TREE-NEXT: "casfs-root-id": "[[T2_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // FULL-TREE-NEXT: "clang-context-hash": "{{[A-Z0-9]+}}", // FULL-TREE-NEXT: "clang-module-deps": [], // FULL-TREE-NEXT: "command-line": [ @@ -56,13 +58,31 @@ // Build with caching // RUN: %deps-to-rsp %t/full_result.json --tu-index 0 > %t/t1.cc1.rsp // RUN: %deps-to-rsp %t/full_result.json --tu-index 1 > %t/t2.cc1.rsp -// RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-MISS +// RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2> %t/t1-miss.err +// RUN: FileCheck %s -input-file=%t/t1-miss.err -check-prefix=CACHE-MISS // RUN: %clang @%t/t1.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-HIT -// RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-MISS +// RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2> %t/t2-miss.err +// RUN: FileCheck %s -input-file=%t/t2-miss.err -check-prefix=CACHE-MISS // RUN: %clang @%t/t2.cc1.rsp -Rcompile-job-cache 2>&1 | FileCheck %s -check-prefix=CACHE-HIT // CACHE-HIT: remark: compile job cache hit // CACHE-MISS: remark: compile job cache miss +// Check cache keys. +// RUN: cp %t/full_result.json %t/combined.txt +// RUN: cat %t/t1-miss.err >> %t/combined.txt +// RUN: cat %t/t2-miss.err >> %t/combined.txt +// RUN: FileCheck %s -input-file=%t/combined.txt -check-prefix=COMBINED + +// COMBINED: "commands": [ +// COMBINED-NEXT: { +// COMBINED-NEXT: "cache-key": "[[T1_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: } +// COMBINED: "commands": [ +// COMBINED: { +// COMBINED-NEXT: "cache-key": "[[T2_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: remark: compile job cache miss for '[[T1_CACHE_KEY]]' +// COMBINED-NEXT: remark: compile job cache miss for '[[T2_CACHE_KEY]]' + // RUN: clang-scan-deps -compilation-database %t/cdb.json -cas-path %t/cas -format experimental-tree -emit-cas-compdb | FileCheck %s -DPREFIX=%/t -check-prefix=COMPDB // COMPDB: [ // COMPDB: { diff --git a/clang/test/ClangScanDeps/include-tree.c b/clang/test/ClangScanDeps/include-tree.c index eed9b3c784015..d21c408650156 100644 --- a/clang/test/ClangScanDeps/include-tree.c +++ b/clang/test/ClangScanDeps/include-tree.c @@ -48,6 +48,7 @@ // FULL-NEXT: { // FULL-NEXT: "commands": [ // FULL-NEXT: { +// FULL-NEXT: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // FULL: "clang-module-deps": [] // FULL: "command-line": [ // FULL-NEXT: "-cc1" @@ -84,7 +85,17 @@ // Build the include-tree command // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp -// RUN: %clang @%t/tu.rsp +// RUN: %clang @%t/tu.rsp -Rcompile-job-cache 2> %t/t.err + +// Check cache key. +// RUN: cp %t/full.txt %t/combined.txt +// RUN: cat %t/t.err >> %t/combined.txt +// RUN: FileCheck %s -input-file=%t/combined.txt -check-prefix=COMBINED + +// COMBINED: "commands": [ +// COMBINED-NEXT: { +// COMBINED-NEXT: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// COMBINED: remark: compile job cache miss for '[[TU_CACHE_KEY]]' //--- cdb.json.template [ diff --git a/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c b/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c index 11a32b40cb1b0..c6ad2068c6c97 100644 --- a/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c +++ b/clang/test/ClangScanDeps/modules-cas-full-by-mod-name.c @@ -35,6 +35,7 @@ module transitive { header "transitive.h" } // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { +// CHECK-NEXT: "cache-key": "[[DIRECT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "clang-module-deps": [ // CHECK-NEXT: { @@ -44,6 +45,9 @@ module transitive { header "transitive.h" } // CHECK-NEXT: ], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "{{.*transitive-.*\.pcm}}" +// CHECK-NEXT: "[[TRANSITIVE_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: ], // CHECK-NEXT: "context-hash": "{{.*}}", // CHECK-NEXT: "file-deps": [ @@ -53,7 +57,8 @@ module transitive { header "transitive.h" } // CHECK-NEXT: "name": "direct" // CHECK-NEXT: }, // CHECK-NEXT: { -// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "cache-key": "[[ROOT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "casfs-root-id": "[[ROOT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK-NEXT: "clang-module-deps": [ // CHECK-NEXT: { // CHECK-NEXT: "context-hash": "{{.*}}", @@ -62,6 +67,9 @@ module transitive { header "transitive.h" } // CHECK-NEXT: ], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "{{.*direct-.*\.pcm}}" +// CHECK-NEXT: "[[DIRECT_CACHE_KEY]]" // CHECK: ], // CHECK-NEXT: "context-hash": "{{.*}}", // CHECK-NEXT: "file-deps": [ @@ -72,7 +80,8 @@ module transitive { header "transitive.h" } // CHECK-NEXT: "name": "root" // CHECK-NEXT: }, // CHECK-NEXT: { -// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" +// CHECK-NEXT: "cache-key": "[[TRANSITIVE_CACHE_KEY]]" +// CHECK-NEXT: "casfs-root-id": "[[LEFT_ROOT_ID]]" // CHECK-NEXT: "clang-module-deps": [], // CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", // CHECK-NEXT: "command-line": [ diff --git a/clang/test/ClangScanDeps/modules-cas-trees.c b/clang/test/ClangScanDeps/modules-cas-trees.c index f3d6cf578e01c..9aad19541aaef 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees.c +++ b/clang/test/ClangScanDeps/modules-cas-trees.c @@ -64,6 +64,7 @@ // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { +// CHECK: "cache-key": "[[LEFT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[LEFT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -92,6 +93,7 @@ // CHECK: "name": "Left" // CHECK: } // CHECK-NEXT: { +// CHECK: "cache-key": "[[RIGHT_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[RIGHT_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -110,7 +112,7 @@ // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[TOP_PCM]]" -// CHECK: "[[TOP_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" +// CHECK: "[[TOP_CACHE_KEY]]" // CHECK: "-fmodule-file={{(Top=)?}}[[TOP_PCM]]" // CHECK: ] // CHECK: "file-deps": [ @@ -120,6 +122,7 @@ // CHECK: "name": "Right" // CHECK: } // CHECK-NEXT: { +// CHECK: "cache-key": "[[TOP_CACHE_KEY]]" // CHECK: "casfs-root-id": "[[TOP_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [] // CHECK: "command-line": [ @@ -144,6 +147,7 @@ // CHECK: { // CHECK: "commands": [ // CHECK: { +// CHECK: "cache-key": "[[TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "casfs-root-id": "[[TU_ROOT_ID:llvmcas://[[:xdigit:]]+]]" // CHECK: "clang-module-deps": [ // CHECK: { @@ -162,10 +166,10 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[LEFT_PCM]]" -// CHECK: "{{llvmcas://[[:xdigit:]]+}}" +// CHECK: "[[LEFT_CACHE_KEY]]" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[RIGHT_PCM]]" -// CHECK: "{{llvmcas://[[:xdigit:]]+}}" +// CHECK: "[[RIGHT_CACHE_KEY]]" // CHECK: "-fmodule-file={{(Left=)?}}[[LEFT_PCM]]" // CHECK: "-fmodule-file={{(Right=)?}}[[RIGHT_PCM]]" // CHECK: ] diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index ecb3d04f80adf..4c0578fc018ca 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -623,6 +623,8 @@ class FullDeps { {"clang-modulemap-file", MD.ClangModuleMapFile}, {"command-line", MD.BuildArguments}, }; + if (MD.ModuleCacheKey) + O.try_emplace("cache-key", MD.ModuleCacheKey); if (MD.CASFileSystemRootID) O.try_emplace("casfs-root-id", MD.CASFileSystemRootID->toString()); if (MD.IncludeTreeID) @@ -643,6 +645,8 @@ class FullDeps { {"executable", Cmd.Executable}, {"command-line", Cmd.Arguments}, }; + if (Cmd.TUCacheKey) + O.try_emplace("cache-key", *Cmd.TUCacheKey); if (I.CASFileSystemRootID) O.try_emplace("casfs-root-id", I.CASFileSystemRootID); if (I.IncludeTreeID) From 7907f5e4d3a19d397133eb1ca45f3afb4022ca8f Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 8 Dec 2022 10:31:16 -0800 Subject: [PATCH 51/88] [clang] Set ShowInSystemHeader for module-build and module-import remarks Without this change, the use of `-Rmodule-build` and `-Rmodule-import` only produces diagnostics for modules built or imported by non-system code. For example, if a project source file requires the Foundation module to be built, then `-Rmodule-build` will show a single diagnostic for Foundation, but not in turn for any of Foundation's (direct or indirect) dependencies. This is because the locations of those transitive module builds are initiated from system headers, which are ignored by default. When wanting to observe module building/importing, the system modules can represent a significant amount of module diagnostics, and I think should be shown by default when `-Rmodule-build` and `-Rmodule-import` are specified. I noticed some other remarks use `ShowInSystemHeader`. Differential Revision: https://reviews.llvm.org/D139653 (cherry picked from commit ba6e747f9b05e541e88822e1dbd6bd5424cfe2fb) --- .../clang/Basic/DiagnosticFrontendKinds.td | 2 ++ .../clang/Basic/DiagnosticSerializationKinds.td | 1 + clang/test/Modules/system-Rmodule-build.m | 16 ++++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 clang/test/Modules/system-Rmodule-build.m diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 81877ea252a7b..9d3af50111fc2 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -252,8 +252,10 @@ def warn_module_config_macro_undef : Warning< def note_module_def_undef_here : Note< "macro was %select{defined|#undef'd}0 here">; def remark_module_build : Remark<"building module '%0' as '%1'">, + ShowInSystemHeader, InGroup; def remark_module_build_done : Remark<"finished building module '%0'">, + ShowInSystemHeader, InGroup; def remark_module_lock : Remark<"locking '%0' to build module '%1'">, InGroup; diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td index f515ea0d9f6dd..ecf2c82b32be3 100644 --- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td +++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td @@ -75,6 +75,7 @@ def note_module_file_conflict : Note< def remark_module_import : Remark< "importing module '%0'%select{| into '%3'}2 from '%1'">, + ShowInSystemHeader, InGroup; def err_imported_module_not_found : Error< diff --git a/clang/test/Modules/system-Rmodule-build.m b/clang/test/Modules/system-Rmodule-build.m new file mode 100644 index 0000000000000..f631487befd20 --- /dev/null +++ b/clang/test/Modules/system-Rmodule-build.m @@ -0,0 +1,16 @@ +// RUN: rm -rf %t +// RUN: mkdir -p %t/sys +// RUN: echo '#include ' > %t/sys/A.h +// RUN: echo '' > %t/sys/B.h +// RUN: echo 'module A { header "A.h" }' > %t/sys/module.modulemap +// RUN: echo 'module B { header "B.h" }' >> %t/sys/module.modulemap + +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fsyntax-only %s \ +// RUN: -isystem %t/sys -Rmodule-build 2>&1 | FileCheck %s + +@import A; + +// CHECK: building module 'A' as +// CHECK: building module 'B' as +// CHECK: finished building module 'B' +// CHECK: finished building module 'A' From a90ad0ecb6949f03b9d10f9632d023680859bc74 Mon Sep 17 00:00:00 2001 From: Augusto Noronha Date: Fri, 31 Mar 2023 17:57:11 -0700 Subject: [PATCH 52/88] [lldb] Add Swift/C++ backward interop expression evaluation tests rdar://100284870 --- .../cxx_interop/backward/expressions/Makefile | 7 ++++ .../TestSwiftBackwardInteropExpressions.py | 21 ++++++++++ .../cxx_interop/backward/expressions/main.cpp | 16 ++++++++ .../backward/expressions/swift-types.swift | 39 +++++++++++++++++++ 4 files changed, 83 insertions(+) create mode 100644 lldb/test/API/lang/swift/cxx_interop/backward/expressions/Makefile create mode 100644 lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py create mode 100644 lldb/test/API/lang/swift/cxx_interop/backward/expressions/main.cpp create mode 100644 lldb/test/API/lang/swift/cxx_interop/backward/expressions/swift-types.swift diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/Makefile b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/Makefile new file mode 100644 index 0000000000000..947094ab0c28f --- /dev/null +++ b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/Makefile @@ -0,0 +1,7 @@ +SWIFT_CXX_HEADER := swift-types.h +SWIFT_SOURCES := swift-types.swift +CXX_SOURCES := main.cpp +SWIFT_CXX_INTEROP := 1 +SWIFTFLAGS_EXTRAS = -Xcc -I$(SRCDIR) -parse-as-library +CFLAGS = -I. -g +include Makefile.rules diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py new file mode 100644 index 0000000000000..4a9f2e630f732 --- /dev/null +++ b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py @@ -0,0 +1,21 @@ + +""" +Test that evaluating expressions works on backward interop mode. +""" +from lldbsuite.test.lldbtest import * +from lldbsuite.test.decorators import * + + +class TestSwiftBackwardInteropExpressions(TestBase): + + @swiftTest + def test_func_step_in(self): + self.build() + self.runCmd('setting set target.experimental.swift-enable-cxx-interop true') + _, _, _, _ = lldbutil.run_to_source_breakpoint( + self, 'Break here', lldb.SBFileSpec('main.cpp')) + self.expect('expr swiftFunc()', substrs=["Inside a Swift function"]) + self.expect('expr swiftClass.swiftMethod()', substrs=["Inside a Swift method"]) + self.expect('expr a::SwiftClass::swiftStaticMethod()', substrs=["In a Swift static method"]) + self.expect('expr swiftClass.getSwiftProperty()', substrs=["This is a class with properties"]) + self.expect('expr swiftSubclassAsClass.overrideableMethod()', substrs=["In subclass"]) diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/main.cpp b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/main.cpp new file mode 100644 index 0000000000000..17431c746188a --- /dev/null +++ b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/main.cpp @@ -0,0 +1,16 @@ +#include "swift-types.h" + +using namespace a; + +int main() { + swiftFunc(); + auto swiftClass = SwiftClass::init(); + swiftClass.swiftMethod(); + SwiftClass::swiftStaticMethod(); + swiftClass.getSwiftProperty(); + auto swiftSubclass = SwiftSubclass::init(); + SwiftClass swiftSubclassAsClass = swiftSubclass; + swiftSubclassAsClass.overrideableMethod(); + return 0; // Break here +} + diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/swift-types.swift b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/swift-types.swift new file mode 100644 index 0000000000000..8a74c3e4c28c6 --- /dev/null +++ b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/swift-types.swift @@ -0,0 +1,39 @@ + +public func swiftFunc() -> String{ + return "Inside a Swift function!" +} + +public class SwiftClass { + var field: Int + var arr: [String] + public init() { + field = 42 + arr = ["An", "array", "of", "strings"] + } + + public func swiftMethod() -> String { + return "Inside a Swift method!" + } + + private var _desc = "This is a class with properties!" + public var swiftProperty: String { + get { + return _desc + } + } + + public static func swiftStaticMethod() -> String { + return "In a Swift static method!" + } + + public func overrideableMethod() -> String { + return "In the base class!" + } +} + +public class SwiftSubclass: SwiftClass { + public override func overrideableMethod() -> String { + return "In subclass!" + } +} + From 1c981654f85a318ecbe7c4a7a46ea3aee9e571ff Mon Sep 17 00:00:00 2001 From: Augusto Noronha Date: Sun, 2 Apr 2023 09:48:17 -0700 Subject: [PATCH 53/88] [lldb] Skip TestSwiftBackwardInteropExpressions in Linux --- .../backward/expressions/TestSwiftBackwardInteropExpressions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py index 4a9f2e630f732..4ca2caeecc6eb 100644 --- a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py +++ b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py @@ -8,6 +8,7 @@ class TestSwiftBackwardInteropExpressions(TestBase): + @skipIfLinux @swiftTest def test_func_step_in(self): self.build() From b3555640bcfe362546879f893163a261d1eb918b Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 3 Apr 2023 10:14:27 -0700 Subject: [PATCH 54/88] Mark Swift test as one --- .../frame/var/direct-ivar/swift/TestFrameVarDirectIvarSwift.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lldb/test/API/commands/frame/var/direct-ivar/swift/TestFrameVarDirectIvarSwift.py b/lldb/test/API/commands/frame/var/direct-ivar/swift/TestFrameVarDirectIvarSwift.py index c45094fe70465..e2288f6e044d3 100644 --- a/lldb/test/API/commands/frame/var/direct-ivar/swift/TestFrameVarDirectIvarSwift.py +++ b/lldb/test/API/commands/frame/var/direct-ivar/swift/TestFrameVarDirectIvarSwift.py @@ -5,12 +5,14 @@ class TestCase(TestBase): + @swiftTest @skipUnlessDarwin def test_objc_self(self): self.build() lldbutil.run_to_source_breakpoint(self, "check self", lldb.SBFileSpec("main.swift")) self.expect("frame variable _prop", startstr="(Int) _prop = 30") + @swiftTest @skipUnlessDarwin def test_objc_self_capture_idiom(self): self.build() From 2d4782c9bc23aa9c7fce7eb40370d4e906fe339a Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 3 Apr 2023 10:14:50 -0700 Subject: [PATCH 55/88] Make test more robust against codegen fluctuations --- .../functionalities/data-formatter/swift-unsafe/main.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lldb/test/API/functionalities/data-formatter/swift-unsafe/main.swift b/lldb/test/API/functionalities/data-formatter/swift-unsafe/main.swift index 0c147b7532608..86db72fb60d54 100644 --- a/lldb/test/API/functionalities/data-formatter/swift-unsafe/main.swift +++ b/lldb/test/API/functionalities/data-formatter/swift-unsafe/main.swift @@ -103,6 +103,7 @@ func main() { colors.withUnsafeBufferPointer { let buf = $0 + print("break") //% self.expect("frame variable -d run-target buf", //% patterns=[ //% '\(UnsafeBufferPointer<(.*)\.ColorCode>\) buf = 2 values \(0[xX][0-9a-fA-F]+\) {', @@ -139,6 +140,7 @@ func main() { numbers.withUnsafeBufferPointer { let buf = $0 + print("break") //% self.expect("frame variable -d run-target buf", //% patterns=[ //% '\(UnsafeBufferPointer<(.*)\.Number>\) buf = 2 values \(0[xX][0-9a-fA-F]+\) {', @@ -153,6 +155,7 @@ func main() { bytes.withUnsafeBufferPointer { let buf = $0 let rawbuf = UnsafeRawBufferPointer(buf) + print("break") //% self.expect("frame variable -d run-target rawbuf", //% patterns=[ //% '\(UnsafeRawBufferPointer\) rawbuf = 256 values \(0[xX][0-9a-fA-F]+\) {', @@ -160,6 +163,7 @@ func main() { //% ]) typealias ByteBuffer = UnsafeRawBufferPointer; let alias = rawbuf as ByteBuffer + print("break") //% self.expect("frame variable -d run-target alias", //% patterns=[ //% '\(ByteBuffer\) alias = 256 values \(0[xX][0-9a-fA-F]+\) {', @@ -167,6 +171,7 @@ func main() { //% ]) typealias ByteBufferAlias = ByteBuffer let secondAlias = alias as ByteBufferAlias + print("break") //% self.expect("frame variable -d run-target secondAlias", //% patterns=[ //% '\(ByteBufferAlias\) secondAlias = 256 values \(0[xX][0-9a-fA-F]+\) {', From 458881b39496a5a40369d07ca2e2517a0e5569d1 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Wed, 22 Mar 2023 16:09:30 -0700 Subject: [PATCH 56/88] [libclang][cas] Expose the module and TU cache keys from the libclang dep-scan API (cherry picked from commit b5ca7144d571408eb402d6017256955d442eea4b) --- clang/include/clang-c/Dependencies.h | 12 ++++++++++++ clang/test/Index/Core/scan-deps-cas.m | 3 +++ clang/tools/c-index-test/core_main.cpp | 21 +++++++++++++++------ clang/tools/libclang/CDependencies.cpp | 16 ++++++++++++++++ clang/tools/libclang/libclang.map | 2 ++ 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 955217fa3301a..2334495605fe6 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -525,6 +525,12 @@ CINDEX_LINKAGE CXCStringArray CINDEX_LINKAGE CXCStringArray clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule); +/** + * \returns the \c ActionCache key for this module, if any. + */ +CINDEX_LINKAGE +const char *clang_experimental_DepGraphModule_getCacheKey(CXDepGraphModule); + /** * \returns the number \c CXDepGraphTUCommand objects in the graph. */ @@ -563,6 +569,12 @@ CINDEX_LINKAGE const char * CINDEX_LINKAGE CXCStringArray clang_experimental_DepGraphTUCommand_getBuildArguments(CXDepGraphTUCommand); +/** + * \returns the \c ActionCache key for this translation unit, if any. + */ +CINDEX_LINKAGE const char * + clang_experimental_DepGraphTUCommand_getCacheKey(CXDepGraphTUCommand); + /** * \returns the list of files which this translation unit directly depends on. * diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index 54b13da1e025f..e9909433e7547 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -33,6 +33,7 @@ // CHECK-NEXT: name: ModA // CHECK-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] // CHECK-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap +// CHECK-NEXT: cache-key: [[CASFS_MODA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // CHECK-NEXT: module-deps: // CHECK-NEXT: file-deps: // CHECK-NEXT: [[PREFIX]]/Inputs/module/ModA.h @@ -51,6 +52,7 @@ // CHECK-NEXT: dependencies: // CHECK-NEXT: command 0: // CHECK-NEXT: context-hash: [[HASH_TU:[A-Z0-9]+]] +// CHECK-NEXT: cache-key: [[CASFS_TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // CHECK-NEXT: module-deps: // CHECK-NEXT: ModA:[[HASH_MOD_A]] // CHECK-NEXT: file-deps: @@ -66,6 +68,7 @@ // INCLUDE_TREE: dependencies: // INCLUDE_TREE-NEXT: command 0: // INCLUDE_TREE-NEXT: context-hash: [[HASH_TU:[A-Z0-9]+]] +// INCLUDE_TREE-NEXT: cache-key: [[INC_TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // INCLUDE_TREE-NEXT: module-deps: // INCLUDE_TREE-NEXT: file-deps: // INCLUDE_TREE-NEXT: [[PREFIX]]/scan-deps-cas.m diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index a7cade1af6ec2..d9aa02fce3228 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -753,10 +753,13 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, unsigned CommandIndex = 0; auto HandleCommand = [&](const char *ContextHash, CXCStringArray ModuleDeps, - CXCStringArray FileDeps, CXCStringArray Args) { + CXCStringArray FileDeps, CXCStringArray Args, + const char *CacheKey) { llvm::outs() << " command " << CommandIndex++ << ":\n"; - llvm::outs() << " context-hash: " << ContextHash << "\n" - << " module-deps:\n"; + llvm::outs() << " context-hash: " << ContextHash << "\n"; + if (CacheKey) + llvm::outs() << " cache-key: " << CacheKey << "\n"; + llvm::outs() << " module-deps:\n"; for (const auto &ModuleName : llvm::makeArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) llvm::outs() << " " << ModuleName << "\n"; @@ -794,6 +797,8 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DepGraphModule_getContextHash(Mod); const char *ModuleMapPath = clang_experimental_DepGraphModule_getModuleMapPath(Mod); + const char *ModuleCacheKey = + clang_experimental_DepGraphModule_getCacheKey(Mod); CXCStringArray ModuleDeps = clang_experimental_DepGraphModule_getModuleDeps(Mod); CXCStringArray FileDeps = @@ -806,8 +811,10 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, << " name: " << Name << "\n" << " context-hash: " << ContextHash << "\n" << " module-map-path: " - << (ModuleMapPath ? ModuleMapPath : "") << "\n" - << " module-deps:\n"; + << (ModuleMapPath ? ModuleMapPath : "") << "\n"; + if (ModuleCacheKey) + llvm::outs() << " cache-key: " << ModuleCacheKey << "\n"; + llvm::outs() << " module-deps:\n"; for (const auto &ModuleName : ArrayRef(ModuleDeps.Strings, ModuleDeps.Count)) llvm::outs() << " " << ModuleName << "\n"; @@ -834,9 +841,11 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DepGraph_getTUCommand(Graph, I); CXCStringArray Args = clang_experimental_DepGraphTUCommand_getBuildArguments(Cmd); + const char *CacheKey = + clang_experimental_DepGraphTUCommand_getCacheKey(Cmd); auto Dispose = llvm::make_scope_exit( [&]() { clang_experimental_DepGraphTUCommand_dispose(Cmd); }); - HandleCommand(TUContextHash, TUModuleDeps, TUFileDeps, Args); + HandleCommand(TUContextHash, TUModuleDeps, TUFileDeps, Args, CacheKey); } return 0; } diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 925901bb09ed5..6208f36cf97a1 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -546,6 +546,14 @@ clang_experimental_DepGraphModule_getBuildArguments(CXDepGraphModule CXDepMod) { return unwrap(CXDepMod)->StrMgr.createCStringsRef(ModDeps.BuildArguments); } +const char * +clang_experimental_DepGraphModule_getCacheKey(CXDepGraphModule CXDepMod) { + ModuleDeps &ModDeps = *unwrap(CXDepMod)->ModDeps; + if (ModDeps.ModuleCacheKey) + return ModDeps.ModuleCacheKey->c_str(); + return nullptr; +} + size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph Graph) { TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; return TUDeps.Commands.size(); @@ -573,6 +581,14 @@ CXCStringArray clang_experimental_DepGraphTUCommand_getBuildArguments( return unwrap(CXCmd)->StrMgr.createCStringsRef(TUCmd.Arguments); } +const char * +clang_experimental_DepGraphTUCommand_getCacheKey(CXDepGraphTUCommand CXCmd) { + Command &TUCmd = *unwrap(CXCmd)->TUCmd; + if (TUCmd.TUCacheKey) + return TUCmd.TUCacheKey->c_str(); + return nullptr; +} + CXCStringArray clang_experimental_DepGraph_getTUFileDeps(CXDepGraph Graph) { TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; return unwrap(Graph)->StrMgr.createCStringsRef(TUDeps.FileDeps); diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 00ded6d77e700..1b7035d732ee1 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -503,6 +503,7 @@ LLVM_16 { clang_experimental_DepGraph_getTUModuleDeps; clang_experimental_DepGraphModule_dispose; clang_experimental_DepGraphModule_getBuildArguments; + clang_experimental_DepGraphModule_getCacheKey; clang_experimental_DepGraphModule_getContextHash; clang_experimental_DepGraphModule_getFileDeps; clang_experimental_DepGraphModule_getModuleDeps; @@ -510,6 +511,7 @@ LLVM_16 { clang_experimental_DepGraphModule_getName; clang_experimental_DepGraphTUCommand_dispose; clang_experimental_DepGraphTUCommand_getBuildArguments; + clang_experimental_DepGraphTUCommand_getCacheKey; clang_experimental_DepGraphTUCommand_getExecutable; clang_getUnqualifiedType; clang_getNonReferenceType; From df651826fcb7b04acaf5f4888e87ac1caa599495 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Mon, 3 Apr 2023 07:54:07 -0700 Subject: [PATCH 57/88] [libclang][cas] Avoid using `CachingOnDiskFileSystem` when include-tree output is requested It has overhead that is not needed for the include-tree mechanism. (cherry picked from commit 32bd17b4dcc20baf665fc94b9cff0a2a4c00e3c6) --- clang/tools/libclang/CDependencies.cpp | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 925901bb09ed5..f3bc1a4eb319c 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -23,6 +23,7 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "llvm/CAS/CASProvidingFileSystem.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/Process.h" @@ -145,7 +146,7 @@ ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const { return ScanningOutputFormat::FullTree; // Note: default caching behaviour is currently cas-fs. - return ConfiguredFormat; + return ScanningOutputFormat::FullTree; } CXDependencyScannerService @@ -155,15 +156,18 @@ clang_experimental_DependencyScannerService_create_v1( std::shared_ptr CAS = unwrap(Opts)->CAS; std::shared_ptr Cache = unwrap(Opts)->Cache; IntrusiveRefCntPtr FS; - if (CAS && Cache) { + ScanningOutputFormat Format = unwrap(Opts)->getFormat(); + bool IsCASFSOutput = Format == ScanningOutputFormat::Tree || + Format == ScanningOutputFormat::FullTree; + if (CAS && Cache && IsCASFSOutput) { assert(unwrap(Opts)->CASOpts.getKind() != CASOptions::UnknownCAS && "CAS and ActionCache must match CASOptions"); FS = llvm::cantFail( llvm::cas::createCachingOnDiskFileSystem(CAS)); } return wrap(new DependencyScanningService( - ScanningMode::DependencyDirectivesScan, unwrap(Opts)->getFormat(), - unwrap(Opts)->CASOpts, std::move(CAS), std::move(Cache), std::move(FS), + ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts, + std::move(CAS), std::move(Cache), std::move(FS), /*ReuseFilemanager=*/false)); } @@ -195,8 +199,16 @@ void clang_experimental_FileDependenciesList_dispose( CXDependencyScannerWorker clang_experimental_DependencyScannerWorker_create_v0( CXDependencyScannerService Service) { - return wrap(new DependencyScanningWorker( - *unwrap(Service), llvm::vfs::createPhysicalFileSystem())); + ScanningOutputFormat Format = unwrap(Service)->getFormat(); + bool IsIncludeTreeOutput = Format == ScanningOutputFormat::IncludeTree || + Format == ScanningOutputFormat::FullIncludeTree; + llvm::IntrusiveRefCntPtr FS = + llvm::vfs::createPhysicalFileSystem(); + if (IsIncludeTreeOutput) + FS = llvm::cas::createCASProvidingFileSystem(unwrap(Service)->getCAS(), + std::move(FS)); + + return wrap(new DependencyScanningWorker(*unwrap(Service), FS)); } void clang_experimental_DependencyScannerWorker_dispose_v0( From 26da45ba1ba35ac0c36242cb371d7fe212948ff2 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 3 Apr 2023 13:54:50 -0700 Subject: [PATCH 58/88] Make test more robust --- lldb/test/API/lang/swift/generic_class/main.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/test/API/lang/swift/generic_class/main.swift b/lldb/test/API/lang/swift/generic_class/main.swift index a50d00f1123a1..972a676818104 100644 --- a/lldb/test/API/lang/swift/generic_class/main.swift +++ b/lldb/test/API/lang/swift/generic_class/main.swift @@ -11,7 +11,7 @@ protocol P { } extension F : P { @inline(never) func method() { - print("break here \(b)") + print("break here \(b) \(self)") } } From de3346a03204c463e7c761613c72d5fcc92177e8 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 3 Apr 2023 13:54:53 -0700 Subject: [PATCH 59/88] Make test more robust. It's not guaranteed that a Variable remains usable after continuing the process --- .../TestSwiftConsumeOperator.py | 165 ++++++++---------- 1 file changed, 71 insertions(+), 94 deletions(-) diff --git a/lldb/test/API/lang/swift/variables/consume_operator/TestSwiftConsumeOperator.py b/lldb/test/API/lang/swift/variables/consume_operator/TestSwiftConsumeOperator.py index f0261c14e8be4..02fc4d7542fca 100644 --- a/lldb/test/API/lang/swift/variables/consume_operator/TestSwiftConsumeOperator.py +++ b/lldb/test/API/lang/swift/variables/consume_operator/TestSwiftConsumeOperator.py @@ -71,95 +71,87 @@ def get_var(self, name): return frame.FindVariable(name) def do_check_copyable_value_test(self): - # We haven't defined varK yet. - varK = self.get_var('k') - - self.assertIsNone(varK.value, "varK initialized too early?!") + # We haven't defined k yet. + self.assertIsNone(self.get_var('k').value, "k initialized too early?!") # Go to break point 2. k should be valid. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to breakpoint 3. k should no longer be valid. self.process.Continue() - self.assertIsNone(varK.value, "K is live but was consumed?!") + self.assertIsNone(self.get_var('k').value, "K is live but was consumed?!") # Run so we hit the next breakpoint to jump to the next test's # breakpoint. self.process.Continue() def do_check_copyable_var_test(self): - # We haven't defined varK yet. - varK = self.get_var('k') - self.assertIsNone(varK.value, "varK initialized too early?!") + # We haven't defined k yet. + self.assertIsNone(self.get_var('k').value, "k initialized too early?!") # Go to break point 2. k should be valid. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to breakpoint 3. We invalidated k self.process.Continue() - self.assertIsNone(varK.value, "K is live but was consumed?!") + self.assertIsNone(self.get_var('k').value, "K is live but was consumed?!") # Go to the last breakpoint and make sure that k is reinitialized # properly. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized") # Run so we hit the next breakpoint to go to the next test. self.process.Continue() def do_check_addressonly_value_test(self): - # We haven't defined varK yet. - varK = self.get_var('k') - + # We haven't defined k yet. # Go to break point 2. k should be valid and m should not be. Since M is # a dbg.declare it is hard to test robustly that it is not initialized # so we don't do so. We have an additional llvm.dbg.addr test where we # move the other variable and show the correct behavior with # llvm.dbg.declare. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "var not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "var not initialized?!") # Go to breakpoint 3. self.process.Continue() - self.assertEqual(varK.unsigned, 0, - "dbg thinks varK is live despite move?!") + self.assertEqual(self.get_var('k').unsigned, 0, + "dbg thinks k is live despite move?!") # Run so we hit the next breakpoint as part of the next test. self.process.Continue() def do_check_addressonly_var_test(self): - varK = self.get_var('k') - # Go to break point 2. k should be valid. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to breakpoint 3. K was invalidated. self.process.Continue() - self.assertIsNone(varK.value, "K is live but was consumed?!") + self.assertIsNone(self.get_var('k').value, "K is live but was consumed?!") # Go to the last breakpoint and make sure that k is reinitialized # properly. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized") # Run so we hit the next breakpoint as part of the next test. self.process.Continue() def do_check_copyable_value_arg_test(self): # k is defined by the argument so it is valid. - varK = self.get_var('k') - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to break point 2. k should be valid. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to breakpoint 3. k should no longer be valid. self.process.Continue() - #self.assertIsNone(varK.value, "K is live but was consumed?!") + #self.assertIsNone(self.get_var('k').value, "K is live but was consumed?!") # Run so we hit the next breakpoint to jump to the next test's # breakpoint. @@ -167,29 +159,27 @@ def do_check_copyable_value_arg_test(self): def do_check_copyable_var_arg_test(self): # k is already defined and is an argument. - varK = self.get_var('k') - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to break point 2. k should be valid. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to breakpoint 3. We invalidated k self.process.Continue() - self.assertIsNone(varK.value, "K is live but was consumed?!") + self.assertIsNone(self.get_var('k').value, "K is live but was consumed?!") # Go to the last breakpoint and make sure that k is reinitialized # properly. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized") # Run so we hit the next breakpoint to go to the next test. self.process.Continue() def do_check_addressonly_value_arg_test(self): # k is defined since it is an argument. - varK = self.get_var('k') - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to break point 2. k should be valid and m should not be. Since M is # a dbg.declare it is hard to test robustly that it is not initialized @@ -197,27 +187,26 @@ def do_check_addressonly_value_arg_test(self): # move the other variable and show the correct behavior with # llvm.dbg.declare. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "var not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "var not initialized?!") # Go to breakpoint 3. self.process.Continue() - self.assertEqual(varK.unsigned, 0, - "dbg thinks varK is live despite move?!") + self.assertEqual(self.get_var('k').unsigned, 0, + "dbg thinks k is live despite move?!") # Run so we hit the next breakpoint as part of the next test. self.process.Continue() def do_check_addressonly_var_arg_test(self): - varK = self.get_var('k') - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to break point 2. k should be valid. self.process.Continue() - self.assertGreater(varK.unsigned, 0, "varK not initialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized?!") # Go to breakpoint 3. K was invalidated. self.process.Continue() - self.assertIsNone(varK.value, "K is live but was consumed?!") + self.assertIsNone(self.get_var('k').value, "K is live but was consumed?!") # Go to the last breakpoint and make sure that k is reinitialized # properly. @@ -225,170 +214,158 @@ def do_check_addressonly_var_arg_test(self): # There is some sort of bug here. We should have the value here. For now # leave the next line commented out and validate we are not seeing the # value so we can detect change in behavior. - self.assertGreater(varK.unsigned, 0, "varK not initialized") + self.assertGreater(self.get_var('k').unsigned, 0, "k not initialized") # Run so we hit the next breakpoint as part of the next test. self.process.Continue() def do_check_copyable_value_ccf_true(self): - varK = self.get_var('k') - - # Check at our start point that we do not have any state for varK and + # Check at our start point that we do not have any state for k and # then continue to our next breakpoint. - self.assertIsNone(varK.value, "varK should not have a value?!") + self.assertIsNone(self.get_var('k').value, "k should not have a value?!") self.process.Continue() # At this breakpoint, k should be defined since we are going to do # something with it. - self.assertIsNotNone(varK.value, "varK should have a value?!") + self.assertIsNotNone(self.get_var('k').value, "k should have a value?!") self.process.Continue() # At this breakpoint, we are now in the conditional control flow part of # the loop. Make sure that we can see k still. - self.assertIsNotNone(varK.value, "varK should have a value?!") + self.assertIsNotNone(self.get_var('k').value, "k should have a value?!") self.process.Continue() # Ok, we just performed the move. k should not be no longer initialized. - self.assertIsNone(varK.value, "varK should not have a value?!") + self.assertIsNone(self.get_var('k').value, "k should not have a value?!") self.process.Continue() # Finally we left the conditional control flow part of the function. k # should still be None. - self.assertIsNone(varK.value, "varK should not have a value!") + self.assertIsNone(self.get_var('k').value, "k should not have a value!") # Run again so we go and run to the next test. self.process.Continue() def do_check_copyable_value_ccf_false(self): - varK = self.get_var('k') - - # Check at our start point that we do not have any state for varK and + # Check at our start point that we do not have any state for k and # then continue to our next breakpoint. - self.assertIsNone(varK.value, "varK should not have a value?!") + self.assertIsNone(self.get_var('k').value, "k should not have a value?!") self.process.Continue() # At this breakpoint, k should be defined since we are going to do # something with it. - self.assertIsNotNone(varK.value, "varK should have a value?!") + self.assertIsNotNone(self.get_var('k').value, "k should have a value?!") self.process.Continue() # At this breakpoint, we are now past the end of the conditional # statement. We know due to the move checking that k can not have any # uses that are reachable from the move. So it is safe to always not # provide the value here. - self.assertIsNone(varK.value, "varK should have a value?!") + self.assertIsNone(self.get_var('k').value, "k should have a value?!") # Run again so we go and run to the next test. self.process.Continue() def do_check_copyable_var_ccf_true_reinit_out_block(self): - varK = self.get_var('k') - # At first we should not have a value for k. - self.assertEqual(varK.unsigned, 0, "varK should be nullptr!") + self.assertEqual(self.get_var('k').unsigned, 0, "k should be nullptr!") self.process.Continue() # Now we are in the conditional true block. K should be defined since we # are on the move itself. - self.assertGreater(varK.unsigned, 0, "varK should not be nullptr!") + self.assertGreater(self.get_var('k').unsigned, 0, "k should not be nullptr!") self.process.Continue() # Now we have executed the move and we are about to run code using # m. Make sure that K is not available! - self.assertEqual(varK.unsigned, 0, - "varK was already consumed! Should be nullptr") + self.assertEqual(self.get_var('k').unsigned, 0, + "k was already consumed! Should be nullptr") self.process.Continue() # We are now out of the conditional lexical block on the line of code # that redefines k. k should still be not available. - self.assertEqual(varK.unsigned, 0, - "varK was already consumed! Should be nullptr") + self.assertEqual(self.get_var('k').unsigned, 0, + "k was already consumed! Should be nullptr") self.process.Continue() # Ok, we have now reinit k and are about to call a method on it. We # should be valid now. - self.assertGreater(varK.unsigned, 0, - "varK should have be reinitialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, + "k should have be reinitialized?!") # Run again so we go and run to the next test. self.process.Continue() def do_check_copyable_var_ccf_true_reinit_in_block(self): - varK = self.get_var('k') - # At first we should not have a value for k. - self.assertEqual(varK.unsigned, 0, "varK should be nullptr!") + self.assertEqual(self.get_var('k').unsigned, 0, "k should be nullptr!") self.process.Continue() # Now we are in the conditional true block. K should be defined since we # are on the move itself. - self.assertGreater(varK.unsigned, 0, "varK should not be nullptr!") + self.assertGreater(self.get_var('k').unsigned, 0, "k should not be nullptr!") self.process.Continue() # Now we have executed the move and we are about to reinit k but have # not yet. Make sure we are not available! - self.assertEqual(varK.unsigned, 0, - "varK was already consumed! Should be nullptr") + self.assertEqual(self.get_var('k').unsigned, 0, + "k was already consumed! Should be nullptr") self.process.Continue() # We are now still inside the conditional part of the code, but have - # reinitialized varK. - self.assertGreater(varK.unsigned, 0, - "varK was reinit! Should be valid value!") + # reinitialized k. + self.assertGreater(self.get_var('k').unsigned, 0, + "k was reinit! Should be valid value!") self.process.Continue() # We now have left the conditional part of the function. k should still # be available. - self.assertGreater(varK.unsigned, 0, - "varK should have be reinitialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, + "k should have be reinitialized?!") # Run again so we go and run to the next test. self.process.Continue() def do_check_copyable_var_ccf_false_reinit_out_block(self): - varK = self.get_var('k') - # At first we should not have a value for k. - self.assertEqual(varK.unsigned, 0, "varK should be nullptr!") + self.assertEqual(self.get_var('k').unsigned, 0, "k should be nullptr!") self.process.Continue() - # Now we are right above the beginning of the false check. varK should + # Now we are right above the beginning of the false check. k should # still be valid. - self.assertGreater(varK.unsigned, 0, "varK should not be nullptr!") + self.assertGreater(self.get_var('k').unsigned, 0, "k should not be nullptr!") self.process.Continue() # Now we are after the conditional part of the code on the reinit # line. Since this is reachable from the move and we haven't reinit yet, # k should not be available. - self.assertEqual(varK.unsigned, 0, - "varK was already consumed! Should be nullptr") + self.assertEqual(self.get_var('k').unsigned, 0, + "k was already consumed! Should be nullptr") self.process.Continue() # Ok, we have now reinit k and are about to call a method on it. We # should be valid now. - self.assertGreater(varK.unsigned, 0, - "varK should have be reinitialized?!") + self.assertGreater(self.get_var('k').unsigned, 0, + "k should have be reinitialized?!") # Run again so we go and run to the next test. self.process.Continue() def do_check_copyable_var_ccf_false_reinit_in_block(self): - varK = self.get_var('k') - # At first we should not have a value for k. - self.assertEqual(varK.unsigned, 0, "varK should be nullptr!") + self.assertEqual(self.get_var('k').unsigned, 0, "k should be nullptr!") self.process.Continue() - # Now we are on the doSomething above the false check. So varK should be + # Now we are on the doSomething above the false check. So k should be # valid. - self.assertGreater(varK.unsigned, 0, "varK should not be nullptr!") + self.assertGreater(self.get_var('k').unsigned, 0, "k should not be nullptr!") self.process.Continue() # Now we are after the conditional scope. Since k was reinitialized in - # the conditional scope, along all paths we are valid so varK should + # the conditional scope, along all paths we are valid so k should # still be available. - self.assertGreater(varK.unsigned, 0, - "varK should not be nullptr?!") + self.assertGreater(self.get_var('k').unsigned, 0, + "k should not be nullptr?!") # Run again so we go and run to the next test. self.process.Continue() From 043436de5c1bc59d47a2645fc651d1d042205f35 Mon Sep 17 00:00:00 2001 From: Jason Molenda Date: Mon, 3 Apr 2023 13:49:51 -0700 Subject: [PATCH 60/88] Using global variable in xnu kernel, set # of addressable bits The kernel has a global variable with the TCR_EL1.T1SZ value, from which was can calculate the number of addressable bits. Find that symbol in DynamicLoaderDarwinKernel and set the bits to that value for this Process. Differential Revision: https://reviews.llvm.org/D147462 rdar://107445318 (cherry picked from commit 8b092714c304defb00876a01995af3b114dadc92) --- .../DynamicLoaderDarwinKernel.cpp | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp index f1b30477b89c3..b7a38e9f41100 100644 --- a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp +++ b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp @@ -1011,6 +1011,7 @@ void DynamicLoaderDarwinKernel::LoadKernelModuleIfNeeded() { if (m_kernel.IsLoaded() && m_kernel.GetModule()) { static ConstString kext_summary_symbol("gLoadedKextSummaries"); + static ConstString arm64_T1Sz_value("gT1Sz"); const Symbol *symbol = m_kernel.GetModule()->FindFirstSymbolWithNameAndType( kext_summary_symbol, eSymbolTypeData); @@ -1019,6 +1020,36 @@ void DynamicLoaderDarwinKernel::LoadKernelModuleIfNeeded() { // Update all image infos ReadAllKextSummaries(); } + // If the kernel global with the T1Sz setting is available, + // update the target.process.virtual-addressable-bits to be correct. + symbol = m_kernel.GetModule()->FindFirstSymbolWithNameAndType( + arm64_T1Sz_value, eSymbolTypeData); + if (symbol) { + const uint32_t orig_bits_value = m_process->GetVirtualAddressableBits(); + // Mark all bits as addressable so we don't strip any from our + // memory read below, with an incorrect default value. + // b55 is the sign extension bit with PAC, b56:63 are TBI, + // don't mark those as addressable. + m_process->SetVirtualAddressableBits(55); + Status error; + // gT1Sz is 8 bytes. We may run on a stripped kernel binary + // where we can't get the size accurately. Hardcode it. + const size_t sym_bytesize = 8; // size of gT1Sz value + uint64_t sym_value = + m_process->GetTarget().ReadUnsignedIntegerFromMemory( + symbol->GetAddress(), sym_bytesize, 0, error); + if (error.Success()) { + // 64 - T1Sz is the highest bit used for auth. + // The value we pass in to SetVirtualAddressableBits is + // the number of bits used for addressing, so if + // T1Sz is 25, then 64-25 == 39, bits 0..38 are used for + // addressing, bits 39..63 are used for PAC/TBI or whatever. + uint32_t virt_addr_bits = 64 - sym_value; + m_process->SetVirtualAddressableBits(virt_addr_bits); + } else { + m_process->SetVirtualAddressableBits(orig_bits_value); + } + } } else { m_kernel.Clear(); } From 636cdde5947e1856a6cbf8b315f585932356688f Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 3 Apr 2023 13:58:49 -0700 Subject: [PATCH 61/88] Make test more robust --- .../swift/variables/inout/TestInOutVariables.py | 16 ++++++++-------- .../API/lang/swift/variables/inout/main.swift | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lldb/test/API/lang/swift/variables/inout/TestInOutVariables.py b/lldb/test/API/lang/swift/variables/inout/TestInOutVariables.py index e22d4b7147b29..aadd864ee7e2b 100644 --- a/lldb/test/API/lang/swift/variables/inout/TestInOutVariables.py +++ b/lldb/test/API/lang/swift/variables/inout/TestInOutVariables.py @@ -72,19 +72,19 @@ def check_class_internal( ivar = x_actual.GetChildAtIndex(0).GetChildAtIndex(0) ovar = x_actual.GetChildAtIndex(1) - self.assertTrue( - ivar.GetName() == "ivar", "Name: %s is not ivar %s" % + self.assertEquals( + ivar.GetName(), "ivar", "Name: %s is not ivar %s" % (ivar.GetName(), message_end)) - self.assertTrue( - ovar.GetName() == "ovar", + self.assertEquals( + ovar.GetName(), "ovar", "ovar is not ovar %s" % (message_end)) - self.assertTrue( - ivar.GetValue() == ivar_value, + self.assertEquals( + ivar.GetValue(), ivar_value, "ivar wrong %s" % (message_end)) - self.assertTrue( - ovar.GetValue() == ovar_value, + self.assertEquals( + ovar.GetValue(), ovar_value, "ovar wrong %s" % (message_end)) diff --git a/lldb/test/API/lang/swift/variables/inout/main.swift b/lldb/test/API/lang/swift/variables/inout/main.swift index 2daecca7b9a1b..9fbd151491ef1 100644 --- a/lldb/test/API/lang/swift/variables/inout/main.swift +++ b/lldb/test/API/lang/swift/variables/inout/main.swift @@ -39,13 +39,13 @@ struct Struct { } func foo (_ x: inout Class) { - print(x.ivar) - x.ivar += 1 // Set breakpoint here for Class access + print(x.ivar) // Set breakpoint here for Class access + x.ivar += 1 } func foo(_ x: inout Struct) { - print(x.ivar) - x.ivar += 1 // Set breakpoint here for Struct access + print(x.ivar) // Set breakpoint here for Struct access + x.ivar += 1 } func fn_ptrs (_ str: Struct) { From af4983e222955bca86edd0a8aa687d20890afac8 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 3 Apr 2023 14:28:59 -0700 Subject: [PATCH 62/88] Relax stepping test --- lldb/test/API/lang/swift/stepping/TestSwiftStepping.py | 9 +++++++-- lldb/test/API/lang/swift/stepping/main.swift | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lldb/test/API/lang/swift/stepping/TestSwiftStepping.py b/lldb/test/API/lang/swift/stepping/TestSwiftStepping.py index 9507c7a4dc302..0e4bc818f4681 100644 --- a/lldb/test/API/lang/swift/stepping/TestSwiftStepping.py +++ b/lldb/test/API/lang/swift/stepping/TestSwiftStepping.py @@ -200,9 +200,10 @@ def do_test(self): thread.StepOver() self.hit_correct_line(thread, "At point initializer.") thread.StepOver() - self.hit_correct_line (thread, "At the beginning of the switch.") + stopped_at_switch = self.hit_correct_line (thread, "At the beginning of the switch.", False) + if stopped_at_switch: + thread.StepOver() - thread.StepOver() stopped_at_case = self.hit_correct_line( thread, "case (let x, let y) where", False) if stopped_at_case: @@ -312,6 +313,10 @@ def do_test(self): if stop_on_partial_apply: thread.StepOver() + stop_at_brace = self.hit_correct_line(thread, "Opening brace", False) + if stop_at_brace: + thread.StepOver() + self.hit_correct_line(thread, "doSomethingWithFunction(cd_maker, 10)") thread.StepInto() diff --git a/lldb/test/API/lang/swift/stepping/main.swift b/lldb/test/API/lang/swift/stepping/main.swift index cd9352805c4cd..535e707aa1bee 100644 --- a/lldb/test/API/lang/swift/stepping/main.swift +++ b/lldb/test/API/lang/swift/stepping/main.swift @@ -205,7 +205,7 @@ func main () -> Void indirect.protocol_func(20) var cd_maker = - { + { // Opening brace (arg : Int) -> ConformsDirectly in // Step into cd_maker stops at closure decl instead. return ConformsDirectly(arg) // Step into should stop here in closure. } From 2d2bb9c7a619033e1de6259383a3821de88fd2c0 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Fri, 31 Mar 2023 16:01:23 -0700 Subject: [PATCH 63/88] [clang][cas] Extend fix for private modules to module builds In 6034ccd1ea9 we fixed an issue where if a public module is imported via PCH we cannot import the private module in a TU because the modulemap is not parsed. This is also an issue when building a module that imports the private module and the TU used a PCH, because even though we don't import the PCH itself, we still use the prebuilt/explicit module found via the PCH instead of building an implicit scanner module. This commit generalizes the original fix to handle all prebuilt/explicit modules. rdar://107446573 (cherry picked from commit 159fe25a5162f09a64add08b39ba6fc5f88043f7) --- .../IncludeTreeActionController.cpp | 92 ++++++++++--------- .../modules-include-tree-pch-with-private.c | 57 ++++++++++++ 2 files changed, 104 insertions(+), 45 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 95cb66b067f5e..563afd75c68d4 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -141,30 +141,33 @@ class IncludeTreeBuilder { Optional ErrorToReport; }; -/// A utility for adding \c PPCallbacks to a compiler instance at the -/// appropriate time. -struct PPCallbacksDependencyCollector : public DependencyCollector { - using MakeCB = +/// A utility for adding \c PPCallbacks and/or \cASTReaderListener to a compiler +/// instance at the appropriate time. +struct AttachOnlyDependencyCollector : public DependencyCollector { + using MakePPCB = llvm::unique_function(Preprocessor &)>; - MakeCB Create; - PPCallbacksDependencyCollector(MakeCB Create) : Create(std::move(Create)) {} + using MakeASTReaderL = + llvm::unique_function(ASTReader &R)>; + MakePPCB CreatePPCB; + MakeASTReaderL CreateASTReaderL; + AttachOnlyDependencyCollector(MakePPCB CreatePPCB, MakeASTReaderL CreateL) + : CreatePPCB(std::move(CreatePPCB)), + CreateASTReaderL(std::move(CreateL)) {} + void attachToPreprocessor(Preprocessor &PP) final { - std::unique_ptr CB = Create(PP); - assert(CB); - PP.addPPCallbacks(std::move(CB)); + if (CreatePPCB) { + std::unique_ptr CB = CreatePPCB(PP); + assert(CB); + PP.addPPCallbacks(std::move(CB)); + } } -}; -/// A utility for adding \c ASTReaderListener to a compiler instance at the -/// appropriate time. -struct ASTReaderListenerDependencyCollector : public DependencyCollector { - using MakeL = - llvm::unique_function(ASTReader &R)>; - MakeL Create; - ASTReaderListenerDependencyCollector(MakeL Create) : Create(std::move(Create)) {} + void attachToASTReader(ASTReader &R) final { - std::unique_ptr L = Create(R); - assert(L); - R.addListener(std::move(L)); + if (CreateASTReaderL) { + std::unique_ptr L = CreateASTReaderL(R); + assert(L); + R.addListener(std::move(L)); + } } }; @@ -243,27 +246,29 @@ struct IncludeTreePPCallbacks : public PPCallbacks { /// MODULE_DIRECTORY record, and ignores the result. class LookupPCHModulesListener : public ASTReaderListener { public: - LookupPCHModulesListener(Preprocessor &PP) : PP(PP) {} + LookupPCHModulesListener(ASTReader &R) : Reader(R) {} private: - void ReadModuleName(StringRef ModuleName) final { - if (PCHFinished) - return; - // Match MODULE_DIRECTORY: allow full search and ignore failure to find - // the module. - (void)PP.getHeaderSearchInfo().lookupModule( - ModuleName, SourceLocation(), /*AllowSearch=*/true, - /*AllowExtraModuleMapSearch=*/true); - } void visitModuleFile(StringRef Filename, serialization::ModuleKind Kind) final { - if (Kind == serialization::MK_PCH) - PCHFinished = true; + // Any prebuilt or explicit modules seen during scanning are "full" modules + // rather than implicitly built scanner modules. + if (Kind == serialization::MK_PrebuiltModule || + Kind == serialization::MK_ExplicitModule) { + serialization::ModuleManager &Manager = Reader.getModuleManager(); + serialization::ModuleFile *MF = Manager.lookupByFileName(Filename); + assert(MF && "module file missing in visitModuleFile"); + // Match MODULE_DIRECTORY: allow full search and ignore failure to find + // the module. + HeaderSearch &HS = Reader.getPreprocessor().getHeaderSearchInfo(); + (void)HS.lookupModule(MF->ModuleName, SourceLocation(), + /*AllowSearch=*/true, + /*AllowExtraModuleMapSearch=*/true); + } } private: - Preprocessor &PP; - bool PCHFinished = false; + ASTReader &Reader; }; } // namespace @@ -316,21 +321,15 @@ Error IncludeTreeActionController::initialize( // Attach callbacks for the IncludeTree of the TU. The preprocessor // does not exist yet, so we need to indirect this via DependencyCollector. - auto DC = std::make_shared( + auto DC = std::make_shared( [&Builder = current()](Preprocessor &PP) { return std::make_unique(Builder, PP); + }, + [](ASTReader &R) { + return std::make_unique(R); }); ScanInstance.addDependencyCollector(std::move(DC)); - // Attach callback for modules loaded via PCH. - if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) { - auto DC = std::make_shared( - [&](ASTReader &R) { - return std::make_unique(R.getPreprocessor()); - }); - ScanInstance.addDependencyCollector(std::move(DC)); - } - // Enable caching in the resulting commands. ScanInstance.getFrontendOpts().CacheCompileJob = true; CASOpts = ScanInstance.getCASOpts(); @@ -366,9 +365,12 @@ Error IncludeTreeActionController::initializeModuleBuild( // Attach callbacks for the IncludeTree of the module. The preprocessor // does not exist yet, so we need to indirect this via DependencyCollector. - auto DC = std::make_shared( + auto DC = std::make_shared( [&Builder = current()](Preprocessor &PP) { return std::make_unique(Builder, PP); + }, + [](ASTReader &R) { + return std::make_unique(R); }); ModuleScanInstance.addDependencyCollector(std::move(DC)); diff --git a/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c index 99910c5bb62f0..1f24620484ace 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c +++ b/clang/test/ClangScanDeps/modules-include-tree-pch-with-private.c @@ -14,8 +14,10 @@ // RUN: > %t/deps_pch.json // RUN: %deps-to-rsp %t/deps_pch.json --module-name Mod > %t/Mod.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Indirect1 > %t/Indirect1.rsp // RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp // RUN: %clang @%t/Mod.rsp +// RUN: %clang @%t/Indirect1.rsp // RUN: %clang @%t/pch.rsp // RUN: clang-scan-deps -compilation-database %t/cdb.json \ @@ -24,10 +26,39 @@ // RUN: > %t/deps.json // RUN: %deps-to-rsp %t/deps.json --module-name Mod_Private > %t/Mod_Private.rsp +// RUN: %deps-to-rsp %t/deps.json --module-name Indirect2 > %t/Indirect2.rsp // RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp // RUN: %clang @%t/Mod_Private.rsp +// RUN: %clang @%t/Indirect2.rsp // RUN: %clang @%t/tu.rsp +// Extract include-tree casids +// RUN: cat %t/Indirect2.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/Indirect.casid +// RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid + +// RUN: echo "MODULE Indirect2" > %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/Indirect.casid >> %t/result.txt +// RUN: echo "TRANSLATION UNIT" >> %t/result.txt +// RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid >> %t/result.txt + +// Explicitly check that Mod_Private is imported as a module and not a header. +// RUN: FileCheck %s -DPREFIX=%/t -input-file %t/result.txt + +// CHECK-LABEL: MODULE Indirect2 +// CHECK: llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 [[PREFIX]]/indirect2.h llvmcas:// +// CHECK: Submodule: Indirect2 +// CHECK: 2:1 (Module) Indirect1 +// CHECK: 3:1 (Module) Mod_Private + +// CHECK-LABEL: TRANSLATION UNIT +// CHECK: (PCH) llvmcas:// +// CHECK: [[PREFIX]]/tu.m llvmcas:// +// CHECK: 1:1 llvmcas:// +// CHECK: 2:1 (Module) Mod_Private +// CHECK: 3:1 (Module) Indirect2 + //--- cdb.json.template [{ "file": "DIR/tu.m", @@ -54,12 +85,38 @@ void pub(void); //--- Mod.framework/PrivateHeaders/Priv.h void priv(void); +//--- module.modulemap +module Indirect1 { + header "indirect1.h" + export * +} +module Indirect2 { + header "indirect2.h" + export * +} + +//--- indirect1.h +#import + +//--- indirect2.h +#import "indirect1.h" +#import + +static inline void indirect(void) { + pub(); + priv(); +} + //--- prefix.h #import +#import "indirect1.h" //--- tu.m #import +#import "indirect2.h" + void tu(void) { pub(); priv(); + indirect(); } From f158ada0d3877a6957cb5f48c6f2ef1dfe393cf7 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 30 Mar 2023 15:47:53 -0700 Subject: [PATCH 64/88] [clang][deps] Remove -coverage-data-file and -coverage-notes-file from modules When not performing codegen, we can strip the coverage-data-file and coverage-notes-file options to improve canonicalization. rdar://107443796 Differential Revision: https://reviews.llvm.org/D147282 (cherry picked from commit 758bca6483853a743297b68bd88a5dba9d5247f2) --- clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp | 2 ++ .../test/ClangScanDeps/Inputs/removed-args/cdb.json.template | 2 +- clang/test/ClangScanDeps/removed-args.c | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index d157350603a91..960a3c7d7bfd9 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -106,6 +106,8 @@ ModuleDepCollector::makeInvocationForModuleBuildWithoutOutputs( if (!CI.getLangOpts()->ModulesCodegen) { CI.getCodeGenOpts().DebugCompilationDir.clear(); CI.getCodeGenOpts().CoverageCompilationDir.clear(); + CI.getCodeGenOpts().CoverageDataFile.clear(); + CI.getCodeGenOpts().CoverageNotesFile.clear(); } // Map output paths that affect behaviour to "-" so their existence is in the diff --git a/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template b/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template index 2bdc340a3fccd..7ae3c88aedd8d 100644 --- a/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template +++ b/clang/test/ClangScanDeps/Inputs/removed-args/cdb.json.template @@ -1,7 +1,7 @@ [ { "directory": "DIR", - "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-module-maps -fmodules-validate-once-per-build-session -fbuild-session-file=DIR/build-session -fmodules-prune-interval=123 -fmodules-prune-after=123 -fmodules-cache-path=DIR/cache -include DIR/header.h -grecord-command-line -fdebug-compilation-dir=DIR/debug -fcoverage-compilation-dir=DIR/coverage -o DIR/tu.o -serialize-diagnostics DIR/tu.diag -MT tu -MD -MF DIR/tu.d", + "command": "clang -fsyntax-only DIR/tu.c -fmodules -fimplicit-module-maps -fmodules-validate-once-per-build-session -fbuild-session-file=DIR/build-session -fmodules-prune-interval=123 -fmodules-prune-after=123 -fmodules-cache-path=DIR/cache -include DIR/header.h -grecord-command-line -fdebug-compilation-dir=DIR/debug -fcoverage-compilation-dir=DIR/coverage -ftest-coverage -o DIR/tu.o -serialize-diagnostics DIR/tu.diag -MT tu -MD -MF DIR/tu.d", "file": "DIR/tu.c" } ] diff --git a/clang/test/ClangScanDeps/removed-args.c b/clang/test/ClangScanDeps/removed-args.c index 02fbb8cacff2c..9a4ef25838e46 100644 --- a/clang/test/ClangScanDeps/removed-args.c +++ b/clang/test/ClangScanDeps/removed-args.c @@ -23,6 +23,8 @@ // CHECK-NEXT: "-cc1" // CHECK-NOT: "-fdebug-compilation-dir=" // CHECK-NOT: "-fcoverage-compilation-dir=" +// CHECK-NOT: "-coverage-notes-file +// CHECK-NOT: "-coverage-data-file // CHECK-NOT: "-dwarf-debug-flags" // CHECK-NOT: "-main-file-name" // CHECK-NOT: "-include" @@ -46,6 +48,8 @@ // CHECK-NEXT: "-cc1" // CHECK-NOT: "-fdebug-compilation-dir= // CHECK-NOT: "-fcoverage-compilation-dir= +// CHECK-NOT: "-coverage-notes-file +// CHECK-NOT: "-coverage-data-file // CHECK-NOT: "-dwarf-debug-flags" // CHECK-NOT: "-main-file-name" // CHECK-NOT: "-include" From 4937431c40b035ce8933e41c75747a90c46d00a5 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 30 Mar 2023 16:15:43 -0700 Subject: [PATCH 65/88] [clang][cas] Strip -coverage-data-file and -coverage-notes-file from PCH Companion to https://reviews.llvm.org/D147282; when caching a PCH, remove the -coverage-data-file and -coverage-notes-file options. These paths cannot currently be prefix-mapped, because they are emitted into the binary for the coverage runtime to use. However, they have no effect on PCH, so remove them. rdar://107443796 (cherry picked from commit 0d086a5125e09158dbab9985702c6a704a786daf) (cherry picked from commit eb9236e51cd666588e88a4c3102a58b4aa8de42a) --- .../lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp | 6 ++++++ clang/test/CAS/depscan-include-tree.c | 6 +++++- clang/test/CAS/fcas-include-tree-with-pch.c | 8 ++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 7b3bd52de3f58..7076c3cdea57a 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -29,6 +29,12 @@ void tooling::dependencies::configureInvocationForCaching( // Clear this otherwise it defeats the purpose of making the compilation key // independent of certain arguments. CI.getCodeGenOpts().DwarfDebugFlags.clear(); + if (FrontendOpts.ProgramAction == frontend::GeneratePCH) { + // Clear paths that are emitted into binaries; they do not affect PCH. + // For modules this is handled in ModuleDepCollector. + CI.getCodeGenOpts().CoverageDataFile.clear(); + CI.getCodeGenOpts().CoverageNotesFile.clear(); + } // "Fix" the CAS options. auto &FileSystemOpts = CI.getFileSystemOpts(); diff --git a/clang/test/CAS/depscan-include-tree.c b/clang/test/CAS/depscan-include-tree.c index f78a027fc2944..ae3f79ea43428 100644 --- a/clang/test/CAS/depscan-include-tree.c +++ b/clang/test/CAS/depscan-include-tree.c @@ -2,7 +2,8 @@ // RUN: split-file %s %t // RUN: %clang -cc1depscan -o %t/inline.rsp -fdepscan=inline -fdepscan-include-tree -cc1-args -cc1 -triple x86_64-apple-macos11.0 \ -// RUN: -emit-obj %t/t.c -o %t/t.o -dwarf-ext-refs -fmodule-format=obj \ +// RUN: -emit-obj %t/t.c -o %t/t.o -dwarf-ext-refs -fmodule-format=obj \ +// RUN: -ftest-coverage -coverage-notes-file %t/t.gcno -coverage-data-file %t/t.gcda \ // RUN: -I %t/includes -isysroot %S/Inputs/SDK -fcas-path %t/cas -DSOME_MACRO -dependency-file %t/inline.d -MT deps // RUN: FileCheck %s -input-file %t/inline.rsp -DPREFIX=%t @@ -13,6 +14,9 @@ // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK: "-fcas-include-tree" // CHECK: "-isysroot" +// CHECK: "-ftest-coverage" +// CHECK: "-coverage-data-file" "[[PREFIX]]/t.gcda" +// CHECK: "-coverage-notes-file" "[[PREFIX]]/t.gcno" // SHOULD-NOT: "-fcas-fs" // SHOULD-NOT: "-fcas-fs-working-directory" // SHOULD-NOT: "-I" diff --git a/clang/test/CAS/fcas-include-tree-with-pch.c b/clang/test/CAS/fcas-include-tree-with-pch.c index 8438e07977f2e..026e4fe785031 100644 --- a/clang/test/CAS/fcas-include-tree-with-pch.c +++ b/clang/test/CAS/fcas-include-tree-with-pch.c @@ -37,6 +37,14 @@ // RUN: %clang @tu2.rsp -emit-llvm -o tree-rel.ll // RUN: diff -u source-rel.ll tree-rel.ll +// Check that -coverage-notes-file and -coverage-data-file are stripped +// RUN: %clang -cc1depscan -o pch3.rsp -fdepscan=inline -fdepscan-include-tree -cc1-args \ +// RUN: -cc1 -emit-pch -x c-header prefix.h -I %t/inc -DCMD_MACRO=1 -fcas-path %t/cas \ +// RUN: -ftest-coverage -coverage-notes-file %t/pch.gcno -coverage-data-file %t/pch.gcda +// RUN: FileCheck %s -check-prefix=COVERAGE -input-file %t/pch3.rsp +// COVERAGE-NOT: -coverage-data-file +// COVERAGE-NOT: -coverage-notes-file + //--- t1.c #if S2_MACRO #include "s2-link.h" From d91b1ce569fff1828f3222595eecb45a99b9ee18 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 29 Mar 2023 11:22:30 -0700 Subject: [PATCH 66/88] [clang][cas] Move -fdepscan-prefix-map-* options to clang driver Centralize the handling of prefix mapping options so that the driver is responsible for lowering `-fdepscan-prefix-map-sdk` and `-fdepscan-prefix-map-toolchain` to use `-fdepscan-prefix-map`, and all the tools take their prefix mappings from `FrontendOptions::PathPrefixMappings` instead of each having their own handling. This makes it easier for new code paths to use prefix mapping, since they get configuration "for free". cc1depscan[d] no longer needs special handling of these options at all, and in tests we just move to the driver or cc1 options. For clang-scan-deps it's move convenient to keep the command-line options, but implement them in the argument adjuster by adding the right driver options. rdar://106307646 (cherry picked from commit 5f5cf44b6f7d2d2ee829576088bebba283b7f23e) --- .../DependencyScanningTool.h | 22 ++--- .../DependencyScanningWorker.h | 7 -- .../DependencyScanning/ScanAndUpdateArgs.h | 14 ++-- clang/lib/Driver/ToolChains/Clang.cpp | 84 +++++++++++++++---- clang/lib/Driver/ToolChains/Clang.h | 8 +- .../CASFSActionController.cpp | 20 ++--- .../DependencyScanning/CachingActions.h | 6 +- .../DependencyScanningTool.cpp | 49 ++++------- .../DependencyScanningWorker.cpp | 13 +-- .../IncludeTreeActionController.cpp | 19 +---- .../DependencyScanning/ScanAndUpdateArgs.cpp | 81 +++++------------- clang/test/CAS/cached-diagnostics.c | 8 +- clang/test/CAS/depscan-prefix-map.c | 60 ++++--------- clang/test/CAS/driver-cache-launcher.c | 5 +- .../CAS/fcas-include-tree-prefix-mapping.c | 24 ++++-- clang/test/CAS/pgo-profile.c | 8 +- clang/test/Driver/fdepscan-prefix-map-sdk.c | 8 ++ .../Driver/fdepscan-prefix-map-toolchain.c | 15 ++++ clang/test/Driver/fdepscan-prefix-map.c | 8 ++ clang/tools/clang-scan-deps/ClangScanDeps.cpp | 45 +++++----- clang/tools/driver/cc1depscanProtocol.cpp | 64 ++------------ clang/tools/driver/cc1depscanProtocol.h | 22 +---- clang/tools/driver/cc1depscan_main.cpp | 56 +++---------- clang/unittests/CAS/IncludeTreeTest.cpp | 4 +- 24 files changed, 261 insertions(+), 389 deletions(-) create mode 100644 clang/test/Driver/fdepscan-prefix-map-sdk.c create mode 100644 clang/test/Driver/fdepscan-prefix-map-toolchain.c create mode 100644 clang/test/Driver/fdepscan-prefix-map.c diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index 87d56443a9e33..52cb75423d911 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -106,8 +106,7 @@ class DependencyScanningTool { /// Collect dependency tree. llvm::Expected - getDependencyTree(const std::vector &CommandLine, StringRef CWD, - DepscanPrefixMapping PrefixMapping = {}); + getDependencyTree(const std::vector &CommandLine, StringRef CWD); /// If \p DiagGenerationAsCompilation is true it will generate error /// diagnostics same way as the normal compilation, with "N errors generated" @@ -117,14 +116,12 @@ class DependencyScanningTool { getDependencyTreeFromCompilerInvocation( std::shared_ptr Invocation, StringRef CWD, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, - bool DiagGenerationAsCompilation, - DepscanPrefixMapping PrefixMapping = {}); + bool DiagGenerationAsCompilation); Expected getIncludeTree(cas::ObjectStore &DB, const std::vector &CommandLine, StringRef CWD, - LookupModuleOutputCallback LookupModuleOutput, - const DepscanPrefixMapping &PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); /// If \p DiagGenerationAsCompilation is true it will generate error /// diagnostics same way as the normal compilation, with "N errors generated" @@ -133,7 +130,6 @@ class DependencyScanningTool { Expected getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, - const DepscanPrefixMapping &PrefixMapping, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation); @@ -155,8 +151,7 @@ class DependencyScanningTool { getTranslationUnitDependencies(const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); /// Given a compilation context specified via the Clang driver command-line, /// gather modular dependencies of module with the given name, and return the @@ -165,8 +160,7 @@ class DependencyScanningTool { getModuleDependencies(StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); ScanningOutputFormat getScanningFormat() const { return Worker.getScanningFormat(); @@ -187,13 +181,11 @@ class DependencyScanningTool { static std::unique_ptr createActionController(DependencyScanningWorker &Worker, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + LookupModuleOutputCallback LookupModuleOutput); private: std::unique_ptr - createActionController(LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping); + createActionController(LookupModuleOutputCallback LookupModuleOutput); private: DependencyScanningWorker Worker; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index fa634e22246a4..eb944a1046213 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -31,7 +31,6 @@ using CachingOnDiskFileSystemPtr = llvm::IntrusiveRefCntPtr; class DependencyScanningWorkerFilesystem; -struct DepscanPrefixMapping; /// A command-line tool invocation that is part of building a TU. /// @@ -99,12 +98,6 @@ class DependencyActionController { const ModuleDeps &MD) { return llvm::Error::success(); } - - /// FIXME: This is temporary until we eliminate the split between consumers in - /// \p DependencyScanningTool and collectors in \p DependencyScanningWorker - /// and have them both in the same file. see FIXME in \p - /// DependencyScanningAction::runInvocation. - virtual const DepscanPrefixMapping *getPrefixMapping() { return nullptr; } }; /// An individual dependency scanning worker that is able to run on its own diff --git a/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h b/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h index 42f0db456e190..792bee4ef0861 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h +++ b/clang/include/clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h @@ -41,14 +41,13 @@ void configureInvocationForCaching(CompilerInvocation &CI, CASOptions CASOpts, bool ProduceIncludeTree); struct DepscanPrefixMapping { - Optional NewSDKPath; - Optional NewToolchainPath; - SmallVector PrefixMap; + /// Add path mappings to the \p Mapper. + static void configurePrefixMapper(const CompilerInvocation &Invocation, + llvm::PrefixMapper &Mapper); - /// Add path mappings from the current path in \p Invocation to the new path - /// from \c DepscanPrefixMapping to the \p Mapper. - llvm::Error configurePrefixMapper(const CompilerInvocation &Invocation, - llvm::PrefixMapper &Mapper) const; + /// Add path mappings to the \p Mapper. + static void configurePrefixMapper(ArrayRef PathPrefixMappings, + llvm::PrefixMapper &Mapper); /// Apply the mappings from \p Mapper to \p Invocation. static void remapInvocationPaths(CompilerInvocation &Invocation, @@ -61,7 +60,6 @@ Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, CompilerInvocation &Invocation, StringRef WorkingDirectory, - const tooling::dependencies::DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB); } // end namespace clang diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index ec21cbfcfcb13..1008c1b376bc4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -1190,7 +1190,8 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA, const Driver &D, const ArgList &Args, ArgStringList &CmdArgs, const InputInfo &Output, - const InputInfoList &Inputs) const { + const InputInfoList &Inputs, + std::optional &Sysroot) const { const bool IsIAMCU = getToolChain().getTriple().isOSIAMCU(); CheckPreprocessingOptions(D, Args); @@ -1428,10 +1429,15 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA, // -isysroot to the CC1 invocation. StringRef sysroot = C.getSysRoot(); if (sysroot != "") { - if (!Args.hasArg(options::OPT_isysroot)) { + if (Arg *A = Args.getLastArg(options::OPT_isysroot)) { + Sysroot = A->getValue(); + } else { CmdArgs.push_back("-isysroot"); CmdArgs.push_back(C.getArgs().MakeArgString(sysroot)); + Sysroot = sysroot; } + } else if (Arg *A = Args.getLastArg(options::OPT_isysroot)) { + Sysroot = A->getValue(); } // Parse additional include paths from environment variables. @@ -4524,6 +4530,52 @@ static void ProcessVSRuntimeLibrary(const ArgList &Args, } } +void Clang::AddPrefixMappingOptions(const ArgList &Args, ArgStringList &CmdArgs, + const Driver &D, + std::optional Sysroot) const { + auto IsPathApplicableAsPrefix = [](std::optional Path) { + return Path && llvm::sys::path::is_absolute(*Path) && + *Path != llvm::sys::path::root_path(*Path); + }; + + if (Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_sdk_EQ)) { + if (IsPathApplicableAsPrefix(Sysroot)) { + CmdArgs.push_back(Args.MakeArgString(Twine("-fdepscan-prefix-map=") + + *Sysroot + "=" + A->getValue())); + } else { + // FIXME: warning if we cannot infer sdk + } + } + + if (Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_toolchain_EQ)) { + StringRef ResourceDir = D.ResourceDir; + StringRef Guess = llvm::sys::path::parent_path(ResourceDir); + for (StringRef Dir : {"clang", "lib", "usr"}) { + if (llvm::sys::path::filename(Guess) != Dir) + break; + Guess = llvm::sys::path::parent_path(Guess); + } + if (IsPathApplicableAsPrefix(Guess)) { + CmdArgs.push_back(Args.MakeArgString(Twine("-fdepscan-prefix-map=") + + Guess + "=" + A->getValue())); + } else { + // FIXME: warning if we cannot infer toolchain + } + } + + for (const Arg *A : Args.filtered(options::OPT_fdepscan_prefix_map_EQ)) { + A->claim(); + StringRef Map = A->getValue(); + StringRef Prefix = Map.split('=').first; + if (Prefix.size() == Map.size() || !IsPathApplicableAsPrefix(Prefix)) { + D.Diag(diag::err_drv_invalid_argument_to_option) + << A->getValue() << A->getOption().getName(); + } else { + A->render(Args, CmdArgs); + } + } +} + void Clang::ConstructJob(Compilation &C, const JobAction &Job, const InputInfo &Output, const InputInfoList &Inputs, const ArgList &Args, const char *LinkingOutput) const { @@ -4640,19 +4692,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &Job, D.Diag(diag::err_drv_clang_unsupported) << "C++ for IAMCU"; { - const OptSpecifier DepScanOpts[] = { - options::OPT_fdepscan_EQ, - options::OPT_fdepscan_include_tree, - options::OPT_fdepscan_share_EQ, - options::OPT_fdepscan_share_identifier, - options::OPT_fdepscan_share_parent, - options::OPT_fdepscan_share_parent_EQ, - options::OPT_fno_depscan_share, - options::OPT_fdepscan_share_stop_EQ, - options::OPT_fdepscan_daemon_EQ, - options::OPT_fdepscan_prefix_map_EQ, - options::OPT_fdepscan_prefix_map_sdk_EQ, - options::OPT_fdepscan_prefix_map_toolchain_EQ}; + const OptSpecifier DepScanOpts[] = {options::OPT_fdepscan_EQ, + options::OPT_fdepscan_include_tree, + options::OPT_fdepscan_share_EQ, + options::OPT_fdepscan_share_identifier, + options::OPT_fdepscan_share_parent, + options::OPT_fdepscan_share_parent_EQ, + options::OPT_fno_depscan_share, + options::OPT_fdepscan_share_stop_EQ, + options::OPT_fdepscan_daemon_EQ}; // Handle depscan. if (Job.getKind() == Action::DepscanJobClass) { @@ -5890,8 +5938,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &Job, // preprocessor. // // FIXME: Support -fpreprocessed + std::optional Sysroot; if (types::getPreprocessedType(InputType) != types::TY_INVALID) - AddPreprocessingOptions(C, JA, D, Args, CmdArgs, Output, Inputs); + AddPreprocessingOptions(C, JA, D, Args, CmdArgs, Output, Inputs, Sysroot); + + // Handle -fdepscan-prefix-map-* options + AddPrefixMappingOptions(Args, CmdArgs, D, Sysroot); // Don't warn about "clang -c -DPIC -fPIC test.i" because libtool.m4 assumes // that "The compiler can only warn and ignore the option if not recognized". diff --git a/clang/lib/Driver/ToolChains/Clang.h b/clang/lib/Driver/ToolChains/Clang.h index d7550665539cc..e5940b45f6620 100644 --- a/clang/lib/Driver/ToolChains/Clang.h +++ b/clang/lib/Driver/ToolChains/Clang.h @@ -43,7 +43,8 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool { const Driver &D, const llvm::opt::ArgList &Args, llvm::opt::ArgStringList &CmdArgs, const InputInfo &Output, - const InputInfoList &Inputs) const; + const InputInfoList &Inputs, + std::optional &Sysroot) const; void RenderTargetOptions(const llvm::Triple &EffectiveTriple, const llvm::opt::ArgList &Args, bool KernelOrKext, @@ -104,6 +105,11 @@ class LLVM_LIBRARY_VISIBILITY Clang : public Tool { StringRef Dir, Compilation &C, StringRef Target, const InputInfo &Output, const InputInfo &Input, const llvm::opt::ArgList &Args) const; + void AddPrefixMappingOptions(const llvm::opt::ArgList &Args, + llvm::opt::ArgStringList &CmdArgs, + const Driver &D, + std::optional Sysroot) const; + public: Clang(const ToolChain &TC, bool HasIntegratedBackend = true); ~Clang() override; diff --git a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp index e1a6167923be3..7f940c6336a2b 100644 --- a/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/CASFSActionController.cpp @@ -22,8 +22,7 @@ namespace { class CASFSActionController : public CallbackActionController { public: CASFSActionController(LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping); + llvm::cas::CachingOnDiskFileSystem &CacheFS); llvm::Error initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) override; @@ -38,7 +37,6 @@ class CASFSActionController : public CallbackActionController { private: llvm::cas::CachingOnDiskFileSystem &CacheFS; - DepscanPrefixMapping PrefixMapping; std::optional Mapper; CASOptions CASOpts; }; @@ -46,17 +44,15 @@ class CASFSActionController : public CallbackActionController { CASFSActionController::CASFSActionController( LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping) - : CallbackActionController(std::move(LookupModuleOutput)), CacheFS(CacheFS), - PrefixMapping(std::move(PrefixMapping)) {} + llvm::cas::CachingOnDiskFileSystem &CacheFS) + : CallbackActionController(std::move(LookupModuleOutput)), + CacheFS(CacheFS) {} Error CASFSActionController::initialize(CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { // Setup prefix mapping. Mapper.emplace(&CacheFS); - if (Error E = PrefixMapping.configurePrefixMapper(NewInvocation, *Mapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(NewInvocation, *Mapper); const PreprocessorOptions &PPOpts = ScanInstance.getPreprocessorOpts(); if (!PPOpts.Includes.empty() || !PPOpts.ImplicitPCHInclude.empty()) @@ -197,8 +193,6 @@ Error CASFSActionController::finalizeModuleInvocation(CompilerInvocation &CI, std::unique_ptr dependencies::createCASFSActionController( LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping) { - return std::make_unique(LookupModuleOutput, CacheFS, - std::move(PrefixMapping)); + llvm::cas::CachingOnDiskFileSystem &CacheFS) { + return std::make_unique(LookupModuleOutput, CacheFS); } diff --git a/clang/lib/Tooling/DependencyScanning/CachingActions.h b/clang/lib/Tooling/DependencyScanning/CachingActions.h index 8dc3e33b61553..8cb39dc649641 100644 --- a/clang/lib/Tooling/DependencyScanning/CachingActions.h +++ b/clang/lib/Tooling/DependencyScanning/CachingActions.h @@ -20,13 +20,11 @@ namespace clang::tooling::dependencies { std::unique_ptr createIncludeTreeActionController(LookupModuleOutputCallback LookupModuleOutput, - cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping); + cas::ObjectStore &DB); std::unique_ptr createCASFSActionController(LookupModuleOutputCallback LookupModuleOutput, - llvm::cas::CachingOnDiskFileSystem &CacheFS, - DepscanPrefixMapping PrefixMapping); + llvm::cas::CachingOnDiskFileSystem &CacheFS); /// The PCH recorded file paths with canonical paths, create a VFS that /// allows remapping back to the non-canonical source paths so that they are diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 90b92d3790cc2..2097a16bbd350 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -162,11 +162,9 @@ class GetIncludeTree : public EmptyDependencyConsumer { llvm::Expected DependencyScanningTool::getDependencyTree( - const std::vector &CommandLine, StringRef CWD, - DepscanPrefixMapping PrefixMapping) { + const std::vector &CommandLine, StringRef CWD) { GetDependencyTree Consumer(*Worker.getCASFS()); - auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS(), - std::move(PrefixMapping)); + auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS()); auto Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -178,10 +176,9 @@ llvm::Expected DependencyScanningTool::getDependencyTreeFromCompilerInvocation( std::shared_ptr Invocation, StringRef CWD, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, - bool DiagGenerationAsCompilation, DepscanPrefixMapping PrefixMapping) { + bool DiagGenerationAsCompilation) { GetDependencyTree Consumer(*Worker.getCASFS()); - auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS(), - std::move(PrefixMapping)); + auto Controller = createCASFSActionController(nullptr, *Worker.getCASFS()); Worker.computeDependenciesFromCompilerInvocation( std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer, VerboseOS, DiagGenerationAsCompilation); @@ -190,11 +187,9 @@ DependencyScanningTool::getDependencyTreeFromCompilerInvocation( Expected DependencyScanningTool::getIncludeTree( cas::ObjectStore &DB, const std::vector &CommandLine, - StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, - const DepscanPrefixMapping &PrefixMapping) { + StringRef CWD, LookupModuleOutputCallback LookupModuleOutput) { GetIncludeTree Consumer(DB); - auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB, - std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -206,12 +201,10 @@ Expected DependencyScanningTool::getIncludeTreeFromCompilerInvocation( cas::ObjectStore &DB, std::shared_ptr Invocation, StringRef CWD, LookupModuleOutputCallback LookupModuleOutput, - const DepscanPrefixMapping &PrefixMapping, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, bool DiagGenerationAsCompilation) { GetIncludeTree Consumer(DB); - auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB, - std::move(PrefixMapping)); + auto Controller = createIncludeTreeActionController(LookupModuleOutput, DB); Worker.computeDependenciesFromCompilerInvocation( std::move(Invocation), CWD, Consumer, *Controller, DiagsConsumer, VerboseOS, DiagGenerationAsCompilation); @@ -222,11 +215,9 @@ llvm::Expected DependencyScanningTool::getTranslationUnitDependencies( const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { FullDependencyConsumer Consumer(AlreadySeen); - auto Controller = - createActionController(LookupModuleOutput, std::move(PrefixMapping)); + auto Controller = createActionController(LookupModuleOutput); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller); if (Result) @@ -237,11 +228,9 @@ DependencyScanningTool::getTranslationUnitDependencies( llvm::Expected DependencyScanningTool::getModuleDependencies( StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::StringSet<> &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { FullDependencyConsumer Consumer(AlreadySeen); - auto Controller = - createActionController(LookupModuleOutput, std::move(PrefixMapping)); + auto Controller = createActionController(LookupModuleOutput); llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller, ModuleName); if (Result) @@ -293,21 +282,17 @@ CallbackActionController::~CallbackActionController() {} std::unique_ptr DependencyScanningTool::createActionController( DependencyScanningWorker &Worker, - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { + LookupModuleOutputCallback LookupModuleOutput) { if (Worker.getScanningFormat() == ScanningOutputFormat::FullIncludeTree) - return createIncludeTreeActionController( - LookupModuleOutput, *Worker.getCAS(), std::move(PrefixMapping)); + return createIncludeTreeActionController(LookupModuleOutput, + *Worker.getCAS()); if (auto CacheFS = Worker.getCASFS()) - return createCASFSActionController(LookupModuleOutput, *CacheFS, - std::move(PrefixMapping)); + return createCASFSActionController(LookupModuleOutput, *CacheFS); return std::make_unique(LookupModuleOutput); } std::unique_ptr DependencyScanningTool::createActionController( - LookupModuleOutputCallback LookupModuleOutput, - DepscanPrefixMapping PrefixMapping) { - return createActionController(Worker, std::move(LookupModuleOutput), - std::move(PrefixMapping)); + LookupModuleOutputCallback LookupModuleOutput) { + return createActionController(Worker, std::move(LookupModuleOutput)); } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 6c95ff872db7e..fe63145171747 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -177,17 +177,14 @@ class ReversePrefixMappingDependencyFileGenerator const DependencyOutputOptions &Opts) : DependencyFileGenerator(Opts) {} - Error initialize(const CompilerInvocation &CI, - const DepscanPrefixMapping &PrefixMapping) { + void initialize(const CompilerInvocation &CI) { llvm::PrefixMapper Mapper; - if (Error E = PrefixMapping.configurePrefixMapper(CI, Mapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(CI, Mapper); if (Mapper.empty()) - return Error::success(); + return; ReverseMapper.addInverseRange(Mapper.getMappings()); ReverseMapper.sort(); - return Error::success(); } void maybeAddDependency(StringRef Filename, bool FromModule, bool IsSystem, @@ -426,9 +423,7 @@ class DependencyScanningAction : public tooling::ToolAction { auto DFG = std::make_shared( *Opts); - if (auto *Mapping = Controller.getPrefixMapping()) - if (auto E = DFG->initialize(ScanInstance.getInvocation(), *Mapping)) - return reportError(std::move(E)); + DFG->initialize(ScanInstance.getInvocation()); ScanInstance.addDependencyCollector(std::move(DFG)); } diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index 563afd75c68d4..1331e19be742e 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -25,10 +25,8 @@ class IncludeTreeBuilder; class IncludeTreeActionController : public CallbackActionController { public: IncludeTreeActionController(cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping, LookupModuleOutputCallback LookupOutput) - : CallbackActionController(LookupOutput), DB(DB), - PrefixMapping(std::move(PrefixMapping)) {} + : CallbackActionController(LookupOutput), DB(DB) {} Expected getIncludeTree(); @@ -43,10 +41,6 @@ class IncludeTreeActionController : public CallbackActionController { Error finalizeModuleInvocation(CompilerInvocation &CI, const ModuleDeps &MD) override; - const DepscanPrefixMapping *getPrefixMapping() override { - return &PrefixMapping; - } - private: IncludeTreeBuilder ¤t() { assert(!BuilderStack.empty()); @@ -56,7 +50,6 @@ class IncludeTreeActionController : public CallbackActionController { private: cas::ObjectStore &DB; CASOptions CASOpts; - DepscanPrefixMapping PrefixMapping; llvm::PrefixMapper PrefixMapper; // IncludeTreePPCallbacks keeps a pointer to the current builder, so use a // pointer so the builder cannot move when resizing. @@ -296,9 +289,7 @@ Expected IncludeTreeActionController::getIncludeTree() { Error IncludeTreeActionController::initialize( CompilerInstance &ScanInstance, CompilerInvocation &NewInvocation) { - if (Error E = - PrefixMapping.configurePrefixMapper(NewInvocation, PrefixMapper)) - return E; + DepscanPrefixMapping::configurePrefixMapper(NewInvocation, PrefixMapper); auto ensurePathRemapping = [&]() { if (PrefixMapper.empty()) @@ -822,8 +813,6 @@ IncludeTreeBuilder::createIncludeFile(StringRef Filename, std::unique_ptr dependencies::createIncludeTreeActionController( - LookupModuleOutputCallback LookupModuleOutput, cas::ObjectStore &DB, - DepscanPrefixMapping PrefixMapping) { - return std::make_unique( - DB, std::move(PrefixMapping), LookupModuleOutput); + LookupModuleOutputCallback LookupModuleOutput, cas::ObjectStore &DB) { + return std::make_unique(DB, LookupModuleOutput); } diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 7076c3cdea57a..ff02143402c7d 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -179,66 +179,28 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation, Mapper.mapInPlace(CodeGenOpts.ProfileRemappingFile); } -Error DepscanPrefixMapping::configurePrefixMapper( - const CompilerInvocation &Invocation, llvm::PrefixMapper &Mapper) const { - auto isPathApplicableAsPrefix = [](StringRef Path) -> bool { - if (Path.empty()) - return false; - if (llvm::sys::path::is_relative(Path)) - return false; - if (Path == llvm::sys::path::root_path(Path)) - return false; - return true; - }; +void DepscanPrefixMapping::configurePrefixMapper(const CompilerInvocation &CI, + llvm::PrefixMapper &Mapper) { + return configurePrefixMapper(CI.getFrontendOpts().PathPrefixMappings, Mapper); +} - const HeaderSearchOptions &HSOpts = Invocation.getHeaderSearchOpts(); +void DepscanPrefixMapping::configurePrefixMapper( + ArrayRef PathPrefixMappings, llvm::PrefixMapper &Mapper) { + if (PathPrefixMappings.empty()) + return; - if (NewSDKPath) { - StringRef SDK = HSOpts.Sysroot; - if (isPathApplicableAsPrefix(SDK)) - // Need a new copy of the string since the invocation will be modified. - Mapper.add({SDK, *NewSDKPath}); - } - if (NewToolchainPath) { - // Look up for the toolchain, assuming resources are at - // /usr/lib/clang/. Return a shallower guess if the - // directories do not match. - // - // FIXME: Should this append ".." instead of calling parent_path? - StringRef ResourceDir = HSOpts.ResourceDir; - StringRef Guess = llvm::sys::path::parent_path(ResourceDir); - for (StringRef Dir : {"clang", "lib", "usr"}) { - if (llvm::sys::path::filename(Guess) != Dir) - break; - Guess = llvm::sys::path::parent_path(Guess); - } - if (isPathApplicableAsPrefix(Guess)) - // Need a new copy of the string since the invocation will be modified. - Mapper.add({Guess, *NewToolchainPath}); - } - if (!PrefixMap.empty()) { - llvm::SmallVector Split; - llvm::MappedPrefix::transformJoinedIfValid(PrefixMap, Split); - for (auto &MappedPrefix : Split) { - if (isPathApplicableAsPrefix(MappedPrefix.Old)) { - Mapper.add(MappedPrefix); - } else { - return createStringError(llvm::errc::invalid_argument, - "invalid prefix map: '" + MappedPrefix.Old + - "=" + MappedPrefix.New + "'"); - } - } - } + llvm::SmallVector Split; + llvm::MappedPrefix::transformJoinedIfValid(PathPrefixMappings, Split); + for (auto &MappedPrefix : Split) + Mapper.add(MappedPrefix); Mapper.sort(); - return Error::success(); } Expected clang::scanAndUpdateCC1InlineWithTool( DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, CompilerInvocation &Invocation, - StringRef WorkingDirectory, const DepscanPrefixMapping &PrefixMapping, - llvm::cas::ObjectStore &DB) { + StringRef WorkingDirectory, llvm::cas::ObjectStore &DB) { // Override the CASOptions. They may match (the caller having sniffed them // out of InputArgs) but if they have been overridden we want the new ones. Invocation.getCASOpts() = Tool.getCASOpts(); @@ -255,8 +217,7 @@ Expected clang::scanAndUpdateCC1InlineWithTool( Tool.getCachingFileSystem()); } llvm::PrefixMapper &Mapper = *MapperPtr; - if (Error E = PrefixMapping.configurePrefixMapper(Invocation, Mapper)) - return std::move(E); + DepscanPrefixMapping::configurePrefixMapper(Invocation, Mapper); auto ScanInvocation = std::make_shared(Invocation); // An error during dep-scanning is treated as if the main compilation has @@ -265,18 +226,18 @@ Expected clang::scanAndUpdateCC1InlineWithTool( Optional Root; if (ProduceIncludeTree) { - if (Error E = Tool.getIncludeTreeFromCompilerInvocation( - DB, std::move(ScanInvocation), WorkingDirectory, - /*LookupModuleOutput=*/nullptr, PrefixMapping, - DiagsConsumer, VerboseOS, - /*DiagGenerationAsCompilation*/ true) - .moveInto(Root)) + if (Error E = + Tool.getIncludeTreeFromCompilerInvocation( + DB, std::move(ScanInvocation), WorkingDirectory, + /*LookupModuleOutput=*/nullptr, DiagsConsumer, VerboseOS, + /*DiagGenerationAsCompilation*/ true) + .moveInto(Root)) return std::move(E); } else { if (Error E = Tool.getDependencyTreeFromCompilerInvocation( std::move(ScanInvocation), WorkingDirectory, DiagsConsumer, VerboseOS, - /*DiagGenerationAsCompilation*/ true, PrefixMapping) + /*DiagGenerationAsCompilation*/ true) .moveInto(Root)) return std::move(E); } diff --git a/clang/test/CAS/cached-diagnostics.c b/clang/test/CAS/cached-diagnostics.c index 509417333d382..6f11adf48f88a 100644 --- a/clang/test/CAS/cached-diagnostics.c +++ b/clang/test/CAS/cached-diagnostics.c @@ -4,8 +4,8 @@ // RUN: %clang_cc1 -triple x86_64-apple-macos12 -fsyntax-only %t/src/main.c -I %t/src/inc -Wunknown-pragmas 2> %t/regular-diags1.txt -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -fdepscan-prefix-map=%t/src=/^src -o %t/t1.rsp -cc1-args \ -// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t1.rsp -cc1-args \ +// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -fdepscan-prefix-map=%t/src=/^src \ // RUN: -emit-obj %t/src/main.c -o %t/out/output.o -I %t/src/inc -Wunknown-pragmas // Compare diagnostics after a miss. @@ -41,8 +41,8 @@ // RUN: %clang_cc1 -triple x86_64-apple-macos12 -fsyntax-only %t/src2/main.c -I %t/src2/inc -Wunknown-pragmas 2> %t/regular-diags2.txt -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -fdepscan-prefix-map=%t/src2=/^src -o %t/t2.rsp -cc1-args \ -// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t/t2.rsp -cc1-args \ +// RUN: -cc1 -triple x86_64-apple-macos12 -fcas-path %t/cas -fdepscan-prefix-map=%t/src2=/^src \ // RUN: -emit-obj %t/src2/main.c -o %t/out2/output.o -I %t/src2/inc -Wunknown-pragmas // RUN: %clang @%t/t2.rsp -Rcompile-job-cache 2> %t/diags-hit2.txt diff --git a/clang/test/CAS/depscan-prefix-map.c b/clang/test/CAS/depscan-prefix-map.c index 062d74b1f0742..1b5f6b5e1cdb0 100644 --- a/clang/test/CAS/depscan-prefix-map.c +++ b/clang/test/CAS/depscan-prefix-map.c @@ -5,46 +5,46 @@ // RUN: rm -rf %t.d // RUN: mkdir %t.d // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=inline \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/usr/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/usr/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=inline \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ // RUN: -cas-args -fcas-path %t.d/cas -- \ // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ // RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=%S=/^source \ -// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ -// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ -// RUN: -fdepscan-prefix-map-toolchain=/^toolchain \ -// RUN: -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ // RUN: -isysroot %S/Inputs/SDK \ // RUN: -resource-dir %S/Inputs/toolchain_dir/usr/lib/clang/1000 \ // RUN: -internal-isystem %S/Inputs/toolchain_dir/usr/lib/clang/1000/include \ // RUN: -working-directory %t.d \ // RUN: -fcas-path %t.d/cas \ +// RUN: -fdepscan-prefix-map=%S=/^source \ +// RUN: -fdepscan-prefix-map=%t.d=/^testdir \ +// RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: | FileCheck %s -DPREFIX=%t.d // // CHECK: "-fcas-path" "[[PREFIX]]/cas" @@ -64,35 +64,5 @@ // CHECK-ROOT-NEXT: file {{.*}} /^source/depscan-prefix-map.c{{$}} // CHECK-ROOT-NEXT: file {{.*}} /^toolchain/usr/lib/clang/1000/include/stdarg.h{{$}} -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=/=/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_ROOT -// ERROR_ROOT: invalid prefix map: '/=/^foo' - -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map==/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_EMPTY -// ERROR_EMPTY: invalid prefix map: '=/^foo' - -// RUN: not %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ -// RUN: -cas-args -fcas-path %t.d/cas -- \ -// RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=daemon \ -// RUN: -fdepscan-daemon=%{clang-daemon-dir}/%basename_t \ -// RUN: -fdepscan-prefix-map=relative=/^foo \ -// RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ -// RUN: -fcas-path %t.d/cas \ -// RUN: 2>&1 | FileCheck %s -DPREFIX=%t.d -check-prefix=ERROR_RELATIVE -// ERROR_RELATIVE: invalid prefix map: 'relative=/^foo' - #include int test() { return 0; } diff --git a/clang/test/CAS/driver-cache-launcher.c b/clang/test/CAS/driver-cache-launcher.c index e12f622830a5d..195e62c86d3ab 100644 --- a/clang/test/CAS/driver-cache-launcher.c +++ b/clang/test/CAS/driver-cache-launcher.c @@ -58,14 +58,15 @@ // RUN: cp -R %S/Inputs/cmake-build %t/cmake-build // RUN: pushd %t/cmake-build // RUN: cache-build-session -prefix-map-cmake -v echo 2>&1 | FileCheck %s -check-prefix=SESSION-CMAKE-PREFIX -// RUN: env LLVM_CACHE_CAS_PATH=%t/cas cache-build-session -prefix-map-cmake %clang-cache %clang -c %s -o %t.o -### 2>&1 | FileCheck %s -check-prefix=CLANG-CMAKE-PREFIX -DPREFIX=%t +// RUN: env LLVM_CACHE_CAS_PATH=%t/cas cache-build-session -prefix-map-cmake %clang-cache %clang -c %s -o %t.o -isysroot %S/Inputs/SDK -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -### 2>&1 | FileCheck %s -check-prefix=CLANG-CMAKE-PREFIX -DPREFIX=%t -DINPUTS=%S/Inputs // RUN: popd // SESSION-CMAKE-PREFIX: note: setting LLVM_CACHE_PREFIX_MAPS=/llvm/build=/^build;/llvm/llvm-project/llvm=/^src;/llvm/llvm-project/clang=/^src-clang;/llvm/llvm-project/clang-tools-extra=/^src-clang-tools-extra;/llvm/llvm-project/third-party/benchmark=/^src-benchmark;/llvm/llvm-project/other/benchmark=/^src-benchmark-1;/llvm/llvm-project/another/benchmark=/^src-benchmark-2{{$}} // SESSION-CMAKE-PREFIX: note: setting LLVM_CACHE_BUILD_SESSION_ID= // CLANG-CMAKE-PREFIX: "-cc1depscan" "-fdepscan=daemon" "-fdepscan-share-identifier" -// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map-sdk=/^sdk" "-fdepscan-prefix-map-toolchain=/^toolchain" +// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=[[INPUTS]]/SDK=/^sdk" +// CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=[[INPUTS]]/toolchain_dir=/^toolchain" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/build=/^build" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/llvm-project/llvm=/^src" // CLANG-CMAKE-PREFIX: "-fdepscan-prefix-map=/llvm/llvm-project/clang=/^src-clang" diff --git a/clang/test/CAS/fcas-include-tree-prefix-mapping.c b/clang/test/CAS/fcas-include-tree-prefix-mapping.c index d6de19be71139..088b8af61480d 100644 --- a/clang/test/CAS/fcas-include-tree-prefix-mapping.c +++ b/clang/test/CAS/fcas-include-tree-prefix-mapping.c @@ -3,11 +3,13 @@ // RUN: mkdir %t/out // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t1.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-llvm %t/src/main.c -o %t/out/output.ll -include %t/src/prefix.h -I %t/src/inc \ // RUN: -MT deps -dependency-file %t/t1.d @@ -41,11 +43,13 @@ // RUN: mkdir %t/out2 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t2.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-llvm %t/src2/main.c -o %t/out2/output.ll -include %t/src2/prefix.h -I %t/src2/inc \ // RUN: -MT deps -dependency-file %t/t2.d @@ -77,22 +81,26 @@ // Check with PCH. // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/pch1.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-pch -x c-header %t/src/prefix.h -o %t/out/prefix.h.pch -include %t/src/prefix.h -I %t/src/inc // RUN: %clang @%t/pch1.rsp // With different cas path to avoid cache hit. // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/pch2.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas2 -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-pch -x c-header %t/src2/prefix.h -o %t/out2/prefix.h.pch -include %t/src2/prefix.h -I %t/src2/inc // RUN: %clang @%t/pch2.rsp @@ -100,11 +108,13 @@ // RUN: diff %t/out/prefix.h.pch %t/out2/prefix.h.pch // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t3.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src=/^src -fdepscan-prefix-map=%t/out=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out \ // RUN: -emit-obj %t/src/main.c -o %t/out/main.o -include-pch %t/out/prefix.h.pch -I %t/src/inc \ // RUN: -MT deps -dependency-file %t/t1.pch.d @@ -115,11 +125,13 @@ // RUN: -e "s/' .*$//" > %t/cache-key3 // RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree \ -// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out -fdepscan-prefix-map-toolchain=/^toolchain -fdepscan-prefix-map-sdk=/^sdk \ // RUN: -o %t/t4.rsp -cc1-args \ // RUN: -cc1 -triple x86_64-apple-macos11 -fcas-path %t/cas -Rcompile-job-cache \ // RUN: -resource-dir %S/Inputs/toolchain_dir/lib/clang/1000 -internal-isystem %S/Inputs/toolchain_dir/lib/clang/1000/include \ // RUN: -isysroot %S/Inputs/SDK -internal-externc-isystem %S/Inputs/SDK/usr/include \ +// RUN: -fdepscan-prefix-map=%t/src2=/^src -fdepscan-prefix-map=%t/out2=/^out \ +// RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ +// RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ // RUN: -debug-info-kind=standalone -dwarf-version=4 -debugger-tuning=lldb -fdebug-compilation-dir=%t/out2 \ // RUN: -emit-obj %t/src2/main.c -o %t/out2/main.o -include-pch %t/out2/prefix.h.pch -I %t/src2/inc \ // RUN: -MT deps -dependency-file %t/t2.pch.d diff --git a/clang/test/CAS/pgo-profile.c b/clang/test/CAS/pgo-profile.c index b64716c34cf98..387b3e8fa7f70 100644 --- a/clang/test/CAS/pgo-profile.c +++ b/clang/test/CAS/pgo-profile.c @@ -37,9 +37,9 @@ // RUN: mkdir -p %t.dir/a && mkdir -p %t.dir/b // RUN: cp %t.profdata %t.dir/a/a.profdata // RUN: cp %t.profdata %t.dir/b/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -o %t4.rsp -fdepscan-prefix-map=%t.dir/a=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -o %t4.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/a=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/a/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -o %t5.rsp -fdepscan-prefix-map=%t.dir/b=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -o %t5.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/b=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/b/a.profdata // RUN: cat %t4.rsp | FileCheck %s --check-prefix=REMAP // RUN: %clang @%t4.rsp 2>&1 | FileCheck %s --check-prefix=CACHE-MISS @@ -54,9 +54,9 @@ // RUN: grep llvmcas %t.dir/cache-key1 // RUN: diff -u %t.dir/cache-key1 %t.dir/cache-key2 -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t4.inc.rsp -fdepscan-prefix-map=%t.dir/a=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t4.inc.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/a=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/a/a.profdata -// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t5.inc.rsp -fdepscan-prefix-map=%t.dir/b=/^testdir -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache \ +// RUN: %clang -cc1depscan -fdepscan=inline -fdepscan-include-tree -o %t5.inc.rsp -cc1-args -cc1 -triple x86_64-apple-macosx12.0.0 -emit-obj -O3 -Rcompile-job-cache -fdepscan-prefix-map=%t.dir/b=/^testdir \ // RUN: -x c %s -o %t.o -fcas-path %t.dir/cas -fprofile-instrument-use-path=%t.dir/b/a.profdata // RUN: cat %t4.inc.rsp | FileCheck %s --check-prefix=REMAP // RUN: %clang @%t4.inc.rsp 2>&1 | FileCheck %s --check-prefix=CACHE-MISS diff --git a/clang/test/Driver/fdepscan-prefix-map-sdk.c b/clang/test/Driver/fdepscan-prefix-map-sdk.c new file mode 100644 index 0000000000000..421148c1a1a6d --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map-sdk.c @@ -0,0 +1,8 @@ +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -isysroot relative -### %s 2>&1 | FileCheck %s -check-prefix=NONE + +// NONE-NOT: -fdepscan-prefix-map + +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk -isysroot /sys/path -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-sdk=/^sdk --sysroot /sys/path -### %s 2>&1 | FileCheck %s +// CHECK: -fdepscan-prefix-map=/sys/path=/^sdk diff --git a/clang/test/Driver/fdepscan-prefix-map-toolchain.c b/clang/test/Driver/fdepscan-prefix-map-toolchain.c new file mode 100644 index 0000000000000..1bdbdfe1ccd78 --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map-toolchain.c @@ -0,0 +1,15 @@ +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir '' -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir relative -### %s 2>&1 | FileCheck %s -check-prefix=NONE +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /lib/clang/10 -### %s 2>&1 | FileCheck %s -check-prefix=NONE + +// NONE-NOT: -fdepscan-prefix-map + +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/10 -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/lib/clang/10 -### %s 2>&1 | FileCheck %s +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -resource-dir /tc/usr/lib/clang/10 -### %s 2>&1 | FileCheck %s + +// CHECK: -fdepscan-prefix-map=/tc=/^tc + +// Implicit resource-dir +// RUN: %clang -fdepscan-prefix-map-toolchain=/^tc -### %s 2>&1 | FileCheck %s -check-prefix=CHECK_IMPLICIT +// CHECK_IMPLICIT: -fdepscan-prefix-map={{.*}}=/^tc diff --git a/clang/test/Driver/fdepscan-prefix-map.c b/clang/test/Driver/fdepscan-prefix-map.c new file mode 100644 index 0000000000000..4c5c34bb98bfd --- /dev/null +++ b/clang/test/Driver/fdepscan-prefix-map.c @@ -0,0 +1,8 @@ +// RUN: %clang -fdepscan-prefix-map=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map==/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map=relative=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// RUN: %clang -fdepscan-prefix-map=/=/^bad -### %s 2>&1 | FileCheck %s -check-prefix=INVALID +// INVALID: error: invalid argument '{{.*}}/^bad' to -fdepscan-prefix-map= + +// RUN: %clang -fdepscan-prefix-map=/good=/^good -### %s 2>&1 | FileCheck %s +// CHECK: "-fdepscan-prefix-map=/good=/^good" diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 4c0578fc018ca..9858575bbbb20 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -265,10 +265,8 @@ llvm::cl::opt Verbose("v", llvm::cl::Optional, static bool emitCompilationDBWithCASTreeArguments( std::shared_ptr DB, std::vector Inputs, - DiagnosticConsumer &DiagsConsumer, - const DepscanPrefixMapping &PrefixMapping, - DependencyScanningService &Service, llvm::ThreadPool &Pool, - llvm::raw_ostream &OS) { + DiagnosticConsumer &DiagsConsumer, DependencyScanningService &Service, + llvm::ThreadPool &Pool, llvm::raw_ostream &OS) { // Follow `-cc1depscan` and also ignore diagnostics. // FIXME: Seems not a good idea to do this.. @@ -329,7 +327,6 @@ static bool emitCompilationDBWithCASTreeArguments( tooling::dependencies::DependencyScanningTool &WorkerTool; DiagnosticConsumer &DiagsConsumer; StringRef CWD; - const DepscanPrefixMapping &PrefixMapping; SmallVectorImpl &OutputArgs; llvm::StringSaver &Saver; @@ -338,12 +335,10 @@ static bool emitCompilationDBWithCASTreeArguments( llvm::cas::ObjectStore &DB, tooling::dependencies::DependencyScanningTool &WorkerTool, DiagnosticConsumer &DiagsConsumer, StringRef CWD, - const DepscanPrefixMapping &PrefixMapping, SmallVectorImpl &OutputArgs, llvm::StringSaver &Saver) : DB(DB), WorkerTool(WorkerTool), DiagsConsumer(DiagsConsumer), - CWD(CWD), PrefixMapping(PrefixMapping), OutputArgs(OutputArgs), - Saver(Saver) {} + CWD(CWD), OutputArgs(OutputArgs), Saver(Saver) {} bool runInvocation(std::shared_ptr Invocation, @@ -352,7 +347,7 @@ static bool emitCompilationDBWithCASTreeArguments( DiagnosticConsumer *DiagConsumer) override { Expected Root = scanAndUpdateCC1InlineWithTool( WorkerTool, DiagsConsumer, /*VerboseOS*/ nullptr, *Invocation, - CWD, PrefixMapping, DB); + CWD, DB); if (!Root) { llvm::consumeError(Root.takeError()); return false; @@ -369,7 +364,7 @@ static bool emitCompilationDBWithCASTreeArguments( llvm::StringSaver &Saver = PerThreadStates[I]->Saver; OutputArgs.push_back(Saver.save(Input->CommandLine.front()).data()); ScanForCC1Action Action(*DB, WorkerTool, *IgnoringDiagsConsumer, CWD, - PrefixMapping, OutputArgs, Saver); + OutputArgs, Saver); llvm::IntrusiveRefCntPtr FileMgr = WorkerTool.getOrCreateFileManager(); @@ -871,6 +866,14 @@ int main(int argc, const char **argv) { } } AdjustedArgs.insert(AdjustedArgs.end(), FlagsEnd, Args.end()); + if (!PrefixMapToolchain.empty()) + AdjustedArgs.push_back("-fdepscan-prefix-map-toolchain=" + + PrefixMapToolchain); + if (!PrefixMapSDK.empty()) + AdjustedArgs.push_back("-fdepscan-prefix-map-sdk=" + PrefixMapSDK); + for (StringRef Map : PrefixMaps) { + AdjustedArgs.push_back("-fdepscan-prefix-map=" + std::string(Map)); + } return AdjustedArgs; }); @@ -902,13 +905,6 @@ int main(int argc, const char **argv) { FS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); } - DepscanPrefixMapping PrefixMapping; - if (!PrefixMapToolchain.empty()) - PrefixMapping.NewToolchainPath = PrefixMapToolchain; - if (!PrefixMapSDK.empty()) - PrefixMapping.NewSDKPath = PrefixMapSDK; - PrefixMapping.PrefixMap.append(PrefixMaps.begin(), PrefixMaps.end()); - DependencyScanningService Service(ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs, EagerLoadModules); llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads)); @@ -920,7 +916,7 @@ int main(int argc, const char **argv) { } return emitCompilationDBWithCASTreeArguments( CAS, AdjustingCompilations->getAllCompileCommands(), *DiagsConsumer, - PrefixMapping, Service, Pool, llvm::outs()); + Service, Pool, llvm::outs()); } std::vector> WorkerTools; @@ -963,7 +959,7 @@ int main(int argc, const char **argv) { << " files using " << Pool.getThreadCount() << " workers\n"; } for (unsigned I = 0; I < Pool.getThreadCount(); ++I) { - Pool.async([I, &CAS, &PrefixMapping, &Lock, &Index, &Inputs, &TreeResults, + Pool.async([I, &CAS, &Lock, &Index, &Inputs, &TreeResults, &HadErrors, &FD, &WorkerTools, &DependencyOS, &Errs]() { llvm::StringSet<> AlreadySeenModules; while (true) { @@ -1000,28 +996,27 @@ int main(int argc, const char **argv) { Errs)) HadErrors = true; } else if (Format == ScanningOutputFormat::Tree) { - auto MaybeTree = WorkerTools[I]->getDependencyTree( - Input->CommandLine, CWD, PrefixMapping); + auto MaybeTree = + WorkerTools[I]->getDependencyTree(Input->CommandLine, CWD); std::unique_lock LockGuard(Lock); TreeResults.emplace_back(LocalIndex, std::move(Filename), std::move(MaybeTree)); } else if (Format == ScanningOutputFormat::IncludeTree) { auto MaybeTree = WorkerTools[I]->getIncludeTree( - *CAS, Input->CommandLine, CWD, LookupOutput, PrefixMapping); + *CAS, Input->CommandLine, CWD, LookupOutput); std::unique_lock LockGuard(Lock); TreeResults.emplace_back(LocalIndex, std::move(Filename), std::move(MaybeTree)); } else if (MaybeModuleName) { auto MaybeFullDeps = WorkerTools[I]->getModuleDependencies( *MaybeModuleName, Input->CommandLine, CWD, AlreadySeenModules, - LookupOutput, PrefixMapping); + LookupOutput); if (handleModuleResult(Filename, MaybeFullDeps, FD, LocalIndex, DependencyOS, Errs)) HadErrors = true; } else { auto MaybeTUDeps = WorkerTools[I]->getTranslationUnitDependencies( - Input->CommandLine, CWD, AlreadySeenModules, LookupOutput, - PrefixMapping); + Input->CommandLine, CWD, AlreadySeenModules, LookupOutput); if (handleTranslationUnitResult(Filename, MaybeTUDeps, FD, LocalIndex, DependencyOS, Errs)) HadErrors = true; diff --git a/clang/tools/driver/cc1depscanProtocol.cpp b/clang/tools/driver/cc1depscanProtocol.cpp index b65d77682c9c9..7e77f4b483f4f 100644 --- a/clang/tools/driver/cc1depscanProtocol.cpp +++ b/clang/tools/driver/cc1depscanProtocol.cpp @@ -280,73 +280,23 @@ llvm::Error CC1DepScanDProtocol::getArgs(llvm::StringSaver &Saver, return Error::success(); } -llvm::Error -CC1DepScanDProtocol::putCommand(StringRef WorkingDirectory, - ArrayRef Args, - const DepscanPrefixMapping &Mapping) { +llvm::Error CC1DepScanDProtocol::putCommand(StringRef WorkingDirectory, + ArrayRef Args) { if (llvm::Error E = putString(WorkingDirectory)) return E; if (llvm::Error E = putArgs(Args)) return E; - return putDepscanPrefixMapping(Mapping); + return llvm::Error::success(); } -llvm::Error CC1DepScanDProtocol::getCommand(llvm::StringSaver &Saver, - StringRef &WorkingDirectory, - SmallVectorImpl &Args, - DepscanPrefixMapping &Mapping) { +llvm::Error +CC1DepScanDProtocol::getCommand(llvm::StringSaver &Saver, + StringRef &WorkingDirectory, + SmallVectorImpl &Args) { if (llvm::Error E = getString(Saver, WorkingDirectory)) return E; if (llvm::Error E = getArgs(Saver, Args)) return E; - return getDepscanPrefixMapping(Saver, Mapping); -} - -llvm::Error CC1DepScanDProtocol::putDepscanPrefixMapping( - const DepscanPrefixMapping &Mapping) { - // Construct the message. - SmallString<256> FullMapping; - if (Mapping.NewSDKPath) - FullMapping.append(*Mapping.NewSDKPath); - FullMapping.push_back(0); - if (Mapping.NewToolchainPath) - FullMapping.append(*Mapping.NewToolchainPath); - FullMapping.push_back(0); - for (StringRef Map : Mapping.PrefixMap) { - FullMapping.append(Map); - FullMapping.push_back(0); - } - return putString(FullMapping); -} - -llvm::Error -CC1DepScanDProtocol::getDepscanPrefixMapping(llvm::StringSaver &Saver, - DepscanPrefixMapping &Mapping) { - StringRef FullMapping; - if (llvm::Error E = getString(Saver, FullMapping)) - return E; - - // Parse the mapping. - size_t Count = 0; - for (auto I = FullMapping.begin(), B = I, E = FullMapping.end(); I != E; - ++I) { - if (I != B && I[-1]) - continue; // Wait for null-terminator. - StringRef Map = I; - switch (Count++) { - case 0: - if (!Map.empty()) - Mapping.NewSDKPath = std::string(Map); - break; - case 1: - if (!Map.empty()) - Mapping.NewToolchainPath = std::string(Map); - break; - default: - Mapping.PrefixMap.push_back(std::string(Map)); - break; - } - } return llvm::Error::success(); } diff --git a/clang/tools/driver/cc1depscanProtocol.h b/clang/tools/driver/cc1depscanProtocol.h index 575ee4f6f36b6..2600306d391c9 100644 --- a/clang/tools/driver/cc1depscanProtocol.h +++ b/clang/tools/driver/cc1depscanProtocol.h @@ -21,14 +21,7 @@ #include // FIXME: Unix-only. Not portable. #include // FIXME: Unix-only. Not portable. -namespace clang { -namespace tooling::dependencies { -struct DepscanPrefixMapping; -} - -namespace cc1depscand { -using tooling::dependencies::DepscanPrefixMapping; - +namespace clang::cc1depscand { struct DepscanSharing { bool OnlyShareParent = false; bool ShareViaIdentifier = false; @@ -137,20 +130,14 @@ class CC1DepScanDProtocol { return putMessage(String.size(), String.begin()); } - llvm::Error putDepscanPrefixMapping(const DepscanPrefixMapping &Mapping); - llvm::Error getDepscanPrefixMapping(llvm::StringSaver &Saver, - DepscanPrefixMapping &Mapping); - llvm::Error putArgs(ArrayRef Args); llvm::Error getArgs(llvm::StringSaver &Saver, SmallVectorImpl &Args); llvm::Error putCommand(StringRef WorkingDirectory, - ArrayRef Args, - const DepscanPrefixMapping &Mapping); + ArrayRef Args); llvm::Error getCommand(llvm::StringSaver &Saver, StringRef &WorkingDirectory, - SmallVectorImpl &Args, - DepscanPrefixMapping &Mapping); + SmallVectorImpl &Args); llvm::Error putScanResultFailed(StringRef Reason); llvm::Error putScanResultSuccess(StringRef RootID, @@ -240,8 +227,7 @@ class ScanDaemon : public OpenSocket { explicit ScanDaemon(OpenSocket S) : OpenSocket(std::move(S)) {} }; -} // namespace cc1depscand -} // namespace clang +} // namespace clang::cc1depscand #endif /* LLVM_ON_UNIX */ #endif // LLVM_CLANG_TOOLS_DRIVER_CC1DEPSCANPROTOCOL_H diff --git a/clang/tools/driver/cc1depscan_main.cpp b/clang/tools/driver/cc1depscan_main.cpp index 030fb270fa02f..85abbe55e31dd 100644 --- a/clang/tools/driver/cc1depscan_main.cpp +++ b/clang/tools/driver/cc1depscan_main.cpp @@ -64,7 +64,6 @@ using namespace clang; using namespace llvm::opt; using cc1depscand::DepscanSharing; -using clang::tooling::dependencies::DepscanPrefixMapping; using llvm::Error; #define DEBUG_TYPE "cc1depscand" @@ -357,7 +356,6 @@ static Expected scanAndUpdateCC1Inline( const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, SmallVectorImpl &OutputArgs, bool ProduceIncludeTree, bool &DiagnosticErrorOccurred, - const DepscanPrefixMapping &PrefixMapping, llvm::function_ref SaveArg, const CASOptions &CASOpts, std::shared_ptr DB, std::shared_ptr Cache); @@ -366,15 +364,14 @@ static Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, - SmallVectorImpl &OutputArgs, - const DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB, + SmallVectorImpl &OutputArgs, llvm::cas::ObjectStore &DB, llvm::function_ref SaveArg); static llvm::Expected scanAndUpdateCC1UsingDaemon( const char *Exec, ArrayRef OldArgs, StringRef WorkingDirectory, SmallVectorImpl &NewArgs, - bool &DiagnosticErrorOccurred, const DepscanPrefixMapping &Mapping, - StringRef Path, const DepscanSharing &Sharing, + bool &DiagnosticErrorOccurred, StringRef Path, + const DepscanSharing &Sharing, llvm::function_ref SaveArg, llvm::cas::ObjectStore &CAS) { using namespace clang::cc1depscand; @@ -391,7 +388,7 @@ static llvm::Expected scanAndUpdateCC1UsingDaemon( CC1DepScanDProtocol Comms(*Daemon); // llvm::dbgs() << "sending request...\n"; - if (auto E = Comms.putCommand(WorkingDirectory, OldArgs, Mapping)) + if (auto E = Comms.putCommand(WorkingDirectory, OldArgs)) return std::move(E); llvm::BumpPtrAllocator Alloc; @@ -441,29 +438,6 @@ static void writeResponseFile(raw_ostream &OS, OS << "\n"; } -static DepscanPrefixMapping -parseCASFSAutoPrefixMappings(DiagnosticsEngine &Diag, const ArgList &Args) { - using namespace clang::driver; - DepscanPrefixMapping Mapping; - for (const Arg *A : Args.filtered(options::OPT_fdepscan_prefix_map_EQ)) { - StringRef Map = A->getValue(); - size_t Equals = Map.find('='); - if (Equals == StringRef::npos) - Diag.Report(diag::err_drv_invalid_argument_to_option) - << Map << A->getOption().getName(); - else - Mapping.PrefixMap.push_back(std::string(Map)); - A->claim(); - } - if (const Arg *A = Args.getLastArg(options::OPT_fdepscan_prefix_map_sdk_EQ)) - Mapping.NewSDKPath = A->getValue(); - if (const Arg *A = - Args.getLastArg(options::OPT_fdepscan_prefix_map_toolchain_EQ)) - Mapping.NewToolchainPath = A->getValue(); - - return Mapping; -} - static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, SmallVectorImpl &NewArgs, DiagnosticsEngine &Diag, @@ -531,8 +505,6 @@ static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, bool ProduceIncludeTree = Args.hasArg(options::OPT_fdepscan_include_tree); - DepscanPrefixMapping PrefixMapping = parseCASFSAutoPrefixMappings(Diag, Args); - auto SaveArg = [&Args](const Twine &T) { return Args.MakeArgString(T); }; CompilerInvocation::GenerateCASArgs(CASOpts, Sharing.CASArgs, SaveArg); if (ProduceIncludeTree) @@ -543,10 +515,10 @@ static int scanAndUpdateCC1(const char *Exec, ArrayRef OldArgs, if (Optional DaemonPath = makeDepscanDaemonPath(Mode, Sharing)) return scanAndUpdateCC1UsingDaemon( Exec, OldArgs, WorkingDirectory, NewArgs, DiagnosticErrorOccurred, - PrefixMapping, *DaemonPath, Sharing, SaveArg, *DB); + *DaemonPath, Sharing, SaveArg, *DB); return scanAndUpdateCC1Inline(Exec, OldArgs, WorkingDirectory, NewArgs, ProduceIncludeTree, DiagnosticErrorOccurred, - PrefixMapping, SaveArg, CASOpts, DB, Cache); + SaveArg, CASOpts, DB, Cache); }; if (llvm::Error E = ScanAndUpdate().moveInto(RootID)) { Diag.Report(diag::err_cas_depscan_failed) << std::move(E); @@ -943,9 +915,7 @@ int ScanServer::listen() { llvm::StringSaver Saver(Alloc); StringRef WorkingDirectory; SmallVector Args; - DepscanPrefixMapping PrefixMapping; - if (llvm::Error E = - Comms.getCommand(Saver, WorkingDirectory, Args, PrefixMapping)) { + if (llvm::Error E = Comms.getCommand(Saver, WorkingDirectory, Args)) { SharedOS.applyLocked([&](raw_ostream &OS) { OS << I << ": failed to get command\n"; logAllUnhandledErrors(std::move(E), OS); @@ -988,8 +958,7 @@ int ScanServer::listen() { SmallVector NewArgs; auto RootID = scanAndUpdateCC1InlineWithTool( *Tool, *DiagsConsumer, &DiagsOS, Argv0, Args, WorkingDirectory, - NewArgs, PrefixMapping, *CAS, - [&](const Twine &T) { return Saver.save(T).data(); }); + NewArgs, *CAS, [&](const Twine &T) { return Saver.save(T).data(); }); if (!RootID) { consumeError(Comms.putScanResultFailed(toString(RootID.takeError()))); SharedOS.applyLocked([&](raw_ostream &OS) { @@ -1071,8 +1040,7 @@ static Expected scanAndUpdateCC1InlineWithTool( tooling::dependencies::DependencyScanningTool &Tool, DiagnosticConsumer &DiagsConsumer, raw_ostream *VerboseOS, const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, - SmallVectorImpl &OutputArgs, - const DepscanPrefixMapping &PrefixMapping, llvm::cas::ObjectStore &DB, + SmallVectorImpl &OutputArgs, llvm::cas::ObjectStore &DB, llvm::function_ref SaveArg) { DiagnosticsEngine Diags(new DiagnosticIDs(), new DiagnosticOptions()); Diags.setClient(&DiagsConsumer, /*ShouldOwnClient=*/false); @@ -1082,8 +1050,7 @@ static Expected scanAndUpdateCC1InlineWithTool( "failed to create compiler invocation"); Expected Root = scanAndUpdateCC1InlineWithTool( - Tool, DiagsConsumer, VerboseOS, *Invocation, WorkingDirectory, - PrefixMapping, DB); + Tool, DiagsConsumer, VerboseOS, *Invocation, WorkingDirectory, DB); if (!Root) return Root; @@ -1097,7 +1064,6 @@ static Expected scanAndUpdateCC1Inline( const char *Exec, ArrayRef InputArgs, StringRef WorkingDirectory, SmallVectorImpl &OutputArgs, bool ProduceIncludeTree, bool &DiagnosticErrorOccurred, - const DepscanPrefixMapping &PrefixMapping, llvm::function_ref SaveArg, const CASOptions &CASOpts, std::shared_ptr DB, std::shared_ptr Cache) { @@ -1128,7 +1094,7 @@ static Expected scanAndUpdateCC1Inline( auto Result = scanAndUpdateCC1InlineWithTool( Tool, *DiagsConsumer, /*VerboseOS*/ nullptr, Exec, InputArgs, - WorkingDirectory, OutputArgs, PrefixMapping, *DB, SaveArg); + WorkingDirectory, OutputArgs, *DB, SaveArg); DiagnosticErrorOccurred = DiagsConsumer->getNumErrors() != 0; return Result; } diff --git a/clang/unittests/CAS/IncludeTreeTest.cpp b/clang/unittests/CAS/IncludeTreeTest.cpp index 1514cba3f89c5..46121711aedd8 100644 --- a/clang/unittests/CAS/IncludeTreeTest.cpp +++ b/clang/unittests/CAS/IncludeTreeTest.cpp @@ -54,10 +54,8 @@ TEST(IncludeTree, IncludeTreeScan) { "-o" "t.cpp.o"}; Optional Root; - DepscanPrefixMapping PrefixMapping; ASSERT_THAT_ERROR( - ScanTool - .getIncludeTree(*DB, CommandLine, /*CWD*/ "", nullptr, PrefixMapping) + ScanTool.getIncludeTree(*DB, CommandLine, /*CWD*/ "", nullptr) .moveInto(Root), llvm::Succeeded()); From e11ea6069bdff4ac189306eb8273d618154d3860 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 4 Apr 2023 09:15:56 -0700 Subject: [PATCH 67/88] [libclang][cas] Update for prefix mapping API change --- clang/tools/libclang/CDependencies.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index 6ad8241030824..d418caaf20b42 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -230,7 +230,7 @@ static CXErrorCode getFullDependencies(DependencyScanningWorker *Worker, llvm::StringSet<> AlreadySeen; FullDependencyConsumer DepConsumer(AlreadySeen); auto Controller = DependencyScanningTool::createActionController( - *Worker, std::move(LookupOutput), /*PrefixMapping=*/{}); + *Worker, std::move(LookupOutput)); bool HasDiagConsumer = DiagConsumer; bool HasError = Error; From c9e56ba2f16a7c09cf9a1a9b300dc960b877e8fc Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 4 Apr 2023 09:17:24 -0700 Subject: [PATCH 68/88] [clang][cas] Add a depscan-prefix-map feature flag For rdar://106307646 --- clang/tools/driver/features.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clang/tools/driver/features.json b/clang/tools/driver/features.json index 770b73c613387..3989988e1bff0 100644 --- a/clang/tools/driver/features.json +++ b/clang/tools/driver/features.json @@ -15,6 +15,9 @@ { "name": "extract-api-ignores" }, + { + "name": "depscan-prefix-map" + }, { "name": "deployment-target-environment-variables", "value": [ From 1a993ea8b3410901dbc7eae25373a9e67935cc48 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 30 Mar 2023 16:28:21 -0700 Subject: [PATCH 69/88] [clang][cas] Prefix map -fdepfile-entry For rdar://106307646 --- .../lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp | 8 ++++++++ clang/test/CAS/depscan-prefix-map.c | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index ff02143402c7d..88457d2f0a8e3 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -177,6 +177,14 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation, Mapper.mapInPlace(CodeGenOpts.ProfileInstrumentUsePath); Mapper.mapInPlace(CodeGenOpts.SampleProfileFile); Mapper.mapInPlace(CodeGenOpts.ProfileRemappingFile); + + // Dependency output options. + // Note: these are not in the cache key, but they are in the module context + // hash, which indirectly impacts the cache key when importing a module. + // In the future we may change how -fmodule-file-cache-key works when + // remapping to avoid needing this. + for (auto &ExtraDep : Invocation.getDependencyOutputOpts().ExtraDeps) + Mapper.mapInPlace(ExtraDep.first); } void DepscanPrefixMapping::configurePrefixMapper(const CompilerInvocation &CI, diff --git a/clang/test/CAS/depscan-prefix-map.c b/clang/test/CAS/depscan-prefix-map.c index 1b5f6b5e1cdb0..25d1fa177b85b 100644 --- a/clang/test/CAS/depscan-prefix-map.c +++ b/clang/test/CAS/depscan-prefix-map.c @@ -16,6 +16,7 @@ // RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ // RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ // RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscan -dump-depscan-tree=%t.root -fdepscan=inline \ // RUN: -cc1-args -triple x86_64-apple-macos11.0 -x c %s -o %t.d/out.o \ @@ -29,6 +30,7 @@ // RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ // RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ // RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // RUN: %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t \ // RUN: -cas-args -fcas-path %t.d/cas -- \ @@ -45,12 +47,14 @@ // RUN: -fdepscan-prefix-map=%{objroot}=/^objroot \ // RUN: -fdepscan-prefix-map=%S/Inputs/toolchain_dir=/^toolchain \ // RUN: -fdepscan-prefix-map=%S/Inputs/SDK=/^sdk \ +// RUN: -fdepfile-entry=%t.d/extra \ // RUN: | FileCheck %s -DPREFIX=%t.d // // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK-SAME: "-working-directory" "/^testdir" // CHECK-SAME: "-x" "c" "/^source/depscan-prefix-map.c" // CHECK-SAME: "-isysroot" "/^sdk" +// CHECK-SAME: "-fdepfile-entry=/^testdir/extra" // RUN: llvm-cas --cas %t.d/cas --ls-tree-recursive @%t.root \ // RUN: | FileCheck %s -check-prefix=CHECK-ROOT From fd871ab9327b1ac6a4eeebd7280cd5345cb906f0 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 3 Apr 2023 15:22:20 -0700 Subject: [PATCH 70/88] [clang][modules] Handle explicit modules when checking for .Private -> _Private While we eventually want to remove the mapping from .Private to _Private modules, until we do, ensure that it behaves the same for explicit modules. rdar://107449872 Differential Revision: https://reviews.llvm.org/D147477 (cherry picked from commit 8ec36e6956cb03d80f3fee8e593808c43a8a1ec3) --- clang/lib/Frontend/CompilerInstance.cpp | 6 +++- ...implicit-private-with-submodule-explicit.m | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 clang/test/Modules/implicit-private-with-submodule-explicit.m diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index f53c34d9bff87..96da881f2b0ea 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -2081,8 +2081,12 @@ CompilerInstance::loadModule(SourceLocation ImportLoc, PrivateModule, PP->getIdentifierInfo(Module->Name)->getTokenID()); PrivPath.push_back(std::make_pair(&II, Path[0].second)); + std::string FileName; + // If there is a modulemap module or prebuilt module, load it. if (PP->getHeaderSearchInfo().lookupModule(PrivateModule, ImportLoc, true, - !IsInclusionDirective)) + !IsInclusionDirective) || + selectModuleSource(nullptr, PrivateModule, FileName, BuiltModules, + PP->getHeaderSearchInfo()) != MS_ModuleNotFound) Sub = loadModule(ImportLoc, PrivPath, Visibility, IsInclusionDirective); if (Sub) { MapPrivateSubModToTopLevel = true; diff --git a/clang/test/Modules/implicit-private-with-submodule-explicit.m b/clang/test/Modules/implicit-private-with-submodule-explicit.m new file mode 100644 index 0000000000000..a2e9950ec3181 --- /dev/null +++ b/clang/test/Modules/implicit-private-with-submodule-explicit.m @@ -0,0 +1,31 @@ +// Checks that the use of .Private to refer to _Private modules works with an +// explicit module. + +// RUN: rm -rf %t +// RUN: split-file %s %t + +// RUN: %clang_cc1 -x objective-c -fmodules -fno-implicit-modules -emit-module -fmodule-name=A %t/module.modulemap -o %t/A.pcm +// RUN: %clang_cc1 -x objective-c -fmodules -fno-implicit-modules -emit-module -fmodule-name=A_Private %t/module.modulemap -o %t/A_Private.pcm + +// Check lazily-loaded module +// RUN: %clang_cc1 -x objective-c -verify -fmodules -fno-implicit-modules -fmodule-file=A=%t/A.pcm -fmodule-file=A_Private=%t/A_Private.pcm -fsyntax-only %t/tu.m + +// Check eagerly-loaded module +// RUN: %clang_cc1 -x objective-c -verify -fmodules -fno-implicit-modules -fmodule-file=%t/A.pcm -fmodule-file=%t/A_Private.pcm -fsyntax-only %t/tu.m + +//--- module.modulemap +module A { header "a.h" } +module A_Private { header "priv.h" } + +//--- a.h + +//--- priv.h +void priv(void); + +//--- tu.m +@import A.Private; // expected-warning{{no submodule named 'Private' in module 'A'; using top level 'A_Private'}} +// expected-note@*:* {{defined here}} + +void tu(void) { + priv(); +} \ No newline at end of file From 0042dc2db812ea94a39184dfe1b563fc037e0d4b Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Mon, 3 Apr 2023 13:56:56 -0700 Subject: [PATCH 71/88] [clang][cas] Switch to include-tree by default As of 3a924f2d4d we have significantly fewer known isuses with include-tree caching than with cas-fs, so it's time to switch the default. rdar://107575958 (cherry picked from commit cfdc12edaef901b2858cd36bf59c83be0526231b) --- clang/test/Index/Core/scan-deps-cas.m | 35 ++++++++++++++++++++------ clang/tools/libclang/CDependencies.cpp | 4 +-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index e9909433e7547..a74c6431364a1 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -1,26 +1,26 @@ -// RUN: rm -rf %t.mcp %t +// RUN: rm -rf %t // RUN: c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcpit \ // RUN: -o FoE.o -x objective-c %s > %t.result -// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t +// RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -check-prefix=INCLUDE_TREE // RUN: env CLANG_CACHE_USE_CASFS_DEPSCAN=1 c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcp \ // RUN: -o FoE.o -x objective-c %s > %t.casfs.result // RUN: cat %t.casfs.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -// FIXME: enable modules when supported. // RUN: env CLANG_CACHE_USE_INCLUDE_TREE=1 c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ +// RUN: -fmodules -fmodules-cache-path=%t/mcpit \ // RUN: -o FoE.o -x objective-c %s > %t.includetree.result // RUN: cat %t.includetree.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -check-prefix=INCLUDE_TREE // RUN: c-index-test core --scan-deps %S -output-dir=%t \ // RUN: -- %clang -c -I %S/Inputs/module \ -// RUN: -fmodules -fmodules-cache-path=%t.mcp \ +// RUN: -fmodules -fmodules-cache-path=%t/mcp \ // RUN: -o FoE.o -x objective-c %s | FileCheck %s -check-prefix=NO_CAS // NO_CAS-NOT: fcas // NO_CAS-NOT: faction-cache @@ -65,16 +65,37 @@ // CHECK-SAME: -fmodule-file-cache-key [[PCM:.*ModA_.*pcm]] llvmcas://{{[[:xdigit:]]+}} // CHECK-SAME: -fmodule-file={{(ModA=)?}}[[PCM]] + +// INCLUDE_TREE: modules: +// INCLUDE_TREE-NEXT: module: +// INCLUDE_TREE-NEXT: name: ModA +// INCLUDE_TREE-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] +// INCLUDE_TREE-NEXT: module-map-path: /Users/blangmuir/src/cas/llvm-project/clang/test/Index/Core/Inputs/module/module.modulemap +// INCLUDE_TREE-NEXT: cache-key: [[ModA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] +// INCLUDE_TREE-NEXT: module-deps: +// INCLUDE_TREE-NEXT: file-deps: +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/ModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/SubModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/SubSubModA.h +// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/module.modulemap +// INCLUDE_TREE-NEXT: build-args: +// INCLUDE_TREE-SAME: -cc1 +// INCLUDE_TREE-SAME: -fcas-path +// INCLUDE_TREE-SAME: -fcas-include-tree llvmcas://{{[[:xdigit:]]+}} +// INCLUDE_TREE-SAME: -fcache-compile-job + // INCLUDE_TREE: dependencies: // INCLUDE_TREE-NEXT: command 0: // INCLUDE_TREE-NEXT: context-hash: [[HASH_TU:[A-Z0-9]+]] // INCLUDE_TREE-NEXT: cache-key: [[INC_TU_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // INCLUDE_TREE-NEXT: module-deps: +// INCLUDE_TREE-NEXT: ModA:[[HASH_MOD_A]] // INCLUDE_TREE-NEXT: file-deps: // INCLUDE_TREE-NEXT: [[PREFIX]]/scan-deps-cas.m -// INCLUDE_TREE-NEXT: [[PREFIX]]/Inputs/module/ModA.h // INCLUDE_TREE-NEXT: build-args: // INCLUDE_TREE-SAME: -cc1 // INCLUDE_TREE-SAME: -fcas-path // INCLUDE_TREE-SAME: -fcas-include-tree llvmcas://{{[[:xdigit:]]+}} // INCLUDE_TREE-SAME: -fcache-compile-job +// INCLUDE_TREE-SAME: -fmodule-file-cache-key [[PCM:.*ModA_.*pcm]] [[ModA_CACHE_KEY]] +// INCLUDE_TREE-SAME: -fmodule-file={{(ModA=)?}}[[PCM]] \ No newline at end of file diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index d418caaf20b42..093b9cba6d952 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -145,8 +145,8 @@ ScanningOutputFormat DependencyScannerServiceOptions::getFormat() const { if (llvm::sys::Process::GetEnv("CLANG_CACHE_USE_CASFS_DEPSCAN")) return ScanningOutputFormat::FullTree; - // Note: default caching behaviour is currently cas-fs. - return ScanningOutputFormat::FullTree; + // Use include-tree by default. + return ScanningOutputFormat::FullIncludeTree; } CXDependencyScannerService From 24e2e4cfe7a4b0de6e0f6dbf509af04f3a108fed Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Tue, 4 Apr 2023 16:17:33 -0700 Subject: [PATCH 72/88] [libclang][cas] Fix local path that showed up in test rdar://107634986 (cherry picked from commit 3c2b29ff2554ddbd4f20934b31863df7d2722528) --- clang/test/Index/Core/scan-deps-cas.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index a74c6431364a1..afeba129d9ecc 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -70,7 +70,7 @@ // INCLUDE_TREE-NEXT: module: // INCLUDE_TREE-NEXT: name: ModA // INCLUDE_TREE-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] -// INCLUDE_TREE-NEXT: module-map-path: /Users/blangmuir/src/cas/llvm-project/clang/test/Index/Core/Inputs/module/module.modulemap +// INCLUDE_TREE-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap // INCLUDE_TREE-NEXT: cache-key: [[ModA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // INCLUDE_TREE-NEXT: module-deps: // INCLUDE_TREE-NEXT: file-deps: From 0c0bb5a2c21bcc661ec2837304ae29cb3361367e Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 4 Apr 2023 16:32:52 -0700 Subject: [PATCH 73/88] [Support] Fix module build for ScopedDurationTimer rdar://107634627 --- llvm/include/llvm/Support/ScopedDurationTimer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/include/llvm/Support/ScopedDurationTimer.h b/llvm/include/llvm/Support/ScopedDurationTimer.h index ad704d20b7b66..4f8e90d6ae19b 100644 --- a/llvm/include/llvm/Support/ScopedDurationTimer.h +++ b/llvm/include/llvm/Support/ScopedDurationTimer.h @@ -11,6 +11,7 @@ #include #include +#include namespace llvm { From 090ef9a463cbc071861da34d6c41593aaf81ead5 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 23 Jan 2023 11:00:47 -0800 Subject: [PATCH 74/88] [VirtualOutput] Add option to only overwrite output if content is different Add new output configuration that checks if Output.keep() will change the file content and skip overwriting when the content are the same. This avoids updating file status for the output file like timestamps. --- .../llvm/Support/VirtualOutputConfig.def | 2 + llvm/lib/Support/VirtualOutputBackends.cpp | 102 ++++++++++++++++++ .../Support/VirtualOutputBackendsTest.cpp | 41 +++++++ .../Support/VirtualOutputConfigTest.cpp | 2 + 4 files changed, 147 insertions(+) diff --git a/llvm/include/llvm/Support/VirtualOutputConfig.def b/llvm/include/llvm/Support/VirtualOutputConfig.def index bf9159ec22eb8..9b88448887e88 100644 --- a/llvm/include/llvm/Support/VirtualOutputConfig.def +++ b/llvm/include/llvm/Support/VirtualOutputConfig.def @@ -19,5 +19,7 @@ HANDLE_OUTPUT_CONFIG_FLAG(CRLF, false) // OF_CRLF. HANDLE_OUTPUT_CONFIG_FLAG(DiscardOnSignal, true) // E.g., RemoveFileOnSignal. HANDLE_OUTPUT_CONFIG_FLAG(AtomicWrite, true) // E.g., use temporaries. HANDLE_OUTPUT_CONFIG_FLAG(ImplyCreateDirectories, true) +// Skip atomic write if existing file content is the same +HANDLE_OUTPUT_CONFIG_FLAG(OnlyIfDifferent, false) #undef HANDLE_OUTPUT_CONFIG_FLAG diff --git a/llvm/lib/Support/VirtualOutputBackends.cpp b/llvm/lib/Support/VirtualOutputBackends.cpp index 6b90c4db9d5bb..e57282a2973df 100644 --- a/llvm/lib/Support/VirtualOutputBackends.cpp +++ b/llvm/lib/Support/VirtualOutputBackends.cpp @@ -337,6 +337,89 @@ Error OnDiskOutputFile::initializeStream() { return Error::success(); } +namespace { +class OpenFileRAII { + static const int InvalidFd = -1; + +public: + int Fd = InvalidFd; + + ~OpenFileRAII() { + if (Fd != InvalidFd) + llvm::sys::Process::SafelyCloseFileDescriptor(Fd); + } +}; + +enum class FileDifference : uint8_t { + /// The source and destination paths refer to the exact same file. + IdenticalFile, + /// The source and destination paths refer to separate files with identical + /// contents. + SameContents, + /// The source and destination paths refer to separate files with different + /// contents. + DifferentContents +}; +} // end anonymous namespace + +static Expected +areFilesDifferent(const llvm::Twine &Source, const llvm::Twine &Destination) { + if (sys::fs::equivalent(Source, Destination)) + return FileDifference::IdenticalFile; + + OpenFileRAII SourceFile; + sys::fs::file_status SourceStatus; + // If we can't open the source file, fail. + if (std::error_code EC = sys::fs::openFileForRead(Source, SourceFile.Fd)) + return convertToOutputError(Source, EC); + + // If we can't stat the source file, fail. + if (std::error_code EC = sys::fs::status(SourceFile.Fd, SourceStatus)) + return convertToOutputError(Source, EC); + + OpenFileRAII DestFile; + sys::fs::file_status DestStatus; + // If we can't open the destination file, report different. + if (std::error_code Error = + sys::fs::openFileForRead(Destination, DestFile.Fd)) + return FileDifference::DifferentContents; + + // If we can't open the destination file, report different. + if (std::error_code Error = sys::fs::status(DestFile.Fd, DestStatus)) + return FileDifference::DifferentContents; + + // If the files are different sizes, they must be different. + uint64_t Size = SourceStatus.getSize(); + if (Size != DestStatus.getSize()) + return FileDifference::DifferentContents; + + // If both files are zero size, they must be the same. + if (Size == 0) + return FileDifference::SameContents; + + // The two files match in size, so we have to compare the bytes to determine + // if they're the same. + std::error_code SourceRegionErr; + sys::fs::mapped_file_region SourceRegion( + sys::fs::convertFDToNativeFile(SourceFile.Fd), + sys::fs::mapped_file_region::readonly, Size, 0, SourceRegionErr); + if (SourceRegionErr) + return convertToOutputError(Source, SourceRegionErr); + + std::error_code DestRegionErr; + sys::fs::mapped_file_region DestRegion( + sys::fs::convertFDToNativeFile(DestFile.Fd), + sys::fs::mapped_file_region::readonly, Size, 0, DestRegionErr); + + if (DestRegionErr) + return FileDifference::DifferentContents; + + if (memcmp(SourceRegion.const_data(), DestRegion.const_data(), Size) != 0) + return FileDifference::DifferentContents; + + return FileDifference::SameContents; +} + Error OnDiskOutputFile::keep() { // Destroy the streams to flush them. BufferOS.reset(); @@ -351,6 +434,25 @@ Error OnDiskOutputFile::keep() { if (!TempPath) return Error::success(); + if (Config.getOnlyIfDifferent()) { + auto Result = areFilesDifferent(*TempPath, OutputPath); + if (!Result) + return Result.takeError(); + switch (*Result) { + case FileDifference::IdenticalFile: + // Do nothing for a self-move. + return Error::success(); + + case FileDifference::SameContents: + // Files are identical; remove the source file. + (void) sys::fs::remove(*TempPath); + return Error::success(); + + case FileDifference::DifferentContents: + break; // Rename the file. + } + } + // Move temporary to the final output path and remove it if that fails. std::error_code RenameEC = sys::fs::rename(*TempPath, OutputPath); if (!RenameEC) diff --git a/llvm/unittests/Support/VirtualOutputBackendsTest.cpp b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp index 1055f448a349c..2369a5937e413 100644 --- a/llvm/unittests/Support/VirtualOutputBackendsTest.cpp +++ b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp @@ -765,4 +765,45 @@ TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendCreateError) { FailedWithMessage(Error1->Msg)); } +TEST(OnDiskBackendTest, OnlyIfDifferent) { + OnDiskOutputBackendProvider Provider; + auto Backend = Provider.createBackend(); + std::string FilePath = Provider.getFilePathToCreate(); + StringRef Data = "some data"; + OutputConfig Config = OutputConfig().setOnlyIfDifferent(); + + OutputFile O1, O2, O3; + sys::fs::file_status Status1, Status2, Status3; + // Write first file. + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1), + Succeeded()); + O1 << Data; + EXPECT_THAT_ERROR(O1.keep(), Succeeded()); + EXPECT_FALSE(O1.isOpen()); + EXPECT_FALSE(sys::fs::status(FilePath, Status1, /*follow=*/false)); + + // Write second with same content. + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2), + Succeeded()); + O2 << Data; + EXPECT_THAT_ERROR(O2.keep(), Succeeded()); + EXPECT_FALSE(O2.isOpen()); + EXPECT_FALSE(sys::fs::status(FilePath, Status2, /*follow=*/false)); + + // Make sure the output path file is not modified with same content. + EXPECT_EQ(Status1.getUniqueID(), Status2.getUniqueID()); + + // Write third with different content. + EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O3), + Succeeded()); + O3 << Data << "\n"; + EXPECT_THAT_ERROR(O3.keep(), Succeeded()); + EXPECT_FALSE(O3.isOpen()); + EXPECT_FALSE(sys::fs::status(FilePath, Status3, /*follow=*/false)); + + // This should overwrite the file and create a different UniqueID. + EXPECT_NE(Status1.getUniqueID(), Status3.getUniqueID()); +} + + } // end namespace diff --git a/llvm/unittests/Support/VirtualOutputConfigTest.cpp b/llvm/unittests/Support/VirtualOutputConfigTest.cpp index 11e0935e9e25a..f5545c10ec752 100644 --- a/llvm/unittests/Support/VirtualOutputConfigTest.cpp +++ b/llvm/unittests/Support/VirtualOutputConfigTest.cpp @@ -22,6 +22,7 @@ TEST(VirtualOutputConfigTest, construct) { EXPECT_TRUE(OutputConfig().getDiscardOnSignal()); EXPECT_TRUE(OutputConfig().getAtomicWrite()); EXPECT_TRUE(OutputConfig().getImplyCreateDirectories()); + EXPECT_FALSE(OutputConfig().getOnlyIfDifferent()); // Test inverted defaults. EXPECT_TRUE(OutputConfig().getNoText()); @@ -29,6 +30,7 @@ TEST(VirtualOutputConfigTest, construct) { EXPECT_FALSE(OutputConfig().getNoDiscardOnSignal()); EXPECT_FALSE(OutputConfig().getNoAtomicWrite()); EXPECT_FALSE(OutputConfig().getNoImplyCreateDirectories()); + EXPECT_TRUE(OutputConfig().getNoOnlyIfDifferent()); } TEST(VirtualOutputConfigTest, set) { From d8c58b212a5019b442a98ed533aa898342d3308e Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 24 Jan 2023 11:11:52 -0800 Subject: [PATCH 75/88] [VirtualOutputBackend] Add HashingOutputBackend Add a hashing outputbackend that can hash the outputs for sanity check. --- .../llvm/Support/HashingOutputBackend.h | 114 ++++++++++++++++++ .../Support/VirtualOutputBackendsTest.cpp | 34 ++++++ 2 files changed, 148 insertions(+) create mode 100644 llvm/include/llvm/Support/HashingOutputBackend.h diff --git a/llvm/include/llvm/Support/HashingOutputBackend.h b/llvm/include/llvm/Support/HashingOutputBackend.h new file mode 100644 index 0000000000000..2c760515e99be --- /dev/null +++ b/llvm/include/llvm/Support/HashingOutputBackend.h @@ -0,0 +1,114 @@ +//===- HashingOutputBackends.h - Hashing output backends --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H +#define LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H + +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/HashBuilder.h" +#include "llvm/Support/VirtualOutputBackend.h" +#include "llvm/Support/VirtualOutputConfig.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace vfs { + +/// raw_pwrite_stream that writes to a hasher. +template +class HashingStream : public llvm::raw_pwrite_stream { +private: + SmallVector Buffer; + raw_svector_ostream OS; + + using HashBuilderT = HashBuilder; + HashBuilderT Builder; + + void write_impl(const char *Ptr, size_t Size) override { + OS.write(Ptr, Size); + } + + void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override { + OS.pwrite(Ptr, Size, Offset); + } + + uint64_t current_pos() const override { return OS.str().size(); } + +public: + HashingStream() : OS(Buffer) { SetUnbuffered(); } + + auto final() { + Builder.update(OS.str()); + return Builder.final(); + } +}; + +template class HashingOutputFile; + +/// An output backend that only generates the hash for outputs. +template class HashingOutputBackend : public OutputBackend { +private: + friend class HashingOutputFile; + void addOutputFile(StringRef Path, StringRef Hash) { + OutputHashes[Path] = std::string(Hash); + } + +protected: + IntrusiveRefCntPtr cloneImpl() const override { + return const_cast *>(this); + } + + Expected> + createFileImpl(StringRef Path, Optional Config) override { + return std::make_unique>(Path, *this); + } + +public: + /// Iterator for all the output file names. + auto outputFiles() const { return OutputHashes.keys(); } + + /// Get hash value for the output files in hex representation. + /// Return None if the requested path is not generated. + Optional getHashValueForFile(StringRef Path) { + auto F = OutputHashes.find(Path); + if (F == OutputHashes.end()) + return None; + return toHex(F->second); + } + +private: + StringMap OutputHashes; +}; + +/// HashingOutputFile. +template +class HashingOutputFile final : public OutputFileImpl { +public: + Error keep() override { + auto Result = OS.final(); + Backend.addOutputFile(OutputPath, toStringRef(Result)); + return Error::success(); + } + Error discard() override { return Error::success(); } + raw_pwrite_stream &getOS() override { return OS; } + + HashingOutputFile(StringRef OutputPath, + HashingOutputBackend &Backend) + : OutputPath(OutputPath.str()), Backend(Backend) {} + +private: + const std::string OutputPath; + HashingStream OS; + HashingOutputBackend &Backend; +}; + +} // namespace vfs +} // namespace llvm + +#endif // LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H diff --git a/llvm/unittests/Support/VirtualOutputBackendsTest.cpp b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp index 2369a5937e413..60c6c458c15fc 100644 --- a/llvm/unittests/Support/VirtualOutputBackendsTest.cpp +++ b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp @@ -8,6 +8,8 @@ #include "llvm/Support/VirtualOutputBackends.h" #include "llvm/ADT/StringMap.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/HashingOutputBackend.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Testing/Support/Error.h" #include "gtest/gtest.h" @@ -805,5 +807,37 @@ TEST(OnDiskBackendTest, OnlyIfDifferent) { EXPECT_NE(Status1.getUniqueID(), Status3.getUniqueID()); } +TEST(HashingBackendTest, HashOutput) { + HashingOutputBackend Backend; + OutputFile O1, O2, O3, O4, O5; + EXPECT_THAT_ERROR(Backend.createFile("file1").moveInto(O1), Succeeded()); + O1 << "some data"; + EXPECT_THAT_ERROR(O1.keep(), Succeeded()); + EXPECT_THAT_ERROR(Backend.createFile("file2").moveInto(O2), Succeeded()); + O2 << "some data"; + EXPECT_THAT_ERROR(O2.keep(), Succeeded()); + EXPECT_EQ(Backend.getHashValueForFile("file1"), + Backend.getHashValueForFile("file2")); + + EXPECT_THAT_ERROR(Backend.createFile("file3").moveInto(O3), Succeeded()); + O3 << "some "; + O3 << "data"; + EXPECT_THAT_ERROR(O3.keep(), Succeeded()); + EXPECT_EQ(Backend.getHashValueForFile("file1"), + Backend.getHashValueForFile("file3")); + + EXPECT_THAT_ERROR(Backend.createFile("file4").moveInto(O4), Succeeded()); + O4 << "same data"; + O4.getOS().pwrite("o", 1, 1); + EXPECT_THAT_ERROR(O4.keep(), Succeeded()); + EXPECT_EQ(Backend.getHashValueForFile("file1"), + Backend.getHashValueForFile("file4")); + + EXPECT_THAT_ERROR(Backend.createFile("file5").moveInto(O5), Succeeded()); + O5 << "different data"; + EXPECT_THAT_ERROR(O5.keep(), Succeeded()); + EXPECT_NE(Backend.getHashValueForFile("file1"), + Backend.getHashValueForFile("file5")); +} } // end namespace From 50a89bccbdb1506c96e3683bdb78ca6ebf10ae28 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 25 Jan 2023 18:02:42 -0800 Subject: [PATCH 76/88] [lldb] Fix lldb build after swift ASTContext change --- lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp index 4862848250398..dbe661052ecf2 100644 --- a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp +++ b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp @@ -2907,7 +2907,7 @@ swift::ASTContext *SwiftASTContext::GetASTContext() { GetLanguageOptions(), GetTypeCheckerOptions(), GetSILOptions(), GetSearchPathOptions(), GetClangImporterOptions(), GetSymbolGraphOptions(), GetSourceManager(), GetDiagnosticEngine(), - ReportModuleLoadingProgress)); + /*OutputBackend=*/nullptr, ReportModuleLoadingProgress)); m_diagnostic_consumer_ap.reset(new StoringDiagnosticConsumer(*this)); if (getenv("LLDB_SWIFT_DUMP_DIAGS")) { From 3980f4d0494d4e9d50f4f49e672e2cabeb9318e1 Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Wed, 5 Apr 2023 12:25:10 -0700 Subject: [PATCH 77/88] [clang][deps] NFC: Don't collect PCH input files Since b4c83a13, PCH input files are no longer necessary. (cherry picked from commit 34f143988f3f18dcca6d26a7a9817d8523300440) --- .../DependencyScanningWorker.cpp | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index af20f1a00dc63..401789d332cda 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -74,27 +74,18 @@ class PrebuiltModuleListener : public ASTReaderListener { public: PrebuiltModuleListener(CompilerInstance &CI, PrebuiltModuleFilesT &PrebuiltModuleFiles, - llvm::StringSet<> &InputFiles, bool VisitInputFiles, llvm::SmallVector &NewModuleFiles) : CI(CI), PrebuiltModuleFiles(PrebuiltModuleFiles), - InputFiles(InputFiles), VisitInputFiles(VisitInputFiles), + NewModuleFiles(NewModuleFiles) {} bool needsImportVisitation() const override { return true; } - bool needsInputFileVisitation() override { return VisitInputFiles; } - bool needsSystemInputFileVisitation() override { return VisitInputFiles; } void visitImport(StringRef ModuleName, StringRef Filename) override { if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second) NewModuleFiles.push_back(Filename.str()); } - bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden, - bool isExplicitModule) override { - InputFiles.insert(Filename); - return true; - } - bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override { CI.getFrontendOpts().ModuleCacheKeys.emplace_back(std::string(Filename), @@ -106,8 +97,6 @@ class PrebuiltModuleListener : public ASTReaderListener { private: CompilerInstance &CI; PrebuiltModuleFilesT &PrebuiltModuleFiles; - llvm::StringSet<> &InputFiles; - bool VisitInputFiles; llvm::SmallVector &NewModuleFiles; }; @@ -115,13 +104,10 @@ class PrebuiltModuleListener : public ASTReaderListener { /// transitively imports and contributing input files. static void visitPrebuiltModule(StringRef PrebuiltModuleFilename, CompilerInstance &CI, - PrebuiltModuleFilesT &ModuleFiles, - llvm::StringSet<> &InputFiles, - bool VisitInputFiles) { + PrebuiltModuleFilesT &ModuleFiles) { // List of module files to be processed. llvm::SmallVector Worklist{PrebuiltModuleFilename.str()}; - PrebuiltModuleListener Listener(CI, ModuleFiles, InputFiles, VisitInputFiles, - Worklist); + PrebuiltModuleListener Listener(CI, ModuleFiles, Worklist); while (!Worklist.empty()) ASTReader::readASTFileControlBlock( @@ -375,16 +361,13 @@ class DependencyScanningAction : public tooling::ToolAction { ScanInstance.createSourceManager(*FileMgr); - llvm::StringSet<> PrebuiltModulesInputFiles; // Store the list of prebuilt module files into header search options. This // will prevent the implicit build to create duplicate modules and will // force reuse of the existing prebuilt module files instead. if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) visitPrebuiltModule( ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance, - ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, - PrebuiltModulesInputFiles, - /*VisitInputFiles=*/getDepScanFS() != nullptr); + ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles); // Use the dependency scanning optimized file system if requested to do so. if (DepFS) { From 799f0e6328bf1d8493d51415cdf8d5efa1001729 Mon Sep 17 00:00:00 2001 From: Jan Svoboda Date: Wed, 5 Apr 2023 12:27:07 -0700 Subject: [PATCH 78/88] [clang][deps] NFC: Make PCH test more debuggable Overwriting CDB files made it difficult to debug earlier clang-scan-deps invocations. This commit creates separate files for each. (cherry picked from commit 65c0134872c1e298ade176b5c84d86b4a058a85c) --- clang/test/ClangScanDeps/modules-pch.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/clang/test/ClangScanDeps/modules-pch.c b/clang/test/ClangScanDeps/modules-pch.c index 1d46c89d57909..e2778dc7c4f6a 100644 --- a/clang/test/ClangScanDeps/modules-pch.c +++ b/clang/test/ClangScanDeps/modules-pch.c @@ -7,8 +7,8 @@ // Scan dependencies of the PCH: // -// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \ +// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_pch.json > %t/cdb_pch.json +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json -format experimental-full \ // RUN: -module-files-dir %t/build > %t/result_pch.json // RUN: cat %t/result_pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-PCH // @@ -94,8 +94,8 @@ // Scan dependencies of the TU: // -// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \ +// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu.json > %t/cdb_tu.json +// RUN: clang-scan-deps -compilation-database %t/cdb_tu.json -format experimental-full \ // RUN: -module-files-dir %t/build > %t/result_tu.json // RUN: cat %t/result_tu.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU // @@ -142,8 +142,8 @@ // Scan dependencies of the TU that has common modules with the PCH: // -// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu_with_common.json > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full \ +// RUN: sed "s|DIR|%/t|g" %S/Inputs/modules-pch/cdb_tu_with_common.json > %t/cdb_tu_with_common.json +// RUN: clang-scan-deps -compilation-database %t/cdb_tu_with_common.json -format experimental-full \ // RUN: -module-files-dir %t/build > %t/result_tu_with_common.json // RUN: cat %t/result_tu_with_common.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=CHECK-TU-WITH-COMMON // From ba36f09961e4ec5f207e3c2feb253c51a1b7bc63 Mon Sep 17 00:00:00 2001 From: Pavel Kosov Date: Wed, 25 Jan 2023 10:50:02 +0300 Subject: [PATCH 79/88] [lldb] Consider all breakpoints in breakpoint detection Currently in some cases lldb reports stop reason as "step out" or "step over" (from thread plan completion) instead of "breakpoint", if the user breakpoint happens to be set on the same address. The part of https://github.com/llvm/llvm-project/commit/f08f5c99262ff9eaa08956334accbb2614b0f7a2 seems to overwrite internal breakpoint detection logic, so that only the last breakpoint for the current stop address is considered. Together with step-out plans not clearing its breakpoint until they are destrouyed, this creates a situation when there is a user breakpoint set for address, but internal breakpoint makes lldb report a plan completion stop reason instead of breakpoint. This patch reverts that internal breakpoint detection logic to consider all breakpoints Reviewed By: jingham Differential Revision: https://reviews.llvm.org/D140368 (cherry picked from commit 2af0a478eaee5e6236e7e9fd9b1e3207228ce2ff) --- lldb/source/Target/StopInfo.cpp | 11 +- .../thread_plan_user_breakpoint/Makefile | 8 ++ .../TestThreadPlanUserBreakpoint.py | 121 ++++++++++++++++++ .../thread_plan_user_breakpoint/main.cpp | 11 ++ 4 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/Makefile create mode 100644 lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/TestThreadPlanUserBreakpoint.py create mode 100644 lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/main.cpp diff --git a/lldb/source/Target/StopInfo.cpp b/lldb/source/Target/StopInfo.cpp index 1c0af2b29d52d..87dccb2f6d409 100644 --- a/lldb/source/Target/StopInfo.cpp +++ b/lldb/source/Target/StopInfo.cpp @@ -256,7 +256,7 @@ class StopInfoBreakpoint : public StopInfo { if (!m_should_perform_action) return; m_should_perform_action = false; - bool internal_breakpoint = true; + bool all_stopping_locs_internal = true; ThreadSP thread_sp(m_thread_wp.lock()); @@ -421,8 +421,6 @@ class StopInfoBreakpoint : public StopInfo { continue; } - internal_breakpoint = bp_loc_sp->GetBreakpoint().IsInternal(); - // First run the precondition, but since the precondition is per // breakpoint, only run it once per breakpoint. std::pair::iterator, bool> result = @@ -509,7 +507,7 @@ class StopInfoBreakpoint : public StopInfo { loc_desc.GetData()); // We want this stop reported, so you will know we auto-continued // but only for external breakpoints: - if (!internal_breakpoint) + if (!bp_loc_sp->GetBreakpoint().IsInternal()) thread_sp->SetShouldReportStop(eVoteYes); auto_continue_says_stop = false; } @@ -539,6 +537,9 @@ class StopInfoBreakpoint : public StopInfo { actually_said_continue = true; } + if (m_should_stop && !bp_loc_sp->GetBreakpoint().IsInternal()) + all_stopping_locs_internal = false; + // If we are going to stop for this breakpoint, then remove the // breakpoint. if (callback_says_stop && bp_loc_sp && @@ -576,7 +577,7 @@ class StopInfoBreakpoint : public StopInfo { __FUNCTION__, m_value); } - if ((!m_should_stop || internal_breakpoint) && + if ((!m_should_stop || all_stopping_locs_internal) && thread_sp->CompletedPlanOverridesBreakpoint()) { // Override should_stop decision when we have completed step plan diff --git a/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/Makefile b/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/Makefile new file mode 100644 index 0000000000000..2c00681fa2280 --- /dev/null +++ b/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/Makefile @@ -0,0 +1,8 @@ +CXX_SOURCES := main.cpp + +ifneq (,$(findstring icc,$(CC))) + CXXFLAGS_EXTRAS := -debug inline-debug-info +endif + + +include Makefile.rules diff --git a/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/TestThreadPlanUserBreakpoint.py b/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/TestThreadPlanUserBreakpoint.py new file mode 100644 index 0000000000000..aaecdff0069f6 --- /dev/null +++ b/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/TestThreadPlanUserBreakpoint.py @@ -0,0 +1,121 @@ +""" +Test that breakpoints (reason = breakpoint) have more priority than +plan completion (reason = step in/out/over) when reporting stop reason after step, +in particular 'step out' and 'step over', and in addition 'step in'. +Check for correct StopReason when stepping to the line with breakpoint, +which should be eStopReasonBreakpoint in general, +and eStopReasonPlanComplete when breakpoint's condition fails or it is disabled. +""" + + +import unittest2 +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil + +class ThreadPlanUserBreakpointsTestCase(TestBase): + + def setUp(self): + TestBase.setUp(self) + + # Build and run to starting breakpoint + self.build() + src = lldb.SBFileSpec('main.cpp') + (self.target, self.process, self.thread, _) = \ + lldbutil.run_to_source_breakpoint(self, '// Start from here', src) + + # Setup two more breakpoints + self.breakpoints = [self.target.BreakpointCreateBySourceRegex('breakpoint_%i' % i, src) + for i in range(2)] + self.assertTrue( + all(bp and bp.GetNumLocations() == 1 for bp in self.breakpoints), + VALID_BREAKPOINT) + + def check_correct_stop_reason(self, breakpoint_idx, condition): + self.assertState(self.process.GetState(), lldb.eStateStopped) + if condition: + # All breakpoints active, stop reason is breakpoint + thread1 = lldbutil.get_one_thread_stopped_at_breakpoint(self.process, self.breakpoints[breakpoint_idx]) + self.assertEquals(self.thread, thread1, "Didn't stop at breakpoint %i." % breakpoint_idx) + else: + # Breakpoints are inactive, stop reason is plan complete + self.assertEquals(self.thread.GetStopReason(), lldb.eStopReasonPlanComplete, + 'Expected stop reason to be step into/over/out for inactive breakpoint %i line.' % breakpoint_idx) + + def change_breakpoints(self, action): + for bp in self.breakpoints: + action(bp) + + def check_thread_plan_user_breakpoint(self, condition, set_up_breakpoint_func): + # Make breakpoints active/inactive in different ways + self.change_breakpoints(lambda bp: set_up_breakpoint_func(condition, bp)) + + self.thread.StepInto() + # We should be stopped at the breakpoint_0 line with the correct stop reason + self.check_correct_stop_reason(0, condition) + + # This step-over creates a step-out from `func_1` plan + self.thread.StepOver() + # We should be stopped at the breakpoint_1 line with the correct stop reason + self.check_correct_stop_reason(1, condition) + + # Check explicit step-out + # Make sure we install the breakpoint at the right address: + # step-out might stop on different lines, if the compiler + # did or did not emit more instructions after the return + return_addr = self.thread.GetFrameAtIndex(1).GetPC() + step_out_breakpoint = self.target.BreakpointCreateByAddress(return_addr) + self.assertTrue(step_out_breakpoint, VALID_BREAKPOINT) + set_up_breakpoint_func(condition, step_out_breakpoint) + self.breakpoints.append(step_out_breakpoint) + self.thread.StepOut() + # We should be stopped somewhere in the main frame with the correct stop reason + self.check_correct_stop_reason(2, condition) + + # Run the process until termination + self.process.Continue() + self.assertState(self.process.GetState(), lldb.eStateExited) + + def set_up_breakpoints_condition(self, condition, bp): + # Set breakpoint condition to true/false + conditionStr = 'true' if condition else 'false' + bp.SetCondition(conditionStr) + + def set_up_breakpoints_enable(self, condition, bp): + # Enable/disable breakpoint + bp.SetEnabled(condition) + + def set_up_breakpoints_callback(self, condition, bp): + # Set breakpoint callback to return True/False + bp.SetScriptCallbackBody('return %s' % condition) + + def test_thread_plan_user_breakpoint_conditional_active(self): + # Test with breakpoints having true condition + self.check_thread_plan_user_breakpoint(condition=True, + set_up_breakpoint_func=self.set_up_breakpoints_condition) + + def test_thread_plan_user_breakpoint_conditional_inactive(self): + # Test with breakpoints having false condition + self.check_thread_plan_user_breakpoint(condition=False, + set_up_breakpoint_func=self.set_up_breakpoints_condition) + + def test_thread_plan_user_breakpoint_unconditional_active(self): + # Test with breakpoints enabled unconditionally + self.check_thread_plan_user_breakpoint(condition=True, + set_up_breakpoint_func=self.set_up_breakpoints_enable) + + def test_thread_plan_user_breakpoint_unconditional_inactive(self): + # Test with breakpoints disabled unconditionally + self.check_thread_plan_user_breakpoint(condition=False, + set_up_breakpoint_func=self.set_up_breakpoints_enable) + + def test_thread_plan_user_breakpoint_callback_active(self): + # Test with breakpoints with callback that returns 'True' + self.check_thread_plan_user_breakpoint(condition=True, + set_up_breakpoint_func=self.set_up_breakpoints_callback) + + def test_thread_plan_user_breakpoint_callback_inactive(self): + # Test with breakpoints with callback that returns 'False' + self.check_thread_plan_user_breakpoint(condition=False, + set_up_breakpoint_func=self.set_up_breakpoints_callback) diff --git a/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/main.cpp b/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/main.cpp new file mode 100644 index 0000000000000..2d7e06eb6da49 --- /dev/null +++ b/lldb/test/API/functionalities/breakpoint/thread_plan_user_breakpoint/main.cpp @@ -0,0 +1,11 @@ +int func_1() { return 1; } + +int func_2() { + func_1(); // breakpoint_0 + return 1 + func_1(); // breakpoint_1 +} + +int main(int argc, char const *argv[]) { + func_2(); // Start from here + return 0; +} From db9ff13b2487ddcd7e050d1b85bb8882f72b2881 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Tue, 4 Apr 2023 10:47:24 -0700 Subject: [PATCH 80/88] [lldb] Log Swift object description messages to expr channel --- .../LanguageRuntime/Swift/SwiftLanguageRuntime.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp index e0c0040e61796..5bf9644ea84b8 100644 --- a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp +++ b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp @@ -35,6 +35,7 @@ #include "lldb/Symbol/Function.h" #include "lldb/Symbol/VariableList.h" #include "lldb/Target/RegisterContext.h" +#include "lldb/Utility/LLDBLog.h" #include "lldb/Utility/Log.h" #include "lldb/Utility/OptionParsing.h" #include "lldb/Utility/Timer.h" @@ -924,7 +925,7 @@ void SwiftLanguageRuntimeImpl::ModulesDidLoad(const ModuleList &module_list) { std::string SwiftLanguageRuntimeImpl::GetObjectDescriptionExpr_Result(ValueObject &object) { - Log *log(GetLog(LLDBLog::DataFormatters)); + Log *log(GetLog(LLDBLog::DataFormatters | LLDBLog::Expressions)); std::string expr_string = llvm::formatv("Swift._DebuggerSupport.stringForPrintObject({0})", object.GetName().GetCString()).str(); @@ -936,7 +937,7 @@ SwiftLanguageRuntimeImpl::GetObjectDescriptionExpr_Result(ValueObject &object) { std::string SwiftLanguageRuntimeImpl::GetObjectDescriptionExpr_Ref(ValueObject &object) { - Log *log(GetLog(LLDBLog::DataFormatters)); + Log *log(GetLog(LLDBLog::DataFormatters | LLDBLog::Expressions)); StreamString expr_string; std::string expr_str @@ -960,7 +961,7 @@ std::string SwiftLanguageRuntimeImpl::GetObjectDescriptionExpr_Copy(ValueObject &object, lldb::addr_t ©_location) { - Log *log(GetLog(LLDBLog::DataFormatters)); + Log *log(GetLog(LLDBLog::DataFormatters | LLDBLog::Expressions)); ValueObjectSP static_sp(object.GetStaticValue()); @@ -1028,7 +1029,7 @@ SwiftLanguageRuntimeImpl::RunObjectDescriptionExpr(ValueObject &object, std::string &expr_string, Stream &result) { - Log *log(GetLog(LLDBLog::DataFormatters)); + Log *log(GetLog(LLDBLog::DataFormatters | LLDBLog::Expressions)); ValueObjectSP result_sp; EvaluateExpressionOptions eval_options; eval_options.SetLanguage(lldb::eLanguageTypeSwift); From c31a24bb88d8c2896ac432200fb11df081bb938c Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Wed, 5 Apr 2023 19:20:31 -0700 Subject: [PATCH 81/88] Fix doxygen comment Fix doxygen warning: '\c' command does not have a valid word argument [-Wdocumentation] --- clang/include/clang/Lex/PPCachedActions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/include/clang/Lex/PPCachedActions.h b/clang/include/clang/Lex/PPCachedActions.h index c741fc022a5c4..a76a2ac1a5302 100644 --- a/clang/include/clang/Lex/PPCachedActions.h +++ b/clang/include/clang/Lex/PPCachedActions.h @@ -36,7 +36,7 @@ class PPCachedActions { FileID FID; Module *Submodule; }; - /// The module that is imported by an \c #include directive or \c @import. + /// The module that is imported by an \c #include directive or \c \@import. struct IncludeModule { SmallVector, 2> ImportPath; // Whether this module should only be "marked visible" rather than imported. From 447383339b2da12363d6af3ee6035eb9f438a2c5 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 16 Mar 2023 08:20:06 -0700 Subject: [PATCH 82/88] [lldb][test] Replace use of p with expression in Shell tests (NFC) In Shell tests, replace use of the `p` alias with the `expression` command. To avoid conflating tests of the alias with tests of the expression command, this patch canonicalizes to the use `expression`. See also D141539 which made the same change to API tests. Differential Revision: https://reviews.llvm.org/D146230 (cherry picked from commit d875838e8b45cb0da5070298d0c1a2d1ee78ce7e) --- lldb/test/Shell/Commands/command-source.test | 2 +- .../Commands/command-stop-hook-no-target.test | 2 +- lldb/test/Shell/Expr/nodefaultlib.cpp | 4 +- lldb/test/Shell/Register/x86-64-fp-read.test | 8 ++-- lldb/test/Shell/Register/x86-db-read.test | 8 ++-- lldb/test/Shell/Register/x86-fp-read.test | 4 +- .../Settings/Inputs/DontStopCommandSource.in | 2 +- .../Settings/Inputs/StopCommandSource.in | 2 +- .../TestStopCommandSourceOnError.test | 4 +- .../DWARF/debug-types-expressions.test | 14 +++---- .../DWARF/split-dwarf-expression-eval-bug.cpp | 4 +- .../x86/DW_TAG_GNU_call_site-DW_AT_low_pc.s | 2 +- .../x86/DW_TAG_variable-DW_AT_const_value.s | 2 +- .../DWARF/x86/debug-types-basic.test | 8 ++-- .../x86/debug-types-missing-signature.test | 4 +- .../SymbolFile/DWARF/x86/dwarf5-line-strp.s | 2 +- .../SymbolFile/DWARF/x86/limit-debug-info.s | 4 +- .../Inputs/inline_sites_live.lldbinit | 6 +-- .../NativePDB/Inputs/local-variables.lldbinit | 38 +++++++++---------- .../NativePDB/inline_sites_live.cpp | 6 +-- .../SymbolFile/NativePDB/local-variables.cpp | 38 +++++++++---------- .../PDB/Inputs/ExpressionsTest0.script | 10 ++--- .../PDB/Inputs/ExpressionsTest1.script | 2 +- .../PDB/Inputs/ExpressionsTest2.script | 4 +- .../Shell/SymbolFile/PDB/Inputs/VBases.script | 4 +- .../Shell/SymbolFile/PDB/expressions.test | 18 ++++----- 26 files changed, 101 insertions(+), 101 deletions(-) diff --git a/lldb/test/Shell/Commands/command-source.test b/lldb/test/Shell/Commands/command-source.test index fa389f2a12889..579e3989f6fd0 100644 --- a/lldb/test/Shell/Commands/command-source.test +++ b/lldb/test/Shell/Commands/command-source.test @@ -6,7 +6,7 @@ # RUN: %lldb -x -b -o 'settings set interpreter.stop-command-source-on-error false' -o "command source %s" 2>&1 | FileCheck %s --check-prefix CONTINUE bogus -p 10+1 +expression 10+1 # CONTINUE: $0 = 11 # STOP-NOT: $0 = 11 diff --git a/lldb/test/Shell/Commands/command-stop-hook-no-target.test b/lldb/test/Shell/Commands/command-stop-hook-no-target.test index ee9ded164d745..cc5e71dff00cd 100644 --- a/lldb/test/Shell/Commands/command-stop-hook-no-target.test +++ b/lldb/test/Shell/Commands/command-stop-hook-no-target.test @@ -1,4 +1,4 @@ # RUN: %clang_host -g %S/Inputs/main.c -o %t -# RUN: %lldb -b -o 'target stop-hook add --name test --shlib test -o "p 95000 + 126"' -o 'file %t' -o 'b main' -o 'r' 2>&1 | FileCheck %s +# RUN: %lldb -b -o 'target stop-hook add --name test --shlib test -o "expression 95000 + 126"' -o 'file %t' -o 'b main' -o 'r' 2>&1 | FileCheck %s # CHECK: Stop hook #1 added # CHECK-NOT: 95126 diff --git a/lldb/test/Shell/Expr/nodefaultlib.cpp b/lldb/test/Shell/Expr/nodefaultlib.cpp index fb53da7fcb08b..f97d8aad9a8db 100644 --- a/lldb/test/Shell/Expr/nodefaultlib.cpp +++ b/lldb/test/Shell/Expr/nodefaultlib.cpp @@ -7,10 +7,10 @@ // XFAIL: system-netbsd || system-freebsd || system-darwin // RUN: %build %s --nodefaultlib -o %t -// RUN: %lldb %t -o "b main" -o run -o "p call_me(5, 6)" -o exit \ +// RUN: %lldb %t -o "b main" -o run -o "expression call_me(5, 6)" -o exit \ // RUN: | FileCheck %s -// CHECK: p call_me(5, 6) +// CHECK: expression call_me(5, 6) // CHECK: (int) $0 = 30 int call_me(int x, long y) { return x * y; } diff --git a/lldb/test/Shell/Register/x86-64-fp-read.test b/lldb/test/Shell/Register/x86-64-fp-read.test index 39fdaf170f3f2..e23e333c204a5 100644 --- a/lldb/test/Shell/Register/x86-64-fp-read.test +++ b/lldb/test/Shell/Register/x86-64-fp-read.test @@ -9,9 +9,9 @@ process launch # CHECK: Process {{.*}} stopped # fdiv (%rbx) gets encoded into 2 bytes, int3 into 1 byte -print (void*)($pc-3) +expression (void*)($pc-3) # CHECK: (void *) $0 = [[FDIV:0x[0-9a-f]*]] -print &zero +expression &zero # CHECK: (uint32_t *) $1 = [[ZERO:0x[0-9a-f]*]] register read --all @@ -32,9 +32,9 @@ register read --all # CHECK-DAG: st{{(mm)?}}7 = {0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00} # legacy approach, superseded by fip/fdp registers -print (void*)($fiseg*0x100000000 + $fioff) +expression (void*)($fiseg*0x100000000 + $fioff) # CHECK: (void *) $2 = [[FDIV]] -print (uint32_t*)($foseg * 0x100000000 + $fooff) +expression (uint32_t*)($foseg * 0x100000000 + $fooff) # CHECK: (uint32_t *) $3 = [[ZERO]] process continue diff --git a/lldb/test/Shell/Register/x86-db-read.test b/lldb/test/Shell/Register/x86-db-read.test index f8380440c6270..85cdcd1cb8104 100644 --- a/lldb/test/Shell/Register/x86-db-read.test +++ b/lldb/test/Shell/Register/x86-db-read.test @@ -15,13 +15,13 @@ watchpoint set variable -w write g_32w watchpoint set variable -w read_write g_32rw # CHECK: Watchpoint created: Watchpoint 4: addr = 0x{{[0-9a-f]*}} size = 4 state = enabled type = rw -print &g_8w +expression &g_8w # CHECK: (uint8_t *) $0 = [[VAR8W:0x[0-9a-f]*]] -print &g_16rw +expression &g_16rw # CHECK: (uint16_t *) $1 = [[VAR16RW:0x[0-9a-f]*]] -print &g_32w +expression &g_32w # CHECK: (uint32_t *) $2 = [[VAR32W:0x[0-9a-f]*]] -print &g_32rw +expression &g_32rw # CHECK: (uint32_t *) $3 = [[VAR64RW:0x[0-9a-f]*]] register read --all diff --git a/lldb/test/Shell/Register/x86-fp-read.test b/lldb/test/Shell/Register/x86-fp-read.test index 566157b938367..f668b8cb78658 100644 --- a/lldb/test/Shell/Register/x86-fp-read.test +++ b/lldb/test/Shell/Register/x86-fp-read.test @@ -9,9 +9,9 @@ process launch # CHECK: Process {{.*}} stopped # fdiv (%rbx) gets encoded into 2 bytes, int3 into 1 byte -print (void*)($pc-3) +expression (void*)($pc-3) # CHECK: (void *) $0 = [[FDIV:0x[0-9a-f]*]] -print &zero +expression &zero # CHECK: (uint32_t *) $1 = [[ZERO:0x[0-9a-f]*]] register read --all diff --git a/lldb/test/Shell/Settings/Inputs/DontStopCommandSource.in b/lldb/test/Shell/Settings/Inputs/DontStopCommandSource.in index e33a1e329cad9..f3ef2a9e7016b 100644 --- a/lldb/test/Shell/Settings/Inputs/DontStopCommandSource.in +++ b/lldb/test/Shell/Settings/Inputs/DontStopCommandSource.in @@ -1,3 +1,3 @@ settings set interpreter.stop-command-source-on-error false bogus -print 123400000 + 56789 +expression 123400000 + 56789 diff --git a/lldb/test/Shell/Settings/Inputs/StopCommandSource.in b/lldb/test/Shell/Settings/Inputs/StopCommandSource.in index 19974c0c444b1..1159554e3425f 100644 --- a/lldb/test/Shell/Settings/Inputs/StopCommandSource.in +++ b/lldb/test/Shell/Settings/Inputs/StopCommandSource.in @@ -1,3 +1,3 @@ settings set interpreter.stop-command-source-on-error true bogus -print 123400000 + 56789 +expression 123400000 + 56789 diff --git a/lldb/test/Shell/Settings/TestStopCommandSourceOnError.test b/lldb/test/Shell/Settings/TestStopCommandSourceOnError.test index d734a0940a2d7..f99c9c204c19c 100644 --- a/lldb/test/Shell/Settings/TestStopCommandSourceOnError.test +++ b/lldb/test/Shell/Settings/TestStopCommandSourceOnError.test @@ -12,10 +12,10 @@ # RUN: %lldb -b -o 'settings set interpreter.stop-command-source-on-error false' -s %S/Inputs/StopCommandSource.in | FileCheck %s --check-prefix CONTINUE # FIXME: Should continue -# RUN: not %lldb -b -s %S/Inputs/DontStopCommandSource.in -o 'bogus' -o 'print 111100000 + 11111' | FileCheck %s --check-prefix STOP +# RUN: not %lldb -b -s %S/Inputs/DontStopCommandSource.in -o 'bogus' -o 'expression 111100000 + 11111' | FileCheck %s --check-prefix STOP # FIXME: Should continue -# RUN: not %lldb -b -o 'settings set interpreter.stop-command-source-on-error false' -o 'bogus' -o 'print 123400000 + 56789' | FileCheck %s --check-prefix STOP +# RUN: not %lldb -b -o 'settings set interpreter.stop-command-source-on-error false' -o 'bogus' -o 'expression 123400000 + 56789' | FileCheck %s --check-prefix STOP # FIXME: Should continue # RUN: not %lldb -b -s %S/Inputs/DontStopCommandSource.in | FileCheck %s --check-prefix STOP diff --git a/lldb/test/Shell/SymbolFile/DWARF/debug-types-expressions.test b/lldb/test/Shell/SymbolFile/DWARF/debug-types-expressions.test index 0442cd8faa3b2..653af7b17c449 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/debug-types-expressions.test +++ b/lldb/test/Shell/SymbolFile/DWARF/debug-types-expressions.test @@ -40,18 +40,18 @@ frame variable *a # CHECK-NEXT: j = 42 # CHECK-NEXT: } -print a->f() -# CHECK-LABEL: print a->f() +expression a->f() +# CHECK-LABEL: expression a->f() # CHECK: (int) $0 = 47 -print ns::A() -# CHECK-LABEL: print ns::A() +expression ns::A() +# CHECK-LABEL: expression ns::A() # CHECK: (ns::A) $1 = (i = 147) -print ns::A().i + a->i -# CHECK-LABEL: print ns::A().i + a->i +expression ns::A().i + a->i +# CHECK-LABEL: expression ns::A().i + a->i # CHECK: (int) $2 = 194 -print ns::A().getA() +expression ns::A().getA() # CHECK-LABEL: ns::A().getA() # CHECK: (A) $3 = (i = 146) diff --git a/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-expression-eval-bug.cpp b/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-expression-eval-bug.cpp index ef1aa257665df..4a8004ddd287f 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-expression-eval-bug.cpp +++ b/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-expression-eval-bug.cpp @@ -11,10 +11,10 @@ // RUN: %clang_host -c -gsplit-dwarf -g %s -o %t2.o -DTWO // RUN: %clang_host -c -gsplit-dwarf -g %s -o %t3.o -DTHREE // RUN: %clang_host %t1.o %t2.o %t3.o -o %t -// RUN: %lldb %t -o "br set -n foo" -o run -o "p bool_in_first_cu" -o exit \ +// RUN: %lldb %t -o "br set -n foo" -o run -o "expression bool_in_first_cu" -o exit \ // RUN: | FileCheck %s -// CHECK: (lldb) p bool_in_first_cu +// CHECK: (lldb) expression bool_in_first_cu // CHECK: (bool) $0 = true diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/DW_TAG_GNU_call_site-DW_AT_low_pc.s b/lldb/test/Shell/SymbolFile/DWARF/x86/DW_TAG_GNU_call_site-DW_AT_low_pc.s index 8cc1cc1ed2f95..c6e5ccda5f2ef 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/x86/DW_TAG_GNU_call_site-DW_AT_low_pc.s +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/DW_TAG_GNU_call_site-DW_AT_low_pc.s @@ -11,7 +11,7 @@ # REQUIRES: target-x86_64, system-linux, lld # RUN: %clang_host -o %t %s -# RUN: %lldb %t -o r -o 'p p' -o exit | FileCheck %s +# RUN: %lldb %t -o r -o 'expression p' -o exit | FileCheck %s # CHECK: (int) $0 = 1 diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/DW_TAG_variable-DW_AT_const_value.s b/lldb/test/Shell/SymbolFile/DWARF/x86/DW_TAG_variable-DW_AT_const_value.s index 7ab9a8c61bafd..1d9bec342de5b 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/x86/DW_TAG_variable-DW_AT_const_value.s +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/DW_TAG_variable-DW_AT_const_value.s @@ -1,7 +1,7 @@ # This tests that lldb is able to print DW_TAG_variable using DW_AT_const_value. # RUN: llvm-mc -triple x86_64-unknown-linux-gnu %s -filetype=obj > %t.o -# RUN: %lldb %t.o -o "p/x magic64" -o exit | FileCheck %s +# RUN: %lldb %t.o -o "expression/x magic64" -o exit | FileCheck %s # CHECK: (const long) $0 = 0xed9a924c00011151 diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-basic.test b/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-basic.test index 947b43ba6b2ff..dc8005f73930e 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-basic.test +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-basic.test @@ -51,12 +51,12 @@ type lookup EC # CHECK-NEXT: e3 # CHECK-NEXT: } -print (E) 1 -# CHECK-LABEL: print (E) 1 +expression (E) 1 +# CHECK-LABEL: expression (E) 1 # CHECK: (E) $0 = e2 -print (EC) 1 -# CHECK-LABEL: print (EC) 1 +expression (EC) 1 +# CHECK-LABEL: expression (EC) 1 # CHECK: (EC) $1 = e2 target variable a e ec diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-missing-signature.test b/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-missing-signature.test index 8bdc2219cc1c9..e94b10a68d4e9 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-missing-signature.test +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/debug-types-missing-signature.test @@ -14,10 +14,10 @@ LOOKUPE: no type was found matching 'E' RUN: %lldb %t -b -o "type lookup EC" | FileCheck --check-prefix=LOOKUPEC %s LOOKUPEC: no type was found matching 'EC' -RUN: not %lldb %t -b -o "print (E) 1" 2>&1 | FileCheck --check-prefix=PRINTE %s +RUN: not %lldb %t -b -o "expression (E) 1" 2>&1 | FileCheck --check-prefix=PRINTE %s PRINTE: use of undeclared identifier 'E' -RUN: not %lldb %t -b -o "print (EC) 1" 2>&1 | FileCheck --check-prefix=PRINTEC %s +RUN: not %lldb %t -b -o "expression (EC) 1" 2>&1 | FileCheck --check-prefix=PRINTEC %s PRINTEC: use of undeclared identifier 'EC' RUN: %lldb %t -b -o "target variable a e ec" | FileCheck --check-prefix=VARS %s diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/dwarf5-line-strp.s b/lldb/test/Shell/SymbolFile/DWARF/x86/dwarf5-line-strp.s index 5c08f4a40528b..67f52454604f6 100755 --- a/lldb/test/Shell/SymbolFile/DWARF/x86/dwarf5-line-strp.s +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/dwarf5-line-strp.s @@ -3,7 +3,7 @@ # UNSUPPORTED: system-darwin, system-windows # RUN: llvm-mc -filetype=obj -o %t -triple x86_64-pc-linux %s -# RUN: %lldb %t -o "p main" \ +# RUN: %lldb %t -o "expression main" \ # RUN: -o exit | FileCheck %s # CHECK: (void (*)()) $0 = 0x0000000000000000 diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/limit-debug-info.s b/lldb/test/Shell/SymbolFile/DWARF/x86/limit-debug-info.s index 6a2140a147921..4bd0fc8a9b55e 100644 --- a/lldb/test/Shell/SymbolFile/DWARF/x86/limit-debug-info.s +++ b/lldb/test/Shell/SymbolFile/DWARF/x86/limit-debug-info.s @@ -3,10 +3,10 @@ # RUN: llvm-mc --triple=x86_64-pc-windows --filetype=obj --defsym EXE=0 %s >%t.exe.o # RUN: lld-link /OUT:%t.dll %t.dll.o /SUBSYSTEM:console /dll /noentry /debug # RUN: lld-link /OUT:%t.exe %t.exe.o /SUBSYSTEM:console /debug /force -# RUN: %lldb %t.exe -o "target modules add %t.dll" -o "p var" \ +# RUN: %lldb %t.exe -o "target modules add %t.dll" -o "expression var" \ # RUN: -o exit 2>&1 | FileCheck %s -# CHECK: (lldb) p var +# CHECK: (lldb) expression var # CHECK: (A) $0 = (member = 47) .section .debug_abbrev,"dr" diff --git a/lldb/test/Shell/SymbolFile/NativePDB/Inputs/inline_sites_live.lldbinit b/lldb/test/Shell/SymbolFile/NativePDB/Inputs/inline_sites_live.lldbinit index 584d835977bf0..2291c7c452717 100644 --- a/lldb/test/Shell/SymbolFile/NativePDB/Inputs/inline_sites_live.lldbinit +++ b/lldb/test/Shell/SymbolFile/NativePDB/Inputs/inline_sites_live.lldbinit @@ -1,7 +1,7 @@ br set -p BP_bar -f inline_sites_live.cpp br set -p BP_foo -f inline_sites_live.cpp run -p param +expression param continue -p param -p local +expression param +expression local diff --git a/lldb/test/Shell/SymbolFile/NativePDB/Inputs/local-variables.lldbinit b/lldb/test/Shell/SymbolFile/NativePDB/Inputs/local-variables.lldbinit index 1071edd25c92b..4c8073e7bd884 100644 --- a/lldb/test/Shell/SymbolFile/NativePDB/Inputs/local-variables.lldbinit +++ b/lldb/test/Shell/SymbolFile/NativePDB/Inputs/local-variables.lldbinit @@ -1,30 +1,30 @@ break set -f local-variables.cpp -l 17 run a b c d e f g -p argc +expression argc step -p SomeLocal +expression SomeLocal step -p Param1 -p Param2 +expression Param1 +expression Param2 step -p Param1 -p Param2 -p Local1 +expression Param1 +expression Param2 +expression Local1 step -p Param1 -p Param2 -p Local1 -p Local2 +expression Param1 +expression Param2 +expression Local1 +expression Local2 step -p Param1 -p Param2 -p Local1 -p Local2 +expression Param1 +expression Param2 +expression Local1 +expression Local2 step -p Param1 -p Param2 -p Local1 -p Local2 +expression Param1 +expression Param2 +expression Local1 +expression Local2 continue target modules dump ast diff --git a/lldb/test/Shell/SymbolFile/NativePDB/inline_sites_live.cpp b/lldb/test/Shell/SymbolFile/NativePDB/inline_sites_live.cpp index bd3f80afd849c..767149ea18c46 100644 --- a/lldb/test/Shell/SymbolFile/NativePDB/inline_sites_live.cpp +++ b/lldb/test/Shell/SymbolFile/NativePDB/inline_sites_live.cpp @@ -24,11 +24,11 @@ int main(int argc, char** argv) { // CHECK: * thread #1, stop reason = breakpoint 1 // CHECK-NEXT: frame #0: {{.*}}`main [inlined] bar(param=2) -// CHECK: (lldb) p param +// CHECK: (lldb) expression param // CHECK-NEXT: (int) $0 = 2 // CHECK: * thread #1, stop reason = breakpoint 2 // CHECK-NEXT: frame #0: {{.*}}`main [inlined] foo(param=1) -// CHECK: (lldb) p param +// CHECK: (lldb) expression param // CHECK-NEXT: (int) $1 = 1 -// CHECK-NEXT: (lldb) p local +// CHECK-NEXT: (lldb) expression local // CHECK-NEXT: (int) $2 = 2 diff --git a/lldb/test/Shell/SymbolFile/NativePDB/local-variables.cpp b/lldb/test/Shell/SymbolFile/NativePDB/local-variables.cpp index 4779a28c8b380..a1f147b4bda6d 100644 --- a/lldb/test/Shell/SymbolFile/NativePDB/local-variables.cpp +++ b/lldb/test/Shell/SymbolFile/NativePDB/local-variables.cpp @@ -37,7 +37,7 @@ int main(int argc, char **argv) { // CHECK-NEXT: 20 // CHECK: Process {{.*}} launched: '{{.*}}local-variables.cpp.tmp.exe' -// CHECK-NEXT: (lldb) p argc +// CHECK-NEXT: (lldb) expression argc // CHECK-NEXT: (int) $0 = 8 // CHECK-NEXT: (lldb) step // CHECK-NEXT: Process {{.*}} stopped @@ -50,7 +50,7 @@ int main(int argc, char **argv) { // CHECK-NEXT: 19 } // CHECK-NEXT: 20 -// CHECK: (lldb) p SomeLocal +// CHECK: (lldb) expression SomeLocal // CHECK-NEXT: (int) $1 = 16 // CHECK-NEXT: (lldb) step // CHECK-NEXT: Process {{.*}} stopped @@ -64,9 +64,9 @@ int main(int argc, char **argv) { // CHECK-NEXT: 11 ++Local1; // CHECK-NEXT: 12 ++Local2; -// CHECK: (lldb) p Param1 +// CHECK: (lldb) expression Param1 // CHECK-NEXT: (int) $2 = 16 -// CHECK-NEXT: (lldb) p Param2 +// CHECK-NEXT: (lldb) expression Param2 // CHECK-NEXT: (char) $3 = 'a' // CHECK-NEXT: (lldb) step // CHECK-NEXT: Process {{.*}} stopped @@ -80,11 +80,11 @@ int main(int argc, char **argv) { // CHECK-NEXT: 12 ++Local2; // CHECK-NEXT: 13 return Local1; -// CHECK: (lldb) p Param1 +// CHECK: (lldb) expression Param1 // CHECK-NEXT: (int) $4 = 16 -// CHECK-NEXT: (lldb) p Param2 +// CHECK-NEXT: (lldb) expression Param2 // CHECK-NEXT: (char) $5 = 'a' -// CHECK-NEXT: (lldb) p Local1 +// CHECK-NEXT: (lldb) expression Local1 // CHECK-NEXT: (unsigned int) $6 = 17 // CHECK-NEXT: (lldb) step // CHECK-NEXT: Process {{.*}} stopped @@ -98,13 +98,13 @@ int main(int argc, char **argv) { // CHECK-NEXT: 13 return Local1; // CHECK-NEXT: 14 } -// CHECK: (lldb) p Param1 +// CHECK: (lldb) expression Param1 // CHECK-NEXT: (int) $7 = 16 -// CHECK-NEXT: (lldb) p Param2 +// CHECK-NEXT: (lldb) expression Param2 // CHECK-NEXT: (char) $8 = 'a' -// CHECK-NEXT: (lldb) p Local1 +// CHECK-NEXT: (lldb) expression Local1 // CHECK-NEXT: (unsigned int) $9 = 17 -// CHECK-NEXT: (lldb) p Local2 +// CHECK-NEXT: (lldb) expression Local2 // CHECK-NEXT: (char) $10 = 'b' // CHECK-NEXT: (lldb) step // CHECK-NEXT: Process {{.*}} stopped @@ -118,13 +118,13 @@ int main(int argc, char **argv) { // CHECK-NEXT: 14 } // CHECK-NEXT: 15 -// CHECK: (lldb) p Param1 +// CHECK: (lldb) expression Param1 // CHECK-NEXT: (int) $11 = 16 -// CHECK-NEXT: (lldb) p Param2 +// CHECK-NEXT: (lldb) expression Param2 // CHECK-NEXT: (char) $12 = 'a' -// CHECK-NEXT: (lldb) p Local1 +// CHECK-NEXT: (lldb) expression Local1 // CHECK-NEXT: (unsigned int) $13 = 18 -// CHECK-NEXT: (lldb) p Local2 +// CHECK-NEXT: (lldb) expression Local2 // CHECK-NEXT: (char) $14 = 'b' // CHECK-NEXT: (lldb) step // CHECK-NEXT: Process {{.*}} stopped @@ -138,13 +138,13 @@ int main(int argc, char **argv) { // CHECK-NEXT: 15 // CHECK-NEXT: 16 int main(int argc, char **argv) { -// CHECK: (lldb) p Param1 +// CHECK: (lldb) expression Param1 // CHECK-NEXT: (int) $15 = 16 -// CHECK-NEXT: (lldb) p Param2 +// CHECK-NEXT: (lldb) expression Param2 // CHECK-NEXT: (char) $16 = 'a' -// CHECK-NEXT: (lldb) p Local1 +// CHECK-NEXT: (lldb) expression Local1 // CHECK-NEXT: (unsigned int) $17 = 18 -// CHECK-NEXT: (lldb) p Local2 +// CHECK-NEXT: (lldb) expression Local2 // CHECK-NEXT: (char) $18 = 'c' // CHECK-NEXT: (lldb) continue // CHECK-NEXT: Process {{.*}} resuming diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest0.script b/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest0.script index d31a2abb68a2f..dcdaa9e1731d2 100644 --- a/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest0.script +++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest0.script @@ -1,7 +1,7 @@ breakpoint set --file ExpressionsTest.cpp --line 19 run -print result -print N0::N1::sum(N0::N1::buf1, sizeof(N0::N1::buf1)) -print N1::sum(N1::buf1, sizeof(N1::buf1)) -print sum(buf1, sizeof(buf1)) -print sum(buf1, 1000000000) +expression result +expression N0::N1::sum(N0::N1::buf1, sizeof(N0::N1::buf1)) +expression N1::sum(N1::buf1, sizeof(N1::buf1)) +expression sum(buf1, sizeof(buf1)) +expression sum(buf1, 1000000000) diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest1.script b/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest1.script index dac887faa5bba..d5d6292d3fdb7 100644 --- a/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest1.script +++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest1.script @@ -1 +1 @@ -print sum(buf0, 1) +expression sum(buf0, 1) diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest2.script b/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest2.script index b19240baf99da..ce6af93463711 100644 --- a/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest2.script +++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/ExpressionsTest2.script @@ -1,2 +1,2 @@ -print sum(buf0, result - 28) -print sum(buf1 + 3, 3) +expression sum(buf0, result - 28) +expression sum(buf1 + 3, 3) diff --git a/lldb/test/Shell/SymbolFile/PDB/Inputs/VBases.script b/lldb/test/Shell/SymbolFile/PDB/Inputs/VBases.script index 8675890b76e1d..9206ccadafdb9 100644 --- a/lldb/test/Shell/SymbolFile/PDB/Inputs/VBases.script +++ b/lldb/test/Shell/SymbolFile/PDB/Inputs/VBases.script @@ -2,6 +2,6 @@ breakpoint set --file VBases.cpp --line 15 run -print c +expression c -frame variable c \ No newline at end of file +frame variable c diff --git a/lldb/test/Shell/SymbolFile/PDB/expressions.test b/lldb/test/Shell/SymbolFile/PDB/expressions.test index 819e2e180945c..89d7c94e7aa06 100644 --- a/lldb/test/Shell/SymbolFile/PDB/expressions.test +++ b/lldb/test/Shell/SymbolFile/PDB/expressions.test @@ -2,34 +2,34 @@ REQUIRES: system-windows, msvc RUN: %build --compiler=msvc --nodefaultlib --output=%t.exe %S/Inputs/ExpressionsTest.cpp RUN: not %lldb -b -s %S/Inputs/ExpressionsTest0.script -s %S/Inputs/ExpressionsTest1.script -s %S/Inputs/ExpressionsTest2.script -- %t.exe 2>&1 | FileCheck %s -// Check the variable value through `print` -CHECK: (lldb) print result +// Check the variable value through `expression` +CHECK: (lldb) expression result CHECK: (char) $0 = '\x1c' // Call the function just like in the code -CHECK: (lldb) print N0::N1::sum(N0::N1::buf1, sizeof(N0::N1::buf1)) +CHECK: (lldb) expression N0::N1::sum(N0::N1::buf1, sizeof(N0::N1::buf1)) CHECK: (char) $1 = '\x1c' // Try the relaxed namespaces search -CHECK: (lldb) print N1::sum(N1::buf1, sizeof(N1::buf1)) +CHECK: (lldb) expression N1::sum(N1::buf1, sizeof(N1::buf1)) CHECK: (char) $2 = '\x1c' // Try the relaxed variables and functions search -CHECK: (lldb) print sum(buf1, sizeof(buf1)) +CHECK: (lldb) expression sum(buf1, sizeof(buf1)) CHECK: (char) $3 = '\x1c' // Make a crash during expression calculation -CHECK: (lldb) print sum(buf1, 1000000000) +CHECK: (lldb) expression sum(buf1, 1000000000) CHECK: The process has been returned to the state before expression evaluation. // Make one more crash -CHECK: (lldb) print sum(buf0, 1) +CHECK: (lldb) expression sum(buf0, 1) CHECK: The process has been returned to the state before expression evaluation. // Check if the process state was restored succesfully -CHECK: (lldb) print sum(buf0, result - 28) +CHECK: (lldb) expression sum(buf0, result - 28) CHECK: (char) $4 = '\0' // Call the function with arbitrary parameters -CHECK: (lldb) print sum(buf1 + 3, 3) +CHECK: (lldb) expression sum(buf1 + 3, 3) CHECK: (char) $5 = '\f' From 538cb04b5d748e0bd7045044206fc7ab0fbae094 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 5 Apr 2023 11:20:13 -0700 Subject: [PATCH 83/88] [lldb] Convert more tests to use `expression` directly (NFC) --- .../clangimporter/Werror/TestSwiftStripWerror.py | 2 +- .../config_macros/TestSwiftDedupMacros.py | 2 +- .../TestSwiftObjCMainConflictingDylibs.py | 4 ++-- ...estSwiftObjCMainConflictingDylibsFailingImport.py | 12 ++++++------ .../remap_sdk_path/TestSwiftRemapSDKPath.py | 2 +- .../TestSwiftStaticArchiveTwoSwiftmodules.py | 4 ++-- lldb/test/API/lang/swift/class_empty/main.swift | 2 +- .../test/API/lang/swift/closure_shortcuts/main.swift | 4 ++-- .../deployment_target/TestSwiftDeploymentTarget.py | 6 +++--- .../expression/allocator/TestSwiftExprAllocator.py | 2 +- .../lang/swift/macCatalyst/TestSwiftMacCatalyst.py | 2 +- .../API/lang/swift/missing_sdk/TestMissingSDK.py | 2 +- .../protocol_extension_computed_property/main.swift | 4 ++-- .../API/lang/swift/protocol_extension_two/main.swift | 4 ++-- .../TestSwiftRuntimeLibraryPath.py | 2 +- .../swift/swift-healthcheck/TestSwiftHealthCheck.py | 2 +- .../tripleDetection/TestSwiftTripleDetection.py | 2 +- .../API/lang/swift/variables/swift/TestSwiftTypes.py | 6 +++--- .../API/lang/swift/xcode_sdk/TestSwiftXcodeSDK.py | 2 +- .../TestAArch64LinuxNonAddressBitMemoryAccess.py | 4 ++-- 20 files changed, 35 insertions(+), 35 deletions(-) diff --git a/lldb/test/API/lang/swift/clangimporter/Werror/TestSwiftStripWerror.py b/lldb/test/API/lang/swift/clangimporter/Werror/TestSwiftStripWerror.py index 4a65a9ab30fbb..2a2889ab26a00 100644 --- a/lldb/test/API/lang/swift/clangimporter/Werror/TestSwiftStripWerror.py +++ b/lldb/test/API/lang/swift/clangimporter/Werror/TestSwiftStripWerror.py @@ -28,7 +28,7 @@ def test(self): log = self.getBuildArtifact("types.log") self.expect("log enable lldb types -f "+log) - self.expect("p foo", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["42"]) + self.expect("expression foo", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["42"]) sanity = 0 import io logfile = io.open(log, "r", encoding='utf-8') diff --git a/lldb/test/API/lang/swift/clangimporter/config_macros/TestSwiftDedupMacros.py b/lldb/test/API/lang/swift/clangimporter/config_macros/TestSwiftDedupMacros.py index 5d17cceda66c9..7cb6b09eac2d2 100644 --- a/lldb/test/API/lang/swift/clangimporter/config_macros/TestSwiftDedupMacros.py +++ b/lldb/test/API/lang/swift/clangimporter/config_macros/TestSwiftDedupMacros.py @@ -49,7 +49,7 @@ def testSwiftDebugMacros(self): log = self.getBuildArtifact("types.log") self.expect("log enable lldb types -f "+log) - self.expect("p foo", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["42"]) + self.expect("expression foo", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["42"]) debug = 0 space = 0 ndebug = 0 diff --git a/lldb/test/API/lang/swift/clangimporter/objcmain_conflicting_dylibs/TestSwiftObjCMainConflictingDylibs.py b/lldb/test/API/lang/swift/clangimporter/objcmain_conflicting_dylibs/TestSwiftObjCMainConflictingDylibs.py index c7c5411a4d3d7..f7c68353949e6 100644 --- a/lldb/test/API/lang/swift/clangimporter/objcmain_conflicting_dylibs/TestSwiftObjCMainConflictingDylibs.py +++ b/lldb/test/API/lang/swift/clangimporter/objcmain_conflicting_dylibs/TestSwiftObjCMainConflictingDylibs.py @@ -53,11 +53,11 @@ def test(self): process = target.LaunchSimple(None, None, os.getcwd()) # This is failing because the Target-SwiftASTContext uses the # amalgamated target header search options from all dylibs. - self.expect("p baz", "wrong baz", substrs=["i_am_from_Foo"]) + self.expect("expression baz", "wrong baz", substrs=["i_am_from_Foo"]) self.expect("fr var baz", "wrong baz", substrs=["i_am_from_Foo"]) process.Continue() - self.expect("p baz", "correct baz", substrs=["i_am_from_Foo"]) + self.expect("expression baz", "correct baz", substrs=["i_am_from_Foo"]) self.expect("fr var baz", "correct baz", substrs=["i_am_from_Foo"]) diff --git a/lldb/test/API/lang/swift/clangimporter/objcmain_conflicting_dylibs_failing_import/TestSwiftObjCMainConflictingDylibsFailingImport.py b/lldb/test/API/lang/swift/clangimporter/objcmain_conflicting_dylibs_failing_import/TestSwiftObjCMainConflictingDylibsFailingImport.py index 3d7f8f048fd97..e68a5aa7bc6fd 100644 --- a/lldb/test/API/lang/swift/clangimporter/objcmain_conflicting_dylibs_failing_import/TestSwiftObjCMainConflictingDylibsFailingImport.py +++ b/lldb/test/API/lang/swift/clangimporter/objcmain_conflicting_dylibs_failing_import/TestSwiftObjCMainConflictingDylibsFailingImport.py @@ -52,9 +52,9 @@ def test(self): # This initially fails with the shared scratch context and is # then retried with the per-dylib scratch context. - # self.expect("p bar", "expected result", substrs=["$R0", "42"]) - # self.expect("p $R0", "expected result", substrs=["$R1", "42"]) - # self.expect("p $R1", "expected result", substrs=["$R2", "42"]) + # self.expect("expression bar", "expected result", substrs=["$R0", "42"]) + # self.expect("expression $R0", "expected result", substrs=["$R1", "42"]) + # self.expect("expression $R1", "expected result", substrs=["$R2", "42"]) # This works by accident because the search paths are in the right order. foo_breakpoint = target.BreakpointCreateBySourceRegex( @@ -64,7 +64,7 @@ def test(self): # FIXME: The following expression evaluator tests are disabled # because it's nondeterministic which one will work. - # self.expect("p foo", "expected result", substrs=["$R3", "23"]) - # self.expect("p $R3", "expected result", substrs=["23"]) - # self.expect("p $R4", "expected result", substrs=["23"]) + # self.expect("expression foo", "expected result", substrs=["$R3", "23"]) + # self.expect("expression $R3", "expected result", substrs=["23"]) + # self.expect("expression $R4", "expected result", substrs=["23"]) diff --git a/lldb/test/API/lang/swift/clangimporter/remap_sdk_path/TestSwiftRemapSDKPath.py b/lldb/test/API/lang/swift/clangimporter/remap_sdk_path/TestSwiftRemapSDKPath.py index 45ada8bed8fe8..9255e7bede1f9 100644 --- a/lldb/test/API/lang/swift/clangimporter/remap_sdk_path/TestSwiftRemapSDKPath.py +++ b/lldb/test/API/lang/swift/clangimporter/remap_sdk_path/TestSwiftRemapSDKPath.py @@ -22,7 +22,7 @@ def test(self): self.runCmd('log enable lldb types -f "%s"' % log) target, process, thread, bkpt = lldbutil.run_to_name_breakpoint( self, 'main') - self.expect("p 1", substrs=["1"]) + self.expect("expression 1", substrs=["1"]) # Scan through the types log. import io diff --git a/lldb/test/API/lang/swift/clangimporter/static_archive/TestSwiftStaticArchiveTwoSwiftmodules.py b/lldb/test/API/lang/swift/clangimporter/static_archive/TestSwiftStaticArchiveTwoSwiftmodules.py index 8fe575e0bfb71..ed74b734850b0 100644 --- a/lldb/test/API/lang/swift/clangimporter/static_archive/TestSwiftStaticArchiveTwoSwiftmodules.py +++ b/lldb/test/API/lang/swift/clangimporter/static_archive/TestSwiftStaticArchiveTwoSwiftmodules.py @@ -52,7 +52,7 @@ def test(self): # This test tests that the search paths from all swiftmodules # that are part of the main binary are honored. self.expect("fr var foo", "expected result", substrs=["23"]) - self.expect("p foo", "expected result", substrs=["$R0", "i", "23"]) + self.expect("expression foo", "expected result", substrs=["$R0", "i", "23"]) process.Continue() self.expect("fr var bar", "expected result", substrs=["42"]) - self.expect("p bar", "expected result", substrs=["j", "42"]) + self.expect("expression bar", "expected result", substrs=["j", "42"]) diff --git a/lldb/test/API/lang/swift/class_empty/main.swift b/lldb/test/API/lang/swift/class_empty/main.swift index a58229dbe7778..bfff59c2ebc02 100644 --- a/lldb/test/API/lang/swift/class_empty/main.swift +++ b/lldb/test/API/lang/swift/class_empty/main.swift @@ -17,7 +17,7 @@ class Empty : CustomStringConvertible { func main() { var e = Empty() - print(e) //% self.expect("p 1", substrs=['1']) + print(e) //% self.expect("expression 1", substrs=['1']) } main() diff --git a/lldb/test/API/lang/swift/closure_shortcuts/main.swift b/lldb/test/API/lang/swift/closure_shortcuts/main.swift index a2d3bb4968e5a..1b696796d3633 100644 --- a/lldb/test/API/lang/swift/closure_shortcuts/main.swift +++ b/lldb/test/API/lang/swift/closure_shortcuts/main.swift @@ -15,8 +15,8 @@ func main() -> Int { let names = ["foo", "patatino"] var reversedNames = names.sorted(by: { - $0 > $1 } //%self.expect('p $0', substrs=['patatino']) - //%self.expect('p $1', substrs=['foo']) + $0 > $1 } //%self.expect('expr $0', substrs=['patatino']) + //%self.expect('expr $1', substrs=['foo']) //%self.expect('frame var $0', substrs=['patatino']) //%self.expect('frame var $1', substrs=['foo']) ) diff --git a/lldb/test/API/lang/swift/deployment_target/TestSwiftDeploymentTarget.py b/lldb/test/API/lang/swift/deployment_target/TestSwiftDeploymentTarget.py index 5f436172ca96a..4c3db53f01234 100644 --- a/lldb/test/API/lang/swift/deployment_target/TestSwiftDeploymentTarget.py +++ b/lldb/test/API/lang/swift/deployment_target/TestSwiftDeploymentTarget.py @@ -34,7 +34,7 @@ def test_swift_deployment_target(self): lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec('main.swift')) - self.expect("p f", substrs=['i = 23']) + self.expect("expression f", substrs=['i = 23']) @skipUnlessDarwin @skipIfDarwinEmbedded # This test uses macOS triples explicitly. @@ -47,7 +47,7 @@ def test_swift_deployment_target_dlopen(self): bkpt = target.BreakpointCreateBySourceRegex( 'break here', lldb.SBFileSpec('NewerTarget.swift')) lldbutil.continue_to_breakpoint(process, bkpt) - self.expect("p self", substrs=['i = 23']) + self.expect("expression self", substrs=['i = 23']) @skipUnlessDarwin @skipIfDarwinEmbedded # This test uses macOS triples explicitly. @@ -61,7 +61,7 @@ def test_swift_deployment_target_from_macho(self): lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec('main.swift')) - self.expect("p f", substrs=['i = 23']) + self.expect("expression f", substrs=['i = 23']) found_no_ast = False found_triple = False diff --git a/lldb/test/API/lang/swift/expression/allocator/TestSwiftExprAllocator.py b/lldb/test/API/lang/swift/expression/allocator/TestSwiftExprAllocator.py index 746b4110670be..14ceb7fe5cf05 100644 --- a/lldb/test/API/lang/swift/expression/allocator/TestSwiftExprAllocator.py +++ b/lldb/test/API/lang/swift/expression/allocator/TestSwiftExprAllocator.py @@ -19,4 +19,4 @@ def test_allocator_self(self): target, process, thread, bkpt = lldbutil.run_to_source_breakpoint( self, 'break here', lldb.SBFileSpec('main.swift')) - self.expect("p x", substrs=['23']) + self.expect("expression x", substrs=['23']) diff --git a/lldb/test/API/lang/swift/macCatalyst/TestSwiftMacCatalyst.py b/lldb/test/API/lang/swift/macCatalyst/TestSwiftMacCatalyst.py index fe29fdaaba1b9..51267cc3a5436 100644 --- a/lldb/test/API/lang/swift/macCatalyst/TestSwiftMacCatalyst.py +++ b/lldb/test/API/lang/swift/macCatalyst/TestSwiftMacCatalyst.py @@ -36,7 +36,7 @@ def test_macCatalyst(self): self.expect("fr v s", "Hello macCatalyst") expr_log = self.getBuildArtifact("expr.log") self.expect('log enable lldb expr -f "%s"' % expr_log) - self.expect("p s", "Hello macCatalyst") + self.expect("expression s", "Hello macCatalyst") import io expr_logfile = io.open(expr_log, "r", encoding='utf-8') diff --git a/lldb/test/API/lang/swift/missing_sdk/TestMissingSDK.py b/lldb/test/API/lang/swift/missing_sdk/TestMissingSDK.py index 67ed05992e7c3..cbd9f12b972ba 100644 --- a/lldb/test/API/lang/swift/missing_sdk/TestMissingSDK.py +++ b/lldb/test/API/lang/swift/missing_sdk/TestMissingSDK.py @@ -26,6 +26,6 @@ def testMissingSDK(self): os.unlink(self.getBuildArtifact("fakesdk")) lldbutil.run_to_source_breakpoint(self, 'break here', lldb.SBFileSpec('main.swift')) - self.expect("p message", VARIABLES_DISPLAYED_CORRECTLY, + self.expect("expression message", VARIABLES_DISPLAYED_CORRECTLY, substrs = ["Hello"]) diff --git a/lldb/test/API/lang/swift/protocol_extension_computed_property/main.swift b/lldb/test/API/lang/swift/protocol_extension_computed_property/main.swift index 4f481e06d0efa..76fdb2801f09c 100644 --- a/lldb/test/API/lang/swift/protocol_extension_computed_property/main.swift +++ b/lldb/test/API/lang/swift/protocol_extension_computed_property/main.swift @@ -13,8 +13,8 @@ extension Measurement where UnitType == UnitAngle { } func f() { - return //%self.expect('p self.radians', substrs=["CGFloat) $R0", "= 1.745"]) - //%self.expect('p self', substrs=["Measurement"]) + return //%self.expect('expression self.radians', substrs=["CGFloat) $R0", "= 1.745"]) + //%self.expect('expression self', substrs=["Measurement"]) } } diff --git a/lldb/test/API/lang/swift/protocol_extension_two/main.swift b/lldb/test/API/lang/swift/protocol_extension_two/main.swift index 2618f01f17257..bc4a86bfa0aa4 100644 --- a/lldb/test/API/lang/swift/protocol_extension_two/main.swift +++ b/lldb/test/API/lang/swift/protocol_extension_two/main.swift @@ -10,8 +10,8 @@ extension Patatino where T == Winky { } func f() { - return //%self.expect('p self.baciotto', substrs=["(Int) $R0 = 0"]) - //%self.expect('p self', substrs=["a.Patatino"]) + return //%self.expect('expression self.baciotto', substrs=["(Int) $R0 = 0"]) + //%self.expect('expression self', substrs=["a.Patatino"]) } } diff --git a/lldb/test/API/lang/swift/runtime_library_path/TestSwiftRuntimeLibraryPath.py b/lldb/test/API/lang/swift/runtime_library_path/TestSwiftRuntimeLibraryPath.py index 28217304aa657..19b34b0941256 100644 --- a/lldb/test/API/lang/swift/runtime_library_path/TestSwiftRuntimeLibraryPath.py +++ b/lldb/test/API/lang/swift/runtime_library_path/TestSwiftRuntimeLibraryPath.py @@ -25,7 +25,7 @@ def test(self): target, process, thread, bkpt = lldbutil.run_to_name_breakpoint( self, 'main') - self.expect("p 1") + self.expect("expression 1") import io logfile = io.open(log, "r", encoding='utf-8') in_expr_log = 0 diff --git a/lldb/test/API/lang/swift/swift-healthcheck/TestSwiftHealthCheck.py b/lldb/test/API/lang/swift/swift-healthcheck/TestSwiftHealthCheck.py index 36adc00421843..8adada198859f 100644 --- a/lldb/test/API/lang/swift/swift-healthcheck/TestSwiftHealthCheck.py +++ b/lldb/test/API/lang/swift/swift-healthcheck/TestSwiftHealthCheck.py @@ -19,7 +19,7 @@ def test_run_healthcheck(self): target, process, thread, bkpt = lldbutil.run_to_name_breakpoint( self, 'main') - self.expect("p 1") + self.expect("expression 1") result = lldb.SBCommandReturnObject() ret_val = self.dbg.GetCommandInterpreter().HandleCommand("swift-healthcheck", result) log = result.GetOutput()[:-1].split(" ")[-1] diff --git a/lldb/test/API/lang/swift/tripleDetection/TestSwiftTripleDetection.py b/lldb/test/API/lang/swift/tripleDetection/TestSwiftTripleDetection.py index 0c2e930d3567e..d73dbee78bf77 100644 --- a/lldb/test/API/lang/swift/tripleDetection/TestSwiftTripleDetection.py +++ b/lldb/test/API/lang/swift/tripleDetection/TestSwiftTripleDetection.py @@ -26,7 +26,7 @@ def test(self): arch+"-apple-macos-unknown") bkpt = target.BreakpointCreateByName("main") process = target.LaunchSimple(None, None, self.get_process_working_directory()) - self.expect("p 1") + self.expect("expression 1") import io types_logfile = io.open(types_log, "r", encoding='utf-8') diff --git a/lldb/test/API/lang/swift/variables/swift/TestSwiftTypes.py b/lldb/test/API/lang/swift/variables/swift/TestSwiftTypes.py index 25ee19418e845..cbe9ca64446a5 100644 --- a/lldb/test/API/lang/swift/variables/swift/TestSwiftTypes.py +++ b/lldb/test/API/lang/swift/variables/swift/TestSwiftTypes.py @@ -177,9 +177,9 @@ def do_test(self): self.expect("frame variable --raw hello", substrs=['String']) - self.expect("p/x int64_minus_two", substrs=['0xfffffffffffffffe']) - self.expect("p/u ~1", substrs=['18446744073709551614']) - self.expect("p/d ~1", substrs=['-2']) + self.expect("expression/x int64_minus_two", substrs=['0xfffffffffffffffe']) + self.expect("expression/u ~1", substrs=['18446744073709551614']) + self.expect("expression/d ~1", substrs=['-2']) self.expect('frame variable uint8_max', substrs=['-1'], matching=False) self.expect( diff --git a/lldb/test/API/lang/swift/xcode_sdk/TestSwiftXcodeSDK.py b/lldb/test/API/lang/swift/xcode_sdk/TestSwiftXcodeSDK.py index b22c22aca2ec1..ee234235edbfa 100644 --- a/lldb/test/API/lang/swift/xcode_sdk/TestSwiftXcodeSDK.py +++ b/lldb/test/API/lang/swift/xcode_sdk/TestSwiftXcodeSDK.py @@ -35,7 +35,7 @@ def test_decode(self): lldbutil.run_to_name_breakpoint(self, 'main') - self.expect("p 1") + self.expect("expression 1") self.check_log(log, "MacOSX") @swiftTest diff --git a/lldb/test/API/linux/aarch64/non_address_bit_memory_access/TestAArch64LinuxNonAddressBitMemoryAccess.py b/lldb/test/API/linux/aarch64/non_address_bit_memory_access/TestAArch64LinuxNonAddressBitMemoryAccess.py index 87953cbad475f..f554d5b39eaf7 100644 --- a/lldb/test/API/linux/aarch64/non_address_bit_memory_access/TestAArch64LinuxNonAddressBitMemoryAccess.py +++ b/lldb/test/API/linux/aarch64/non_address_bit_memory_access/TestAArch64LinuxNonAddressBitMemoryAccess.py @@ -153,10 +153,10 @@ def test_non_address_bit_memory_caching(self): # This should fill the cache by doing a read of buf_with_non_address # with the non-address bits removed (which is == buf). - self.runCmd("p buf_with_non_address") + self.runCmd("expression buf_with_non_address") # This will read from the cache since the two pointers point to the # same place. - self.runCmd("p buf") + self.runCmd("expression buf") # Open log ignoring utf-8 decode errors with open(log_file, 'r', errors='ignore') as f: From c1b4d2db7c4b36804020adcb9a5da257ccb7779e Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 6 Apr 2023 11:19:50 -0700 Subject: [PATCH 84/88] [lldb] Replace use of p with expression in (Swift) shell tests --- lldb/test/Shell/Swift/DeserializationFailure.test | 2 +- lldb/test/Shell/Swift/DynamicTyperesolutionConflict.test | 4 ++-- lldb/test/Shell/Swift/MissingVFSOverlay.test | 2 +- lldb/test/Shell/Swift/RemoteASTImport.test | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lldb/test/Shell/Swift/DeserializationFailure.test b/lldb/test/Shell/Swift/DeserializationFailure.test index c54e200342f65..a2a15c6e68e58 100644 --- a/lldb/test/Shell/Swift/DeserializationFailure.test +++ b/lldb/test/Shell/Swift/DeserializationFailure.test @@ -15,7 +15,7 @@ b main run # Create a SwiftASTContext, to induce error output. -p 1 +expression 1 # The {{ }} avoids accidentally matching the input script! # CHECK: a.out:{{ }}{{.*}}The serialized module is corrupted. diff --git a/lldb/test/Shell/Swift/DynamicTyperesolutionConflict.test b/lldb/test/Shell/Swift/DynamicTyperesolutionConflict.test index b2e5f8b3e53ee..d229f99273f5b 100644 --- a/lldb/test/Shell/Swift/DynamicTyperesolutionConflict.test +++ b/lldb/test/Shell/Swift/DynamicTyperesolutionConflict.test @@ -21,8 +21,8 @@ b Dylib.swift:9 run target v foofoo -p input -p input +expression input +expression input # The {{ }} avoids accidentally matching the input script! # CHECK: {{Swift}} error in fallback scratch context diff --git a/lldb/test/Shell/Swift/MissingVFSOverlay.test b/lldb/test/Shell/Swift/MissingVFSOverlay.test index 5a4decc34f2f5..e79d439487f95 100644 --- a/lldb/test/Shell/Swift/MissingVFSOverlay.test +++ b/lldb/test/Shell/Swift/MissingVFSOverlay.test @@ -12,7 +12,7 @@ b main run # Create a SwiftASTContext, to induce error output. -p 1 +expression 1 # The {{ }} avoids accidentally matching the input script! # CHECK: a.out:{{ }} diff --git a/lldb/test/Shell/Swift/RemoteASTImport.test b/lldb/test/Shell/Swift/RemoteASTImport.test index 06522707862a4..7c6d5094bb3aa 100644 --- a/lldb/test/Shell/Swift/RemoteASTImport.test +++ b/lldb/test/Shell/Swift/RemoteASTImport.test @@ -45,7 +45,7 @@ b Library.swift:10 run -p input +expression input # The {{ }} avoids accidentally matching the input script! # CHECK-NOT: undeclared identifier {{'SYNTAX_ERROR'}} From b4f5ea309a618de2b9a0b8f3eeda650682b708ac Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Wed, 8 Mar 2023 13:22:00 -0800 Subject: [PATCH 85/88] [lldb] Change dwim-print to default to disabled persistent results Change `dwim-print` to now disable persistent results by default, unless requested by the user with the `--persistent-result` flag. Ex: ``` (lldb) dwim-print 1 + 1 (int) 2 (lldb) dwim-print --persistent-result on -- 1 + 1 (int) $0 = 2 ``` Users who wish to enable persistent results can make and use an alias that includes `--persistent-result on`. Differential Revision: https://reviews.llvm.org/D145609 (cherry picked from commit 385496385476fc9735da5fa4acabc34654e8b30d) --- .../Commands/CommandObjectDWIMPrint.cpp | 4 +++ .../Commands/CommandObjectExpression.cpp | 9 +++--- .../source/Commands/CommandObjectExpression.h | 2 +- .../API/commands/dwim-print/TestDWIMPrint.py | 31 ++++++++----------- .../persistent_result/TestPersistentResult.py | 8 ++--- .../API/functionalities/alias/TestPAlias.py | 2 +- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp index 437082ca80aa8..9955e7a0cf0ee 100644 --- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp +++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp @@ -75,6 +75,10 @@ bool CommandObjectDWIMPrint::DoExecute(StringRef command, return false; } + // If the user has not specified, default to disabling persistent results. + if (m_expr_options.suppress_persistent_result == eLazyBoolCalculate) + m_expr_options.suppress_persistent_result = eLazyBoolYes; + auto verbosity = GetDebugger().GetDWIMPrintVerbosity(); Target *target_ptr = m_exe_ctx.GetTargetPtr(); diff --git a/lldb/source/Commands/CommandObjectExpression.cpp b/lldb/source/Commands/CommandObjectExpression.cpp index a513b3fdfc897..19a7a7319c79e 100644 --- a/lldb/source/Commands/CommandObjectExpression.cpp +++ b/lldb/source/Commands/CommandObjectExpression.cpp @@ -150,7 +150,7 @@ Status CommandObjectExpression::CommandOptions::SetOptionValue( bool persist_result = OptionArgParser::ToBoolean(option_arg, true, &success); if (success) - suppress_persistent_result = !persist_result; + suppress_persistent_result = !persist_result ? eLazyBoolYes : eLazyBoolNo; else error.SetErrorStringWithFormat( "could not convert \"%s\" to a boolean value.", @@ -197,7 +197,7 @@ void CommandObjectExpression::CommandOptions::OptionParsingStarting( auto_apply_fixits = eLazyBoolCalculate; top_level = false; allow_jit = true; - suppress_persistent_result = false; + suppress_persistent_result = eLazyBoolCalculate; // BEGIN SWIFT bind_generic_types = eBindAuto; // END SWIFT @@ -215,8 +215,9 @@ CommandObjectExpression::CommandOptions::GetEvaluateExpressionOptions( options.SetCoerceToId(display_opts.use_objc); // Explicitly disabling persistent results takes precedence over the // m_verbosity/use_objc logic. - if (suppress_persistent_result) - options.SetSuppressPersistentResult(true); + if (suppress_persistent_result != eLazyBoolCalculate) + options.SetSuppressPersistentResult(suppress_persistent_result == + eLazyBoolYes); else if (m_verbosity == eLanguageRuntimeDescriptionDisplayVerbosityCompact) options.SetSuppressPersistentResult(display_opts.use_objc); options.SetUnwindOnError(unwind_on_error); diff --git a/lldb/source/Commands/CommandObjectExpression.h b/lldb/source/Commands/CommandObjectExpression.h index a685f4a8108dd..e30b94db1012b 100644 --- a/lldb/source/Commands/CommandObjectExpression.h +++ b/lldb/source/Commands/CommandObjectExpression.h @@ -53,7 +53,7 @@ class CommandObjectExpression : public CommandObjectRaw, lldb::LanguageType language; LanguageRuntimeDescriptionDisplayVerbosity m_verbosity; LazyBool auto_apply_fixits; - bool suppress_persistent_result; + LazyBool suppress_persistent_result; // BEGIN SWIFT lldb::BindGenericTypes bind_generic_types; // END SWIFT diff --git a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py index 61aa7f2284e8c..26736d5127386 100644 --- a/lldb/test/API/commands/dwim-print/TestDWIMPrint.py +++ b/lldb/test/API/commands/dwim-print/TestDWIMPrint.py @@ -16,18 +16,16 @@ def _run_cmd(self, cmd: str) -> str: self.ci.HandleCommand(cmd, result) return result.GetOutput().rstrip() - VAR_IDENT_RAW = r"(?:\$\d+|\w+) = " - VAR_IDENT = re.compile(VAR_IDENT_RAW) + VAR_IDENT = re.compile(r"(?:\$\d+|\w+) = ") - def _mask_persistent_var(self, string: str) -> str: + def _strip_result_var(self, string: str) -> str: """ - Replace persistent result variables (ex '$0', '$1', etc) with a regex - that matches any persistent result (r'\$\d+'). The returned string can - be matched against other `expression` results. + Strip (persistent) result variables (ex '$0 = ', or 'someVar = ', etc). + + This allows for using the output of `expression`/`frame variable`, to + compare it to `dwim-print` output, which disables result variables. """ - before, after = self.VAR_IDENT.split(string, maxsplit=1) - # Support either a frame variable (\w+) or a persistent result (\$\d+). - return re.escape(before) + self.VAR_IDENT_RAW + re.escape(after) + return self.VAR_IDENT.subn("", string, 1)[0] def _expect_cmd( self, @@ -46,19 +44,16 @@ def _expect_cmd( if actual_cmd == "frame variable": resolved_cmd = resolved_cmd.replace(" -- ", " ", 1) - expected_output = self._run_cmd(resolved_cmd) + resolved_cmd_output = self._run_cmd(resolved_cmd) + dwim_cmd_output = self._strip_result_var(resolved_cmd_output) # Verify dwim-print chose the expected command. self.runCmd("settings set dwim-print-verbosity full") - substrs = [f"note: ran `{resolved_cmd}`"] - patterns = [] - - if self.VAR_IDENT.search(expected_output): - patterns.append(self._mask_persistent_var(expected_output)) - else: - substrs.append(expected_output) - self.expect(dwim_cmd, substrs=substrs, patterns=patterns) + self.expect(dwim_cmd, substrs=[ + f"note: ran `{resolved_cmd}`", + dwim_cmd_output, + ]) def test_variables(self): """Test dwim-print with variables.""" diff --git a/lldb/test/API/commands/expression/persistent_result/TestPersistentResult.py b/lldb/test/API/commands/expression/persistent_result/TestPersistentResult.py index 10eb100bac37b..911b8f605939b 100644 --- a/lldb/test/API/commands/expression/persistent_result/TestPersistentResult.py +++ b/lldb/test/API/commands/expression/persistent_result/TestPersistentResult.py @@ -31,7 +31,7 @@ def test_expression_persists_result(self): self.expect("expression i", substrs=["(int) $0 = 30"]) self.expect("expression $0", substrs=["(int) $0 = 30"]) - def test_p_persists_result(self): - """Test `p` does persist a result variable.""" - self.expect("p i", substrs=["(int) $0 = 30"]) - self.expect("p $0", substrs=["(int) $0 = 30"]) + def test_p_does_not_persist_results(self): + """Test `p` does not persist a result variable.""" + self.expect("p i", startstr="(int) 30") + self.expect("p $0", error=True) diff --git a/lldb/test/API/functionalities/alias/TestPAlias.py b/lldb/test/API/functionalities/alias/TestPAlias.py index b694e903b9f00..e1f00b91e0149 100644 --- a/lldb/test/API/functionalities/alias/TestPAlias.py +++ b/lldb/test/API/functionalities/alias/TestPAlias.py @@ -7,5 +7,5 @@ class TestCase(TestBase): def test(self): self.build() lldbutil.run_to_source_breakpoint(self, "return", lldb.SBFileSpec("main.c")) - self.expect("p -g", substrs=["$0 = -"]) + self.expect("p -g", startstr="(int) -41") self.expect("p -i0 -g", error=True) From 35e4364439bc24248792b405d38b128a4c236383 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 10 Apr 2023 14:52:49 -0700 Subject: [PATCH 86/88] Revert "[lldb] Fix lldb build after swift ASTContext change" This reverts commit 50a89bccbdb1506c96e3683bdb78ca6ebf10ae28. --- lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp index dbe661052ecf2..4862848250398 100644 --- a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp +++ b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp @@ -2907,7 +2907,7 @@ swift::ASTContext *SwiftASTContext::GetASTContext() { GetLanguageOptions(), GetTypeCheckerOptions(), GetSILOptions(), GetSearchPathOptions(), GetClangImporterOptions(), GetSymbolGraphOptions(), GetSourceManager(), GetDiagnosticEngine(), - /*OutputBackend=*/nullptr, ReportModuleLoadingProgress)); + ReportModuleLoadingProgress)); m_diagnostic_consumer_ap.reset(new StoringDiagnosticConsumer(*this)); if (getenv("LLDB_SWIFT_DUMP_DIAGS")) { From 44c656aa85b0077348d203bee21d51a89020e5c4 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Tue, 11 Apr 2023 08:55:37 -0700 Subject: [PATCH 87/88] Revert "[lldb] Skip TestSwiftBackwardInteropExpressions in Linux" This reverts commit 1c981654f85a318ecbe7c4a7a46ea3aee9e571ff. --- .../backward/expressions/TestSwiftBackwardInteropExpressions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py index 4ca2caeecc6eb..4a9f2e630f732 100644 --- a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py +++ b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py @@ -8,7 +8,6 @@ class TestSwiftBackwardInteropExpressions(TestBase): - @skipIfLinux @swiftTest def test_func_step_in(self): self.build() From d707f3245890ae38096ec9d88971af155c0ba649 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Tue, 11 Apr 2023 08:55:40 -0700 Subject: [PATCH 88/88] Revert "[lldb] Add Swift/C++ backward interop expression evaluation tests" This reverts commit a90ad0ecb6949f03b9d10f9632d023680859bc74. --- .../cxx_interop/backward/expressions/Makefile | 7 ---- .../TestSwiftBackwardInteropExpressions.py | 21 ---------- .../cxx_interop/backward/expressions/main.cpp | 16 -------- .../backward/expressions/swift-types.swift | 39 ------------------- 4 files changed, 83 deletions(-) delete mode 100644 lldb/test/API/lang/swift/cxx_interop/backward/expressions/Makefile delete mode 100644 lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py delete mode 100644 lldb/test/API/lang/swift/cxx_interop/backward/expressions/main.cpp delete mode 100644 lldb/test/API/lang/swift/cxx_interop/backward/expressions/swift-types.swift diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/Makefile b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/Makefile deleted file mode 100644 index 947094ab0c28f..0000000000000 --- a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -SWIFT_CXX_HEADER := swift-types.h -SWIFT_SOURCES := swift-types.swift -CXX_SOURCES := main.cpp -SWIFT_CXX_INTEROP := 1 -SWIFTFLAGS_EXTRAS = -Xcc -I$(SRCDIR) -parse-as-library -CFLAGS = -I. -g -include Makefile.rules diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py deleted file mode 100644 index 4a9f2e630f732..0000000000000 --- a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/TestSwiftBackwardInteropExpressions.py +++ /dev/null @@ -1,21 +0,0 @@ - -""" -Test that evaluating expressions works on backward interop mode. -""" -from lldbsuite.test.lldbtest import * -from lldbsuite.test.decorators import * - - -class TestSwiftBackwardInteropExpressions(TestBase): - - @swiftTest - def test_func_step_in(self): - self.build() - self.runCmd('setting set target.experimental.swift-enable-cxx-interop true') - _, _, _, _ = lldbutil.run_to_source_breakpoint( - self, 'Break here', lldb.SBFileSpec('main.cpp')) - self.expect('expr swiftFunc()', substrs=["Inside a Swift function"]) - self.expect('expr swiftClass.swiftMethod()', substrs=["Inside a Swift method"]) - self.expect('expr a::SwiftClass::swiftStaticMethod()', substrs=["In a Swift static method"]) - self.expect('expr swiftClass.getSwiftProperty()', substrs=["This is a class with properties"]) - self.expect('expr swiftSubclassAsClass.overrideableMethod()', substrs=["In subclass"]) diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/main.cpp b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/main.cpp deleted file mode 100644 index 17431c746188a..0000000000000 --- a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/main.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "swift-types.h" - -using namespace a; - -int main() { - swiftFunc(); - auto swiftClass = SwiftClass::init(); - swiftClass.swiftMethod(); - SwiftClass::swiftStaticMethod(); - swiftClass.getSwiftProperty(); - auto swiftSubclass = SwiftSubclass::init(); - SwiftClass swiftSubclassAsClass = swiftSubclass; - swiftSubclassAsClass.overrideableMethod(); - return 0; // Break here -} - diff --git a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/swift-types.swift b/lldb/test/API/lang/swift/cxx_interop/backward/expressions/swift-types.swift deleted file mode 100644 index 8a74c3e4c28c6..0000000000000 --- a/lldb/test/API/lang/swift/cxx_interop/backward/expressions/swift-types.swift +++ /dev/null @@ -1,39 +0,0 @@ - -public func swiftFunc() -> String{ - return "Inside a Swift function!" -} - -public class SwiftClass { - var field: Int - var arr: [String] - public init() { - field = 42 - arr = ["An", "array", "of", "strings"] - } - - public func swiftMethod() -> String { - return "Inside a Swift method!" - } - - private var _desc = "This is a class with properties!" - public var swiftProperty: String { - get { - return _desc - } - } - - public static func swiftStaticMethod() -> String { - return "In a Swift static method!" - } - - public func overrideableMethod() -> String { - return "In the base class!" - } -} - -public class SwiftSubclass: SwiftClass { - public override func overrideableMethod() -> String { - return "In subclass!" - } -} -