diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h index eb840ccbc8ba2..3b1a603f4bb66 100644 --- a/lldb/include/lldb/Core/ModuleList.h +++ b/lldb/include/lldb/Core/ModuleList.h @@ -65,6 +65,12 @@ class ModuleListProperties : public Properties { bool GetSwiftValidateTypeSystem() const; SwiftModuleLoadingMode GetSwiftModuleLoadingMode() const; bool SetSwiftModuleLoadingMode(SwiftModuleLoadingMode); + + bool GetEnableSwiftMetadataCache() const; + uint64_t GetSwiftMetadataCacheMaxByteSize(); + uint64_t GetSwiftMetadataCacheExpirationDays(); + FileSpec GetSwiftMetadataCachePath() const; + bool SetSwiftMetadataCachePath(const FileSpec &path); // END SWIFT FileSpec GetClangModulesCachePath() const; diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py index cdf583ec22861..05b6e198bc44f 100644 --- a/lldb/packages/Python/lldbsuite/test/lldbtest.py +++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py @@ -735,6 +735,12 @@ def setUpCommands(cls): 'settings set symbols.clang-modules-cache-path "{}"'.format( configuration.lldb_module_cache_dir), + # Enable the swift metadata cache in order to speed up tests. + 'settings set symbols.enable-swift-metadata-cache true', + + 'settings set symbols.swift-metadata-cache-path "{}"'.format( + configuration.lldb_module_cache_dir), + # Enable expensive validations in TypeSystemSwiftTypeRef. 'settings set symbols.swift-validate-typesystem true', diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td index 725209c2ca818..3cb4fbdfc4d0c 100644 --- a/lldb/source/Core/CoreProperties.td +++ b/lldb/source/Core/CoreProperties.td @@ -30,6 +30,22 @@ let Definition = "modulelist" in { DefaultEnumValue<"eSwiftModuleLoadingModePreferSerialized">, EnumValues<"OptionEnumValues(g_swift_module_loading_mode_enums)">, Desc<"The module loading mode to use when loading modules for Swift.">; + def EnableSwiftMetadataCache: Property<"enable-swift-metadata-cache", "Boolean">, + Global, + DefaultFalse, + Desc<"Enable caching for Swift reflection metadata in LLDB.">; + def SwiftMetadataCachePath: Property<"swift-metadata-cache-path", "FileSpec">, + Global, + DefaultStringValue<"">, + Desc<"The path to LLDB's Swift reflection cache directory.">; + def SwiftMetadataCacheMaxByteSize: Property<"swift-metadata-cache-max-byte-size", "UInt64">, + Global, + DefaultUnsignedValue<0>, + Desc<"The maximum size for LLDB's Swift reflection cache directory in bytes. A value over the amount of available space on the disk will be reduced to the amount of available space. A value of 0 disables the absolute size-based pruning.">; + def SwiftMetadataCacheExpirationDays: Property<"swift-metadata-cache-expiration-days", "UInt64">, + Global, + DefaultUnsignedValue<7>, + Desc<"The expiration time in days for a Swift reflection cache file. When a file hasn't been accessed for the specified amount of days, it is removed from the cache. A value of 0 disables the expiration-based pruning.">; // END SWIFT def SymLinkPaths: Property<"debug-info-symlink-paths", "FileSpecList">, Global, diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp index 212073d329470..f13b15507ed92 100644 --- a/lldb/source/Core/ModuleList.cpp +++ b/lldb/source/Core/ModuleList.cpp @@ -108,6 +108,13 @@ ModuleListProperties::ModuleListProperties() { // BEGIN SWIFT SetSwiftModuleLoadingMode(eSwiftModuleLoadingModePreferSerialized); + + path.clear(); + if (llvm::sys::path::cache_directory(path)) { + llvm::sys::path::append(path, "lldb"); + llvm::sys::path::append(path, "SwiftMetadataCache"); + lldbassert(SetLLDBIndexCachePath(FileSpec(path))); + } // END SWIFT path.clear(); @@ -194,6 +201,36 @@ bool ModuleListProperties::SetSwiftModuleLoadingMode(SwiftModuleLoadingMode mode return m_collection_sp->SetPropertyAtIndexAsEnumeration( nullptr, ePropertySwiftModuleLoadingMode, mode); } + +FileSpec ModuleListProperties::GetSwiftMetadataCachePath() const { + return m_collection_sp + ->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false, + ePropertySwiftMetadataCachePath) + ->GetCurrentValue(); +} + +bool ModuleListProperties::SetSwiftMetadataCachePath(const FileSpec &path) { + return m_collection_sp->SetPropertyAtIndexAsFileSpec( + nullptr, ePropertySwiftMetadataCachePath, path); +} + +bool ModuleListProperties::GetEnableSwiftMetadataCache() const { + const uint32_t idx = ePropertyEnableSwiftMetadataCache; + return m_collection_sp->GetPropertyAtIndexAsBoolean( + nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0); +} + +uint64_t ModuleListProperties::GetSwiftMetadataCacheMaxByteSize() { + const uint32_t idx = ePropertySwiftMetadataCacheMaxByteSize; + return m_collection_sp->GetPropertyAtIndexAsUInt64( + nullptr, idx, g_modulelist_properties[idx].default_uint_value); +} + +uint64_t ModuleListProperties::GetSwiftMetadataCacheExpirationDays() { + const uint32_t idx = ePropertySwiftMetadataCacheExpirationDays; + return m_collection_sp->GetPropertyAtIndexAsUInt64( + nullptr, idx, g_modulelist_properties[idx].default_uint_value); +} // END SWIFT FileSpec ModuleListProperties::GetLLDBIndexCachePath() const { diff --git a/lldb/source/Plugins/LanguageRuntime/Swift/CMakeLists.txt b/lldb/source/Plugins/LanguageRuntime/Swift/CMakeLists.txt index 96e580f0148c0..f4c30e8cf93ff 100644 --- a/lldb/source/Plugins/LanguageRuntime/Swift/CMakeLists.txt +++ b/lldb/source/Plugins/LanguageRuntime/Swift/CMakeLists.txt @@ -3,6 +3,7 @@ add_lldb_library(lldbPluginSwiftLanguageRuntime PLUGIN SwiftLanguageRuntime.cpp SwiftLanguageRuntimeDynamicTypeResolution.cpp SwiftLanguageRuntimeNames.cpp + SwiftMetadataCache.cpp LINK_LIBS swiftAST diff --git a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp index 42df7950f6bd7..f969820018a0d 100644 --- a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp +++ b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp @@ -13,6 +13,7 @@ #include "SwiftLanguageRuntime.h" #include "Plugins/LanguageRuntime/Swift/LLDBMemoryReader.h" #include "SwiftLanguageRuntimeImpl.h" +#include "SwiftMetadataCache.h" #include "Plugins/Process/Utility/RegisterContext_x86.h" #include "Plugins/TypeSystem/Clang/TypeSystemClang.h" @@ -442,6 +443,13 @@ SwiftLanguageRuntimeImpl::GetReflectionContext() { return m_reflection_ctx.get(); } +SwiftMetadataCache * +SwiftLanguageRuntimeImpl::GetSwiftMetadataCache() { + if (!m_swift_metadata_cache.is_enabled()) + return {}; + return &m_swift_metadata_cache; +} + void SwiftLanguageRuntimeImpl::SetupReflection() { LLDB_SCOPED_TIMER(); @@ -481,13 +489,13 @@ void SwiftLanguageRuntimeImpl::SetupReflection() { "Initializing a 64-bit reflection context (%s) for \"%s\"", triple.str().c_str(), objc_interop_msg); m_reflection_ctx = ReflectionContextInterface::CreateReflectionContext64( - this->GetMemoryReader(), objc_interop); + this->GetMemoryReader(), objc_interop, GetSwiftMetadataCache()); } else if (triple.isArch32Bit()) { LLDB_LOGF(GetLog(LLDBLog::Types), "Initializing a 32-bit reflection context (%s) for \"%s\"", triple.str().c_str(), objc_interop_msg); m_reflection_ctx = ReflectionContextInterface::CreateReflectionContext32( - this->GetMemoryReader(), objc_interop); + this->GetMemoryReader(), objc_interop, GetSwiftMetadataCache()); } else { LLDB_LOGF(GetLog(LLDBLog::Types), "Could not initialize reflection context for \"%s\"", @@ -649,7 +657,7 @@ bool SwiftLanguageRuntimeImpl::AddJitObjectFileToReflectionContext( if (!obj_file_format) return false; - return m_reflection_ctx->addImage( + auto reflection_info_id = m_reflection_ctx->addImage( [&](swift::ReflectionSectionKind section_kind) -> std::pair, uint64_t> { auto section_name = obj_file_format->getSectionName(section_kind); @@ -676,9 +684,13 @@ bool SwiftLanguageRuntimeImpl::AddJitObjectFileToReflectionContext( return {}; }, likely_module_names); + // We don't care to cache modules generated by the jit, because they will + // only be used by the current process. + return reflection_info_id.hasValue(); } -bool SwiftLanguageRuntimeImpl::AddObjectFileToReflectionContext( +llvm::Optional +SwiftLanguageRuntimeImpl::AddObjectFileToReflectionContext( ModuleSP module, llvm::SmallVector likely_module_names) { auto obj_format_type = @@ -856,6 +868,8 @@ bool SwiftLanguageRuntimeImpl::AddModuleToReflectionContext( auto read_from_file_cache = GetMemoryReader()->readMetadataFromFileCacheEnabled(); + + llvm::Optional info_id; // When dealing with ELF, we need to pass in the contents of the on-disk // file, since the Section Header Table is not present in the child process if (obj_file->GetPluginName().equals("elf")) { @@ -863,20 +877,26 @@ bool SwiftLanguageRuntimeImpl::AddModuleToReflectionContext( auto size = obj_file->GetData(0, obj_file->GetByteSize(), extractor); const uint8_t *file_data = extractor.GetDataStart(); llvm::sys::MemoryBlock file_buffer((void *)file_data, size); - m_reflection_ctx->readELF( + info_id = m_reflection_ctx->readELF( swift::remote::RemoteAddress(load_ptr), llvm::Optional(file_buffer), likely_module_names); } else if (read_from_file_cache && obj_file->GetPluginName().equals("mach-o")) { - if (!AddObjectFileToReflectionContext(module_sp, likely_module_names)) { - m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr), + info_id = AddObjectFileToReflectionContext(module_sp, likely_module_names); + if (!info_id) + info_id = m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr), likely_module_names); - } } else { - m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr), + info_id = m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr), likely_module_names); } + + if (info_id) + if (auto *swift_metadata_cache = GetSwiftMetadataCache()) + swift_metadata_cache->registerModuleWithReflectionInfoID(module_sp, + *info_id); + return true; } diff --git a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeDynamicTypeResolution.cpp b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeDynamicTypeResolution.cpp index 231e61515d9be..5693d45b8055e 100644 --- a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeDynamicTypeResolution.cpp +++ b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeDynamicTypeResolution.cpp @@ -11,8 +11,9 @@ //===----------------------------------------------------------------------===// #include "LLDBMemoryReader.h" -#include "SwiftLanguageRuntimeImpl.h" #include "SwiftLanguageRuntime.h" +#include "SwiftLanguageRuntimeImpl.h" +#include "SwiftMetadataCache.h" #include "Plugins/ExpressionParser/Clang/ClangUtil.h" #include "Plugins/TypeSystem/Clang/TypeSystemClang.h" @@ -24,12 +25,13 @@ #include "lldb/Utility/LLDBLog.h" #include "lldb/Utility/Log.h" #include "lldb/Utility/Timer.h" -#include "swift/AST/Types.h" +#include "llvm/ADT/STLExtras.h" #include "swift/AST/ASTContext.h" #include "swift/AST/ASTMangler.h" #include "swift/AST/ASTWalker.h" #include "swift/AST/Decl.h" +#include "swift/AST/Types.h" #include "swift/Demangling/Demangle.h" #include "swift/Reflection/ReflectionContext.h" #include "swift/Reflection/TypeRefBuilder.h" @@ -241,10 +243,11 @@ class TargetReflectionContext public: TargetReflectionContext( - std::shared_ptr reader) - : m_reflection_ctx(reader) {} + std::shared_ptr reader, + SwiftMetadataCache *swift_metadata_cache) + : m_reflection_ctx(reader, swift_metadata_cache) {} - bool addImage( + llvm::Optional addImage( llvm::function_ref, uint64_t>( swift::ReflectionSectionKind)> find_section, @@ -252,13 +255,13 @@ class TargetReflectionContext return m_reflection_ctx.addImage(find_section, likely_module_names); } - bool + llvm::Optional addImage(swift::remote::RemoteAddress image_start, llvm::SmallVector likely_module_names) override { return m_reflection_ctx.addImage(image_start, likely_module_names); } - bool readELF( + llvm::Optional readELF( swift::remote::RemoteAddress ImageStart, llvm::Optional FileBuffer, llvm::SmallVector likely_module_names = {}) override { @@ -361,21 +364,25 @@ class TargetReflectionContext std::unique_ptr SwiftLanguageRuntimeImpl::ReflectionContextInterface::CreateReflectionContext32( - std::shared_ptr reader, bool ObjCInterop) { + std::shared_ptr reader, bool ObjCInterop, + SwiftMetadataCache *swift_metadata_cache) { using ReflectionContext32ObjCInterop = TargetReflectionContext>>>>; using ReflectionContext32NoObjCInterop = TargetReflectionContext>>>>; - if (ObjCInterop) - return std::make_unique(reader); - return std::make_unique(reader); + if (ObjCInterop) + return std::make_unique( + reader, swift_metadata_cache); + return std::make_unique( + reader, swift_metadata_cache); } std::unique_ptr SwiftLanguageRuntimeImpl::ReflectionContextInterface::CreateReflectionContext64( - std::shared_ptr reader, bool ObjCInterop) { + std::shared_ptr reader, bool ObjCInterop, + SwiftMetadataCache *swift_metadata_cache) { using ReflectionContext64ObjCInterop = TargetReflectionContext>>>>; @@ -383,8 +390,10 @@ SwiftLanguageRuntimeImpl::ReflectionContextInterface::CreateReflectionContext64( TargetReflectionContext>>>>; if (ObjCInterop) - return std::make_unique(reader); - return std::make_unique(reader); + return std::make_unique( + reader, swift_metadata_cache); + return std::make_unique( + reader, swift_metadata_cache); } SwiftLanguageRuntimeImpl::ReflectionContextInterface:: diff --git a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeImpl.h b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeImpl.h index 000c1e88857a1..4dcd578cb0916 100644 --- a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeImpl.h +++ b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeImpl.h @@ -15,6 +15,7 @@ #include "LLDBMemoryReader.h" #include "SwiftLanguageRuntime.h" +#include "SwiftMetadataCache.h" #include "swift/Reflection/TypeLowering.h" #include "llvm/Support/Memory.h" @@ -201,24 +202,26 @@ class SwiftLanguageRuntimeImpl { /// Return a 32-bit reflection context. static std::unique_ptr CreateReflectionContext32( - std::shared_ptr reader, bool ObjCInterop); + std::shared_ptr reader, bool ObjCInterop, + SwiftMetadataCache *swift_metadata_cache); /// Return a 64-bit reflection context. static std::unique_ptr CreateReflectionContext64( - std::shared_ptr reader, bool ObjCInterop); + std::shared_ptr reader, bool ObjCInterop, + SwiftMetadataCache *swift_metadata_cache); virtual ~ReflectionContextInterface(); - virtual bool addImage( + virtual llvm::Optional addImage( llvm::function_ref, uint64_t>( swift::ReflectionSectionKind)> find_section, llvm::SmallVector likely_module_names = {}) = 0; - virtual bool addImage( + virtual llvm::Optional addImage( swift::remote::RemoteAddress image_start, llvm::SmallVector likely_module_names = {}) = 0; - virtual bool + virtual llvm::Optional readELF(swift::remote::RemoteAddress ImageStart, llvm::Optional FileBuffer, llvm::SmallVector likely_module_names = {}) = 0; @@ -365,6 +368,8 @@ class SwiftLanguageRuntimeImpl { /// Lazily initialize and return \p m_SwiftNativeNSErrorISA. llvm::Optional GetSwiftNativeNSErrorISA(); + SwiftMetadataCache *GetSwiftMetadataCache(); + /// These members are used to track and toggle the state of the "dynamic /// exclusivity enforcement flag" in the swift runtime. This flag is set to /// true when an LLDB expression starts running, and reset to its original @@ -382,6 +387,8 @@ class SwiftLanguageRuntimeImpl { /// \{ std::unique_ptr m_reflection_ctx; + SwiftMetadataCache m_swift_metadata_cache; + /// Record modules added through ModulesDidLoad, which are to be /// added to the reflection context once it's being initialized. ModuleList m_modules_to_add; @@ -400,8 +407,9 @@ class SwiftLanguageRuntimeImpl { /// Add the reflections sections to the reflection context by extracting /// the directly from the object file. - /// \return true on success. - bool AddObjectFileToReflectionContext( + /// \return the info id of the newly registered reflection info on success, or + /// llvm::None otherwise. + llvm::Optional AddObjectFileToReflectionContext( lldb::ModuleSP module, llvm::SmallVector likely_module_names); diff --git a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftMetadataCache.cpp b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftMetadataCache.cpp new file mode 100644 index 0000000000000..4a137d24a8dac --- /dev/null +++ b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftMetadataCache.cpp @@ -0,0 +1,279 @@ +#include "SwiftMetadataCache.h" + +#include "lldb/Utility/DataEncoder.h" +#include "lldb/Utility/LLDBLog.h" +#include "lldb/Utility/Log.h" +#include "lldb/Version/Version.h" +#include "llvm/CodeGen/AccelTable.h" +#include "llvm/CodeGen/AsmPrinter.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/CachePruning.h" +#include "llvm/Support/Compression.h" + +using namespace lldb; +using namespace lldb_private; +using namespace swift::reflection; +using namespace swift::remote; + +SwiftMetadataCache::SwiftMetadataCache() { + if (ModuleList::GetGlobalModuleListProperties() + .GetEnableSwiftMetadataCache()) { + llvm::CachePruningPolicy policy; + ModuleListProperties &properties = + ModuleList::GetGlobalModuleListProperties(); + policy.Interval = std::chrono::hours(1); + policy.MaxSizeBytes = properties.GetSwiftMetadataCacheMaxByteSize(); + policy.Expiration = std::chrono::hours( + properties.GetSwiftMetadataCacheExpirationDays() * 24); + m_data_file_cache.emplace(ModuleList::GetGlobalModuleListProperties() + .GetSwiftMetadataCachePath() + .GetPath(), + policy); + } +} + +bool SwiftMetadataCache::is_enabled() { + return llvm::zlib::isAvailable() && m_data_file_cache.hasValue(); +} + +void SwiftMetadataCache::registerModuleWithReflectionInfoID(ModuleSP module, + uint64_t info_id) { + std::lock_guard guard(m_mutex); + Log *log = GetLog(LLDBLog::Types); + + if (!is_enabled()) + return; + + m_info_to_module[info_id] = {module, false}; + + // Attempt to load the cached file. + auto module_name = getTyperefCacheFileNameForModule(module); + auto mem_buffer_up = m_data_file_cache->GetCachedData(module_name); + + // Nothing cached. + if (!mem_buffer_up) { + LLDB_LOGV(log, "[SwiftMetadataCache] No cached file found for module {0}.", + module->GetFileSpec().GetFilename()); + return; + } + + // Extractor used to extract the header information (see the .h file for + // details on the format). + DataExtractor header_extractor(mem_buffer_up->getBufferStart(), + mem_buffer_up->getBufferSize(), + module->GetObjectFile()->GetByteOrder(), + module->GetObjectFile()->GetAddressByteSize()); + + lldb::offset_t read_offset = 0; + + std::string UUID = module->GetUUID().GetAsString(); + std::string cached_UUID = header_extractor.GetCStr(&read_offset); + // If no uuid in the file something is wrong with the cache. + if (cached_UUID.empty()) { + LLDB_LOG(log, + "[SwiftMetadataCache] Failed to read cached UUID for module {0}.", + module->GetFileSpec().GetFilename()); + m_data_file_cache->RemoveCacheFile(module_name); + return; + } + + // If the UUIDs don't match this is most likely a stale cache. + if (cached_UUID != UUID) { + LLDB_LOGV(log, "[SwiftMetadataCache] Module UUID mismatch for {0}.", + module->GetFileSpec().GetFilename()); + m_data_file_cache->RemoveCacheFile(module_name); + return; + } + + uint64_t expanded_size = 0; + if (!header_extractor.GetU64(&read_offset, &expanded_size, 1)) { + LLDB_LOGV(log, + "[SwiftMetadataCache] Failed to read decompressed cache size for " + "module {0}.", + module->GetFileSpec().GetFilename()); + m_data_file_cache->RemoveCacheFile(module_name); + return; + } + + const auto *start = (const char *)header_extractor.GetData(&read_offset, 0); + // Create a reference to the compressed data. + llvm::StringRef string_buffer(start, (uint64_t)mem_buffer_up->getBufferEnd() - + (uint64_t)start); + + llvm::SmallString<0> decompressed; + auto error = + llvm::zlib::uncompress(string_buffer, decompressed, expanded_size); + if (error) { + auto error_string = llvm::toString(std::move(error)); + LLDB_LOG(log, + "[SwiftMetadataCache] Cache decompression failed with error: {0}. " + "Deleting cached file.", + error_string); + m_data_file_cache->RemoveCacheFile(module_name); + return; + } + + // Extractor to extract the body of the cached file (see SwiftMetadataCache.h + // for more details of the format). + DataExtractor body_extractor(decompressed.data(), decompressed.size(), + module->GetObjectFile()->GetByteOrder(), + module->GetObjectFile()->GetAddressByteSize()); + read_offset = 0; + auto num_entries = body_extractor.GetU64(&read_offset); + + // Map to extract the encoded data to. Since extraction can fail we don't want + // to insert values into the final map in case we have to abort midway. + llvm::StringMap temp_map; + for (size_t i = 0; i < num_entries; i++) { + const auto *mangled_name = body_extractor.GetCStr(&read_offset); + if (!mangled_name) { + LLDB_LOG(log, + "[SwiftMetadataCache] Failed to read mangled name {0} at offset " + "{1} for module {2}.", + i, read_offset, module->GetFileSpec().GetFilename()); + m_data_file_cache->RemoveCacheFile(module_name); + return; + } + uint64_t offset = 0; + if (!body_extractor.GetU64(&read_offset, &offset, 1)) { + LLDB_LOG(log, + "[SwiftMetadataCache] Failed to read mangled name {0} at offset " + "{1} for module {2}.", + i, read_offset, module->GetFileSpec().GetFilename()); + m_data_file_cache->RemoveCacheFile(module_name); + return; + } + temp_map[mangled_name] = {info_id, offset}; + } + + // Move the values to the actual map now that we know that it's safe. + for (auto &p : temp_map) + m_mangled_name_to_offset.try_emplace(p.getKey(), p.second); + + // Mark this reflection info as processed. + m_info_to_module[info_id] = {module, true}; + LLDB_LOGV(log, "[SwiftMetadataCache] Loaded cache for module {0}.", + module->GetFileSpec().GetFilename()); +} + +static bool areMangledNamesAndFieldSectionSameSize( + const swift::reflection::FieldSection &field_descriptors, + const std::vector &mangled_names) { + // FieldSection is not random access, so we have to iterate over it in it's + // entirety to find out it's true size + uint64_t field_descriptors_size = + std::distance(field_descriptors.begin(), field_descriptors.end()); + + return field_descriptors_size == mangled_names.size(); +} + +bool SwiftMetadataCache::writeMangledNamesAndOffsetsToEncoder( + uint64_t info_id, const swift::reflection::FieldSection &field_descriptors, + const std::vector &mangled_names, DataEncoder &encoder) { + Log *log = GetLog(LLDBLog::Types); + auto num_entries = mangled_names.size(); + encoder.AppendU64(num_entries); + + // If the amount of mangled names and field descriptors don't match something + // unexpected happened. + if (!areMangledNamesAndFieldSectionSameSize(field_descriptors, + mangled_names)) { + LLDB_LOG(log, "[SwiftMetadataCache] Mismatch between number of mangled " + "names and field descriptors passed in."); + return false; + } + + for (auto pair : llvm::zip(field_descriptors, mangled_names)) { + auto field_descriptor = std::get<0>(pair); + auto &mangled_name = std::get<1>(pair); + if (mangled_name.empty()) + continue; + auto offset = field_descriptor.getAddressData() - + field_descriptors.startAddress().getAddressData(); + encoder.AppendCString(mangled_name.data()); + encoder.AppendU64(offset); + } + return true; +} + +void SwiftMetadataCache::cacheFieldDescriptors( + uint64_t info_id, const swift::reflection::FieldSection &field_descriptors, + llvm::ArrayRef mangled_names) { + std::lock_guard guard(m_mutex); + Log *log = GetLog(LLDBLog::Types); + + if (!is_enabled()) + return; + + auto it = m_info_to_module.find(info_id); + if (it == m_info_to_module.end()) { + LLDB_LOGV(log, "[SwiftMetadataCache] No module found with module id {0}.", + info_id); + return; + } + + auto module = std::get(it->second); + // Write the data to the body encoder with the format expected by the current + // cache version. + DataEncoder body_encoder; + if (!writeMangledNamesAndOffsetsToEncoder(info_id, field_descriptors, + mangled_names, body_encoder)) + return; + + uint64_t typeref_buffer_size = body_encoder.GetData().size(); + llvm::StringRef typeref_buffer((const char *)body_encoder.GetData().data(), + typeref_buffer_size); + + llvm::SmallString<0> compressed_buffer; + llvm::zlib::compress(typeref_buffer, compressed_buffer); + + // Write the header followed by the body. + DataEncoder encoder; + encoder.AppendCString(module->GetUUID().GetAsString()); + encoder.AppendU64(typeref_buffer_size); + encoder.AppendData(compressed_buffer); + + auto filename = getTyperefCacheFileNameForModule(module); + + m_data_file_cache->SetCachedData(filename, encoder.GetData()); + LLDB_LOGV(log, "[SwiftMetadataCache] Cache file written for module {0}.", + module->GetFileSpec().GetFilename()); +} + +llvm::Optional +SwiftMetadataCache::getFieldDescriptorLocator(const std::string &Name) { + std::lock_guard guard(m_mutex); + Log *log = GetLog(LLDBLog::Types); + auto it = m_mangled_name_to_offset.find(Name); + if (it != m_mangled_name_to_offset.end()) { + LLDB_LOGV( + log, + "[SwiftMetadataCache] Returning field descriptor for mangled name {0}", + Name); + return it->second; + } + return {}; +} + +bool SwiftMetadataCache::isReflectionInfoCached(uint64_t info_id) { + std::lock_guard guard(m_mutex); + auto it = m_info_to_module.find(info_id); + // First check if we've registered the reflection info with that id. + if (it != m_info_to_module.end()) + // Then check whether we've already parsed it or not. + return std::get(it->second); + return false; +} + +std::string SwiftMetadataCache::getTyperefCacheFileNameForModule( + const lldb::ModuleSP &module) { + // We hash the lldb string version (so we don't run into the risk of two lldbs + // invalidating each other's cache), and the modules path (so we clean up + // stale caches when the module changes) as the typeref cache file name. + llvm::BLAKE3 blake3; + const char *version = lldb_private::GetVersion(); + blake3.update(version); + blake3.update(module->GetFileSpec().GetPath()); + auto hashed_result = llvm::toHex(blake3.final()); + return "typeref-" + hashed_result; +} diff --git a/lldb/source/Plugins/LanguageRuntime/Swift/SwiftMetadataCache.h b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftMetadataCache.h new file mode 100644 index 0000000000000..325e94c91e858 --- /dev/null +++ b/lldb/source/Plugins/LanguageRuntime/Swift/SwiftMetadataCache.h @@ -0,0 +1,83 @@ + +#ifndef liblldb_TypeRefCacher_h_ +#define liblldb_TypeRefCacher_h_ + +#include +#include + +#include "lldb/Core/DataFileCache.h" +#include "lldb/Core/Module.h" + +#include "llvm/ADT/STLExtras.h" + +#include "swift/Reflection/ReflectionContext.h" +#include "swift/Remote/ExternalTypeRefCache.h" + +namespace lldb_private { + +/// A cache for data used to speed up retrieving swift metadata. +/// Currently we only cache typeref information. +/// This caches files in the directory specified by the SwiftMetadataCachePath +/// setting. The file format is the following: A header consisting of: +/// - Version number (uint16_t). +/// - Module signature. +/// - Size of the remainder of the contents when decompressed (uint64_t). +/// A body that is zlib compressed containing: +/// - The number (N) of name/offset pairs (uint64_t). +/// - N pairs composed of a c-string mangled name followed by the offset +/// (uint32_t) where the corresponding field descriptor can be found in +// the fieldmd section. +struct SwiftMetadataCache : swift::remote::ExternalTypeRefCache { +public: + SwiftMetadataCache(); + + /// Whether the cache is enabled or disabled. This is controlled by the + /// enable-swift-metadata-cache setting. + bool is_enabled(); + + /// Relate the lldb module with the reflection info id. + void registerModuleWithReflectionInfoID(lldb::ModuleSP module, + uint64_t info_id); + + /// Cache the field descriptors in the info with the mangled names + /// passed in. The number of field descriptors and mangled names passed should + /// be the same, otherwise caching is aborted. + void cacheFieldDescriptors( + uint64_t info_id, + const swift::reflection::FieldSection &field_descriptors, + llvm::ArrayRef mangled_names) override; + + llvm::Optional + getFieldDescriptorLocator(const std::string &mangled_name) override; + + bool isReflectionInfoCached(uint64_t info_id) override; + +private: + bool writeMangledNamesAndOffsetsToEncoder( + uint64_t info_id, + const swift::reflection::FieldSection &field_descriptors, + const std::vector &mangled_names, DataEncoder &encoder); + + void + fillMangledNameToOffsetMap(uint64_t info_id, + const swift::reflection::ReflectionInfo &info, + const std::vector &mangled_names); + + /// Gets the file name that the cache file should use for a given module. + std::string getTyperefCacheFileNameForModule(const lldb::ModuleSP &module); + + /// A map from mangled names to field descriptor locators. + llvm::StringMap + m_mangled_name_to_offset; + + /// A map from reflection infos ids to a pair constituting of its + /// corresponding module and whether or not we've inserted the cached metadata + /// for that reflection info already. + llvm::DenseMap> m_info_to_module; + + std::recursive_mutex m_mutex; + + llvm::Optional m_data_file_cache; +}; +} // namespace lldb_private +#endif diff --git a/lldb/test/API/lang/swift/metadata_cache/Makefile b/lldb/test/API/lang/swift/metadata_cache/Makefile new file mode 100644 index 0000000000000..0e4afbf4b1703 --- /dev/null +++ b/lldb/test/API/lang/swift/metadata_cache/Makefile @@ -0,0 +1,2 @@ + +include Makefile.rules diff --git a/lldb/test/API/lang/swift/metadata_cache/TestSwiftMetadataCache.py b/lldb/test/API/lang/swift/metadata_cache/TestSwiftMetadataCache.py new file mode 100644 index 0000000000000..36235adb8b4e5 --- /dev/null +++ b/lldb/test/API/lang/swift/metadata_cache/TestSwiftMetadataCache.py @@ -0,0 +1,110 @@ +""" +Test the swift metadata cache works correctly +""" +import glob +import io +import os +import lldb +from lldbsuite.test.lldbtest import * +from lldbsuite.test.decorators import * +import lldbsuite.test.lldbutil as lldbutil +import unittest2 + + +class TestSwiftMetadataCache(TestBase): + + def setUp(self): + # Call super's setUp(). + TestBase.setUp(self) + + cache_dir = self.getBuildArtifact('swift-metadata-cache') + # Set the lldb module cache directory to a directory inside the build + # artifacts directory so no other tests are interfered with. + self.runCmd('settings set symbols.swift-metadata-cache-path "%s"' % cache_dir) + self.runCmd('settings set symbols.enable-swift-metadata-cache true') + + def check_strings_in_log(self, log_path, strings): + types_log = io.open(log_path, "r", encoding='utf-8') + for line in types_log: + # copy to remove from original + copy = strings[:] + for s in copy: + if s in line: + try: + strings.remove(s) + # In case the string shows up more than once + except ValueError: + pass + + if len(strings) == 0: + return True + return False + + @skipUnlessDarwin + @swiftTest + def test_swift_metadata_cache(self): + """Test the swift metadata cache.""" + + # This test runs does three runs: + + # In the first run, we make sure we build the cache. + + # In the second run, we make sure we use the cache. + + # In the third run, we emulate a source file modification by building + # main_modified.swift instead of main.swift. We then make sure that + # we invalidate the modified cache and rebuild it. + + # Build with the "original" main file. + self.build(dictionary={'SWIFT_SOURCES': 'main.swift'}) + + types_log = self.getBuildDir() + "/first_run_types.txt" + self.runCmd('log enable lldb types -v -f "%s"' % types_log) + target, process, thread, bkpt = lldbutil.run_to_source_breakpoint( + self, 'Set breakpoint here', lldb.SBFileSpec('main.swift')) + + # Run frame variable for the first time to build the cache. + self.expect('v v', substrs=['SomeTypeWeWillLookUp']) + + # Check that we wrote the cache for the main module. + self.assertTrue(self.check_strings_in_log(types_log, [ + 'Cache file written for module a.out.'])) + # Finish run. + self.runCmd('c') + + + types_log = self.getBuildDir() + "/second_run_types.txt" + self.runCmd('log enable lldb types -v -f "%s"' % types_log) + target, process, thread, bkpt = lldbutil.run_to_source_breakpoint( + self, 'Set breakpoint here', lldb.SBFileSpec('main.swift')) + + # Run frame variable for the second time to check if the cache was queried. + self.expect('v v', substrs=['SomeTypeWeWillLookUp']) + + # Check that we have a cache and that we found the type of 'v' in it. + self.assertTrue(self.check_strings_in_log(types_log, [ + 'Loaded cache for module a.out.', + 'Returning field descriptor for mangled name 1a20SomeTypeWeWillLookUpV'])) + + # Finish run. + self.runCmd('c') + + + # Rebuild the "modified" program. This should have the exact same path + # as the original one. + self.build(dictionary={'SWIFT_SOURCES': 'main_modified.swift'}) + + types_log = self.getBuildDir() + "/third_run_types.txt" + self.runCmd('log enable lldb types -v -f "%s"' % types_log) + target, process, thread, bkpt = lldbutil.run_to_source_breakpoint( + self, 'Set breakpoint here', lldb.SBFileSpec('main_modified.swift')) + + + # Run frame variable for the third time to check that the cache is invalidated + # and that we rebuild it. + self.expect('v v', substrs=['SomeTypeWeWillLookUp']) + + # Check that we found the type of 'v' in the cache. + self.assertTrue(self.check_strings_in_log(types_log, [ + 'Module UUID mismatch for a.out.', + 'Cache file written for module a.out.'])) diff --git a/lldb/test/API/lang/swift/metadata_cache/main.swift b/lldb/test/API/lang/swift/metadata_cache/main.swift new file mode 100644 index 0000000000000..2981a4d5473e8 --- /dev/null +++ b/lldb/test/API/lang/swift/metadata_cache/main.swift @@ -0,0 +1,5 @@ +struct SomeTypeWeWillLookUp { +} + +let v = SomeTypeWeWillLookUp() +print(v) // Set breakpoint here diff --git a/lldb/test/API/lang/swift/metadata_cache/main_modified.swift b/lldb/test/API/lang/swift/metadata_cache/main_modified.swift new file mode 100644 index 0000000000000..52f9a20bf2a3d --- /dev/null +++ b/lldb/test/API/lang/swift/metadata_cache/main_modified.swift @@ -0,0 +1,8 @@ +struct SomeTypeWeWillLookUp { +} + +struct SomeNewTypeToBreakTheCache { +} + +let v = SomeTypeWeWillLookUp() +print(v) // Set breakpoint here