From 2d3fab0175de83ce22c55354837b160fa9a19158 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Wed, 29 May 2024 14:15:26 -0700 Subject: [PATCH 1/2] [memprof] Use CallStackRadixTreeBuilder in the V3 format This patch integrates CallStackRadixTreeBuilder into the V3 format, reducing the profile size to about 27% of the V2 profile size. - Serialization: writeMemProfCallStackArray just needs to write out the radix tree array prepared by CallStackRadixTreeBuilder. Mappings from CallStackIds to LinearCallStackIds are moved by new function CallStackRadixTreeBuilder::takeCallStackPos. - Deserialization: Deserializing a call stack is the same as deserializing an array encoded in the obvious manner -- the length followed by the payload, except that we need to follow a pointer to the parent to take advantage of common prefixes once in a while. This patch teaches LinearCallStackIdConverter to how to handle those pointers. --- llvm/include/llvm/ProfileData/MemProf.h | 16 ++++++++++++++-- llvm/lib/ProfileData/InstrProfWriter.cpp | 22 +++++++--------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 6b0fa6cd6541c..9db0113be5de9 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -900,9 +900,17 @@ struct LinearCallStackIdConverter { Frames.reserve(NumFrames); for (; NumFrames; --NumFrames) { LinearFrameId Elem = - support::endian::readNext( - Ptr); + support::endian::read(Ptr); + // Follow a pointer to the parent, if any. + if (static_cast>(Elem) < 0) { + Ptr += (-Elem) * sizeof(LinearFrameId); + Elem = + support::endian::read(Ptr); + } + // We shouldn't encounter another pointer. + assert(static_cast>(Elem) >= 0); Frames.push_back(FrameIdToFrame(Elem)); + Ptr += sizeof(LinearFrameId); } return Frames; @@ -1028,6 +1036,10 @@ class CallStackRadixTreeBuilder { getCallStackPos() const { return CallStackPos; } + + llvm::DenseMap takeCallStackPos() { + return std::move(CallStackPos); + } }; // Verify that each CallStackId is computed with hashCallStack. This function diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index e58e6b8acfc81..a73f72a534f16 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -547,19 +547,11 @@ writeMemProfCallStackArray( llvm::DenseMap MemProfCallStackIndexes; - MemProfCallStackIndexes.reserve(MemProfCallStackData.size()); - uint64_t CallStackBase = OS.tell(); - for (const auto &[CSId, CallStack] : MemProfCallStackData) { - memprof::LinearCallStackId CallStackIndex = - (OS.tell() - CallStackBase) / sizeof(memprof::LinearCallStackId); - MemProfCallStackIndexes.insert({CSId, CallStackIndex}); - const llvm::SmallVector CS = CallStack; - OS.write32(CS.size()); - for (const auto F : CS) { - assert(MemProfFrameIndexes.contains(F)); - OS.write32(MemProfFrameIndexes[F]); - } - } + memprof::CallStackRadixTreeBuilder Builder; + Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes); + for (auto I : Builder.getRadixArray()) + OS.write32(I); + MemProfCallStackIndexes = Builder.takeCallStackPos(); // Release the memory of this vector as it is no longer needed. MemProfCallStackData.clear(); @@ -695,8 +687,8 @@ static Error writeMemProfV2(ProfOStream &OS, // uint64_t Schema entry 1 // .... // uint64_t Schema entry N - 1 -// OnDiskChainedHashTable MemProfFrameData -// OnDiskChainedHashTable MemProfCallStackData +// Frames serialized one after another +// Call stacks encoded as a radix tree // OnDiskChainedHashTable MemProfRecordData static Error writeMemProfV3(ProfOStream &OS, memprof::IndexedMemProfData &MemProfData, From 06bc0ee05ecc139e4ef12290f571ba57a20f60b6 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 6 Jun 2024 23:55:54 -0700 Subject: [PATCH 2/2] Migrate existing users of getCallStackPos to takeCallStackPos. Expand a comment to point to CallStackRadixTreeBuilder. --- llvm/include/llvm/ProfileData/MemProf.h | 8 ++------ llvm/unittests/ProfileData/MemProfTest.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h index 9db0113be5de9..0e6245acb77e8 100644 --- a/llvm/include/llvm/ProfileData/MemProf.h +++ b/llvm/include/llvm/ProfileData/MemProf.h @@ -901,7 +901,8 @@ struct LinearCallStackIdConverter { for (; NumFrames; --NumFrames) { LinearFrameId Elem = support::endian::read(Ptr); - // Follow a pointer to the parent, if any. + // Follow a pointer to the parent, if any. See comments below on + // CallStackRadixTreeBuilder for the description of the radix tree format. if (static_cast>(Elem) < 0) { Ptr += (-Elem) * sizeof(LinearFrameId); Elem = @@ -1032,11 +1033,6 @@ class CallStackRadixTreeBuilder { const std::vector &getRadixArray() const { return RadixArray; } - const llvm::DenseMap & - getCallStackPos() const { - return CallStackPos; - } - llvm::DenseMap takeCallStackPos() { return std::move(CallStackPos); } diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp index e120bc3e35947..26421200e1a11 100644 --- a/llvm/unittests/ProfileData/MemProfTest.cpp +++ b/llvm/unittests/ProfileData/MemProfTest.cpp @@ -670,7 +670,7 @@ TEST(MemProf, RadixTreeBuilderEmpty) { llvm::memprof::CallStackRadixTreeBuilder Builder; Builder.build(std::move(MemProfCallStackData), MemProfFrameIndexes); ASSERT_THAT(Builder.getRadixArray(), testing::IsEmpty()); - const auto &Mappings = Builder.getCallStackPos(); + const auto Mappings = Builder.takeCallStackPos(); ASSERT_THAT(Mappings, testing::IsEmpty()); } @@ -689,7 +689,7 @@ TEST(MemProf, RadixTreeBuilderOne) { 2U, // MemProfFrameIndexes[12] 1U // MemProfFrameIndexes[11] })); - const auto &Mappings = Builder.getCallStackPos(); + const auto Mappings = Builder.takeCallStackPos(); ASSERT_THAT(Mappings, SizeIs(1)); EXPECT_THAT(Mappings, testing::Contains(testing::Pair( llvm::memprof::hashCallStack(CS1), 0U))); @@ -715,7 +715,7 @@ TEST(MemProf, RadixTreeBuilderTwo) { 2U, // MemProfFrameIndexes[12] 1U // MemProfFrameIndexes[11] })); - const auto &Mappings = Builder.getCallStackPos(); + const auto Mappings = Builder.takeCallStackPos(); ASSERT_THAT(Mappings, SizeIs(2)); EXPECT_THAT(Mappings, testing::Contains(testing::Pair( llvm::memprof::hashCallStack(CS1), 0U))); @@ -758,7 +758,7 @@ TEST(MemProf, RadixTreeBuilderSuccessiveJumps) { 2U, // MemProfFrameIndexes[12] 1U // MemProfFrameIndexes[11] })); - const auto &Mappings = Builder.getCallStackPos(); + const auto Mappings = Builder.takeCallStackPos(); ASSERT_THAT(Mappings, SizeIs(4)); EXPECT_THAT(Mappings, testing::Contains(testing::Pair( llvm::memprof::hashCallStack(CS1), 0U)));