From e7b5843b336cdb18a1a661346736f34687bad9e2 Mon Sep 17 00:00:00 2001 From: Erik Eckstein Date: Wed, 2 Dec 2020 17:08:49 +0100 Subject: [PATCH 1/2] runtime: add a StackAllocator utility A StackAllocator performs fast allocation and deallocation of memory by implementing a bump-pointer allocation strategy. In contrast to a pure bump-pointer allocator, it's possible to free memory. Allocations and deallocations must follow a strict stack discipline. In general, slabs which become unused are _not_ freed, but reused for subsequent allocations. The first slab can be placed into pre-allocated memory. --- stdlib/public/runtime/StackAllocator.h | 280 +++++++++++++++++++++++++ unittests/runtime/CMakeLists.txt | 1 + unittests/runtime/StackAllocator.cpp | 116 ++++++++++ 3 files changed, 397 insertions(+) create mode 100644 stdlib/public/runtime/StackAllocator.h create mode 100644 unittests/runtime/StackAllocator.cpp diff --git a/stdlib/public/runtime/StackAllocator.h b/stdlib/public/runtime/StackAllocator.h new file mode 100644 index 0000000000000..b9c821690ba31 --- /dev/null +++ b/stdlib/public/runtime/StackAllocator.h @@ -0,0 +1,280 @@ +//===--- StackAllocator.h - A stack allocator -----------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// A bump-pointer allocator that obeys a stack discipline. +// +//===----------------------------------------------------------------------===// + +#include "swift/Runtime/Debug.h" +#include "llvm/Support/Alignment.h" +#include + +namespace swift { + +/// A bump-pointer allocator that obeys a stack discipline. +/// +/// StackAllocator performs fast allocation and deallocation of memory by +/// implementing a bump-pointer allocation strategy. +/// +/// This isn't strictly a bump-pointer allocator as it uses backing slabs of +/// memory rather than relying on a boundless contiguous heap. However, it has +/// bump-pointer semantics in that it is a monotonically growing pool of memory +/// where every allocation is found by merely allocating the next N bytes in +/// the slab, or the next N bytes in the next slab. +/// +/// In contrast to a pure bump-pointer allocator, it's possible to free memory. +/// Allocations and deallocations must follow a strict stack discipline. In +/// general, slabs which become unused are _not_ freed, but reused for +/// subsequent allocations. +/// +/// It's possible to place the first slab into pre-allocated memory. +/// +/// The SlabCapacity specifies the capacity for newly allocated slabs. +template +class StackAllocator { +private: + + struct Allocation; + struct Slab; + + /// The last active allocation. + /// + /// A deallocate() must free this allocation. + Allocation *lastAllocation = nullptr; + + /// The first slab. + Slab *firstSlab; + + /// Used for unit testing. + int32_t numAllocatedSlabs = 0; + + /// True if the first slab is pre-allocated. + bool firstSlabIsPreallocated; + + /// If set to true, memory allocations are checked for buffer overflows and + /// use-after-free, similar to guard-malloc. + static constexpr bool guardAllocations = +#ifdef NDEBUG + false; +#else + true; +#endif + + static constexpr uintptr_t magicUninitialized = (uintptr_t)0xcdcdcdcdcdcdcdcdull; + static constexpr uintptr_t magicEndOfAllocation = (uintptr_t)0xdeadbeafdeadbeafull; + + /// A memory slab holding multiple allocations. + /// + /// This struct is actually just the slab header. The slab buffer is tail + /// allocated after Slab. + struct Slab { + /// A single linked list of all allocated slabs. + Slab *next = nullptr; + + // Capacity and offset do not include these header fields. + uint32_t capacity; + uint32_t currentOffset = 0; + + // Here starts the tail allocated memory buffer of the slab. + + Slab(size_t newCapacity) : capacity(newCapacity) { + assert((size_t)capacity == newCapacity && "capacity overflow"); + } + + /// Return the payload buffer address at \p atOffset. + /// + /// Note: it's valid to call this function on a not-yet-constructed slab. + char *getAddr(size_t atOffset) { + return (char *)(this + 1) + atOffset; + } + + /// Return true if this slab can fit an allocation of \p size. + /// + /// \p size does not include the allocation header, but must include the + /// overhead for guardAllocations (if enabled). + inline bool canAllocate(size_t size) const { + return currentOffset + Allocation::includingHeader(size) <= capacity; + } + + /// Return true, if no memory is allocated in this slab. + bool isEmpty() const { return currentOffset == 0; } + + /// Allocate \p alignedSize of bytes in this slab. + /// + /// \p alignedSize does not include the allocation header, but must include + /// the overhead for guardAllocations (if enabled). + /// + /// Precondition: \p alignedSize must be aligned up to + /// StackAllocator::alignment. + /// Precondition: there must be enough space in this slab to fit the + /// allocation. + Allocation *allocate(size_t alignedSize, Allocation *lastAllocation) { + assert(llvm::isAligned(llvm::Align(alignment), alignedSize)); + assert(canAllocate(alignedSize)); + void *buffer = getAddr(currentOffset); + auto *allocation = new (buffer) Allocation(lastAllocation, this); + currentOffset += Allocation::includingHeader(alignedSize); + if (guardAllocations) { + uintptr_t *endOfCurrentAllocation = (uintptr_t *)getAddr(currentOffset); + endOfCurrentAllocation[-1] = magicEndOfAllocation; + } + return allocation; + } + + /// Deallocate \p allocation. + /// + /// Precondition: \p allocation must be an allocation in this slab. + void deallocate(Allocation *allocation) { + assert(allocation->slab == this); + if (guardAllocations) { + auto *endOfAllocation = (uintptr_t *)getAddr(currentOffset); + if (endOfAllocation[-1] != magicEndOfAllocation) + fatalError(0, "Buffer overflow in StackAllocator"); + for (auto *p = (uintptr_t *)allocation; p < endOfAllocation; ++p) + *p = magicUninitialized; + } + currentOffset = (char *)allocation - getAddr(0); + } + }; + + /// A single memory allocation. + /// + /// This struct is actually just the allocation header. The allocated + /// memory buffer is located after Allocation. + struct Allocation { + /// A single linked list of previous allocations. + Allocation *previous; + /// The containing slab. + Slab *slab; + + // Here starts the tail allocated memory. + + Allocation(Allocation *previous, Slab *slab) : + previous(previous), slab(slab) {} + + void *getAllocatedMemory() { + return (void *)(this + 1); + } + + /// Return \p size with the added overhead of the allocation header. + static size_t includingHeader(size_t size) { + return size + sizeof(Allocation); + } + }; + + static constexpr size_t alignment = alignof(std::max_align_t); + + static_assert(sizeof(Slab) % StackAllocator::alignment == 0, + "Slab size must be a multiple of the max allocation alignment"); + + static_assert(sizeof(Allocation) % StackAllocator::alignment == 0, + "Allocation size must be a multiple of the max allocation alignment"); + + // Return a slab which is suitable to allocate \p size memory. + Slab *getSlabForAllocation(size_t size) { + Slab *slab = (lastAllocation ? lastAllocation->slab : firstSlab); + if (slab) { + // Is there enough space in the current slab? + if (slab->canAllocate(size)) + return slab; + + // Is there a successor slab, which we allocated before (and became free + // in the meantime)? + if (Slab *nextSlab = slab->next) { + assert(nextSlab->isEmpty()); + if (nextSlab->canAllocate(size)) + return nextSlab; + + // No space in the next slab. Although it's empty, the size exceeds its + // capacity. + // As we have to allocate a new slab anyway, free all successor slabs + // and allocate a new one with the accumulated capacity. + size_t alreadyAllocatedCapacity = freeAllSlabs(slab->next); + size = std::max(size, alreadyAllocatedCapacity); + } + } + size_t capacity = std::max(SlabCapacity, + Allocation::includingHeader(size)); + void *slabBuffer = malloc(sizeof(Slab) + capacity); + Slab *newSlab = new (slabBuffer) Slab(capacity); + if (slab) + slab->next = newSlab; + else + firstSlab = newSlab; + numAllocatedSlabs++; + return newSlab; + } + + /// Deallocate all slabs after \p first and set \p first to null. + size_t freeAllSlabs(Slab *&first) { + size_t freedCapacity = 0; + Slab *slab = first; + first = nullptr; + while (slab) { + Slab *next = slab->next; + freedCapacity += slab->capacity; + free(slab); + numAllocatedSlabs--; + slab = next; + } + return freedCapacity; + } + +public: + /// Construct a StackAllocator without a pre-allocated first slab. + StackAllocator() : firstSlab(nullptr), firstSlabIsPreallocated(false) { } + + /// Construct a StackAllocator with a pre-allocated first slab. + StackAllocator(void *firstSlabBuffer, size_t bufferCapacity) { + char *start = (char *)llvm::alignAddr(firstSlabBuffer, llvm::Align(alignment)); + char *end = (char *)firstSlabBuffer + bufferCapacity; + assert(start + sizeof(Slab) <= end && "buffer for first slab too small"); + firstSlab = new (start) Slab(end - start - sizeof(Slab)); + firstSlabIsPreallocated = true; + } + + ~StackAllocator() { + if (lastAllocation) + fatalError(0, "not all allocations are deallocated"); + (void)freeAllSlabs(firstSlabIsPreallocated ? firstSlab->next : firstSlab); + assert(getNumAllocatedSlabs() == 0); + } + + /// Allocate a memory buffer of \p size. + void *alloc(size_t size) { + if (guardAllocations) + size += sizeof(uintptr_t); + size_t alignedSize = llvm::alignTo(size, llvm::Align(alignment)); + Slab *slab = getSlabForAllocation(alignedSize); + Allocation *allocation = slab->allocate(alignedSize, lastAllocation); + lastAllocation = allocation; + assert(llvm::isAddrAligned(llvm::Align(alignment), + allocation->getAllocatedMemory())); + return allocation->getAllocatedMemory(); + } + + /// Deallocate memory \p ptr. + void dealloc(void *ptr) { + if (!lastAllocation || lastAllocation->getAllocatedMemory() != ptr) + fatalError(0, "freed pointer was not the last allocation"); + + Allocation *prev = lastAllocation->previous; + lastAllocation->slab->deallocate(lastAllocation); + lastAllocation = prev; + } + + /// For unit testing. + int getNumAllocatedSlabs() { return numAllocatedSlabs; } +}; + +} // namespace swift + diff --git a/unittests/runtime/CMakeLists.txt b/unittests/runtime/CMakeLists.txt index f5413a88837ef..60f9944e1e64f 100644 --- a/unittests/runtime/CMakeLists.txt +++ b/unittests/runtime/CMakeLists.txt @@ -79,6 +79,7 @@ if(("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "${SWIFT_PRIMARY_VARIANT_SDK}") AND Enum.cpp Refcounting.cpp Stdlib.cpp + StackAllocator.cpp ${PLATFORM_SOURCES} # The runtime tests link to internal runtime symbols, which aren't exported diff --git a/unittests/runtime/StackAllocator.cpp b/unittests/runtime/StackAllocator.cpp new file mode 100644 index 0000000000000..1989f37e4ce0f --- /dev/null +++ b/unittests/runtime/StackAllocator.cpp @@ -0,0 +1,116 @@ +//===--- StackAllocator.cpp - Unit tests for the StackAllocator -----------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#include "../../stdlib/public/runtime/StackAllocator.h" +#include "gtest/gtest.h" + +using namespace swift; + +static constexpr size_t slabCapacity = 256; +static constexpr size_t firstSlabBufferCapacity = 140; +static constexpr size_t fitsIntoFirstSlab = 16; +static constexpr size_t fitsIntoSlab = slabCapacity - 16; +static constexpr size_t twoFitIntoSlab = slabCapacity / 2 - 32; +static constexpr size_t exceedsSlab = slabCapacity + 16; + +TEST(StackAllocatorTest, withPreallocatedSlab) { + + char firstSlab[firstSlabBufferCapacity]; + StackAllocator allocator(firstSlab, firstSlabBufferCapacity); + + char *mem1 = (char *)allocator.alloc(fitsIntoFirstSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 0); + char *mem1a = (char *)allocator.alloc(fitsIntoFirstSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 0); + + char *mem2 = (char *)allocator.alloc(exceedsSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 1); + + char *mem3 = (char *)allocator.alloc(fitsIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 2); + + char *mem4 = (char *)allocator.alloc(fitsIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 3); + + allocator.dealloc(mem4); + allocator.dealloc(mem3); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 3); + + char *mem5 = (char *)allocator.alloc(twoFitIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 3); + char *mem6 = (char *)allocator.alloc(twoFitIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 3); + char *mem7 = (char *)allocator.alloc(twoFitIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 3); + + allocator.dealloc(mem7); + allocator.dealloc(mem6); + allocator.dealloc(mem5); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 3); + + char *mem8 = (char *)allocator.alloc(exceedsSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 2); + + allocator.dealloc(mem8); + allocator.dealloc(mem2); + allocator.dealloc(mem1a); + allocator.dealloc(mem1); +} + +TEST(StackAllocatorTest, withoutPreallocatedSlab) { + + constexpr size_t slabCapacity = 256; + + StackAllocator allocator; + + size_t fitsIntoSlab = slabCapacity - 16; + size_t twoFitIntoSlab = slabCapacity / 2 - 32; + size_t exceedsSlab = slabCapacity + 16; + + char *mem1 = (char *)allocator.alloc(twoFitIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 1); + char *mem1a = (char *)allocator.alloc(twoFitIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 1); + + char *mem2 = (char *)allocator.alloc(exceedsSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 2); + + char *mem3 = (char *)allocator.alloc(fitsIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 3); + + char *mem4 = (char *)allocator.alloc(fitsIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 4); + + allocator.dealloc(mem4); + allocator.dealloc(mem3); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 4); + + char *mem5 = (char *)allocator.alloc(twoFitIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 4); + char *mem6 = (char *)allocator.alloc(twoFitIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 4); + char *mem7 = (char *)allocator.alloc(twoFitIntoSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 4); + + allocator.dealloc(mem7); + allocator.dealloc(mem6); + allocator.dealloc(mem5); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 4); + + char *mem8 = (char *)allocator.alloc(exceedsSlab); + EXPECT_EQ(allocator.getNumAllocatedSlabs(), 3); + + allocator.dealloc(mem8); + allocator.dealloc(mem2); + allocator.dealloc(mem1a); + allocator.dealloc(mem1); +} From ec1490d06ee0d0e32ee0abd0ffb4b5d577738ab6 Mon Sep 17 00:00:00 2001 From: Erik Eckstein Date: Tue, 8 Dec 2020 15:04:37 +0100 Subject: [PATCH 2/2] [concurrency] Implement the Task allocator as bump-pointer allocator. Use the StackAllocator as task allocator. TODO: we could pass an initial pre-allocated first slab to the allocator, which is allocated on the stack or with the parent task's allocator. rdar://problem/71157018 --- stdlib/public/Concurrency/Task.cpp | 2 ++ stdlib/public/Concurrency/TaskAlloc.cpp | 37 +++++++--------------- stdlib/public/runtime/StackAllocator.h | 42 ++++++++++++++++--------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/stdlib/public/Concurrency/Task.cpp b/stdlib/public/Concurrency/Task.cpp index 235adbe3ed747..4b30da41e6f3a 100644 --- a/stdlib/public/Concurrency/Task.cpp +++ b/stdlib/public/Concurrency/Task.cpp @@ -311,6 +311,8 @@ AsyncTaskAndContext swift::swift_task_create_future_f( initialContext->Flags.setShouldNotDeallocateInCallee(true); // Initialize the task-local allocator. + // TODO: consider providing an initial pre-allocated first slab to the + // allocator. _swift_task_alloc_initialize(task); return {task, initialContext}; diff --git a/stdlib/public/Concurrency/TaskAlloc.cpp b/stdlib/public/Concurrency/TaskAlloc.cpp index 37858b9312e80..4afe29167fed0 100644 --- a/stdlib/public/Concurrency/TaskAlloc.cpp +++ b/stdlib/public/Concurrency/TaskAlloc.cpp @@ -19,40 +19,27 @@ #include "TaskPrivate.h" #include "swift/Runtime/Concurrency.h" -#include "swift/Runtime/Debug.h" +#include "../runtime/StackAllocator.h" #include -#include using namespace swift; namespace { -class TaskAllocator { - // Just keep track of all allocations in a vector so that we can - // verify stack discipline. We should make sure the allocator - // implementation strictly verifies allocation order at least - // until we've stabilized the compiler implementation. - std::vector Allocations; +/// The size of an allocator slab. +/// +/// TODO: find the optimal value by experiment. +static constexpr size_t SlabCapacity = 1024; -public: - void *alloc(size_t size) { - void *ptr = malloc(size); - Allocations.push_back(ptr); - return ptr; - } +using TaskAllocator = StackAllocator; - void dealloc(void *ptr) { - if (Allocations.empty() || Allocations.back() != ptr) - fatalError(0, "pointer was not the last allocation on this task"); +struct GlobalAllocator { + TaskAllocator allocator; + void *spaceForFirstSlab[64]; - Allocations.pop_back(); - free(ptr); - } + GlobalAllocator() : allocator(spaceForFirstSlab, sizeof(spaceForFirstSlab)) {} }; -static_assert(sizeof(TaskAllocator) <= sizeof(AsyncTask::AllocatorPrivate), - "task allocator must fit in allocator-private slot"); - static_assert(alignof(TaskAllocator) <= alignof(decltype(AsyncTask::AllocatorPrivate)), "task allocator must not be more aligned than " "allocator-private slot"); @@ -70,8 +57,8 @@ static TaskAllocator &allocator(AsyncTask *task) { // FIXME: this fall-back shouldn't be necessary, but it's useful // for now, since the current execution tests aren't setting up a task // properly. - static TaskAllocator global; - return global; + static GlobalAllocator global; + return global.allocator; } void swift::_swift_task_alloc_destroy(AsyncTask *task) { diff --git a/stdlib/public/runtime/StackAllocator.h b/stdlib/public/runtime/StackAllocator.h index b9c821690ba31..01fd89c794daf 100644 --- a/stdlib/public/runtime/StackAllocator.h +++ b/stdlib/public/runtime/StackAllocator.h @@ -60,6 +60,9 @@ class StackAllocator { /// True if the first slab is pre-allocated. bool firstSlabIsPreallocated; + /// The minimal alignment of allocated memory. + static constexpr size_t alignment = alignof(std::max_align_t); + /// If set to true, memory allocations are checked for buffer overflows and /// use-after-free, similar to guard-malloc. static constexpr bool guardAllocations = @@ -90,11 +93,21 @@ class StackAllocator { assert((size_t)capacity == newCapacity && "capacity overflow"); } + /// The size of the slab header. + static size_t headerSize() { + return llvm::alignTo(sizeof(Slab), llvm::Align(alignment)); + } + + /// Return \p size with the added overhead of the slab header. + static size_t includingHeader(size_t size) { + return headerSize() + size; + } + /// Return the payload buffer address at \p atOffset. /// /// Note: it's valid to call this function on a not-yet-constructed slab. char *getAddr(size_t atOffset) { - return (char *)(this + 1) + atOffset; + return (char *)this + headerSize() + atOffset; } /// Return true if this slab can fit an allocation of \p size. @@ -162,23 +175,20 @@ class StackAllocator { previous(previous), slab(slab) {} void *getAllocatedMemory() { - return (void *)(this + 1); + return (char *)this + headerSize(); + } + + /// The size of the allocation header. + static size_t headerSize() { + return llvm::alignTo(sizeof(Allocation), llvm::Align(alignment)); } /// Return \p size with the added overhead of the allocation header. static size_t includingHeader(size_t size) { - return size + sizeof(Allocation); + return headerSize() + size; } }; - static constexpr size_t alignment = alignof(std::max_align_t); - - static_assert(sizeof(Slab) % StackAllocator::alignment == 0, - "Slab size must be a multiple of the max allocation alignment"); - - static_assert(sizeof(Allocation) % StackAllocator::alignment == 0, - "Allocation size must be a multiple of the max allocation alignment"); - // Return a slab which is suitable to allocate \p size memory. Slab *getSlabForAllocation(size_t size) { Slab *slab = (lastAllocation ? lastAllocation->slab : firstSlab); @@ -204,7 +214,7 @@ class StackAllocator { } size_t capacity = std::max(SlabCapacity, Allocation::includingHeader(size)); - void *slabBuffer = malloc(sizeof(Slab) + capacity); + void *slabBuffer = malloc(Slab::includingHeader(capacity)); Slab *newSlab = new (slabBuffer) Slab(capacity); if (slab) slab->next = newSlab; @@ -235,10 +245,12 @@ class StackAllocator { /// Construct a StackAllocator with a pre-allocated first slab. StackAllocator(void *firstSlabBuffer, size_t bufferCapacity) { - char *start = (char *)llvm::alignAddr(firstSlabBuffer, llvm::Align(alignment)); + char *start = (char *)llvm::alignAddr(firstSlabBuffer, + llvm::Align(alignment)); char *end = (char *)firstSlabBuffer + bufferCapacity; - assert(start + sizeof(Slab) <= end && "buffer for first slab too small"); - firstSlab = new (start) Slab(end - start - sizeof(Slab)); + assert(start + Slab::headerSize() <= end && + "buffer for first slab too small"); + firstSlab = new (start) Slab(end - start - Slab::headerSize()); firstSlabIsPreallocated = true; }