From c25b3b43a2640f730abb0cff408a7b7782fca39b Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:52:24 -0700 Subject: [PATCH 01/26] [Runtime] Restructure BytecodeLayouts --- include/swift/ABI/Metadata.h | 14 +- stdlib/public/runtime/BytecodeLayouts.cpp | 824 +++++++++++++++++- stdlib/public/runtime/BytecodeLayouts.h | 25 + .../layout_string_witnesses_types.swift | 5 + 4 files changed, 847 insertions(+), 21 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index 38511c67533ca..9b2cc2b04d014 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -323,13 +323,13 @@ struct TargetMetadata { } const uint8_t *getLayoutString() const { - assert(hasLayoutString()); - if (isAnyClass()) { - return asFullMetadata( - reinterpret_cast *>( - this)) - ->layoutString; - } + //assert(hasLayoutString()); + // if (isAnyClass()) { + // return asFullMetadata( + // reinterpret_cast *>( + // this)) + // ->layoutString; + // } return asFullMetadata(this)->layoutString; } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index ba1a87219b7b5..645a07f241e12 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -91,7 +91,7 @@ static void existential_destroy(OpaqueValue* object) { } template -inline static bool handleNextRefCount(const Metadata *metadata, +inline bool handleNextRefCount(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, Params... params) { uint64_t skip = reader.readBytes(); @@ -139,7 +139,7 @@ inline static bool handleNextRefCount(const Metadata *metadata, } template -inline static void handleRefCounts(const Metadata *metadata, +inline void handleRefCounts(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, Params... params) { if (N == 0) { @@ -155,7 +155,7 @@ inline static void handleRefCounts(const Metadata *metadata, } template -inline static void handleRefCounts(const Metadata *metadata, Params... params) { +inline void handleRefCounts(const Metadata *metadata, Params... params) { LayoutStringReader reader{metadata->getLayoutString(), layoutStringHeaderSize}; uintptr_t addrOffset = 0; @@ -359,10 +359,16 @@ static void handleMultiPayloadEnumGeneric(const Metadata *metadata, addrOffset += enumSize; } +void swift_release_masked(void* ptr) { + HeapObject *object = (HeapObject*)(((uintptr_t)ptr) & ~_swift_abi_SwiftSpareBitsMask); + if (object != nullptr) + object->refCounts.decrementAndMaybeDeinit(1); +} + const DestroyFuncAndMask destroyTable[] = { {(DestrFn)&skipDestroy, false}, {(DestrFn)&swift_errorRelease, true}, - {(DestrFn)&swift_release, true}, + {(DestrFn)&swift_release_masked, true}, {(DestrFn)&swift_unownedRelease, true}, {(DestrFn)&swift_weakDestroy, false}, {(DestrFn)&swift_unknownObjectRelease, true}, @@ -437,9 +443,789 @@ struct DestroyHandler { } }; +static void handleRefCountsDestroy(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr); + +static void endDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + return; +} + +static void errorDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + SwiftError *error = *(SwiftError**)(addr + addrOffset); + swift_errorRelease(error); +} + +static void nativeStrongDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + swift_release(object); + // if (object != nullptr) + // object->refCounts.decrementAndMaybeDeinit(1); +} + +static void unownedDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + swift_unownedRelease(object); +} + +static void weakDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + swift_weakDestroy((WeakReference *)(addr + addrOffset)); +} + +static void unknownDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + void *object = *(void**)(addr + addrOffset); + swift_unknownObjectRelease(object); +} + +static void unknownUnownedDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + UnownedReference *object = (UnownedReference*)(addr + addrOffset); + swift_unknownObjectUnownedDestroy(object); +} + +static void unknownWeakDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + swift_unknownObjectWeakDestroy((WeakReference *)(addr + addrOffset)); +} + +static void bridgeDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); +} + +static void singlePayloadEnumSimpleDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (extraTagBytesPattern) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = + readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); + if (tagBytes) { + xiTagBytesPattern = 0; + } + } + + if (xiTagBytesPattern) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + uint64_t tagBytes = + readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + if (tagBytes >= xiTagValues) { + return; + } + } + + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag == 0) { + reader.skip(sizeof(size_t) * 2); + } else { + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag != 0) { + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumGenericDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (extraTagBytesPattern) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); + + if (tagBytes) { + xiType = nullptr; + } + } + + if (xiType) { + auto tag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset), + numEmptyCases); + if (tag == 0) { + return; + } + } + + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void multiPayloadEnumFNDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(numPayloads, refCountBytes, enumSize); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + +static void multiPayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(numPayloads, refCountBytes, enumSize); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + + +static void multiPayloadEnumGenericDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + auto tagBytesOffset = enumSize - tagBytes; + + auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + +#if SWIFT_OBJC_INTEROP +static void blockDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + _Block_release((void *)(addr + addrOffset)); +} +#endif + +static void metatypeDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *type = reader.readBytes(); + type->vw_destroy((OpaqueValue *)(addr + addrOffset)); +} + +static void existentialDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); + auto* type = getExistentialTypeMetadata(object); + if (type->getValueWitnesses()->isValueInline()) { + type->vw_destroy(object); + } else { + swift_release(*(HeapObject**)object); + } +} + +static void resilientDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *type = getResilientTypeMetadata(metadata, reader); + type->vw_destroy((OpaqueValue *)(addr + addrOffset)); +} + +typedef void (*DestrFnBranchless)(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr); + +static const DestrFnBranchless destroyTableBranchless[] = { + &endDestroyBranchless, + &errorDestroyBranchless, + &nativeStrongDestroyBranchless, + &unownedDestroyBranchless, + &weakDestroyBranchless, + &unknownDestroyBranchless, + &unknownUnownedDestroyBranchless, + &unknownWeakDestroyBranchless, + &bridgeDestroyBranchless, +#if SWIFT_OBJC_INTEROP + &blockDestroyBranchless, + &unknownDestroyBranchless, +#else + nullptr, + nullptr, +#endif + nullptr, // Custom + &metatypeDestroyBranchless, + nullptr, // Generic + &existentialDestroyBranchless, + &resilientDestroyBranchless, + &singlePayloadEnumSimpleDestroyBranchless, + &singlePayloadEnumFNDestroyBranchless, + &singlePayloadEnumFNResolvedDestroyBranchless, + &singlePayloadEnumGenericDestroyBranchless, + &multiPayloadEnumFNDestroyBranchless, + &multiPayloadEnumFNResolvedDestroyBranchless, + &multiPayloadEnumGenericDestroyBranchless, +}; + +static void handleRefCountsDestroy(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + uint64_t tag = 0; + do { + tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + + destroyTableBranchless[tag](metadata, reader, addrOffset, addr); + } while (tag != 0); +} + extern "C" void swift_generic_destroy(swift::OpaqueValue *address, const Metadata *metadata) { - handleRefCounts<0, DestroyHandler>(metadata, (uint8_t *)address); + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsDestroy(metadata, reader, addrOffset, (uint8_t *)address); +} + +static void handleRefCountsInitWithCopy(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); + +static void endRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + return; +} + +static void errorRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_errorRetain(*(SwiftError**)(dest + addrOffset)); +} + +static void nativeStrongRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_retain(*(HeapObject **)(dest + addrOffset)); +} + +static void unownedRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset))); + swift_unownedRetain(object); +} + +static void weakCopyInitBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_weakCopyInit((WeakReference *)(dest + addrOffset), (WeakReference *)(src + addrOffset)); +} + +static void unknownRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + void *object = *(void**)(dest + addrOffset); + swift_unknownObjectRetain(object); +} + +static void unknownUnownedCopyInitBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + UnownedReference *objectDest = (UnownedReference*)(dest + addrOffset); + UnownedReference *objectSrc = (UnownedReference*)(src + addrOffset); + swift_unknownObjectUnownedCopyInit(objectDest, objectSrc); +} + +static void unknownWeakCopyInitBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_unknownObjectWeakCopyInit((WeakReference *)(dest + addrOffset), + (WeakReference *)(src + addrOffset)); +} + +static void bridgeRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_bridgeObjectRetain(*(void **)(dest + addrOffset)); +} + +static void singlePayloadEnumSimpleRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (extraTagBytesPattern) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = + readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + if (tagBytes) { + xiTagBytesPattern = 0; + } + } + + if (xiTagBytesPattern) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + uint64_t tagBytes = + readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + if (tagBytes >= xiTagValues) { + return; + } + } + + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (enumTag == 0) { + reader.skip(sizeof(size_t) * 2); + } else { + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumFNResolvedRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (enumTag != 0) { + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumGenericRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (extraTagBytesPattern) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + + if (tagBytes) { + xiType = nullptr; + } + } + + if (xiType) { + auto tag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), + numEmptyCases); + if (tag == 0) { + return; + } + } + + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void multiPayloadEnumFNRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(numPayloads, refCountBytes, enumSize); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + +static void multiPayloadEnumFNResolvedRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(numPayloads, refCountBytes, enumSize); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + + +static void multiPayloadEnumGenericRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + auto tagBytesOffset = enumSize - tagBytes; + + auto enumTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + +#if SWIFT_OBJC_INTEROP +static void blockCopyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + *(void**)dest = _Block_copy(*(void**)(src + addrOffset)); +} +#endif + +static void metatypeInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *type = reader.readBytes(); + type->vw_initializeWithCopy((OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); +} + +static void existentialInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + type->vw_initializeBufferWithCopyOfBuffer((ValueBuffer*)(dest + addrOffset), + (ValueBuffer*)(src + addrOffset)); +} + +static void resilientInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *type = getResilientTypeMetadata(metadata, reader); + type->vw_initializeWithCopy((OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); +} + +typedef void (*InitWithCopyFnBranchless)(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); + +static const InitWithCopyFnBranchless initWithCopyTableBranchless[] = { + &endRetainBranchless, + &errorRetainBranchless, + &nativeStrongRetainBranchless, + &unownedRetainBranchless, + &weakCopyInitBranchless, + &unknownRetainBranchless, + &unknownUnownedCopyInitBranchless, + &unknownWeakCopyInitBranchless, + &bridgeRetainBranchless, +#if SWIFT_OBJC_INTEROP + &blockCopyBranchless, + &unknownRetainBranchless, +#else + nullptr, + nullptr, +#endif + nullptr, // Custom + &metatypeInitWithCopyBranchless, + nullptr, // Generic + &existentialInitWithCopyBranchless, + &resilientInitWithCopyBranchless, + &singlePayloadEnumSimpleRetainBranchless, + &singlePayloadEnumFNRetainBranchless, + &singlePayloadEnumFNResolvedRetainBranchless, + &singlePayloadEnumGenericRetainBranchless, + &multiPayloadEnumFNRetainBranchless, + &multiPayloadEnumFNResolvedRetainBranchless, + &multiPayloadEnumGenericRetainBranchless, +}; + +static void handleRefCountsInitWithCopy(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uint64_t tag = 0; + do { + tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + + initWithCopyTableBranchless[tag](metadata, reader, addrOffset, dest, src); + } while (tag != 0); +} + +extern "C" swift::OpaqueValue * +swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, + const Metadata *metadata) { + if (dest == src) + llvm_unreachable("WOOT"); + size_t size = metadata->vw_size(); + memcpy(dest, src, size); + + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsInitWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); + + return dest; } struct RetainFuncAndMask { @@ -464,10 +1250,20 @@ void* existential_initializeWithCopy(OpaqueValue* dest, OpaqueValue* src) { (ValueBuffer*)src); } +void* swift_retain_inline(HeapObject* object) { + if (object != nullptr) { + // Return the result of increment() to make the eventual call to + // incrementSlow a tail call, which avoids pushing a stack frame on the fast + // path on ARM64. + return object->refCounts.increment(1); + } + return object; +} + const RetainFuncAndMask retainTable[] = { {(void*)&skipRetain, true}, {(void*)&swift_errorRetain, true}, - {(void*)&swift_retain, true}, + {(void*)&swift_retain_inline, true}, {(void*)&swift_unownedRetain, true}, {(void*)&swift_weakCopyInit, false}, {(void*)&swift_unknownObjectRetain, true}, @@ -546,16 +1342,16 @@ struct CopyHandler { } }; -extern "C" swift::OpaqueValue * -swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, - const Metadata *metadata) { - size_t size = metadata->vw_size(); - memcpy(dest, src, size); +// extern "C" swift::OpaqueValue * +// swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, +// const Metadata *metadata) { +// size_t size = metadata->vw_size(); +// memcpy(dest, src, size); - handleRefCounts<0, CopyHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); +// handleRefCounts<0, CopyHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); - return dest; -} +// return dest; +// } struct TakeHandler { static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, diff --git a/stdlib/public/runtime/BytecodeLayouts.h b/stdlib/public/runtime/BytecodeLayouts.h index e3747e3edbeef..a3d9e14cd1799 100644 --- a/stdlib/public/runtime/BytecodeLayouts.h +++ b/stdlib/public/runtime/BytecodeLayouts.h @@ -70,6 +70,31 @@ struct LayoutStringReader { return returnVal; } + template + inline void readBytes(T&... result) { + uintptr_t additionalOffset = 0; + ([&] { + memcpy(&result, layoutStr + offset + additionalOffset, sizeof(T)); + additionalOffset += sizeof(T); + }(), ...); + offset += additionalOffset; + } + + template + inline T modify(F &&f) { + LayoutStringReader readerCopy = *this; + T res = f(readerCopy); + offset = readerCopy.offset; + return res; + } + + template + inline void modify(F &&f) { + LayoutStringReader readerCopy = *this; + f(readerCopy); + offset = readerCopy.offset; + } + template inline T peekBytes(size_t peekOffset = 0) const { T returnVal; diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index c21a467dd3931..224db34e51b1a 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -535,6 +535,11 @@ public func testAssign(_ ptr: UnsafeMutablePointer, from x: T) { ptr.pointee = x } +@inline(never) +public func testAssign(_ ptr: UnsafeMutablePointer, from x: UnsafeMutablePointer) { + ptr.assign(from: x, count: 1) +} + @inline(never) public func testInit(_ ptr: UnsafeMutablePointer, to x: T) { ptr.initialize(to: x) From ec3a2dc3704fe913c087f215fcaf7dbd636046c9 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:52:39 -0700 Subject: [PATCH 02/26] [Runtime] Templatize shared functionality in BytecodeLayouts --- stdlib/public/runtime/BytecodeLayouts.cpp | 296 +++------------------- 1 file changed, 39 insertions(+), 257 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 645a07f241e12..292a20ae77df4 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -72,24 +72,6 @@ static const Metadata *getResilientTypeMetadata(const Metadata *metadata, return fn(metadata->getGenericArgs()); } -typedef void (*DestrFn)(void*); - -struct DestroyFuncAndMask { - DestrFn fn; - bool isIndirect; -}; - -static void skipDestroy(void* ignore) { } - -static void existential_destroy(OpaqueValue* object) { - auto* metadata = getExistentialTypeMetadata(object); - if (metadata->getValueWitnesses()->isValueInline()) { - metadata->vw_destroy(object); - } else { - swift_release(*(HeapObject**)object); - } -} - template inline bool handleNextRefCount(const Metadata *metadata, LayoutStringReader &reader, @@ -359,90 +341,6 @@ static void handleMultiPayloadEnumGeneric(const Metadata *metadata, addrOffset += enumSize; } -void swift_release_masked(void* ptr) { - HeapObject *object = (HeapObject*)(((uintptr_t)ptr) & ~_swift_abi_SwiftSpareBitsMask); - if (object != nullptr) - object->refCounts.decrementAndMaybeDeinit(1); -} - -const DestroyFuncAndMask destroyTable[] = { - {(DestrFn)&skipDestroy, false}, - {(DestrFn)&swift_errorRelease, true}, - {(DestrFn)&swift_release_masked, true}, - {(DestrFn)&swift_unownedRelease, true}, - {(DestrFn)&swift_weakDestroy, false}, - {(DestrFn)&swift_unknownObjectRelease, true}, - {(DestrFn)&swift_unknownObjectUnownedDestroy, false}, - {(DestrFn)&swift_unknownObjectWeakDestroy, false}, - {(DestrFn)&swift_bridgeObjectRelease, true}, -#if SWIFT_OBJC_INTEROP - {(DestrFn)&_Block_release, true}, - {(DestrFn)&swift_unknownObjectRelease, true}, -#else - {nullptr, true}, - {nullptr, true}, -#endif - // TODO: how to handle Custom? - {nullptr, true}, - {nullptr, true}, - {nullptr, true}, - {(DestrFn)&existential_destroy, false}, -}; - -struct DestroyHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *addr) { - type->vw_destroy((OpaqueValue *)(addr + addrOffset)); - } - - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumSimple(reader, addr, addrOffset); - } - - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumFN(reader, resolved, addr, addrOffset); - } - - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumGeneric(reader, addr, addrOffset); - } - - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, addr); - } - - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleMultiPayloadEnumGeneric(metadata, reader, - addrOffset, addr); - } - - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *addr) { - const auto &destroyFunc = destroyTable[static_cast(tag)]; - if (SWIFT_LIKELY(destroyFunc.isIndirect)) { - destroyFunc.fn( - (void *)((*(uintptr_t *)(addr + addrOffset)))); - } else { - destroyFunc.fn(((void *)(addr + addrOffset))); - } - } -}; - static void handleRefCountsDestroy(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, @@ -469,8 +367,6 @@ static void nativeStrongDestroyBranchless(const Metadata *metadata, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); swift_release(object); - // if (object != nullptr) - // object->refCounts.decrementAndMaybeDeinit(1); } static void unownedDestroyBranchless(const Metadata *metadata, @@ -518,10 +414,12 @@ static void bridgeDestroyBranchless(const Metadata *metadata, swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); } -static void singlePayloadEnumSimpleDestroyBranchless(const Metadata *metadata, +template +static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T... param) { reader.modify([&](LayoutStringReader &reader) { uint64_t byteCountsAndOffset; size_t payloadSize; @@ -561,10 +459,12 @@ static void singlePayloadEnumSimpleDestroyBranchless(const Metadata *metadata, }); } -static void singlePayloadEnumFNDestroyBranchless(const Metadata *metadata, +template +static void singlePayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -582,10 +482,12 @@ static void singlePayloadEnumFNDestroyBranchless(const Metadata *metadata, }); } -static void singlePayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata, +template +static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag; size_t refCountBytes; @@ -601,10 +503,12 @@ static void singlePayloadEnumFNResolvedDestroyBranchless(const Metadata *metadat }); } -static void singlePayloadEnumGenericDestroyBranchless(const Metadata *metadata, +template +static void singlePayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { auto tagBytesAndOffset = reader.readBytes(); auto payloadSize = reader.readBytes(); @@ -640,10 +544,12 @@ static void singlePayloadEnumGenericDestroyBranchless(const Metadata *metadata, }); } -static void multiPayloadEnumFNDestroyBranchless(const Metadata *metadata, +template +static void multiPayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -660,7 +566,7 @@ static void multiPayloadEnumFNDestroyBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -668,10 +574,12 @@ static void multiPayloadEnumFNDestroyBranchless(const Metadata *metadata, }); } -static void multiPayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata, +template +static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = reader.readBytes(); @@ -688,7 +596,7 @@ static void multiPayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -696,11 +604,12 @@ static void multiPayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata }); } - -static void multiPayloadEnumGenericDestroyBranchless(const Metadata *metadata, +template +static void multiPayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { size_t tagBytes; size_t numPayloads; @@ -718,7 +627,7 @@ static void multiPayloadEnumGenericDestroyBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -769,7 +678,7 @@ typedef void (*DestrFnBranchless)(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr); -static const DestrFnBranchless destroyTableBranchless[] = { +const DestrFnBranchless destroyTableBranchless[] = { &endDestroyBranchless, &errorDestroyBranchless, &nativeStrongDestroyBranchless, @@ -791,13 +700,13 @@ static const DestrFnBranchless destroyTableBranchless[] = { nullptr, // Generic &existentialDestroyBranchless, &resilientDestroyBranchless, - &singlePayloadEnumSimpleDestroyBranchless, - &singlePayloadEnumFNDestroyBranchless, - &singlePayloadEnumFNResolvedDestroyBranchless, - &singlePayloadEnumGenericDestroyBranchless, - &multiPayloadEnumFNDestroyBranchless, - &multiPayloadEnumFNResolvedDestroyBranchless, - &multiPayloadEnumGenericDestroyBranchless, + &singlePayloadEnumSimpleBranchless<>, + &singlePayloadEnumFNBranchless<>, + &singlePayloadEnumFNResolvedBranchless<>, + &singlePayloadEnumGenericBranchless<>, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, }; static void handleRefCountsDestroy(const Metadata *metadata, @@ -1215,8 +1124,6 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, extern "C" swift::OpaqueValue * swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - if (dest == src) - llvm_unreachable("WOOT"); size_t size = metadata->vw_size(); memcpy(dest, src, size); @@ -1228,131 +1135,6 @@ swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, return dest; } -struct RetainFuncAndMask { - void* fn; - bool isSingle; -}; - -#if SWIFT_OBJC_INTEROP -void* Block_copyForwarder(void** dest, const void** src) { - *dest = _Block_copy(*src); - return *dest; -} -#endif - -typedef void* (*RetainFn)(void*); -typedef void* (*CopyInitFn)(void*, void*); - -void* skipRetain(void* ignore) { return nullptr; } -void* existential_initializeWithCopy(OpaqueValue* dest, OpaqueValue* src) { - auto* metadata = getExistentialTypeMetadata(src); - return metadata->vw_initializeBufferWithCopyOfBuffer((ValueBuffer*)dest, - (ValueBuffer*)src); -} - -void* swift_retain_inline(HeapObject* object) { - if (object != nullptr) { - // Return the result of increment() to make the eventual call to - // incrementSlow a tail call, which avoids pushing a stack frame on the fast - // path on ARM64. - return object->refCounts.increment(1); - } - return object; -} - -const RetainFuncAndMask retainTable[] = { - {(void*)&skipRetain, true}, - {(void*)&swift_errorRetain, true}, - {(void*)&swift_retain_inline, true}, - {(void*)&swift_unownedRetain, true}, - {(void*)&swift_weakCopyInit, false}, - {(void*)&swift_unknownObjectRetain, true}, - {(void*)&swift_unknownObjectUnownedCopyInit, false}, - {(void*)&swift_unknownObjectWeakCopyInit, false}, - {(void*)&swift_bridgeObjectRetain, true}, -#if SWIFT_OBJC_INTEROP - {(void*)&Block_copyForwarder, false}, - {(void*)&objc_retain, true}, -#else - {nullptr, true}, - {nullptr, true}, -#endif - // TODO: how to handle Custom? - {nullptr, true}, - {nullptr, true}, - {nullptr, true}, - {(void*)&existential_initializeWithCopy, false}, -}; - -struct CopyHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - type->vw_initializeWithCopy((OpaqueValue*)((uintptr_t)dest + addrOffset), - (OpaqueValue*)((uintptr_t)src + addrOffset)); - } - - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumSimple(reader, src, addrOffset); - } - - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleSinglePayloadEnumFN(reader, resolved, src, addrOffset); - } - - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumGeneric(reader, src, addrOffset); - } - - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, dest, src); - } - - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - dest, src); - } - - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - const auto &retainFunc = retainTable[static_cast(tag)]; - if (SWIFT_LIKELY(retainFunc.isSingle)) { - ((RetainFn)retainFunc.fn)(*(void**)(((uintptr_t)dest + addrOffset))); - } else { - ((CopyInitFn)retainFunc.fn)((void*)((uintptr_t)dest + addrOffset), - (void*)((uintptr_t)src + addrOffset)); - } - } -}; - -// extern "C" swift::OpaqueValue * -// swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, -// const Metadata *metadata) { -// size_t size = metadata->vw_size(); -// memcpy(dest, src, size); - -// handleRefCounts<0, CopyHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); - -// return dest; -// } - struct TakeHandler { static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, uint8_t *dest, uint8_t *src) { From 2f61e2a9c958534aeb3bd7fcc800a55cfde3ab98 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:53:14 -0700 Subject: [PATCH 03/26] [Runtime] Further abstract shared functionality in BytecodeLayouts --- stdlib/public/runtime/BytecodeLayouts.cpp | 709 +++++----------------- 1 file changed, 147 insertions(+), 562 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 292a20ae77df4..753165c39dc20 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -72,79 +72,6 @@ static const Metadata *getResilientTypeMetadata(const Metadata *metadata, return fn(metadata->getGenericArgs()); } -template -inline bool handleNextRefCount(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, Params... params) { - uint64_t skip = reader.readBytes(); - auto tag = static_cast(skip >> 56); - skip &= ~(0xffULL << 56); - addrOffset += skip; - - if (SWIFT_UNLIKELY(tag == RefCountingKind::End)) { - return false; - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::Metatype)) { - auto *type = reader.readBytes(); - Handler::handleMetatype(type, addrOffset, std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::Resilient)) { - auto *type = getResilientTypeMetadata(metadata, reader); - Handler::handleMetatype(type, addrOffset, std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::SinglePayloadEnumSimple)) { - Handler::handleSinglePayloadEnumSimple(reader, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumFN)) { - Handler::handleSinglePayloadEnumFN(reader, false, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::SinglePayloadEnumFNResolved)) { - Handler::handleSinglePayloadEnumFN(reader, true, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumGeneric)) { - Handler::handleSinglePayloadEnumGeneric(reader, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumFN)) { - Handler::handleMultiPayloadEnumFN(metadata, reader, false, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::MultiPayloadEnumFNResolved)) { - Handler::handleMultiPayloadEnumFN(metadata, reader, true, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumGeneric)) { - Handler::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - std::forward(params)...); - } else { - Handler::handleReference(tag, addrOffset, std::forward(params)...); - } - - return true; -} - -template -inline void handleRefCounts(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, Params... params) { - if (N == 0) { - while (handleNextRefCount(metadata, reader, addrOffset, - std::forward(params)...)) { - } - } else { - for (unsigned i = 0; i < N; i++) { - handleNextRefCount(metadata, reader, addrOffset, - std::forward(params)...); - } - } -} - -template -inline void handleRefCounts(const Metadata *metadata, Params... params) { - LayoutStringReader reader{metadata->getLayoutString(), - layoutStringHeaderSize}; - uintptr_t addrOffset = 0; - handleRefCounts(metadata, reader, addrOffset, - std::forward(params)...); -} - static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) { switch (byteCount) { case 1: @@ -169,187 +96,17 @@ static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) { } } -static void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uint8_t *addr, - uintptr_t &addrOffset) { - auto byteCountsAndOffset = reader.readBytes(); - auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); - auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; - auto xiTagBytesOffset = - byteCountsAndOffset & std::numeric_limits::max(); - - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto payloadSize = reader.readBytes(); - auto tagBytes = - readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); - if (tagBytes) { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - goto noPayload; - } - } else { - reader.skip(sizeof(size_t)); - } - - if (xiTagBytesPattern) { - auto zeroTagValue = reader.readBytes(); - auto xiTagValues = reader.readBytes(); - - auto xiTagBytes = 1 << (xiTagBytesPattern - 1); - uint64_t tagBytes = - readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) - - zeroTagValue; - if (tagBytes >= xiTagValues) { - reader.skip(sizeof(size_t) * 2); - return; - } - } else { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - } - -noPayload: - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - reader.skip(refCountBytes); - addrOffset += skip; -} - -typedef unsigned (*GetEnumTagFn)(const uint8_t *); - -static void handleSinglePayloadEnumFN(LayoutStringReader &reader, bool resolved, - uint8_t *addr, uintptr_t &addrOffset) { - GetEnumTagFn getEnumTag; - if (resolved) { - getEnumTag = reader.readBytes(); - } else { - getEnumTag = readRelativeFunctionPointer(reader); - } - - unsigned enumTag = getEnumTag(addr + addrOffset); - - if (enumTag == 0) { - reader.skip(sizeof(size_t) * 2); - } else { - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - reader.skip(refCountBytes); - addrOffset += skip; - } -} - -static void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uint8_t *addr, - uintptr_t &addrOffset) { - auto tagBytesAndOffset = reader.readBytes(); - auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); - auto xiTagBytesOffset = - tagBytesAndOffset & std::numeric_limits::max(); - const Metadata *xiType = nullptr; - - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto payloadSize = reader.readBytes(); - auto tagBytes = - readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); - if (tagBytes) { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - goto noPayload; - } - } else { - reader.skip(sizeof(size_t)); - } - - xiType = reader.readBytes(); - - if (xiType) { - auto numEmptyCases = reader.readBytes(); - - auto tag = xiType->vw_getEnumTagSinglePayload( - (const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset), - numEmptyCases); - if (tag == 0) { - reader.skip(sizeof(size_t) * 2); - return; - } - } else { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - } - -noPayload: - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - reader.skip(refCountBytes); - addrOffset += skip; -} - -template -static void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, bool resolved, - uintptr_t &addrOffset, uint8_t *addr, - Params... params) { - GetEnumTagFn getEnumTag; - if (resolved) { - getEnumTag = reader.readBytes(); - } else { - getEnumTag = readRelativeFunctionPointer(reader); - } - - size_t numPayloads = reader.readBytes(); - size_t refCountBytes = reader.readBytes(); - size_t enumSize = reader.readBytes(); - - unsigned enumTag = getEnumTag(addr + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCounts<0, Handler>(metadata, nestedReader, nestedAddrOffset, addr, - std::forward(params)...); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; -} - -template -static void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, uint8_t *addr, - Params... params) { - auto tagBytes = reader.readBytes(); - auto numPayloads = reader.readBytes(); - auto refCountBytes = reader.readBytes(); - auto enumSize = reader.readBytes(); - auto tagBytesOffset = enumSize - tagBytes; - - auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCounts<0, Handler>(metadata, nestedReader, nestedAddrOffset, addr, - std::forward(params)...); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; -} - static void handleRefCountsDestroy(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr); -static void endDestroyBranchless(const Metadata *metadata, +template +static void handleEnd(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + Params ...params) { return; } @@ -414,12 +171,12 @@ static void bridgeDestroyBranchless(const Metadata *metadata, swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); } -template +template static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T... param) { + Params... params) { reader.modify([&](LayoutStringReader &reader) { uint64_t byteCountsAndOffset; size_t payloadSize; @@ -435,7 +192,7 @@ static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, auto xiTagBytesOffset = byteCountsAndOffset & std::numeric_limits::max(); - if (extraTagBytesPattern) { + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { auto extraTagBytes = 1 << (extraTagBytesPattern - 1); auto tagBytes = readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); @@ -444,7 +201,7 @@ static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, } } - if (xiTagBytesPattern) { + if (SWIFT_LIKELY(xiTagBytesPattern)) { auto xiTagBytes = 1 << (xiTagBytesPattern - 1); uint64_t tagBytes = readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) - @@ -459,12 +216,14 @@ static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, }); } -template +typedef unsigned (*GetEnumTagFn)(const uint8_t *); + +template static void singlePayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -482,12 +241,12 @@ static void singlePayloadEnumFNBranchless(const Metadata *metadata, }); } -template +template static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag; size_t refCountBytes; @@ -503,12 +262,12 @@ static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, }); } -template +template static void singlePayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { auto tagBytesAndOffset = reader.readBytes(); auto payloadSize = reader.readBytes(); @@ -544,12 +303,12 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, }); } -template +template static void multiPayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -566,7 +325,7 @@ static void multiPayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -574,12 +333,12 @@ static void multiPayloadEnumFNBranchless(const Metadata *metadata, }); } -template +template static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = reader.readBytes(); @@ -596,7 +355,7 @@ static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -604,12 +363,12 @@ static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, }); } -template +template static void multiPayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { size_t tagBytes; size_t numPayloads; @@ -627,7 +386,7 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -642,6 +401,14 @@ static void blockDestroyBranchless(const Metadata *metadata, uint8_t *addr) { _Block_release((void *)(addr + addrOffset)); } + +static void objcStrongDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + objc_object *object = (objc_object*)(*(uintptr_t *)(addr + addrOffset)); + objc_release(object); +} #endif static void metatypeDestroyBranchless(const Metadata *metadata, @@ -679,7 +446,7 @@ typedef void (*DestrFnBranchless)(const Metadata *metadata, uint8_t *addr); const DestrFnBranchless destroyTableBranchless[] = { - &endDestroyBranchless, + &handleEnd, &errorDestroyBranchless, &nativeStrongDestroyBranchless, &unownedDestroyBranchless, @@ -690,7 +457,7 @@ const DestrFnBranchless destroyTableBranchless[] = { &bridgeDestroyBranchless, #if SWIFT_OBJC_INTEROP &blockDestroyBranchless, - &unknownDestroyBranchless, + &objcStrongDestroyBranchless, #else nullptr, nullptr, @@ -737,14 +504,6 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, uint8_t *dest, uint8_t *src); -static void endRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - return; -} - static void errorRetainBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, @@ -814,228 +573,22 @@ static void bridgeRetainBranchless(const Metadata *metadata, swift_bridgeObjectRetain(*(void **)(dest + addrOffset)); } -static void singlePayloadEnumSimpleRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - uint64_t byteCountsAndOffset; - size_t payloadSize; - uint64_t zeroTagValue; - size_t xiTagValues; - size_t refCountBytes; - size_t skip; - - reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); - - auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); - auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; - auto xiTagBytesOffset = - byteCountsAndOffset & std::numeric_limits::max(); - - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto tagBytes = - readTagBytes(src + addrOffset + payloadSize, extraTagBytes); - if (tagBytes) { - xiTagBytesPattern = 0; - } - } - - if (xiTagBytesPattern) { - auto xiTagBytes = 1 << (xiTagBytesPattern - 1); - uint64_t tagBytes = - readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - - zeroTagValue; - if (tagBytes >= xiTagValues) { - return; - } - } - - reader.skip(refCountBytes); - addrOffset += skip; - }); -} - -static void singlePayloadEnumFNRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); - - unsigned enumTag = getEnumTag(src + addrOffset); - - if (enumTag == 0) { - reader.skip(sizeof(size_t) * 2); - } else { - size_t refCountBytes; - size_t skip; - reader.readBytes(refCountBytes, skip); - reader.skip(refCountBytes); - addrOffset += skip; - } - }); -} - -static void singlePayloadEnumFNResolvedRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag; - size_t refCountBytes; - size_t skip; - reader.readBytes(getEnumTag, refCountBytes, skip); - - unsigned enumTag = getEnumTag(src + addrOffset); - - if (enumTag != 0) { - reader.skip(refCountBytes); - addrOffset += skip; - } - }); -} - -static void singlePayloadEnumGenericRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - auto tagBytesAndOffset = reader.readBytes(); - auto payloadSize = reader.readBytes(); - auto *xiType = reader.readBytes(); - auto numEmptyCases = reader.readBytes(); - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - - auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); - auto xiTagBytesOffset = - tagBytesAndOffset & std::numeric_limits::max(); - - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto tagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); - - if (tagBytes) { - xiType = nullptr; - } - } - - if (xiType) { - auto tag = xiType->vw_getEnumTagSinglePayload( - (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), - numEmptyCases); - if (tag == 0) { - return; - } - } - - reader.skip(refCountBytes); - addrOffset += skip; - }); -} - -static void multiPayloadEnumFNRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); - - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; - reader.readBytes(numPayloads, refCountBytes, enumSize); - - unsigned enumTag = getEnumTag(src + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; - }); -} - -static void multiPayloadEnumFNResolvedRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = reader.readBytes(); - - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; - reader.readBytes(numPayloads, refCountBytes, enumSize); - - unsigned enumTag = getEnumTag(src + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; - }); -} - - -static void multiPayloadEnumGenericRetainBranchless(const Metadata *metadata, +#if SWIFT_OBJC_INTEROP +static void blockCopyBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - size_t tagBytes; - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; - reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); - - auto tagBytesOffset = enumSize - tagBytes; - - auto enumTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; - }); + *(void**)dest = _Block_copy(*(void**)(src + addrOffset)); } -#if SWIFT_OBJC_INTEROP -static void blockCopyBranchless(const Metadata *metadata, +static void objcStrongRetainBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - *(void**)dest = _Block_copy(*(void**)(src + addrOffset)); + objc_object *object = (objc_object*)(*(uintptr_t *)(src + addrOffset)); + objc_retain(object); } #endif @@ -1069,14 +622,14 @@ static void resilientInitWithCopyBranchless(const Metadata *metadata, (OpaqueValue *)(src + addrOffset)); } -typedef void (*InitWithCopyFnBranchless)(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src); +typedef void (*InitFn)(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); -static const InitWithCopyFnBranchless initWithCopyTableBranchless[] = { - &endRetainBranchless, +static const InitFn initWithCopyTable[] = { + &handleEnd, &errorRetainBranchless, &nativeStrongRetainBranchless, &unownedRetainBranchless, @@ -1087,7 +640,7 @@ static const InitWithCopyFnBranchless initWithCopyTableBranchless[] = { &bridgeRetainBranchless, #if SWIFT_OBJC_INTEROP &blockCopyBranchless, - &unknownRetainBranchless, + &objcStrongRetainBranchless, #else nullptr, nullptr, @@ -1097,13 +650,13 @@ static const InitWithCopyFnBranchless initWithCopyTableBranchless[] = { nullptr, // Generic &existentialInitWithCopyBranchless, &resilientInitWithCopyBranchless, - &singlePayloadEnumSimpleRetainBranchless, - &singlePayloadEnumFNRetainBranchless, - &singlePayloadEnumFNResolvedRetainBranchless, - &singlePayloadEnumGenericRetainBranchless, - &multiPayloadEnumFNRetainBranchless, - &multiPayloadEnumFNResolvedRetainBranchless, - &multiPayloadEnumGenericRetainBranchless, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, }; static void handleRefCountsInitWithCopy(const Metadata *metadata, @@ -1117,7 +670,7 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, addrOffset += (tag & ~(0xFFULL << 56)); tag >>= 56; - initWithCopyTableBranchless[tag](metadata, reader, addrOffset, dest, src); + initWithCopyTable[tag](metadata, reader, addrOffset, dest, src); } while (tag != 0); } @@ -1135,73 +688,101 @@ swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, return dest; } -struct TakeHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue*)((uintptr_t)dest + addrOffset), - (OpaqueValue*)((uintptr_t)src + addrOffset)); - } - } - - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumSimple(reader, src, addrOffset); - } +static void handleRefCountsInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleSinglePayloadEnumFN(reader, resolved, src, addrOffset); - } +static void unknownWeakInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_unknownObjectWeakTakeInit((WeakReference *)(dest + addrOffset), + (WeakReference *)(src + addrOffset)); +} - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumGeneric(reader, src, addrOffset); +static inline void metatypeInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *type = reader.readBytes(); + if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { + type->vw_initializeWithTake( + (OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); } +} - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, dest, src); +static void existentialInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { + type->vw_initializeWithTake((OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); } +} - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - dest, src); +static inline void resilientInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *type = getResilientTypeMetadata(metadata, reader); + if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { + type->vw_initializeWithTake( + (OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); } +} - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - if (tag == RefCountingKind::UnknownWeak) { - swift_unknownObjectWeakTakeInit( - (WeakReference*)((uintptr_t)dest + addrOffset), - (WeakReference*)((uintptr_t)src + addrOffset)); - } else if (tag == RefCountingKind::Existential) { - auto *type = getExistentialTypeMetadata( - (OpaqueValue*)((uintptr_t)src + addrOffset)); - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue *)((uintptr_t)dest + addrOffset), - (OpaqueValue *)((uintptr_t)src + addrOffset)); - } - } - } +static const InitFn initWithTakeTable[] = { + &handleEnd, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + &unknownWeakInitWithTake, + &bridgeRetainBranchless, + nullptr, + nullptr, + nullptr, // Custom + &metatypeInitWithTake, + nullptr, // Generic + &existentialInitWithTake, + &resilientInitWithTake, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, }; +static void handleRefCountsInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uint64_t tag = 0; + do { + tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + + if (auto handler = initWithTakeTable[tag]) + handler(metadata, reader, addrOffset, dest, src); + } while (tag != 0); +} + extern "C" swift::OpaqueValue * swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { @@ -1213,7 +794,11 @@ swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, return dest; } - handleRefCounts<0, TakeHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + + handleRefCountsInitWithTake(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); return dest; } From 4a395cb8d3af426810a526980071f0f661ca92ce Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:53:44 -0700 Subject: [PATCH 04/26] [Runtime+IRGen] Some fixes and optimizations for layout strings --- include/swift/ABI/Metadata.h | 16 +++--- lib/IRGen/GenMeta.cpp | 15 +++-- stdlib/public/runtime/BytecodeLayouts.cpp | 12 ++-- stdlib/public/runtime/Metadata.cpp | 69 +++++++++++------------ stdlib/public/runtime/MetadataImpl.h | 5 ++ 5 files changed, 64 insertions(+), 53 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index 9b2cc2b04d014..3eaff50606d03 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -323,13 +323,13 @@ struct TargetMetadata { } const uint8_t *getLayoutString() const { - //assert(hasLayoutString()); - // if (isAnyClass()) { - // return asFullMetadata( - // reinterpret_cast *>( - // this)) - // ->layoutString; - // } + assert(hasLayoutString()); + if (isAnyClass()) { + return asFullMetadata( + reinterpret_cast *>( + this)) + ->layoutString; + } return asFullMetadata(this)->layoutString; } @@ -3778,7 +3778,7 @@ class swift_ptrauth_struct_context_descriptor(TypeContextDescriptor) } bool hasLayoutString() const { - return getTypeContextDescriptorFlags().hasLayoutString(); + return !hasForeignMetadataInitialization() && getTypeContextDescriptorFlags().hasLayoutString(); } /// Given that this type has foreign metadata initialization, return the diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index d8cab7583483d..8cc25d227c5a8 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -2580,7 +2580,8 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, auto genericSig = lowered.getNominalOrBoundGenericNominal()->getGenericSignature(); - hasLayoutString = !!typeLayoutEntry->layoutString(IGM, genericSig); + hasLayoutString = !ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal) && + !!typeLayoutEntry->layoutString(IGM, genericSig); if (!hasLayoutString && IGM.Context.LangOpts.hasFeature( @@ -4647,7 +4648,9 @@ namespace { return false; } - return !!getLayoutString(); + auto &ti = IGM.getTypeInfo(getLoweredType()); + + return !ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal) && !!getLayoutString(); } ConstantReference emitValueWitnessTable(bool relativeReference) { @@ -5163,10 +5166,10 @@ namespace { if (IGM.Context.LangOpts.hasFeature(Feature::LayoutStringValueWitnessesInstantiation) && IGM.getOptions().EnableLayoutStringValueWitnessesInstantiation) { - return !!getLayoutString() || needsSingletonMetadataInitialization(IGM, Target); + return (!!getLayoutString() || needsSingletonMetadataInitialization(IGM, Target)) && !needsForeignMetadataCompletionFunction(IGM, Target); } - return !!getLayoutString(); + return !!getLayoutString() && !needsForeignMetadataCompletionFunction(IGM, Target); } llvm::Constant *emitNominalTypeDescriptor() { @@ -6247,6 +6250,10 @@ namespace { return Target->getDeclaredType()->getCanonicalType(); } + bool hasLayoutString() { + return false; + } + void createMetadataCompletionFunction() { llvm_unreachable("foreign structs never require completion"); } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 753165c39dc20..54590686703b5 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -380,7 +380,7 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); - if (enumTag < numPayloads) { + if (SWIFT_LIKELY(enumTag < numPayloads)) { size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); LayoutStringReader nestedReader = reader; @@ -517,7 +517,8 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_retain(*(HeapObject **)(dest + addrOffset)); + HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + swift_retain(object); } static void unownedRetainBranchless(const Metadata *metadata, @@ -525,7 +526,7 @@ static void unownedRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset))); + HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); swift_unownedRetain(object); } @@ -1244,13 +1245,12 @@ void swift::swift_resolve_resilientAccessors(uint8_t *layoutStr, reader.skip(sizeof(size_t)); size_t casesBeginOffset = layoutStrOffset + reader.offset + - layoutStringHeaderSize + (numCases * sizeof(size_t)); + auto fieldCasesBeginOffset = fieldLayoutStr + (numCases * sizeof(size_t)) + reader.offset; for (size_t j = 0; j < numCases; j++) { size_t caseOffset = reader.readBytes(); - const uint8_t *caseLayoutString = fieldLayoutStr + reader.offset + - (numCases * sizeof(size_t)) + + const uint8_t *caseLayoutString = fieldCasesBeginOffset + caseOffset; swift_resolve_resilientAccessors(layoutStr, casesBeginOffset + caseOffset, diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index b39417bd967a7..3f1f7e0f480f3 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2783,10 +2783,6 @@ void swift::swift_initStructMetadataWithLayoutString( size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { if (type->vw_size() == 0 || type->getValueWitnesses()->isPOD()) { return 0; - } else if (type->hasLayoutString()) { - size_t offset = sizeof(uint64_t); - return LayoutStringReader{type->getLayoutString(), offset} - .readBytes(); } else if (auto *tuple = dyn_cast(type)) { size_t res = 0; for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { @@ -2803,6 +2799,10 @@ size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { } } return sizeof(uint64_t); + } else if (type->hasLayoutString()) { + size_t offset = sizeof(uint64_t); + return LayoutStringReader{type->getLayoutString(), offset} + .readBytes(); } else if (type->isAnyExistentialType()) { return sizeof(uint64_t); } else { @@ -2825,37 +2825,6 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, // No need to handle PODs previousFieldOffset = offset + fieldType->vw_size(); fullOffset += fieldType->vw_size(); - } else if (fieldType->hasLayoutString()) { - LayoutStringReader reader{fieldType->getLayoutString(), 0}; - const auto fieldFlags = reader.readBytes(); - const auto fieldRefCountBytes = reader.readBytes(); - if (fieldRefCountBytes > 0) { - flags |= fieldFlags; - memcpy(writer.layoutStr + writer.offset, - reader.layoutStr + layoutStringHeaderSize, fieldRefCountBytes); - - if (fieldFlags & LayoutStringFlags::HasRelativePointers) { - swift_resolve_resilientAccessors( - writer.layoutStr, writer.offset, - reader.layoutStr + layoutStringHeaderSize, fieldType); - } - - if (offset) { - LayoutStringReader tagReader {writer.layoutStr, writer.offset}; - auto writerOffsetCopy = writer.offset; - auto firstTagAndOffset = tagReader.readBytes(); - firstTagAndOffset += offset; - writer.writeBytes(firstTagAndOffset); - writer.offset = writerOffsetCopy; - } - - reader.offset = layoutStringHeaderSize + fieldRefCountBytes; - previousFieldOffset = reader.readBytes(); - writer.skip(fieldRefCountBytes); - } else { - previousFieldOffset += fieldType->vw_size(); - } - fullOffset += fieldType->vw_size(); } else if (auto *tuple = dyn_cast(fieldType)) { for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { _swift_addRefCountStringForMetatype(writer, flags, @@ -2890,6 +2859,36 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, writer.writeBytes(((uint64_t)tag << 56) | offset); previousFieldOffset = fieldType->vw_size(); fullOffset += previousFieldOffset; + } else if (fieldType->hasLayoutString()) { + LayoutStringReader reader{fieldType->getLayoutString(), 0}; + const auto fieldFlags = reader.readBytes(); + const auto fieldRefCountBytes = reader.readBytes(); + if (fieldRefCountBytes > 0) { + flags |= fieldFlags; + memcpy(writer.layoutStr + writer.offset, + reader.layoutStr + layoutStringHeaderSize, fieldRefCountBytes); + + if (fieldFlags & LayoutStringFlags::HasRelativePointers) { + swift_resolve_resilientAccessors(writer.layoutStr, writer.offset, + reader.layoutStr + layoutStringHeaderSize, fieldType); + } + + if (offset) { + LayoutStringReader tagReader {writer.layoutStr, writer.offset}; + auto writerOffsetCopy = writer.offset; + auto firstTagAndOffset = tagReader.readBytes(); + firstTagAndOffset += offset; + writer.writeBytes(firstTagAndOffset); + writer.offset = writerOffsetCopy; + } + + reader.offset = layoutStringHeaderSize + fieldRefCountBytes; + previousFieldOffset = reader.readBytes(); + writer.skip(fieldRefCountBytes); + } else { + previousFieldOffset += fieldType->vw_size(); + } + fullOffset += fieldType->vw_size(); } else if (fieldType->isAnyExistentialType()) { auto *existential = dyn_cast(fieldType); assert(existential); diff --git a/stdlib/public/runtime/MetadataImpl.h b/stdlib/public/runtime/MetadataImpl.h index 1365e24d6f783..727d1f60be3b8 100644 --- a/stdlib/public/runtime/MetadataImpl.h +++ b/stdlib/public/runtime/MetadataImpl.h @@ -94,6 +94,7 @@ struct NativeBox { static constexpr size_t isPOD = std::is_pod::value; static constexpr bool isBitwiseTakable = isPOD; static constexpr unsigned numExtraInhabitants = 0; + static constexpr bool hasLayoutString = false; static void destroy(T *value) { value->T::~T(); @@ -142,6 +143,7 @@ template struct RetainableBoxBase { #else static constexpr bool isAtomic = true; #endif + static constexpr bool hasLayoutString = false; static void destroy(T *addr) { Impl::release(*addr); @@ -254,6 +256,7 @@ struct WeakRetainableBoxBase { static constexpr bool isPOD = false; static constexpr bool isBitwiseTakable = false; static constexpr unsigned numExtraInhabitants = 0; + static constexpr bool hasLayoutString = false; // The implementation must provide implementations of: // static void destroy(T *); @@ -777,6 +780,8 @@ struct ValueWitnesses .withPOD(isPOD) .withBitwiseTakable(isBitwiseTakable); + static constexpr bool hasLayoutString = Box::hasLayoutString; + static void destroy(OpaqueValue *value, const Metadata *self) { return Box::destroy((typename Box::type*) value); } From 31de67c71370e50c793b2d0d86f6370353bfb5dd Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:54:14 -0700 Subject: [PATCH 05/26] [IRGen+Runtime] Further fixes for layout strings --- include/swift/ABI/Metadata.h | 16 +++++----- lib/IRGen/GenMeta.cpp | 24 +++++++------- stdlib/public/runtime/BytecodeLayouts.cpp | 38 +++++++++++++---------- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index 3eaff50606d03..ca9096a132574 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -323,13 +323,13 @@ struct TargetMetadata { } const uint8_t *getLayoutString() const { - assert(hasLayoutString()); - if (isAnyClass()) { - return asFullMetadata( - reinterpret_cast *>( - this)) - ->layoutString; - } + // assert(hasLayoutString()); + // if (isAnyClass()) { + // return asFullMetadata( + // reinterpret_cast *>( + // this)) + // ->layoutString; + // } return asFullMetadata(this)->layoutString; } @@ -3778,7 +3778,7 @@ class swift_ptrauth_struct_context_descriptor(TypeContextDescriptor) } bool hasLayoutString() const { - return !hasForeignMetadataInitialization() && getTypeContextDescriptorFlags().hasLayoutString(); + return getTypeContextDescriptorFlags().hasLayoutString(); } /// Given that this type has foreign metadata initialization, return the diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index 8cc25d227c5a8..3d161cef6345d 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -2580,9 +2580,10 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, auto genericSig = lowered.getNominalOrBoundGenericNominal()->getGenericSignature(); - hasLayoutString = !ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal) && - !!typeLayoutEntry->layoutString(IGM, genericSig); + hasLayoutString = !!typeLayoutEntry->layoutString(IGM, genericSig); + } + if (auto sd = dyn_cast(type)) { if (!hasLayoutString && IGM.Context.LangOpts.hasFeature( Feature::LayoutStringValueWitnessesInstantiation) && @@ -2591,9 +2592,6 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, needsSingletonMetadataInitialization(IGM, type) || (type->isGenericContext() && !isa(ti)); } - } - - if (auto sd = dyn_cast(type)) { StructContextDescriptorBuilder(IGM, sd, requireMetadata, hasLayoutString).emit(); } else if (auto ed = dyn_cast(type)) { @@ -4650,7 +4648,7 @@ namespace { auto &ti = IGM.getTypeInfo(getLoweredType()); - return !ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal) && !!getLayoutString(); + return !!getLayoutString(); } ConstantReference emitValueWitnessTable(bool relativeReference) { @@ -5166,10 +5164,10 @@ namespace { if (IGM.Context.LangOpts.hasFeature(Feature::LayoutStringValueWitnessesInstantiation) && IGM.getOptions().EnableLayoutStringValueWitnessesInstantiation) { - return (!!getLayoutString() || needsSingletonMetadataInitialization(IGM, Target)) && !needsForeignMetadataCompletionFunction(IGM, Target); + return !!getLayoutString() || needsSingletonMetadataInitialization(IGM, Target); } - return !!getLayoutString() && !needsForeignMetadataCompletionFunction(IGM, Target); + return !!getLayoutString(); } llvm::Constant *emitNominalTypeDescriptor() { @@ -5679,7 +5677,7 @@ namespace { llvm::Constant *emitNominalTypeDescriptor() { auto descriptor = EnumContextDescriptorBuilder( - IGM, Target, RequireMetadata, hasLayoutString()) + IGM, Target, RequireMetadata, !!getLayoutString()) .emit(); return descriptor; } @@ -5728,7 +5726,7 @@ namespace { } bool canBeConstant() { - return !HasUnfilledPayloadSize && !hasInstantiatedLayoutString(); + return !HasUnfilledPayloadSize;// && !hasInstantiatedLayoutString(); } }; @@ -6250,9 +6248,9 @@ namespace { return Target->getDeclaredType()->getCanonicalType(); } - bool hasLayoutString() { - return false; - } + // bool hasLayoutString() { + // return false; + // } void createMetadataCompletionFunction() { llvm_unreachable("foreign structs never require completion"); diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 54590686703b5..2f20957a350c8 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -369,29 +369,33 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t enumTag; + uintptr_t nestedAddrOffset; + LayoutStringReader nestedReader; reader.modify([&](LayoutStringReader &reader) { - size_t tagBytes; - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; auto tagBytesOffset = enumSize - tagBytes; - auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); - - if (SWIFT_LIKELY(enumTag < numPayloads)) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); - } - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); + addrOffset += enumSize; }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + } } #if SWIFT_OBJC_INTEROP @@ -704,7 +708,7 @@ static void unknownWeakInitWithTake(const Metadata *metadata, (WeakReference *)(src + addrOffset)); } -static inline void metatypeInitWithTake(const Metadata *metadata, +static void metatypeInitWithTake(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *dest, @@ -729,7 +733,7 @@ static void existentialInitWithTake(const Metadata *metadata, } } -static inline void resilientInitWithTake(const Metadata *metadata, +static void resilientInitWithTake(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *dest, From 27b1764d131331bd8c53e73b941e70972e01e594 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:54:35 -0700 Subject: [PATCH 06/26] [Runtime+IRGen] Fix layout string flag in type layout and add array functions --- lib/IRGen/GenMeta.cpp | 14 +- lib/IRGen/GenStruct.cpp | 8 +- stdlib/public/runtime/Array.cpp | 9 + stdlib/public/runtime/BytecodeLayouts.cpp | 256 ++++++++++++---------- stdlib/public/runtime/BytecodeLayouts.h | 65 ++++++ 5 files changed, 227 insertions(+), 125 deletions(-) diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index 3d161cef6345d..b4b4198116c65 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -2581,17 +2581,17 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, auto genericSig = lowered.getNominalOrBoundGenericNominal()->getGenericSignature(); hasLayoutString = !!typeLayoutEntry->layoutString(IGM, genericSig); - } - if (auto sd = dyn_cast(type)) { if (!hasLayoutString && IGM.Context.LangOpts.hasFeature( Feature::LayoutStringValueWitnessesInstantiation) && IGM.getOptions().EnableLayoutStringValueWitnessesInstantiation) { - hasLayoutString |= requiresForeignTypeMetadata(type) || - needsSingletonMetadataInitialization(IGM, type) || + hasLayoutString |= needsSingletonMetadataInitialization(IGM, type) || (type->isGenericContext() && !isa(ti)); } + } + + if (auto sd = dyn_cast(type)) { StructContextDescriptorBuilder(IGM, sd, requireMetadata, hasLayoutString).emit(); } else if (auto ed = dyn_cast(type)) { @@ -4646,8 +4646,6 @@ namespace { return false; } - auto &ti = IGM.getTypeInfo(getLoweredType()); - return !!getLayoutString(); } @@ -6248,10 +6246,6 @@ namespace { return Target->getDeclaredType()->getCanonicalType(); } - // bool hasLayoutString() { - // return false; - // } - void createMetadataCompletionFunction() { llvm_unreachable("foreign structs never require completion"); } diff --git a/lib/IRGen/GenStruct.cpp b/lib/IRGen/GenStruct.cpp index b11c960bdce61..cc2828d841a3a 100644 --- a/lib/IRGen/GenStruct.cpp +++ b/lib/IRGen/GenStruct.cpp @@ -919,10 +919,10 @@ namespace { field.getTypeInfo().buildTypeLayoutEntry(IGM, fieldTy, useStructLayouts)); } - if (fields.size() == 1 && isFixedSize() && - getBestKnownAlignment() == *fields[0]->fixedAlignment(IGM)) { - return fields[0]; - } + // if (fields.size() == 1 && isFixedSize() && + // getBestKnownAlignment() == *fields[0]->fixedAlignment(IGM)) { + // return fields[0]; + // } return IGM.typeLayoutCache.getOrCreateAlignedGroupEntry( fields, T, getBestKnownAlignment().getValue(), *this); diff --git a/stdlib/public/runtime/Array.cpp b/stdlib/public/runtime/Array.cpp index ec9b5de9fac96..84e57199468fb 100644 --- a/stdlib/public/runtime/Array.cpp +++ b/stdlib/public/runtime/Array.cpp @@ -22,6 +22,7 @@ // //===----------------------------------------------------------------------===// +#include "BytecodeLayouts.h" #include "swift/Runtime/Config.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Metadata.h" @@ -124,6 +125,10 @@ static void array_copy_operation(OpaqueValue *dest, OpaqueValue *src, assert(copyKind == ArrayCopy::BackToFront); assert(count != 0); + if (self->hasLayoutString() && destOp == ArrayDest::Init && srcOp == ArraySource::Copy) { + return swift_generic_arrayInitWithCopy(dest, src, count, stride, self); + } + auto copy = get_witness_function(wtable); size_t i = count; do { @@ -202,6 +207,10 @@ void swift_arrayDestroy(OpaqueValue *begin, size_t count, const Metadata *self) return; auto stride = wtable->getStride(); + if (self->hasLayoutString()) { + return swift_generic_arrayDestroy(begin, count, stride, self); + } + for (size_t i = 0; i < count; ++i) { auto offset = i * stride; auto *obj = reinterpret_cast((char *)begin + offset); diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 2f20957a350c8..fb5959edfea8b 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -44,13 +44,13 @@ static Metadata *getExistentialTypeMetadata(OpaqueValue *object) { return reinterpret_cast(object)[NumWords_ValueBuffer]; } -template -static const FnTy readRelativeFunctionPointer(LayoutStringReader &reader) { +template +static const FnTy readRelativeFunctionPointer(Reader &reader) { static_assert(std::is_pointer::value); - auto absolute = reader.layoutStr + reader.offset; + auto absolute = reader.getAbsolute(); auto relativeOffset = - (uintptr_t)(intptr_t)(int32_t)reader.readBytes(); + (uintptr_t)(intptr_t)(int32_t)reader.template readBytes(); FnTy fn; #if SWIFT_PTRAUTH @@ -66,8 +66,9 @@ static const FnTy readRelativeFunctionPointer(LayoutStringReader &reader) { typedef Metadata *(*MetadataAccessor)(const Metadata *const *); +template static const Metadata *getResilientTypeMetadata(const Metadata *metadata, - LayoutStringReader &reader) { + Reader &reader) { auto fn = readRelativeFunctionPointer(reader); return fn(metadata->getGenericArgs()); } @@ -97,13 +98,13 @@ static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) { } static void handleRefCountsDestroy(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr); template static void handleEnd(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { @@ -111,7 +112,7 @@ static void handleEnd(const Metadata *metadata, } static void errorDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { SwiftError *error = *(SwiftError**)(addr + addrOffset); @@ -119,7 +120,7 @@ static void errorDestroyBranchless(const Metadata *metadata, } static void nativeStrongDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); @@ -127,7 +128,7 @@ static void nativeStrongDestroyBranchless(const Metadata *metadata, } static void unownedDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); @@ -135,14 +136,14 @@ static void unownedDestroyBranchless(const Metadata *metadata, } static void weakDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { swift_weakDestroy((WeakReference *)(addr + addrOffset)); } static void unknownDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { void *object = *(void**)(addr + addrOffset); @@ -150,7 +151,7 @@ static void unknownDestroyBranchless(const Metadata *metadata, } static void unknownUnownedDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { UnownedReference *object = (UnownedReference*)(addr + addrOffset); @@ -158,14 +159,14 @@ static void unknownUnownedDestroyBranchless(const Metadata *metadata, } static void unknownWeakDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { swift_unknownObjectWeakDestroy((WeakReference *)(addr + addrOffset)); } static void bridgeDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); @@ -173,11 +174,11 @@ static void bridgeDestroyBranchless(const Metadata *metadata, template static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params... params) { - reader.modify([&](LayoutStringReader &reader) { + reader.modify([&](LayoutStringReader1 &reader) { uint64_t byteCountsAndOffset; size_t payloadSize; uint64_t zeroTagValue; @@ -220,16 +221,16 @@ typedef unsigned (*GetEnumTagFn)(const uint8_t *); template static void singlePayloadEnumFNBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { + reader.modify([&](LayoutStringReader1 &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); unsigned enumTag = getEnumTag(addr + addrOffset); - if (enumTag == 0) { + if (SWIFT_LIKELY(enumTag == 0)) { reader.skip(sizeof(size_t) * 2); } else { size_t refCountBytes; @@ -243,11 +244,11 @@ static void singlePayloadEnumFNBranchless(const Metadata *metadata, template static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { + reader.modify([&](LayoutStringReader1 &reader) { GetEnumTagFn getEnumTag; size_t refCountBytes; size_t skip; @@ -255,7 +256,7 @@ static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, unsigned enumTag = getEnumTag(addr + addrOffset); - if (enumTag != 0) { + if (SWIFT_UNLIKELY(enumTag != 0)) { reader.skip(refCountBytes); addrOffset += skip; } @@ -264,11 +265,11 @@ static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, template static void singlePayloadEnumGenericBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { + reader.modify([&](LayoutStringReader1 &reader) { auto tagBytesAndOffset = reader.readBytes(); auto payloadSize = reader.readBytes(); auto *xiType = reader.readBytes(); @@ -280,7 +281,7 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, auto xiTagBytesOffset = tagBytesAndOffset & std::numeric_limits::max(); - if (extraTagBytesPattern) { + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { auto extraTagBytes = 1 << (extraTagBytesPattern - 1); auto tagBytes = readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); @@ -289,11 +290,11 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, } } - if (xiType) { + if (SWIFT_LIKELY(xiType)) { auto tag = xiType->vw_getEnumTagSinglePayload( (const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset), numEmptyCases); - if (tag == 0) { + if (SWIFT_LIKELY(tag == 0)) { return; } } @@ -305,67 +306,69 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, template static void multiPayloadEnumFNBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; - unsigned enumTag = getEnumTag(addr + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); - } - + enumTag = getEnumTag(addr + addrOffset); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); addrOffset += enumSize; }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + } } template static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = reader.readBytes(); + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; - unsigned enumTag = getEnumTag(addr + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); - } - + enumTag = getEnumTag(addr + addrOffset); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); addrOffset += enumSize; }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + } } template static void multiPayloadEnumGenericBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { @@ -375,17 +378,17 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, size_t enumSize; uint64_t enumTag; uintptr_t nestedAddrOffset; - LayoutStringReader nestedReader; - reader.modify([&](LayoutStringReader &reader) { + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); nestedReader = reader; nestedAddrOffset = addrOffset; auto tagBytesOffset = enumSize - tagBytes; - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); addrOffset += enumSize; }); @@ -400,14 +403,14 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, #if SWIFT_OBJC_INTEROP static void blockDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { _Block_release((void *)(addr + addrOffset)); } static void objcStrongDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { objc_object *object = (objc_object*)(*(uintptr_t *)(addr + addrOffset)); @@ -416,7 +419,7 @@ static void objcStrongDestroyBranchless(const Metadata *metadata, #endif static void metatypeDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { auto *type = reader.readBytes(); @@ -424,7 +427,7 @@ static void metatypeDestroyBranchless(const Metadata *metadata, } static void existentialDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); @@ -437,7 +440,7 @@ static void existentialDestroyBranchless(const Metadata *metadata, } static void resilientDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { auto *type = getResilientTypeMetadata(metadata, reader); @@ -445,7 +448,7 @@ static void resilientDestroyBranchless(const Metadata *metadata, } typedef void (*DestrFnBranchless)(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr); @@ -481,35 +484,46 @@ const DestrFnBranchless destroyTableBranchless[] = { }; static void handleRefCountsDestroy(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - uint64_t tag = 0; - do { - tag = reader.readBytes(); + while (true) { + auto tag = reader.readBytes(); addrOffset += (tag & ~(0xFFULL << 56)); tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } destroyTableBranchless[tag](metadata, reader, addrOffset, addr); - } while (tag != 0); + } } extern "C" void swift_generic_destroy(swift::OpaqueValue *address, const Metadata *metadata) { const uint8_t *layoutStr = metadata->getLayoutString(); - LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; uintptr_t addrOffset = 0; handleRefCountsDestroy(metadata, reader, addrOffset, (uint8_t *)address); } +void swift::swift_generic_arrayDestroy(swift::OpaqueValue *address, size_t count, size_t stride, const Metadata *metadata) { + const uint8_t *layoutStr = metadata->getLayoutString(); + for (size_t i = 0; i < count; i++) { + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = i * stride; + handleRefCountsDestroy(metadata, reader, addrOffset, (uint8_t *)address); + } +} + static void handleRefCountsInitWithCopy(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src); static void errorRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -517,7 +531,7 @@ static void errorRetainBranchless(const Metadata *metadata, } static void nativeStrongRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -526,7 +540,7 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, } static void unownedRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -535,7 +549,7 @@ static void unownedRetainBranchless(const Metadata *metadata, } static void weakCopyInitBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -543,7 +557,7 @@ static void weakCopyInitBranchless(const Metadata *metadata, } static void unknownRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -552,7 +566,7 @@ static void unknownRetainBranchless(const Metadata *metadata, } static void unknownUnownedCopyInitBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -562,7 +576,7 @@ static void unknownUnownedCopyInitBranchless(const Metadata *metadata, } static void unknownWeakCopyInitBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -571,7 +585,7 @@ static void unknownWeakCopyInitBranchless(const Metadata *metadata, } static void bridgeRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -580,7 +594,7 @@ static void bridgeRetainBranchless(const Metadata *metadata, #if SWIFT_OBJC_INTEROP static void blockCopyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -588,7 +602,7 @@ static void blockCopyBranchless(const Metadata *metadata, } static void objcStrongRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -598,7 +612,7 @@ static void objcStrongRetainBranchless(const Metadata *metadata, #endif static void metatypeInitWithCopyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -608,7 +622,7 @@ static void metatypeInitWithCopyBranchless(const Metadata *metadata, } static void existentialInitWithCopyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -618,7 +632,7 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, } static void resilientInitWithCopyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -628,7 +642,7 @@ static void resilientInitWithCopyBranchless(const Metadata *metadata, } typedef void (*InitFn)(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src); @@ -665,18 +679,20 @@ static const InitFn initWithCopyTable[] = { }; static void handleRefCountsInitWithCopy(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - uint64_t tag = 0; - do { - tag = reader.readBytes(); + while (true) { + auto tag = reader.readBytes(); addrOffset += (tag & ~(0xFFULL << 56)); tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } initWithCopyTable[tag](metadata, reader, addrOffset, dest, src); - } while (tag != 0); + } } extern "C" swift::OpaqueValue * @@ -686,21 +702,37 @@ swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, memcpy(dest, src, size); const uint8_t *layoutStr = metadata->getLayoutString(); - LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; uintptr_t addrOffset = 0; handleRefCountsInitWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); return dest; } +void swift::swift_generic_arrayInitWithCopy(swift::OpaqueValue *dest, + swift::OpaqueValue *src, + size_t count, + size_t stride, + const Metadata *metadata) { + size_t size = stride * count; + memcpy(dest, src, size); + + const uint8_t *layoutStr = metadata->getLayoutString(); + for (size_t i = 0; i < count; i++) { + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = i * stride; + handleRefCountsInitWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); + } +} + static void handleRefCountsInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src); static void unknownWeakInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -709,7 +741,7 @@ static void unknownWeakInitWithTake(const Metadata *metadata, } static void metatypeInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -722,7 +754,7 @@ static void metatypeInitWithTake(const Metadata *metadata, } static void existentialInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -734,7 +766,7 @@ static void existentialInitWithTake(const Metadata *metadata, } static void resilientInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -773,19 +805,21 @@ static const InitFn initWithTakeTable[] = { }; static void handleRefCountsInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - uint64_t tag = 0; - do { - tag = reader.readBytes(); + while (true) { + auto tag = reader.readBytes(); addrOffset += (tag & ~(0xFFULL << 56)); tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } if (auto handler = initWithTakeTable[tag]) handler(metadata, reader, addrOffset, dest, src); - } while (tag != 0); + } } extern "C" swift::OpaqueValue * @@ -800,7 +834,7 @@ swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, } const uint8_t *layoutStr = metadata->getLayoutString(); - LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; uintptr_t addrOffset = 0; handleRefCountsInitWithTake(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); @@ -973,7 +1007,7 @@ extern "C" unsigned swift_multiPayloadEnumGeneric_getEnumTag(swift::OpaqueValue *address, const Metadata *metadata) { auto addr = reinterpret_cast(address); - LayoutStringReader reader{metadata->getLayoutString(), + LayoutStringReader1 reader{metadata->getLayoutString() + layoutStringHeaderSize + sizeof(uint64_t)}; auto tagBytes = reader.readBytes(); diff --git a/stdlib/public/runtime/BytecodeLayouts.h b/stdlib/public/runtime/BytecodeLayouts.h index a3d9e14cd1799..66519530ae2b3 100644 --- a/stdlib/public/runtime/BytecodeLayouts.h +++ b/stdlib/public/runtime/BytecodeLayouts.h @@ -58,6 +58,56 @@ enum class RefCountingKind : uint8_t { // Reserved: 0x81 - 0xFF }; +struct LayoutStringReader1 { + const uint8_t *layoutStr; + + template + inline T readBytes() { + T returnVal; + memcpy(&returnVal, layoutStr, sizeof(T)); + layoutStr += sizeof(T); + return returnVal; + } + + template + inline void readBytes(T&... result) { + uintptr_t additionalOffset = 0; + ([&] { + memcpy(&result, layoutStr + additionalOffset, sizeof(T)); + additionalOffset += sizeof(T); + }(), ...); + layoutStr += additionalOffset; + } + + template + inline T modify(F &&f) { + LayoutStringReader1 readerCopy = *this; + T res = f(readerCopy); + layoutStr = readerCopy.layoutStr; + return res; + } + + template + inline void modify(F &&f) { + LayoutStringReader1 readerCopy = *this; + f(readerCopy); + layoutStr = readerCopy.layoutStr; + } + + template + inline T peekBytes(size_t peekOffset = 0) const { + T returnVal; + memcpy(&returnVal, layoutStr + peekOffset, sizeof(T)); + return returnVal; + } + + inline void skip(size_t n) { layoutStr += n; } + + inline uintptr_t getAbsolute() { + return (uintptr_t) layoutStr; + } +}; + struct LayoutStringReader { const uint8_t *layoutStr; size_t offset; @@ -103,6 +153,10 @@ struct LayoutStringReader { } inline void skip(size_t n) { offset += n; } + + inline uintptr_t getAbsolute() { + return (uintptr_t) layoutStr + offset; + } }; struct LayoutStringWriter { @@ -180,6 +234,17 @@ void swift_resolve_resilientAccessors(uint8_t *layoutStr, const uint8_t *fieldLayoutStr, const Metadata *fieldType); +void swift_generic_arrayDestroy(swift::OpaqueValue *addr, + size_t count, + size_t stride, + const Metadata *metadata); + +void swift_generic_arrayInitWithCopy(swift::OpaqueValue *dest, + swift::OpaqueValue *src, + size_t count, + size_t stride, + const Metadata *metadata); + constexpr size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t); } // namespace swift From 30d628270b34d7c5532ab1acc730f2ab18b44c2d Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:55:02 -0700 Subject: [PATCH 07/26] [IRGen+Runtime] Make more fine grained copies in layout string runtime functions --- lib/IRGen/TypeLayout.cpp | 3 +- stdlib/public/runtime/BytecodeLayouts.cpp | 424 ++++++++++++++++++---- stdlib/public/runtime/Metadata.cpp | 12 +- 3 files changed, 365 insertions(+), 74 deletions(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index 3a84c92267d10..39af07bef177c 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -364,8 +364,7 @@ class LayoutStringBuilder { uint64_t op = (static_cast(refCounting.kind) << 56) | skip; B.addInt64(op); refCountBytes += sizeof(uint64_t); - - skip = refCounting.size; + skip = 0; break; } } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index fb5959edfea8b..57a3d74b72a97 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -116,6 +116,7 @@ static void errorDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { SwiftError *error = *(SwiftError**)(addr + addrOffset); + addrOffset += sizeof(SwiftError*); swift_errorRelease(error); } @@ -124,6 +125,7 @@ static void nativeStrongDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(HeapObject*); swift_release(object); } @@ -132,6 +134,7 @@ static void unownedDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(HeapObject*); swift_unownedRelease(object); } @@ -139,7 +142,9 @@ static void weakDestroyBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - swift_weakDestroy((WeakReference *)(addr + addrOffset)); + auto *object = (WeakReference *)(addr + addrOffset); + addrOffset += sizeof(WeakReference); + swift_weakDestroy(object); } static void unknownDestroyBranchless(const Metadata *metadata, @@ -147,6 +152,7 @@ static void unknownDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { void *object = *(void**)(addr + addrOffset); + addrOffset += sizeof(void*); swift_unknownObjectRelease(object); } @@ -155,6 +161,7 @@ static void unknownUnownedDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { UnownedReference *object = (UnownedReference*)(addr + addrOffset); + addrOffset += sizeof(UnownedReference); swift_unknownObjectUnownedDestroy(object); } @@ -162,22 +169,24 @@ static void unknownWeakDestroyBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - swift_unknownObjectWeakDestroy((WeakReference *)(addr + addrOffset)); + auto *object = (WeakReference *)(addr + addrOffset); + addrOffset += sizeof(WeakReference); + swift_unknownObjectWeakDestroy(object); } static void bridgeDestroyBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); + auto *object = *(void **)(addr + addrOffset); + addrOffset += sizeof(void*); + swift_bridgeObjectRelease(object); } -template static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params... params) { + uint8_t *addr) { reader.modify([&](LayoutStringReader1 &reader) { uint64_t byteCountsAndOffset; size_t payloadSize; @@ -219,12 +228,10 @@ static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, typedef unsigned (*GetEnumTagFn)(const uint8_t *); -template static void singlePayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { reader.modify([&](LayoutStringReader1 &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -242,12 +249,10 @@ static void singlePayloadEnumFNBranchless(const Metadata *metadata, }); } -template static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { reader.modify([&](LayoutStringReader1 &reader) { GetEnumTagFn getEnumTag; size_t refCountBytes; @@ -263,12 +268,10 @@ static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, }); } -template static void singlePayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { reader.modify([&](LayoutStringReader1 &reader) { auto tagBytesAndOffset = reader.readBytes(); auto payloadSize = reader.readBytes(); @@ -304,12 +307,11 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, }); } -template +template static void multiPayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { size_t numPayloads; size_t refCountBytes; size_t enumSize; @@ -325,22 +327,23 @@ static void multiPayloadEnumFNBranchless(const Metadata *metadata, enumTag = getEnumTag(addr + addrOffset); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; }); if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); + } else { + addrOffset += enumSize; } } -template +template static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { size_t numPayloads; size_t refCountBytes; size_t enumSize; @@ -356,22 +359,23 @@ static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, enumTag = getEnumTag(addr + addrOffset); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; }); if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); + } else { + addrOffset += enumSize; } } -template +template static void multiPayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { size_t tagBytes; size_t numPayloads; size_t refCountBytes; @@ -389,15 +393,258 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); + } else { addrOffset += enumSize; + } +} + +static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = + readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + if (tagBytes) { + xiTagBytesPattern = 0; + } + } + + if (SWIFT_LIKELY(xiTagBytesPattern)) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + uint64_t tagBytes = + readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + if (tagBytes >= xiTagValues) { + return; + } + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (SWIFT_LIKELY(enumTag == 0)) { + reader.skip(sizeof(size_t) * 2); + } else { + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (SWIFT_UNLIKELY(enumTag != 0)) { + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumGenericBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + + if (tagBytes) { + xiType = nullptr; + } + } + + if (SWIFT_LIKELY(xiType)) { + auto tag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), + numEmptyCases); + if (SWIFT_LIKELY(tag == 0)) { + return; + } + } + + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +template +static void multiPayloadEnumFNBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + enumTag = getEnumTag(src + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); }); if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; + } +} +template +static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + enumTag = getEnumTag(src + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; + } +} + +template +static void multiPayloadEnumGenericBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t enumTag; + uintptr_t nestedAddrOffset; + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + nestedReader = reader; + nestedAddrOffset = addrOffset; + auto tagBytesOffset = enumSize - tagBytes; + + enumTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; } } @@ -406,7 +653,9 @@ static void blockDestroyBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - _Block_release((void *)(addr + addrOffset)); + void* object = (void *)(addr + addrOffset); + addrOffset += sizeof(void*); + _Block_release(object); } static void objcStrongDestroyBranchless(const Metadata *metadata, @@ -414,6 +663,7 @@ static void objcStrongDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { objc_object *object = (objc_object*)(*(uintptr_t *)(addr + addrOffset)); + addrOffset += sizeof(objc_object*); objc_release(object); } #endif @@ -423,7 +673,9 @@ static void metatypeDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { auto *type = reader.readBytes(); - type->vw_destroy((OpaqueValue *)(addr + addrOffset)); + auto *object = (OpaqueValue *)(addr + addrOffset); + addrOffset += type->vw_size(); + type->vw_destroy(object); } static void existentialDestroyBranchless(const Metadata *metadata, @@ -432,6 +684,7 @@ static void existentialDestroyBranchless(const Metadata *metadata, uint8_t *addr) { OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); auto* type = getExistentialTypeMetadata(object); + addrOffset += type->vw_size(); if (type->getValueWitnesses()->isValueInline()) { type->vw_destroy(object); } else { @@ -474,10 +727,10 @@ const DestrFnBranchless destroyTableBranchless[] = { nullptr, // Generic &existentialDestroyBranchless, &resilientDestroyBranchless, - &singlePayloadEnumSimpleBranchless<>, - &singlePayloadEnumFNBranchless<>, - &singlePayloadEnumFNResolvedBranchless<>, - &singlePayloadEnumGenericBranchless<>, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, &multiPayloadEnumFNBranchless, &multiPayloadEnumFNResolvedBranchless, &multiPayloadEnumGenericBranchless, @@ -527,7 +780,10 @@ static void errorRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_errorRetain(*(SwiftError**)(dest + addrOffset)); + memcpy(dest + addrOffset, src + addrOffset, sizeof(SwiftError*)); + auto *object = *(SwiftError**)(dest + addrOffset); + addrOffset += sizeof(object); + swift_errorRetain(object); } static void nativeStrongRetainBranchless(const Metadata *metadata, @@ -535,7 +791,9 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + memcpy(dest + addrOffset, src + addrOffset, sizeof(HeapObject*)); HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(object); swift_retain(object); } @@ -544,7 +802,9 @@ static void unownedRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + memcpy(dest + addrOffset, src + addrOffset, sizeof(HeapObject*)); HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(object); swift_unownedRetain(object); } @@ -553,7 +813,10 @@ static void weakCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_weakCopyInit((WeakReference *)(dest + addrOffset), (WeakReference *)(src + addrOffset)); + auto *destObject = (WeakReference *)(dest + addrOffset); + auto *srcObject = (WeakReference *)(src + addrOffset); + addrOffset += sizeof(WeakReference); + swift_weakCopyInit(destObject, srcObject); } static void unknownRetainBranchless(const Metadata *metadata, @@ -561,7 +824,9 @@ static void unknownRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + memcpy(dest + addrOffset, src + addrOffset, sizeof(void*)); void *object = *(void**)(dest + addrOffset); + addrOffset += sizeof(void*); swift_unknownObjectRetain(object); } @@ -572,6 +837,7 @@ static void unknownUnownedCopyInitBranchless(const Metadata *metadata, uint8_t *src) { UnownedReference *objectDest = (UnownedReference*)(dest + addrOffset); UnownedReference *objectSrc = (UnownedReference*)(src + addrOffset); + addrOffset += sizeof(UnownedReference); swift_unknownObjectUnownedCopyInit(objectDest, objectSrc); } @@ -580,8 +846,10 @@ static void unknownWeakCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_unknownObjectWeakCopyInit((WeakReference *)(dest + addrOffset), - (WeakReference *)(src + addrOffset)); + auto *destObject = (WeakReference *)(dest + addrOffset); + auto *srcObject = (WeakReference *)(src + addrOffset); + addrOffset += sizeof(WeakReference); + swift_unknownObjectWeakCopyInit(destObject, srcObject); } static void bridgeRetainBranchless(const Metadata *metadata, @@ -589,7 +857,10 @@ static void bridgeRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_bridgeObjectRetain(*(void **)(dest + addrOffset)); + memcpy(dest + addrOffset, src + addrOffset, sizeof(void*)); + auto *object = *(void **)(dest + addrOffset); + addrOffset += sizeof(void*); + swift_bridgeObjectRetain(object); } #if SWIFT_OBJC_INTEROP @@ -598,7 +869,9 @@ static void blockCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - *(void**)dest = _Block_copy(*(void**)(src + addrOffset)); + auto *copy = _Block_copy(*(void**)(src + addrOffset)); + memcpy(dest + addrOffset, ©, sizeof(void*)); + addrOffset += sizeof(void*); } static void objcStrongRetainBranchless(const Metadata *metadata, @@ -606,7 +879,9 @@ static void objcStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + memcpy(dest + addrOffset, src + addrOffset, sizeof(objc_object*)); objc_object *object = (objc_object*)(*(uintptr_t *)(src + addrOffset)); + addrOffset += sizeof(objc_object*); objc_retain(object); } #endif @@ -617,8 +892,10 @@ static void metatypeInitWithCopyBranchless(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = reader.readBytes(); - type->vw_initializeWithCopy((OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); + type->vw_initializeWithCopy(destObject, srcObject); } static void existentialInitWithCopyBranchless(const Metadata *metadata, @@ -626,9 +903,14 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); - type->vw_initializeBufferWithCopyOfBuffer((ValueBuffer*)(dest + addrOffset), - (ValueBuffer*)(src + addrOffset)); + auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *destObject = (ValueBuffer *)(dest + addrOffset); + auto *srcObject = (ValueBuffer *)(src + addrOffset); + memcpy(dest + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), + src + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), + sizeof(uintptr_t) * 2); + addrOffset += type->vw_size(); + type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); } static void resilientInitWithCopyBranchless(const Metadata *metadata, @@ -637,8 +919,10 @@ static void resilientInitWithCopyBranchless(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = getResilientTypeMetadata(metadata, reader); - type->vw_initializeWithCopy((OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); + type->vw_initializeWithCopy(destObject, srcObject); } typedef void (*InitFn)(const Metadata *metadata, @@ -685,7 +969,11 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, uint8_t *src) { while (true) { auto tag = reader.readBytes(); - addrOffset += (tag & ~(0xFFULL << 56)); + auto offset = (tag & ~(0xFFULL << 56)); + if (offset) { + memcpy(dest + addrOffset, src + addrOffset, offset); + } + addrOffset += offset; tag >>= 56; if (SWIFT_UNLIKELY(tag == 0)) { return; @@ -698,9 +986,6 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, extern "C" swift::OpaqueValue * swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - size_t size = metadata->vw_size(); - memcpy(dest, src, size); - const uint8_t *layoutStr = metadata->getLayoutString(); LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; uintptr_t addrOffset = 0; @@ -714,9 +999,6 @@ void swift::swift_generic_arrayInitWithCopy(swift::OpaqueValue *dest, size_t count, size_t stride, const Metadata *metadata) { - size_t size = stride * count; - memcpy(dest, src, size); - const uint8_t *layoutStr = metadata->getLayoutString(); for (size_t i = 0; i < count; i++) { LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; @@ -736,8 +1018,11 @@ static void unknownWeakInitWithTake(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_unknownObjectWeakTakeInit((WeakReference *)(dest + addrOffset), - (WeakReference *)(src + addrOffset)); + auto *destObject = (WeakReference *)(dest + addrOffset); + auto *srcObject = (WeakReference *)(src + addrOffset); + addrOffset += sizeof(WeakReference); + + swift_unknownObjectWeakTakeInit(destObject, srcObject); } static void metatypeInitWithTake(const Metadata *metadata, @@ -746,10 +1031,11 @@ static void metatypeInitWithTake(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = reader.readBytes(); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + type->vw_initializeWithTake(destObject, srcObject); } } @@ -759,9 +1045,11 @@ static void existentialInitWithTake(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake((OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + type->vw_initializeWithTake(destObject, srcObject); } } @@ -771,10 +1059,11 @@ static void resilientInitWithTake(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = getResilientTypeMetadata(metadata, reader); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + type->vw_initializeWithTake(destObject, srcObject); } } @@ -817,8 +1106,11 @@ static void handleRefCountsInitWithTake(const Metadata *metadata, return; } - if (auto handler = initWithTakeTable[tag]) + if (auto handler = initWithTakeTable[tag]) { handler(metadata, reader, addrOffset, dest, src); + } else { + addrOffset += sizeof(uintptr_t); + } } } diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 3f1f7e0f480f3..015511fc4c48c 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2857,8 +2857,8 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, } writer.writeBytes(((uint64_t)tag << 56) | offset); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); } else if (fieldType->hasLayoutString()) { LayoutStringReader reader{fieldType->getLayoutString(), 0}; const auto fieldFlags = reader.readBytes(); @@ -2895,14 +2895,14 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, auto tag = existential->isClassBounded() ? RefCountingKind::Unknown : RefCountingKind::Existential; writer.writeBytes(((uint64_t)tag << 56) | offset); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; + previousFieldOffset = existential->isClassBounded() ? fieldType->vw_size() - sizeof(uintptr_t) : 0; + fullOffset += fieldType->vw_size(); } else { metadata: writer.writeBytes(((uint64_t)RefCountingKind::Metatype << 56) | offset); writer.writeBytes(fieldType); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); } } From 4f6e69b35f1f006d406b31ad5c4e7d2370038933 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:55:25 -0700 Subject: [PATCH 08/26] [Runtime] Reduce instruction count in simple layout string operations --- stdlib/public/runtime/BytecodeLayouts.cpp | 42 ++++++++++++----------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 57a3d74b72a97..615346bb3c776 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -780,9 +780,9 @@ static void errorRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(SwiftError*)); - auto *object = *(SwiftError**)(dest + addrOffset); - addrOffset += sizeof(object); + SwiftError *object = *(SwiftError **)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(SwiftError*)); + addrOffset += sizeof(SwiftError *); swift_errorRetain(object); } @@ -791,10 +791,11 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(HeapObject*)); - HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); - addrOffset += sizeof(object); - swift_retain(object); + uintptr_t object = *(uintptr_t *)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(HeapObject*)); + object &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset += sizeof(HeapObject *); + swift_retain((HeapObject *)object); } static void unownedRetainBranchless(const Metadata *metadata, @@ -802,10 +803,11 @@ static void unownedRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(HeapObject*)); - HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); - addrOffset += sizeof(object); - swift_unownedRetain(object); + uintptr_t object = *(uintptr_t *)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(HeapObject*)); + object &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset += sizeof(HeapObject *); + swift_unownedRetain((HeapObject *)object); } static void weakCopyInitBranchless(const Metadata *metadata, @@ -824,9 +826,9 @@ static void unknownRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(void*)); - void *object = *(void**)(dest + addrOffset); - addrOffset += sizeof(void*); + void *object = *(void **)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(void*)); + addrOffset += sizeof(void *); swift_unknownObjectRetain(object); } @@ -857,8 +859,8 @@ static void bridgeRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(void*)); - auto *object = *(void **)(dest + addrOffset); + void *object = *(void **)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(void*)); addrOffset += sizeof(void*); swift_bridgeObjectRetain(object); } @@ -879,8 +881,8 @@ static void objcStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(objc_object*)); objc_object *object = (objc_object*)(*(uintptr_t *)(src + addrOffset)); + memcpy(dest + addrOffset, &object, sizeof(objc_object*)); addrOffset += sizeof(objc_object*); objc_retain(object); } @@ -904,11 +906,11 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *witness = *(void**)(src + addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); + memcpy(dest + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); + memcpy(dest + addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); auto *destObject = (ValueBuffer *)(dest + addrOffset); auto *srcObject = (ValueBuffer *)(src + addrOffset); - memcpy(dest + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), - src + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), - sizeof(uintptr_t) * 2); addrOffset += type->vw_size(); type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); } From 8ab845fdca870d56680f9ba08df3dd0f45d68768 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:55:51 -0700 Subject: [PATCH 09/26] [Runtime+IRGen] Fix offsets of existentials in layout strings --- lib/IRGen/TypeLayout.cpp | 10 +- stdlib/public/runtime/BytecodeLayouts.cpp | 143 ++++++++++-------- stdlib/public/runtime/Metadata.cpp | 2 +- .../layout_string_witnesses_types.swift | 41 +++++ .../layout_string_witnesses_dynamic.swift | 34 +++++ 5 files changed, 168 insertions(+), 62 deletions(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index 39af07bef177c..e788838212b27 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -360,13 +360,21 @@ class LayoutStringBuilder { break; } - default: { + case RefCountingKind::Existential: { uint64_t op = (static_cast(refCounting.kind) << 56) | skip; B.addInt64(op); refCountBytes += sizeof(uint64_t); skip = 0; break; } + + default: { + uint64_t op = (static_cast(refCounting.kind) << 56) | skip; + B.addInt64(op); + refCountBytes += sizeof(uint64_t); + skip = refCounting.size - IGM.getPointerSize().getValue(); + break; + } } } } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 615346bb3c776..c33d1aaf61ce9 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -684,7 +684,7 @@ static void existentialDestroyBranchless(const Metadata *metadata, uint8_t *addr) { OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); auto* type = getExistentialTypeMetadata(object); - addrOffset += type->vw_size(); + addrOffset += sizeof(uintptr_t) * (NumWords_ValueBuffer + 2); if (type->getValueWitnesses()->isValueInline()) { type->vw_destroy(object); } else { @@ -780,9 +780,10 @@ static void errorRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - SwiftError *object = *(SwiftError **)(src + addrOffset); + uintptr_t _addrOffset = addrOffset; + SwiftError *object = *(SwiftError **)(src + _addrOffset); memcpy(dest + addrOffset, &object, sizeof(SwiftError*)); - addrOffset += sizeof(SwiftError *); + addrOffset = _addrOffset + sizeof(SwiftError *); swift_errorRetain(object); } @@ -791,10 +792,11 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - uintptr_t object = *(uintptr_t *)(src + addrOffset); - memcpy(dest + addrOffset, &object, sizeof(HeapObject*)); + uintptr_t _addrOffset = addrOffset; + uintptr_t object = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(HeapObject*)); object &= ~_swift_abi_SwiftSpareBitsMask; - addrOffset += sizeof(HeapObject *); + addrOffset = _addrOffset + sizeof(HeapObject *); swift_retain((HeapObject *)object); } @@ -803,10 +805,11 @@ static void unownedRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - uintptr_t object = *(uintptr_t *)(src + addrOffset); - memcpy(dest + addrOffset, &object, sizeof(HeapObject*)); + uintptr_t _addrOffset = addrOffset; + uintptr_t object = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(HeapObject*)); object &= ~_swift_abi_SwiftSpareBitsMask; - addrOffset += sizeof(HeapObject *); + addrOffset = _addrOffset + sizeof(HeapObject *); swift_unownedRetain((HeapObject *)object); } @@ -815,9 +818,10 @@ static void weakCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto *destObject = (WeakReference *)(dest + addrOffset); - auto *srcObject = (WeakReference *)(src + addrOffset); - addrOffset += sizeof(WeakReference); + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); swift_weakCopyInit(destObject, srcObject); } @@ -826,9 +830,10 @@ static void unknownRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - void *object = *(void **)(src + addrOffset); - memcpy(dest + addrOffset, &object, sizeof(void*)); - addrOffset += sizeof(void *); + uintptr_t _addrOffset = addrOffset; + void *object = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void *); swift_unknownObjectRetain(object); } @@ -837,9 +842,10 @@ static void unknownUnownedCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - UnownedReference *objectDest = (UnownedReference*)(dest + addrOffset); - UnownedReference *objectSrc = (UnownedReference*)(src + addrOffset); - addrOffset += sizeof(UnownedReference); + uintptr_t _addrOffset = addrOffset; + UnownedReference *objectDest = (UnownedReference*)(dest + _addrOffset); + UnownedReference *objectSrc = (UnownedReference*)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(UnownedReference); swift_unknownObjectUnownedCopyInit(objectDest, objectSrc); } @@ -848,9 +854,10 @@ static void unknownWeakCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto *destObject = (WeakReference *)(dest + addrOffset); - auto *srcObject = (WeakReference *)(src + addrOffset); - addrOffset += sizeof(WeakReference); + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); swift_unknownObjectWeakCopyInit(destObject, srcObject); } @@ -859,9 +866,10 @@ static void bridgeRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - void *object = *(void **)(src + addrOffset); - memcpy(dest + addrOffset, &object, sizeof(void*)); - addrOffset += sizeof(void*); + uintptr_t _addrOffset = addrOffset; + void *object = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); swift_bridgeObjectRetain(object); } @@ -871,9 +879,10 @@ static void blockCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto *copy = _Block_copy(*(void**)(src + addrOffset)); - memcpy(dest + addrOffset, ©, sizeof(void*)); - addrOffset += sizeof(void*); + uintptr_t _addrOffset = addrOffset; + auto *copy = _Block_copy(*(void**)(src + _addrOffset)); + memcpy(dest + _addrOffset, ©, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); } static void objcStrongRetainBranchless(const Metadata *metadata, @@ -881,9 +890,10 @@ static void objcStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - objc_object *object = (objc_object*)(*(uintptr_t *)(src + addrOffset)); - memcpy(dest + addrOffset, &object, sizeof(objc_object*)); - addrOffset += sizeof(objc_object*); + uintptr_t _addrOffset = addrOffset; + objc_object *object = (objc_object*)(*(uintptr_t *)(src + _addrOffset)); + memcpy(dest + _addrOffset, &object, sizeof(objc_object*)); + addrOffset = _addrOffset + sizeof(objc_object*); objc_retain(object); } #endif @@ -893,10 +903,11 @@ static void metatypeInitWithCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto *type = reader.readBytes(); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); type->vw_initializeWithCopy(destObject, srcObject); } @@ -905,13 +916,14 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); - auto *witness = *(void**)(src + addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); - memcpy(dest + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); - memcpy(dest + addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); - auto *destObject = (ValueBuffer *)(dest + addrOffset); - auto *srcObject = (ValueBuffer *)(src + addrOffset); - addrOffset += type->vw_size(); + uintptr_t _addrOffset = addrOffset; + auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); + auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); + memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); + memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); + auto *destObject = (ValueBuffer *)(dest + _addrOffset); + auto *srcObject = (ValueBuffer *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); } @@ -920,10 +932,11 @@ static void resilientInitWithCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto *type = getResilientTypeMetadata(metadata, reader); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); type->vw_initializeWithCopy(destObject, srcObject); } @@ -970,12 +983,13 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, uint8_t *dest, uint8_t *src) { while (true) { + uintptr_t _addrOffset = addrOffset; auto tag = reader.readBytes(); auto offset = (tag & ~(0xFFULL << 56)); if (offset) { - memcpy(dest + addrOffset, src + addrOffset, offset); + memcpy(dest + _addrOffset, src + _addrOffset, offset); } - addrOffset += offset; + addrOffset = _addrOffset + offset; tag >>= 56; if (SWIFT_UNLIKELY(tag == 0)) { return; @@ -1032,10 +1046,11 @@ static void metatypeInitWithTake(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto *type = reader.readBytes(); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); } @@ -1046,10 +1061,11 @@ static void existentialInitWithTake(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); } @@ -1060,10 +1076,11 @@ static void resilientInitWithTake(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto *type = getResilientTypeMetadata(metadata, reader); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); } @@ -1101,17 +1118,25 @@ static void handleRefCountsInitWithTake(const Metadata *metadata, uint8_t *dest, uint8_t *src) { while (true) { + uintptr_t _addrOffset = addrOffset; auto tag = reader.readBytes(); - addrOffset += (tag & ~(0xFFULL << 56)); + auto offset = (tag & ~(0xFFULL << 56)); + if (offset) { + memcpy(dest + _addrOffset, src + _addrOffset, offset); + } + _addrOffset += offset; tag >>= 56; if (SWIFT_UNLIKELY(tag == 0)) { + addrOffset = _addrOffset; return; } if (auto handler = initWithTakeTable[tag]) { + addrOffset = _addrOffset; handler(metadata, reader, addrOffset, dest, src); } else { - addrOffset += sizeof(uintptr_t); + memcpy(dest + _addrOffset, src + _addrOffset, sizeof(uintptr_t)); + addrOffset = _addrOffset + sizeof(uintptr_t); } } } @@ -1119,11 +1144,9 @@ static void handleRefCountsInitWithTake(const Metadata *metadata, extern "C" swift::OpaqueValue * swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - size_t size = metadata->vw_size(); - - memcpy(dest, src, size); - if (SWIFT_LIKELY(metadata->getValueWitnesses()->isBitwiseTakable())) { + size_t size = metadata->vw_size(); + memcpy(dest, src, size); return dest; } diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 015511fc4c48c..559f41acb9f99 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2743,7 +2743,7 @@ void swift::swift_initStructMetadataWithLayoutString( } fullOffset += fieldType->size; - previousFieldOffset = fieldType->size; + previousFieldOffset = fieldType->size - sizeof(uintptr_t); continue; } diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index 224db34e51b1a..4b2334b244959 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -223,6 +223,47 @@ public struct Recursive { } } +public protocol Foo: AnyObject {} + +public enum Repro { + public struct R1 { + let x: R2? + let y: Int + weak var z: Foo? + let w: [NSNotification.Name] + + public init(x: R2?, + y: Int, + z: Foo?, + w: [NSNotification.Name]) { + self.x = x + self.y = y + self.z = z + self.w = w + } + } + + public struct R2 { + let x: NSNotification.Name? + let y: Bool + let z: E1 + + public init(x: NSNotification.Name?, + y: Bool, + z: E1) { + self.x = x + self.y = y + self.z = z + } + } + + public enum E1: Int { + case x + case y + case z + } +} + #if os(macOS) import Foundation diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index af95e974e8cdf..6a1db82293aa7 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -24,6 +24,40 @@ class TestClass { } } +class XX: Foo { + deinit { + print("XX deinitialized!") + } +} + +func testRepro() { + let ptr = allocateInternalGenericPtr(of: Repro.R1.self) + + do { + let x = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: XX(), w: []) + testGenericInit(ptr, to: x) + } + + do { + let y = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: XX(), w: []) + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: XX deinitialized! + testGenericAssign(ptr, from: y) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: XX deinitialized! + testGenericDestroy(ptr, of: Repro.R1.self) + + ptr.deallocate() +} + +testRepro() + func testGeneric() { let ptr = allocateInternalGenericPtr(of: TestClass.self) From 7c0203dd550db14a7ff34d08d3f069a4250dfb1b Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:56:19 -0700 Subject: [PATCH 10/26] [Runtime] Combine copy and destroy in assignWithCopy --- stdlib/public/runtime/BytecodeLayouts.cpp | 643 +++++++++++++++++- .../layout_string_witnesses_dynamic.swift | 10 +- 2 files changed, 641 insertions(+), 12 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index c33d1aaf61ce9..90a7136c44bd0 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -986,7 +986,7 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, uintptr_t _addrOffset = addrOffset; auto tag = reader.readBytes(); auto offset = (tag & ~(0xFFULL << 56)); - if (offset) { + if (SWIFT_UNLIKELY(offset)) { memcpy(dest + _addrOffset, src + _addrOffset, offset); } addrOffset = _addrOffset + offset; @@ -1051,9 +1051,8 @@ static void metatypeInitWithTake(const Metadata *metadata, auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); addrOffset = _addrOffset + type->vw_size(); - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake(destObject, srcObject); - } + + type->vw_initializeWithTake(destObject, srcObject); } static void existentialInitWithTake(const Metadata *metadata, @@ -1063,8 +1062,11 @@ static void existentialInitWithTake(const Metadata *metadata, uint8_t *src) { uintptr_t _addrOffset = addrOffset; auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); + memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); + memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); @@ -1081,9 +1083,7 @@ static void resilientInitWithTake(const Metadata *metadata, auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); addrOffset = _addrOffset + type->vw_size(); - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake(destObject, srcObject); - } + type->vw_initializeWithTake(destObject, srcObject); } static const InitFn initWithTakeTable[] = { @@ -1121,7 +1121,7 @@ static void handleRefCountsInitWithTake(const Metadata *metadata, uintptr_t _addrOffset = addrOffset; auto tag = reader.readBytes(); auto offset = (tag & ~(0xFFULL << 56)); - if (offset) { + if (SWIFT_UNLIKELY(offset)) { memcpy(dest + _addrOffset, src + _addrOffset, offset); } _addrOffset += offset; @@ -1159,11 +1159,634 @@ swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, return dest; } +static void errorAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + SwiftError *destObject = *(SwiftError **)(dest + _addrOffset); + SwiftError *srcObject = *(SwiftError **)(src + _addrOffset); + memcpy(dest + addrOffset, &srcObject, sizeof(SwiftError*)); + addrOffset = _addrOffset + sizeof(SwiftError *); + swift_errorRelease(destObject); + swift_errorRetain(srcObject); +} + +static void nativeStrongAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset); + uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(HeapObject*)); + srcObject &= ~_swift_abi_SwiftSpareBitsMask; + destObject &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset = _addrOffset + sizeof(HeapObject *); + swift_release((HeapObject *)destObject); + swift_retain((HeapObject *)srcObject); +} + +static void unownedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset); + uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(HeapObject*)); + destObject &= ~_swift_abi_SwiftSpareBitsMask; + srcObject &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset = _addrOffset + sizeof(HeapObject *); + swift_unownedRelease((HeapObject *)destObject); + swift_unownedRetain((HeapObject *)srcObject); +} + +static void weakAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); + swift_weakCopyAssign(destObject, srcObject); +} + +static void unknownAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + void *destObject = *(void **)(dest + _addrOffset); + void *srcObject = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void *); + swift_unknownObjectRelease(destObject); + swift_unknownObjectRetain(srcObject); +} + +static void unknownUnownedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + UnownedReference *objectDest = (UnownedReference*)(dest + _addrOffset); + UnownedReference *objectSrc = (UnownedReference*)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(UnownedReference); + swift_unknownObjectUnownedCopyAssign(objectDest, objectSrc); +} + +static void unknownWeakAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); + swift_unknownObjectWeakCopyAssign(destObject, srcObject); +} + +static void bridgeAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + void *destObject = *(void **)(dest + _addrOffset); + void *srcObject = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); + swift_bridgeObjectRelease(destObject); + swift_bridgeObjectRetain(srcObject); +} + +#if SWIFT_OBJC_INTEROP +static void blockAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + _Block_release(*(void **)(dest + _addrOffset)); + auto *copy = _Block_copy(*(void **)(src + _addrOffset)); + memcpy(dest + _addrOffset, ©, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); +} + +static void objcStrongAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + objc_object *destObject = (objc_object*)(*(uintptr_t *)(dest + _addrOffset)); + objc_object *srcObject = (objc_object*)(*(uintptr_t *)(src + _addrOffset)); + memcpy(dest + _addrOffset, &srcObject, sizeof(objc_object*)); + addrOffset = _addrOffset + sizeof(objc_object*); + objc_release(destObject); + objc_retain(srcObject); +} +#endif + +static void metatypeAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = reader.readBytes(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_assignWithCopy(destObject, srcObject); +} + +static void existentialAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); + auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); + memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); + memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); + if (type->getValueWitnesses()->isValueInline()) { + type->vw_assignWithCopy(destObject, srcObject); + } else { + swift_release(*(HeapObject**)destObject); + memcpy(destObject, srcObject, sizeof(uintptr_t)); + swift_retain(*(HeapObject**)srcObject); + } +} + +static void resilientAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getResilientTypeMetadata(metadata, reader); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_assignWithCopy(destObject, srcObject); +} + +static void handleSingleRefCountDestroy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } + destroyTableBranchless[tag](metadata, reader, addrOffset, addr); +} + +static void handleSingleRefCountInitWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } + initWithCopyTable[tag](metadata, reader, addrOffset, dest, src); +} + +static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t srcTagBytes = 0; + uint64_t destTagBytes = 0; + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + + srcTagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + destTagBytes = readTagBytes(dest + addrOffset + payloadSize, extraTagBytes); + } + + if (SWIFT_LIKELY(xiTagBytesPattern)) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + srcTagBytes = srcTagBytes ? 0 : + readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + destTagBytes = destTagBytes ? 0 : + readTagBytes(dest + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + } + + if (srcTagBytes >= xiTagValues && destTagBytes >= xiTagValues) { + return; + } else if (destTagBytes >= xiTagValues) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (srcTagBytes >= xiTagValues) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + + unsigned srcTag = getEnumTag(src + addrOffset); + unsigned destTag = getEnumTag(dest + addrOffset); + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned srcTag = getEnumTag(src + addrOffset); + unsigned destTag = getEnumTag(dest + addrOffset); + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumGenericAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t srcTag = 0; + uint64_t destTag = 0; + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + srcTag = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + destTag = readTagBytes(dest + addrOffset + payloadSize, extraTagBytes); + } + + if (SWIFT_LIKELY(xiType)) { + if (!srcTag) { + srcTag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), + numEmptyCases); + } + + if (!destTag) { + destTag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(dest + addrOffset + xiTagBytesOffset), + numEmptyCases); + } + } + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void multiPayloadEnumFNAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned srcTag; + unsigned destTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + srcTag = getEnumTag(src + addrOffset); + destTag = getEnumTag(dest + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); + } + + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} + +static void multiPayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned srcTag; + unsigned destTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + srcTag = getEnumTag(src + addrOffset); + destTag = getEnumTag(dest + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); + } + + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} + +static void multiPayloadEnumGenericAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t srcTag; + uint64_t destTag; + uintptr_t nestedAddrOffset; + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + nestedReader = reader; + nestedAddrOffset = addrOffset; + auto tagBytesOffset = enumSize - tagBytes; + + srcTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); + destTag = readTagBytes(dest + addrOffset + tagBytesOffset, tagBytes); + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); + } + + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} + +static const InitFn assignWithCopyTable[] = { + &handleEnd, + &errorAssignWithCopy, + &nativeStrongAssignWithCopy, + &unownedAssignWithCopy, + &weakAssignWithCopy, + &unknownAssignWithCopy, + &unknownUnownedAssignWithCopy, + &unknownWeakAssignWithCopy, + &bridgeAssignWithCopy, +#if SWIFT_OBJC_INTEROP + &blockAssignWithCopy, + &objcStrongAssignWithCopy, +#else + nullptr, + nullptr, +#endif + nullptr, // Custom + &metatypeAssignWithCopy, + nullptr, // Generic + &existentialAssignWithCopy, + &resilientAssignWithCopy, + &singlePayloadEnumSimpleAssignWithCopy, + &singlePayloadEnumFNAssignWithCopy, + &singlePayloadEnumFNResolvedAssignWithCopy, + &singlePayloadEnumGenericAssignWithCopy, + &multiPayloadEnumFNAssignWithCopy, + &multiPayloadEnumFNResolvedAssignWithCopy, + &multiPayloadEnumGenericAssignWithCopy, +}; + +static void handleRefCountsAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + while (true) { + uintptr_t _addrOffset = addrOffset; + auto tag = reader.readBytes(); + auto offset = (tag & ~(0xFFULL << 56)); + if (offset) { + memcpy(dest + _addrOffset, src + _addrOffset, offset); + } + addrOffset = _addrOffset + offset; + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } + + assignWithCopyTable[tag](metadata, reader, addrOffset, dest, src); + } +} + extern "C" swift::OpaqueValue * swift_generic_assignWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - swift_generic_destroy(dest, metadata); - return swift_generic_initWithCopy(dest, src, metadata); + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsAssignWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); + + return dest; } extern "C" swift::OpaqueValue * diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index 6a1db82293aa7..2d9dd91754703 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -33,18 +33,23 @@ class XX: Foo { func testRepro() { let ptr = allocateInternalGenericPtr(of: Repro.R1.self) + var xx: XX? = XX() + do { - let x = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: XX(), w: []) + let x = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: xx!, w: []) testGenericInit(ptr, to: x) } + var yy: XX? = XX() + do { - let y = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: XX(), w: []) + let y = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: yy!, w: []) // CHECK: Before deinit print("Before deinit") // CHECK-NEXT: XX deinitialized! testGenericAssign(ptr, from: y) + xx = nil } // CHECK-NEXT: Before deinit @@ -52,6 +57,7 @@ func testRepro() { // CHECK-NEXT: XX deinitialized! testGenericDestroy(ptr, of: Repro.R1.self) + yy = nil ptr.deallocate() } From 8b3aaf54236bccf110bc35c9e7c8b3e3d65e8936 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:56:42 -0700 Subject: [PATCH 11/26] [Runtime] Compute end before looping over single payload ref count --- stdlib/public/runtime/BytecodeLayouts.cpp | 24 +++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 90a7136c44bd0..694f335af9f6b 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -1414,12 +1414,14 @@ static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, if (srcTagBytes >= xiTagValues && destTagBytes >= xiTagValues) { return; } else if (destTagBytes >= xiTagValues) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; } else if (srcTagBytes >= xiTagValues) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); } } else { @@ -1448,12 +1450,14 @@ static void singlePayloadEnumFNAssignWithCopy(const Metadata *metadata, if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { return; } else if (srcTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; } else if (destTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); } } else { @@ -1482,12 +1486,14 @@ static void singlePayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { return; } else if (srcTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; } else if (destTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); } } else { @@ -1541,12 +1547,14 @@ static void singlePayloadEnumGenericAssignWithCopy(const Metadata *metadata, if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { return; } else if (srcTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; } else if (destTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); } } else { From 56048ac19fea6ce8c55b1cebaab94c0367dbceac Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:57:06 -0700 Subject: [PATCH 12/26] [Runtime+IRGen] Fix existential offset for multiple protocol witnesses --- lib/IRGen/TypeLayout.cpp | 2 +- stdlib/public/runtime/BytecodeLayouts.cpp | 17 ++------- .../layout_string_witnesses_types.swift | 15 ++++++++ .../layout_string_witnesses_static.swift | 37 +++++++++++++++++++ 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index e788838212b27..06c5934464d62 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -364,7 +364,7 @@ class LayoutStringBuilder { uint64_t op = (static_cast(refCounting.kind) << 56) | skip; B.addInt64(op); refCountBytes += sizeof(uint64_t); - skip = 0; + skip = refCounting.size - getFixedBufferSize(IGM).getValue(); break; } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 694f335af9f6b..9edb27a78f0a3 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -684,7 +684,7 @@ static void existentialDestroyBranchless(const Metadata *metadata, uint8_t *addr) { OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); auto* type = getExistentialTypeMetadata(object); - addrOffset += sizeof(uintptr_t) * (NumWords_ValueBuffer + 2); + addrOffset += sizeof(uintptr_t) * NumWords_ValueBuffer; if (type->getValueWitnesses()->isValueInline()) { type->vw_destroy(object); } else { @@ -918,12 +918,9 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, uint8_t *src) { uintptr_t _addrOffset = addrOffset; auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); - auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); - memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); - memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); auto *destObject = (ValueBuffer *)(dest + _addrOffset); auto *srcObject = (ValueBuffer *)(src + _addrOffset); - addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); } @@ -1062,12 +1059,9 @@ static void existentialInitWithTake(const Metadata *metadata, uint8_t *src) { uintptr_t _addrOffset = addrOffset; auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); - auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); - memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); - memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); - addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); } @@ -1317,12 +1311,9 @@ static void existentialAssignWithCopy(const Metadata *metadata, uint8_t *src) { uintptr_t _addrOffset = addrOffset; auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); - auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); - memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); - memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); - addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); if (type->getValueWitnesses()->isValueInline()) { type->vw_assignWithCopy(destObject, srcObject); } else { diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index 4b2334b244959..9190d06cc3d37 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -264,6 +264,21 @@ public enum Repro { } } +public protocol A {} +public protocol B {} +public protocol C {} + +public struct MultiProtocolExistentialWrapper { + let x: Int = 0 + let y: any (A&B&C) + let z: AnyObject + + public init(y: any (A&B&C), z: AnyObject) { + self.y = y + self.z = z + } +} + #if os(macOS) import Foundation diff --git a/test/Interpreter/layout_string_witnesses_static.swift b/test/Interpreter/layout_string_witnesses_static.swift index 861c53ab87a96..738c21e47f114 100644 --- a/test/Interpreter/layout_string_witnesses_static.swift +++ b/test/Interpreter/layout_string_witnesses_static.swift @@ -293,6 +293,43 @@ func testExistentialStructBox() { testExistentialStructBox() +class ClassWithABC: A, B, C { + deinit { + print("ClassWithABC deinitialized!") + } +} + +func testMultiProtocolExistential() { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + + do { + let x = ClassWithABC() + testInit(ptr, to: MultiProtocolExistentialWrapper(y: x, z: SimpleClass(x: 23))) + } + + do { + let y = ClassWithABC() + + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: ClassWithABC deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testAssign(ptr, from: MultiProtocolExistentialWrapper(y: y, z: SimpleClass(x: 32))) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: ClassWithABC deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testDestroy(ptr) + + ptr.deallocate() +} + +testMultiProtocolExistential() + class ClassWithSomeClassProtocol: SomeClassProtocol { deinit { print("ClassWithSomeClassProtocol deinitialized!") From 7e3f56de242e8e63ae4bc56880709ea1b6eab761 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:57:33 -0700 Subject: [PATCH 13/26] [Runtime] Fix generic existentials in layout strings --- stdlib/public/runtime/Metadata.cpp | 2 +- .../layout_string_witnesses_types.swift | 19 +++++++--- .../layout_string_witnesses_dynamic.swift | 36 ++++++++++++++++--- .../layout_string_witnesses_static.swift | 31 ++++++++++++++++ 4 files changed, 79 insertions(+), 9 deletions(-) diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 559f41acb9f99..e7fff9ccb92a6 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2895,7 +2895,7 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, auto tag = existential->isClassBounded() ? RefCountingKind::Unknown : RefCountingKind::Existential; writer.writeBytes(((uint64_t)tag << 56) | offset); - previousFieldOffset = existential->isClassBounded() ? fieldType->vw_size() - sizeof(uintptr_t) : 0; + previousFieldOffset = fieldType->vw_size() - (existential->isClassBounded() ? sizeof(uintptr_t) : (NumWords_ValueBuffer * sizeof(uintptr_t))); fullOffset += fieldType->vw_size(); } else { metadata: diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index 9190d06cc3d37..a0fc4f63144f8 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -279,6 +279,17 @@ public struct MultiProtocolExistentialWrapper { } } +public struct AnyWrapper { + let x: Int = 0 + let y: Any + let z: AnyObject + + public init(y: Any, z: AnyObject) { + self.y = y + self.z = z + } +} + #if os(macOS) import Foundation @@ -343,8 +354,8 @@ public struct NestedWrapper { } struct InternalGeneric { - let x: T let y: Int + let x: T } public enum SinglePayloadSimpleClassEnum { @@ -615,19 +626,19 @@ public func allocateInternalGenericPtr(of tpe: T.Type) -> UnsafeMutableRawPoi @inline(never) public func testGenericAssign(_ ptr: __owned UnsafeMutableRawPointer, from x: T) { let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) - let x = InternalGeneric(x: x, y: 23) + let x = InternalGeneric(y: 23, x: x) testAssign(ptr, from: x) } @inline(never) public func testGenericInit(_ ptr: __owned UnsafeMutableRawPointer, to x: T) { let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) - let x = InternalGeneric(x: x, y: 23) + let x = InternalGeneric(y: 23, x: x) testInit(ptr, to: x) } @inline(never) public func testGenericDestroy(_ ptr: __owned UnsafeMutableRawPointer, of tpe: T.Type) { - let ptr = ptr.assumingMemoryBound(to: tpe) + let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) testDestroy(ptr) } diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index 2d9dd91754703..932b19612f79d 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -4,7 +4,7 @@ // RUN: %target-codesign %t/%target-library-name(layout_string_witnesses_types) // RUN: %target-swift-frontend -enable-experimental-feature LayoutStringValueWitnesses -enable-experimental-feature LayoutStringValueWitnessesInstantiation -enable-layout-string-value-witnesses -enable-layout-string-value-witnesses-instantiation -enable-library-evolution -enable-autolinking-runtime-compatibility-bytecode-layouts -emit-module -emit-module-path=%t/layout_string_witnesses_types_resilient.swiftmodule %S/Inputs/layout_string_witnesses_types_resilient.swift // RUN: %target-build-swift -g -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnesses -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnessesInstantiation -Xfrontend -enable-layout-string-value-witnesses -Xfrontend -enable-layout-string-value-witnesses-instantiation -Xfrontend -enable-library-evolution -c -parse-as-library -o %t/layout_string_witnesses_types_resilient.o %S/Inputs/layout_string_witnesses_types_resilient.swift -// RUN: %target-build-swift -g -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnesses -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnessesInstantiation -Xfrontend -enable-layout-string-value-witnesses -Xfrontend -enable-layout-string-value-witnesses-instantiation -Xfrontend -enable-type-layout -parse-stdlib -module-name layout_string_witnesses_dynamic -llayout_string_witnesses_types -L%t %t/layout_string_witnesses_types_resilient.o -I %t -o %t/main %s %target-rpath(%t) +// RUN: %target-build-swift -g -parse-stdlib -module-name layout_string_witnesses_dynamic -llayout_string_witnesses_types -L%t %t/layout_string_witnesses_types_resilient.o -I %t -o %t/main %s %target-rpath(%t) // RUN: %target-codesign %t/main // RUN: %target-run %t/main %t/%target-library-name(layout_string_witnesses_types) | %FileCheck %s --check-prefix=CHECK -check-prefix=CHECK-%target-os @@ -92,7 +92,35 @@ func testGeneric() { testGeneric() -func testPrespecializedStructAnyObject() { +func testGenericAny() { + let ptr = allocateInternalGenericPtr(of: Any.self) + + do { + let x: Any = TestClass() + testGenericInit(ptr, to: x as Any) + } + + do { + let y: Any = TestClass() + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + testGenericAssign(ptr, from: y as Any) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + testGenericDestroy(ptr, of: Any.self) + + ptr.deallocate() +} + +testGenericAny() + +func testPrespecializedAnyObject() { let ptr = UnsafeMutablePointer>.allocate(capacity: 1) do { @@ -120,7 +148,7 @@ func testPrespecializedStructAnyObject() { ptr.deallocate() } -testPrespecializedStructAnyObject() +testPrespecializedAnyObject() func testPrespecializedStructSimpleClass() { let ptr = UnsafeMutablePointer>.allocate(capacity: 1) @@ -866,7 +894,7 @@ func testResilientSinglePayloadEnumComplexTag() { testResilientSinglePayloadEnumComplexTag() func testResilientMultiPayloadEnumTag() { - let x = switch getResilientMultiPayloadEnumEmpty0() { + let x = switch getResilientMultiPayloadEnumEmpty0(AnyObject.self) { case .nonEmpty0: 0 case .nonEmpty1: 1 case .empty0: 2 diff --git a/test/Interpreter/layout_string_witnesses_static.swift b/test/Interpreter/layout_string_witnesses_static.swift index 738c21e47f114..8ecab346e4ea1 100644 --- a/test/Interpreter/layout_string_witnesses_static.swift +++ b/test/Interpreter/layout_string_witnesses_static.swift @@ -293,6 +293,37 @@ func testExistentialStructBox() { testExistentialStructBox() +func testAnyWrapper() { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + + do { + let x = TestClass() + testInit(ptr, to: AnyWrapper(y: x, z: SimpleClass(x: 23))) + } + + do { + let y = TestClass() + + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testAssign(ptr, from: AnyWrapper(y: y, z: SimpleClass(x: 32))) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testDestroy(ptr) + + ptr.deallocate() +} + +testAnyWrapper() + class ClassWithABC: A, B, C { deinit { print("ClassWithABC deinitialized!") From 734e9b8ed4cf0ed7b0d86a877407ce26bf6d8d89 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:57:57 -0700 Subject: [PATCH 14/26] [IRGen] Make enum metadata non-const for instantiated layout strings --- lib/IRGen/GenMeta.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index b4b4198116c65..bda693170b7de 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -5675,7 +5675,7 @@ namespace { llvm::Constant *emitNominalTypeDescriptor() { auto descriptor = EnumContextDescriptorBuilder( - IGM, Target, RequireMetadata, !!getLayoutString()) + IGM, Target, RequireMetadata, hasLayoutString()) .emit(); return descriptor; } @@ -5724,7 +5724,7 @@ namespace { } bool canBeConstant() { - return !HasUnfilledPayloadSize;// && !hasInstantiatedLayoutString(); + return !HasUnfilledPayloadSize && !hasInstantiatedLayoutString(); } }; From fd967fd9e3817e51dad8bd3d61484ad259cdccbb Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:58:25 -0700 Subject: [PATCH 15/26] [Runtime] Properly handle boxed references in layout string instantiation --- stdlib/public/runtime/Metadata.cpp | 51 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index e7fff9ccb92a6..724b11bfb6d5c 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2781,7 +2781,8 @@ void swift::swift_initStructMetadataWithLayoutString( } size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { - if (type->vw_size() == 0 || type->getValueWitnesses()->isPOD()) { + auto *vwt = type->getValueWitnesses(); + if (type->vw_size() == 0 || vwt->isPOD()) { return 0; } else if (auto *tuple = dyn_cast(type)) { size_t res = 0; @@ -2789,14 +2790,13 @@ size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { res += _swift_refCountBytesForMetatype(tuple->getElement(i).Type); } return res; + } else if (vwt == &VALUE_WITNESS_SYM(Bo) || + vwt == &VALUE_WITNESS_SYM(BO) || + vwt == &VALUE_WITNESS_SYM(Bb)) { + return sizeof(uint64_t); } else if (auto *cls = type->getClassObject()) { if (cls->isTypeMetadata()) { - auto *vwt = cls->getValueWitnesses(); - if (vwt != &VALUE_WITNESS_SYM(Bo) && - vwt != &VALUE_WITNESS_SYM(BO) && - vwt != &VALUE_WITNESS_SYM(Bb)) { - goto metadata; - } + goto metadata; } return sizeof(uint64_t); } else if (type->hasLayoutString()) { @@ -2819,9 +2819,10 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, size_t unalignedOffset = fullOffset; fullOffset = roundUpToAlignMask(fullOffset, fieldType->vw_alignment() - 1); size_t offset = fullOffset - unalignedOffset + previousFieldOffset; + auto *vwt = fieldType->getValueWitnesses(); if (fieldType->vw_size() == 0) { return; - } else if (fieldType->getValueWitnesses()->isPOD()) { + } else if (vwt->isPOD()) { // No need to handle PODs previousFieldOffset = offset + fieldType->vw_size(); fullOffset += fieldType->vw_size(); @@ -2831,6 +2832,25 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, tuple->getElement(i).Type, fullOffset, previousFieldOffset); } + } else if (vwt == &VALUE_WITNESS_SYM(Bo)) { + auto tag = RefCountingKind::NativeStrong; + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); + } else if (vwt == &VALUE_WITNESS_SYM(BO)) { + #if SWIFT_OBJC_INTEROP + auto tag = RefCountingKind::ObjC; + #else + auto tag = RefCountingKind::Unknown; + #endif + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); + } else if (vwt == &VALUE_WITNESS_SYM(Bb)) { + auto tag = RefCountingKind::Bridge; + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); } else if (auto *cls = fieldType->getClassObject()) { RefCountingKind tag; if (!cls->isTypeMetadata()) { @@ -2840,20 +2860,7 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, tag = RefCountingKind::Unknown; #endif } else { - auto *vwt = cls->getValueWitnesses(); - if (vwt == &VALUE_WITNESS_SYM(Bo)) { - tag = RefCountingKind::NativeStrong; - } else if (vwt == &VALUE_WITNESS_SYM(BO)) { - #if SWIFT_OBJC_INTEROP - tag = RefCountingKind::ObjC; - #else - tag = RefCountingKind::Unknown; - #endif - } else if (vwt == &VALUE_WITNESS_SYM(Bb)) { - tag = RefCountingKind::Bridge; - } else { - goto metadata; - }; + goto metadata; } writer.writeBytes(((uint64_t)tag << 56) | offset); From 0ff7f1db232617e861e267b8b56582296019fe5d Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:58:47 -0700 Subject: [PATCH 16/26] [IRGen] Don't generate layout strings for moveonly types --- lib/IRGen/TypeLayout.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index 06c5934464d62..f7e995679a6a2 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -1741,7 +1741,7 @@ AlignedGroupEntry::layoutString(IRGenModule &IGM, bool AlignedGroupEntry::refCountString(IRGenModule &IGM, LayoutStringBuilder &B, GenericSignature genericSig) const { - if (!isFixedSize(IGM)) { + if (!isFixedSize(IGM) || ty.isMoveOnly()) { return false; } From 21f40643802419cc1706a5598471f0f952d7ea7e Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:59:15 -0700 Subject: [PATCH 17/26] [Runtime] Fix singlePayloadEnumSimpleAssignWithCopy --- stdlib/public/runtime/BytecodeLayouts.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 9edb27a78f0a3..a651966e57e30 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -1404,13 +1404,13 @@ static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, if (srcTagBytes >= xiTagValues && destTagBytes >= xiTagValues) { return; - } else if (destTagBytes >= xiTagValues) { + } else if (srcTagBytes >= xiTagValues) { const uint8_t *end = (reader.layoutStr + refCountBytes); while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; - } else if (srcTagBytes >= xiTagValues) { + } else if (destTagBytes >= xiTagValues) { const uint8_t *end = (reader.layoutStr + refCountBytes); while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); From a0e48a33623d92f5c177cc29a62a928096880563 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 11:02:59 -0700 Subject: [PATCH 18/26] [Runtime] Assert when trying to get layout string from class --- include/swift/ABI/Metadata.h | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index ca9096a132574..99fb5ac17e786 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -323,13 +323,9 @@ struct TargetMetadata { } const uint8_t *getLayoutString() const { - // assert(hasLayoutString()); - // if (isAnyClass()) { - // return asFullMetadata( - // reinterpret_cast *>( - // this)) - // ->layoutString; - // } + assert(hasLayoutString()); + // Classes should not have layout strings + assert(!isAnyClass()); return asFullMetadata(this)->layoutString; } From 904418f517633f409cb2d90ebd3ba393fa37f105 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 13:16:00 -0700 Subject: [PATCH 19/26] [Test] Remove accidentally commited broken test code --- .../layout_string_witnesses_types.swift | 41 ------------------ .../layout_string_witnesses_dynamic.swift | 42 +------------------ 2 files changed, 1 insertion(+), 82 deletions(-) diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index a0fc4f63144f8..60312f3d0635b 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -223,47 +223,6 @@ public struct Recursive { } } -public protocol Foo: AnyObject {} - -public enum Repro { - public struct R1 { - let x: R2? - let y: Int - weak var z: Foo? - let w: [NSNotification.Name] - - public init(x: R2?, - y: Int, - z: Foo?, - w: [NSNotification.Name]) { - self.x = x - self.y = y - self.z = z - self.w = w - } - } - - public struct R2 { - let x: NSNotification.Name? - let y: Bool - let z: E1 - - public init(x: NSNotification.Name?, - y: Bool, - z: E1) { - self.x = x - self.y = y - self.z = z - } - } - - public enum E1: Int { - case x - case y - case z - } -} - public protocol A {} public protocol B {} public protocol C {} diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index 932b19612f79d..e7de40bb7fe5c 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -24,46 +24,6 @@ class TestClass { } } -class XX: Foo { - deinit { - print("XX deinitialized!") - } -} - -func testRepro() { - let ptr = allocateInternalGenericPtr(of: Repro.R1.self) - - var xx: XX? = XX() - - do { - let x = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: xx!, w: []) - testGenericInit(ptr, to: x) - } - - var yy: XX? = XX() - - do { - let y = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: yy!, w: []) - // CHECK: Before deinit - print("Before deinit") - - // CHECK-NEXT: XX deinitialized! - testGenericAssign(ptr, from: y) - xx = nil - } - - // CHECK-NEXT: Before deinit - print("Before deinit") - - // CHECK-NEXT: XX deinitialized! - testGenericDestroy(ptr, of: Repro.R1.self) - yy = nil - - ptr.deallocate() -} - -testRepro() - func testGeneric() { let ptr = allocateInternalGenericPtr(of: TestClass.self) @@ -894,7 +854,7 @@ func testResilientSinglePayloadEnumComplexTag() { testResilientSinglePayloadEnumComplexTag() func testResilientMultiPayloadEnumTag() { - let x = switch getResilientMultiPayloadEnumEmpty0(AnyObject.self) { + let x = switch getResilientMultiPayloadEnumEmpty0() { case .nonEmpty0: 0 case .nonEmpty1: 1 case .empty0: 2 From 4a8d23d30362889b6cbbc6a74bd616f0c10e81ff Mon Sep 17 00:00:00 2001 From: Karoy Lorentey Date: Tue, 22 Aug 2023 13:44:51 -0700 Subject: [PATCH 20/26] =?UTF-8?q?[stdlib]=20Add=20@discardableResult=20to?= =?UTF-8?q?=20Set.insert=E2=80=99s=20AnyHashable=20overload?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unlike the core Set.insert, this utility declaration wasn’t declared @discardableOverload, introducing an undesirable inconsistency. rdar://114008680 --- stdlib/public/core/SetAnyHashableExtensions.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/stdlib/public/core/SetAnyHashableExtensions.swift b/stdlib/public/core/SetAnyHashableExtensions.swift index bff5639cc9ae7..bab822cf0dfe3 100644 --- a/stdlib/public/core/SetAnyHashableExtensions.swift +++ b/stdlib/public/core/SetAnyHashableExtensions.swift @@ -16,6 +16,7 @@ extension Set where Element == AnyHashable { @inlinable + @discardableResult public mutating func insert( _ newMember: __owned ConcreteElement ) -> (inserted: Bool, memberAfterInsert: ConcreteElement) { From 3e952142e250376e283e5274e7a9717fe9cf60dc Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Wed, 23 Aug 2023 10:22:09 -0700 Subject: [PATCH 21/26] [CodeCompletion] Add test case for some/any type relations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test case used to fail at some point but it’s passing now. Add it so that we don’t break it in the future. rdar://97897857 --- .../IDE/complete_type_relation_any_some.swift | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/IDE/complete_type_relation_any_some.swift diff --git a/test/IDE/complete_type_relation_any_some.swift b/test/IDE/complete_type_relation_any_some.swift new file mode 100644 index 0000000000000..c14ed956f1154 --- /dev/null +++ b/test/IDE/complete_type_relation_any_some.swift @@ -0,0 +1,23 @@ +// RUN: %empty-directory(%t) +// RUN: %swift-ide-test -batch-code-completion -source-filename %s -filecheck %raw-FileCheck --completion-output-dir %t + +protocol Shape {} + +struct Square : Shape {} + +struct Test { + let protocolType: Shape + let structType: Square + + func testAny() -> any Shape { + return self.#^WITH_ANY_CONTEXTUAL_TYPE^# +// WITH_ANY_CONTEXTUAL_TYPE-DAG: Decl[InstanceVar]/CurrNominal/TypeRelation[Convertible]: protocolType[#any Shape#]; +// WITH_ANY_CONTEXTUAL_TYPE-DAG: Decl[InstanceVar]/CurrNominal/TypeRelation[Convertible]: structType[#Square#]; + } + + func testSome() -> some Shape { + return self.#^WITH_SOME_CONTEXTUAL_TYPE^# +// WITH_SOME_CONTEXTUAL_TYPE-DAG: Decl[InstanceVar]/CurrNominal: protocolType[#any Shape#]; +// WITH_SOME_CONTEXTUAL_TYPE-DAG: Decl[InstanceVar]/CurrNominal/TypeRelation[Convertible]: structType[#Square#]; + } +} From 7d13da43ded184e85d4ae6f83b9c08fe2279cb4e Mon Sep 17 00:00:00 2001 From: Tristan Labelle Date: Wed, 23 Aug 2023 13:30:12 -0400 Subject: [PATCH 22/26] Remove realpath in lit site config. --- test/lit.site.cfg.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lit.site.cfg.in b/test/lit.site.cfg.in index 4471ca0901149..5bf5facd3eaba 100644 --- a/test/lit.site.cfg.in +++ b/test/lit.site.cfg.in @@ -13,6 +13,7 @@ import os import platform import sys +import lit.util config.cmake = "@CMAKE_COMMAND@" config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@" @@ -172,6 +173,6 @@ if '@SWIFT_SWIFT_PARSER@' == 'TRUE': # Let the main config do the real work. if config.test_exec_root is None: - config.test_exec_root = os.path.dirname(os.path.realpath(__file__)) + config.test_exec_root = os.path.dirname(lit.util.abs_path_preserve_drive(__file__)) lit_config.load_config( config, os.path.join(config.swift_src_root, "test", "lit.cfg")) From e3a1bac8596739242921bed88b69519ce758f00c Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Thu, 24 Aug 2023 11:01:14 -0700 Subject: [PATCH 23/26] [Runtime] Use copy of addrOffset in single payload assign with copy --- stdlib/public/runtime/BytecodeLayouts.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index a651966e57e30..aac184ad30b7a 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -1412,8 +1412,9 @@ static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, return; } else if (destTagBytes >= xiTagValues) { const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; while (reader.layoutStr < end) { - handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); } } else { reader.skip(refCountBytes); @@ -1448,8 +1449,9 @@ static void singlePayloadEnumFNAssignWithCopy(const Metadata *metadata, return; } else if (destTag == 0) { const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; while (reader.layoutStr < end) { - handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); } } else { reader.skip(refCountBytes); @@ -1484,8 +1486,9 @@ static void singlePayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, return; } else if (destTag == 0) { const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; while (reader.layoutStr < end) { - handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); } } else { reader.skip(refCountBytes); @@ -1545,8 +1548,9 @@ static void singlePayloadEnumGenericAssignWithCopy(const Metadata *metadata, return; } else if (destTag == 0) { const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; while (reader.layoutStr < end) { - handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); } } else { reader.skip(refCountBytes); From 97b2dc722e1ee736638313a118fa263e8fba8d95 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Thu, 24 Aug 2023 11:01:55 -0700 Subject: [PATCH 24/26] [IRGen] Assign extraTagByteCount to the correct union field in addSinglePayloadEnumFN --- lib/IRGen/TypeLayout.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index f7e995679a6a2..33ff42e6eb661 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -160,7 +160,7 @@ class LayoutStringBuilder { RefCounting op; op.kind = RefCountingKind::SinglePayloadEnumFN; op.singlePayloadEnumFN.tagFn = tagFn; - op.singlePayloadEnumSimple.extraTagByteCount = extraTagByteCount; + op.singlePayloadEnumFN.extraTagByteCount = extraTagByteCount; op.singlePayloadEnumFN.payload = payload; refCountings.push_back(op); } From 4e22193d9e6211e936fc7dea2f2661a36d827f7c Mon Sep 17 00:00:00 2001 From: Nate Chandler Date: Thu, 24 Aug 2023 13:45:18 -0700 Subject: [PATCH 25/26] [Test] Require asserts for some experimental-feature tests. For LazyImmediate and TupleConformances. --- test/IRGen/tuple_conformances.swift | 3 +++ test/Interpreter/lazy/deferred_syntax_errors.swift | 3 +++ test/Interpreter/lazy/deferred_type_errors.swift | 3 +++ test/Interpreter/lazy/globals.swift | 3 +++ test/Interpreter/lazy/linkage_promotion.swift | 3 +++ 5 files changed, 15 insertions(+) diff --git a/test/IRGen/tuple_conformances.swift b/test/IRGen/tuple_conformances.swift index bcb9075da4849..a85fb8c4a53de 100644 --- a/test/IRGen/tuple_conformances.swift +++ b/test/IRGen/tuple_conformances.swift @@ -1,5 +1,8 @@ // RUN: %target-swift-frontend -emit-ir -parse-stdlib -primary-file %s -enable-experimental-feature TupleConformances -parse-as-library | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + import Swift protocol P { diff --git a/test/Interpreter/lazy/deferred_syntax_errors.swift b/test/Interpreter/lazy/deferred_syntax_errors.swift index 6788805237b04..5e112c69994b8 100644 --- a/test/Interpreter/lazy/deferred_syntax_errors.swift +++ b/test/Interpreter/lazy/deferred_syntax_errors.swift @@ -1,6 +1,9 @@ // REQUIRES: OS=macosx // RUN: %target-jit-run %s -enable-experimental-feature LazyImmediate | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + // Tests that parsing of function bodies is deferred // to runtime when the interpreter is invoked using // experimental lazy compilation. Syntax errors in diff --git a/test/Interpreter/lazy/deferred_type_errors.swift b/test/Interpreter/lazy/deferred_type_errors.swift index 8b68308b09282..b905f932264dd 100644 --- a/test/Interpreter/lazy/deferred_type_errors.swift +++ b/test/Interpreter/lazy/deferred_type_errors.swift @@ -1,6 +1,9 @@ // REQUIRES: OS=macosx // RUN: %target-jit-run %s -enable-experimental-feature LazyImmediate | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + // Tests that type checking of function bodies is // deferred to runtime when the interpreter is invoked // using experimental lazy compilation. Type errors in diff --git a/test/Interpreter/lazy/globals.swift b/test/Interpreter/lazy/globals.swift index e4b2b9381ae30..17da6a282d42c 100644 --- a/test/Interpreter/lazy/globals.swift +++ b/test/Interpreter/lazy/globals.swift @@ -1,6 +1,9 @@ // REQUIRES: OS=macosx // RUN: %target-jit-run %s -enable-experimental-feature LazyImmediate | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + // Tests that piecewise compilation works with global variables let x = 1 diff --git a/test/Interpreter/lazy/linkage_promotion.swift b/test/Interpreter/lazy/linkage_promotion.swift index 72d4f3d5c095f..1dbfcc4134633 100644 --- a/test/Interpreter/lazy/linkage_promotion.swift +++ b/test/Interpreter/lazy/linkage_promotion.swift @@ -2,6 +2,9 @@ // REQUIRES: swift_interpreter // RUN: %target-jit-run %s -enable-experimental-feature LazyImmediate | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + // Tests that the linkage of private symbols is // promoted to hidden external, allowing // single-function compilation of non-public symbols. From d5a3e2e630443c72e3d836556bfed5e66f744106 Mon Sep 17 00:00:00 2001 From: Kshitij Jain <8564203+jkshtj@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:57:10 -0600 Subject: [PATCH 26/26] [AutoDiff] Fixes memory leaks in autodiff linear map context allocation builtins (#67944) When the differentiating a function containing loops, we allocate a linear map context object on the heap. This context object may store non-trivial objects, such as closures, that need to be released explicitly. Fix the autodiff linear map context allocation builtins to correctly release such objects and not just free the memory they occupy. --- include/swift/AST/Builtins.def | 8 +- include/swift/Runtime/RuntimeFunctions.def | 16 ++-- lib/AST/Builtins.cpp | 11 +-- lib/IRGen/GenBuiltin.cpp | 16 ++-- lib/IRGen/GenCall.cpp | 20 +++-- lib/IRGen/GenCall.h | 11 ++- lib/SIL/IR/OperandOwnership.cpp | 5 +- lib/SIL/IR/ValueOwnership.cpp | 4 +- lib/SIL/Utils/MemAccessUtils.cpp | 4 +- lib/SILGen/SILGenBuiltin.cpp | 16 ++-- .../Differentiation/VJPCloner.cpp | 41 +++++++--- .../AccessEnforcementReleaseSinking.cpp | 4 +- stdlib/public/runtime/AutoDiffSupport.cpp | 43 +++++++++- stdlib/public/runtime/AutoDiffSupport.h | 82 ++++++++++++++++++- test/AutoDiff/IRGen/runtime.swift | 23 ++---- test/AutoDiff/SILGen/autodiff_builtins.swift | 28 +++---- ...ential_not_leaked_in_func_with_loops.swift | 60 ++++++++++++++ 17 files changed, 289 insertions(+), 103 deletions(-) create mode 100644 test/AutoDiff/stdlib/callee_differential_not_leaked_in_func_with_loops.swift diff --git a/include/swift/AST/Builtins.def b/include/swift/AST/Builtins.def index fb271e7e1ba88..286ebdb1bfd6a 100644 --- a/include/swift/AST/Builtins.def +++ b/include/swift/AST/Builtins.def @@ -985,14 +985,14 @@ BUILTIN_MISC_OPERATION_WITH_SILGEN(CreateAsyncTaskInGroup, /// is a pure value and therefore we can consider it as readnone). BUILTIN_MISC_OPERATION_WITH_SILGEN(GlobalStringTablePointer, "globalStringTablePointer", "n", Special) -// autoDiffCreateLinearMapContext: (Builtin.Word) -> Builtin.NativeObject -BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffCreateLinearMapContext, "autoDiffCreateLinearMapContext", "", Special) +// autoDiffCreateLinearMapContextWithType: (T.Type) -> Builtin.NativeObject +BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffCreateLinearMapContextWithType, "autoDiffCreateLinearMapContextWithType", "", Special) // autoDiffProjectTopLevelSubcontext: (Builtin.NativeObject) -> Builtin.RawPointer BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffProjectTopLevelSubcontext, "autoDiffProjectTopLevelSubcontext", "n", Special) -// autoDiffAllocateSubcontext: (Builtin.NativeObject, Builtin.Word) -> Builtin.RawPointer -BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffAllocateSubcontext, "autoDiffAllocateSubcontext", "", Special) +// autoDiffAllocateSubcontextWithType: (Builtin.NativeObject, T.Type) -> Builtin.RawPointer +BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffAllocateSubcontextWithType, "autoDiffAllocateSubcontextWithType", "", Special) /// Build a Builtin.Executor value from an "ordinary" serial executor /// reference. diff --git a/include/swift/Runtime/RuntimeFunctions.def b/include/swift/Runtime/RuntimeFunctions.def index 812f807d971ac..5a788b02bf7a7 100644 --- a/include/swift/Runtime/RuntimeFunctions.def +++ b/include/swift/Runtime/RuntimeFunctions.def @@ -2273,12 +2273,12 @@ FUNCTION(TaskGroupDestroy, ATTRS(NoUnwind), EFFECT(Concurrency)) -// AutoDiffLinearMapContext *swift_autoDiffCreateLinearMapContext(size_t); -FUNCTION(AutoDiffCreateLinearMapContext, - swift_autoDiffCreateLinearMapContext, SwiftCC, +// AutoDiffLinearMapContext *swift_autoDiffCreateLinearMapContextWithType(const Metadata *); +FUNCTION(AutoDiffCreateLinearMapContextWithType, + swift_autoDiffCreateLinearMapContextWithType, SwiftCC, DifferentiationAvailability, RETURNS(RefCountedPtrTy), - ARGS(SizeTy), + ARGS(TypeMetadataPtrTy), ATTRS(NoUnwind, ArgMemOnly), EFFECT(AutoDiff)) @@ -2291,12 +2291,12 @@ FUNCTION(AutoDiffProjectTopLevelSubcontext, ATTRS(NoUnwind, ArgMemOnly), EFFECT(AutoDiff)) -// void *swift_autoDiffAllocateSubcontext(AutoDiffLinearMapContext *, size_t); -FUNCTION(AutoDiffAllocateSubcontext, - swift_autoDiffAllocateSubcontext, SwiftCC, +// void *swift_autoDiffAllocateSubcontextWithType(AutoDiffLinearMapContext *, const Metadata *); +FUNCTION(AutoDiffAllocateSubcontextWithType, + swift_autoDiffAllocateSubcontextWithType, SwiftCC, DifferentiationAvailability, RETURNS(Int8PtrTy), - ARGS(RefCountedPtrTy, SizeTy), + ARGS(RefCountedPtrTy, TypeMetadataPtrTy), ATTRS(NoUnwind, ArgMemOnly), EFFECT(AutoDiff)) diff --git a/lib/AST/Builtins.cpp b/lib/AST/Builtins.cpp index 035cbf9beee5a..fe168edc629bc 100644 --- a/lib/AST/Builtins.cpp +++ b/lib/AST/Builtins.cpp @@ -1609,7 +1609,8 @@ static ValueDecl *getBuildComplexEqualitySerialExecutorRef(ASTContext &ctx, static ValueDecl *getAutoDiffCreateLinearMapContext(ASTContext &ctx, Identifier id) { return getBuiltinFunction( - id, {BuiltinIntegerType::getWordType(ctx)}, ctx.TheNativeObjectType); + ctx, id, _thin, _generics(_unrestricted), + _parameters(_metatype(_typeparam(0))), _nativeObject); } static ValueDecl *getAutoDiffProjectTopLevelSubcontext(ASTContext &ctx, @@ -1621,8 +1622,8 @@ static ValueDecl *getAutoDiffProjectTopLevelSubcontext(ASTContext &ctx, static ValueDecl *getAutoDiffAllocateSubcontext(ASTContext &ctx, Identifier id) { return getBuiltinFunction( - id, {ctx.TheNativeObjectType, BuiltinIntegerType::getWordType(ctx)}, - ctx.TheRawPointerType); + ctx, id, _thin, _generics(_unrestricted), + _parameters(_nativeObject, _metatype(_typeparam(0))), _rawPointer); } static ValueDecl *getPoundAssert(ASTContext &Context, Identifier Id) { @@ -2966,13 +2967,13 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) { case BuiltinValueKind::HopToActor: return getHopToActor(Context, Id); - case BuiltinValueKind::AutoDiffCreateLinearMapContext: + case BuiltinValueKind::AutoDiffCreateLinearMapContextWithType: return getAutoDiffCreateLinearMapContext(Context, Id); case BuiltinValueKind::AutoDiffProjectTopLevelSubcontext: return getAutoDiffProjectTopLevelSubcontext(Context, Id); - case BuiltinValueKind::AutoDiffAllocateSubcontext: + case BuiltinValueKind::AutoDiffAllocateSubcontextWithType: return getAutoDiffAllocateSubcontext(Context, Id); } diff --git a/lib/IRGen/GenBuiltin.cpp b/lib/IRGen/GenBuiltin.cpp index c697557a38684..5dfaebc1d0318 100644 --- a/lib/IRGen/GenBuiltin.cpp +++ b/lib/IRGen/GenBuiltin.cpp @@ -1307,9 +1307,10 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin, return; } - if (Builtin.ID == BuiltinValueKind::AutoDiffCreateLinearMapContext) { - auto topLevelSubcontextSize = args.claimNext(); - out.add(emitAutoDiffCreateLinearMapContext(IGF, topLevelSubcontextSize) + if (Builtin.ID == BuiltinValueKind::AutoDiffCreateLinearMapContextWithType) { + auto topLevelSubcontextMetaType = args.claimNext(); + out.add(emitAutoDiffCreateLinearMapContextWithType( + IGF, topLevelSubcontextMetaType) .getAddress()); return; } @@ -1322,12 +1323,13 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin, return; } - if (Builtin.ID == BuiltinValueKind::AutoDiffAllocateSubcontext) { + if (Builtin.ID == BuiltinValueKind::AutoDiffAllocateSubcontextWithType) { Address allocatorAddr(args.claimNext(), IGF.IGM.RefCountedStructTy, IGF.IGM.getPointerAlignment()); - auto size = args.claimNext(); - out.add( - emitAutoDiffAllocateSubcontext(IGF, allocatorAddr, size).getAddress()); + auto subcontextMetatype = args.claimNext(); + out.add(emitAutoDiffAllocateSubcontextWithType(IGF, allocatorAddr, + subcontextMetatype) + .getAddress()); return; } diff --git a/lib/IRGen/GenCall.cpp b/lib/IRGen/GenCall.cpp index c5f264d0e2c8f..c92ac7e86887c 100644 --- a/lib/IRGen/GenCall.cpp +++ b/lib/IRGen/GenCall.cpp @@ -5479,11 +5479,13 @@ IRGenFunction::getFunctionPointerForResumeIntrinsic(llvm::Value *resume) { return fnPtr; } -Address irgen::emitAutoDiffCreateLinearMapContext( - IRGenFunction &IGF, llvm::Value *topLevelSubcontextSize) { +Address irgen::emitAutoDiffCreateLinearMapContextWithType( + IRGenFunction &IGF, llvm::Value *topLevelSubcontextMetatype) { + topLevelSubcontextMetatype = IGF.Builder.CreateBitCast( + topLevelSubcontextMetatype, IGF.IGM.TypeMetadataPtrTy); auto *call = IGF.Builder.CreateCall( - IGF.IGM.getAutoDiffCreateLinearMapContextFunctionPointer(), - {topLevelSubcontextSize}); + IGF.IGM.getAutoDiffCreateLinearMapContextWithTypeFunctionPointer(), + {topLevelSubcontextMetatype}); call->setDoesNotThrow(); call->setCallingConv(IGF.IGM.SwiftCC); return Address(call, IGF.IGM.RefCountedStructTy, @@ -5500,11 +5502,13 @@ Address irgen::emitAutoDiffProjectTopLevelSubcontext( return Address(call, IGF.IGM.Int8Ty, IGF.IGM.getPointerAlignment()); } -Address irgen::emitAutoDiffAllocateSubcontext( - IRGenFunction &IGF, Address context, llvm::Value *size) { +Address irgen::emitAutoDiffAllocateSubcontextWithType( + IRGenFunction &IGF, Address context, llvm::Value *subcontextMetatype) { + subcontextMetatype = + IGF.Builder.CreateBitCast(subcontextMetatype, IGF.IGM.TypeMetadataPtrTy); auto *call = IGF.Builder.CreateCall( - IGF.IGM.getAutoDiffAllocateSubcontextFunctionPointer(), - {context.getAddress(), size}); + IGF.IGM.getAutoDiffAllocateSubcontextWithTypeFunctionPointer(), + {context.getAddress(), subcontextMetatype}); call->setDoesNotThrow(); call->setCallingConv(IGF.IGM.SwiftCC); return Address(call, IGF.IGM.Int8Ty, IGF.IGM.getPointerAlignment()); diff --git a/lib/IRGen/GenCall.h b/lib/IRGen/GenCall.h index c64e97dc8a264..70879ce628d05 100644 --- a/lib/IRGen/GenCall.h +++ b/lib/IRGen/GenCall.h @@ -261,12 +261,15 @@ namespace irgen { CanSILFunctionType fnType, Explosion &result, Explosion &error); - Address emitAutoDiffCreateLinearMapContext( - IRGenFunction &IGF, llvm::Value *topLevelSubcontextSize); + Address emitAutoDiffCreateLinearMapContextWithType( + IRGenFunction &IGF, llvm::Value *topLevelSubcontextMetatype); + Address emitAutoDiffProjectTopLevelSubcontext( IRGenFunction &IGF, Address context); - Address emitAutoDiffAllocateSubcontext( - IRGenFunction &IGF, Address context, llvm::Value *size); + + Address + emitAutoDiffAllocateSubcontextWithType(IRGenFunction &IGF, Address context, + llvm::Value *subcontextMetatype); FunctionPointer getFunctionPointerForDispatchCall(IRGenModule &IGM, const FunctionPointer &fn); diff --git a/lib/SIL/IR/OperandOwnership.cpp b/lib/SIL/IR/OperandOwnership.cpp index 825ea48e9e85b..b61fa8dfc6e51 100644 --- a/lib/SIL/IR/OperandOwnership.cpp +++ b/lib/SIL/IR/OperandOwnership.cpp @@ -944,7 +944,7 @@ BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, InitializeDistributedRemoteActor) BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, InitializeNonDefaultDistributedActor) -BUILTIN_OPERAND_OWNERSHIP(PointerEscape, AutoDiffAllocateSubcontext) +BUILTIN_OPERAND_OWNERSHIP(PointerEscape, AutoDiffAllocateSubcontextWithType) BUILTIN_OPERAND_OWNERSHIP(PointerEscape, AutoDiffProjectTopLevelSubcontext) // FIXME: ConvertTaskToJob is documented as taking NativePointer. It's operand's @@ -956,8 +956,7 @@ BUILTIN_OPERAND_OWNERSHIP(BitwiseEscape, BuildComplexEqualitySerialExecutorRef) BUILTIN_OPERAND_OWNERSHIP(BitwiseEscape, BuildDefaultActorExecutorRef) BUILTIN_OPERAND_OWNERSHIP(BitwiseEscape, BuildMainActorExecutorRef) -BUILTIN_OPERAND_OWNERSHIP(TrivialUse, AutoDiffCreateLinearMapContext) - +BUILTIN_OPERAND_OWNERSHIP(TrivialUse, AutoDiffCreateLinearMapContextWithType) #undef BUILTIN_OPERAND_OWNERSHIP #define SHOULD_NEVER_VISIT_BUILTIN(ID) \ diff --git a/lib/SIL/IR/ValueOwnership.cpp b/lib/SIL/IR/ValueOwnership.cpp index 104348cf8d862..572b158e6ca29 100644 --- a/lib/SIL/IR/ValueOwnership.cpp +++ b/lib/SIL/IR/ValueOwnership.cpp @@ -572,9 +572,9 @@ CONSTANT_OWNERSHIP_BUILTIN(None, InitializeDefaultActor) CONSTANT_OWNERSHIP_BUILTIN(None, DestroyDefaultActor) CONSTANT_OWNERSHIP_BUILTIN(None, InitializeDistributedRemoteActor) CONSTANT_OWNERSHIP_BUILTIN(None, InitializeNonDefaultDistributedActor) -CONSTANT_OWNERSHIP_BUILTIN(Owned, AutoDiffCreateLinearMapContext) +CONSTANT_OWNERSHIP_BUILTIN(Owned, AutoDiffCreateLinearMapContextWithType) CONSTANT_OWNERSHIP_BUILTIN(None, AutoDiffProjectTopLevelSubcontext) -CONSTANT_OWNERSHIP_BUILTIN(None, AutoDiffAllocateSubcontext) +CONSTANT_OWNERSHIP_BUILTIN(None, AutoDiffAllocateSubcontextWithType) CONSTANT_OWNERSHIP_BUILTIN(None, GetCurrentExecutor) CONSTANT_OWNERSHIP_BUILTIN(None, ResumeNonThrowingContinuationReturning) CONSTANT_OWNERSHIP_BUILTIN(None, ResumeThrowingContinuationReturning) diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index 6cf1a89f14f3a..b672f69550c50 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -2567,8 +2567,8 @@ static void visitBuiltinAddress(BuiltinInst *builtin, case BuiltinValueKind::CancelAsyncTask: case BuiltinValueKind::CreateAsyncTask: case BuiltinValueKind::CreateAsyncTaskInGroup: - case BuiltinValueKind::AutoDiffCreateLinearMapContext: - case BuiltinValueKind::AutoDiffAllocateSubcontext: + case BuiltinValueKind::AutoDiffCreateLinearMapContextWithType: + case BuiltinValueKind::AutoDiffAllocateSubcontextWithType: case BuiltinValueKind::InitializeDefaultActor: case BuiltinValueKind::InitializeDistributedRemoteActor: case BuiltinValueKind::InitializeNonDefaultDistributedActor: diff --git a/lib/SILGen/SILGenBuiltin.cpp b/lib/SILGen/SILGenBuiltin.cpp index 2dbcde3be080a..d907e73ef0e90 100644 --- a/lib/SILGen/SILGenBuiltin.cpp +++ b/lib/SILGen/SILGenBuiltin.cpp @@ -1708,16 +1708,15 @@ static ManagedValue emitBuiltinHopToActor(SILGenFunction &SGF, SILLocation loc, return ManagedValue::forObjectRValueWithoutOwnership(SGF.emitEmptyTuple(loc)); } -static ManagedValue emitBuiltinAutoDiffCreateLinearMapContext( +static ManagedValue emitBuiltinAutoDiffCreateLinearMapContextWithType( SILGenFunction &SGF, SILLocation loc, SubstitutionMap subs, ArrayRef args, SGFContext C) { ASTContext &ctx = SGF.getASTContext(); auto *builtinApply = SGF.B.createBuiltin( loc, - ctx.getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffCreateLinearMapContext)), - SILType::getNativeObjectType(ctx), - subs, + ctx.getIdentifier(getBuiltinName( + BuiltinValueKind::AutoDiffCreateLinearMapContextWithType)), + SILType::getNativeObjectType(ctx), subs, /*args*/ {args[0].getValue()}); return SGF.emitManagedRValueWithCleanup(builtinApply); } @@ -1736,16 +1735,15 @@ static ManagedValue emitBuiltinAutoDiffProjectTopLevelSubcontext( return ManagedValue::forObjectRValueWithoutOwnership(builtinApply); } -static ManagedValue emitBuiltinAutoDiffAllocateSubcontext( +static ManagedValue emitBuiltinAutoDiffAllocateSubcontextWithType( SILGenFunction &SGF, SILLocation loc, SubstitutionMap subs, ArrayRef args, SGFContext C) { ASTContext &ctx = SGF.getASTContext(); auto *builtinApply = SGF.B.createBuiltin( loc, ctx.getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffAllocateSubcontext)), - SILType::getRawPointerType(ctx), - subs, + getBuiltinName(BuiltinValueKind::AutoDiffAllocateSubcontextWithType)), + SILType::getRawPointerType(ctx), subs, /*args*/ {args[0].borrow(SGF, loc).getValue(), args[1].getValue()}); return ManagedValue::forObjectRValueWithoutOwnership(builtinApply); } diff --git a/lib/SILOptimizer/Differentiation/VJPCloner.cpp b/lib/SILOptimizer/Differentiation/VJPCloner.cpp index 7cb715b278553..50098dda2c173 100644 --- a/lib/SILOptimizer/Differentiation/VJPCloner.cpp +++ b/lib/SILOptimizer/Differentiation/VJPCloner.cpp @@ -17,6 +17,8 @@ #define DEBUG_TYPE "differentiation" +#include "swift/AST/Types.h" + #include "swift/SILOptimizer/Differentiation/VJPCloner.h" #include "swift/SILOptimizer/Analysis/DifferentiableActivityAnalysis.h" #include "swift/SILOptimizer/Differentiation/ADContext.h" @@ -118,15 +120,21 @@ class VJPCloner::Implementation final auto pullbackTupleType = remapASTType(pullbackInfo.getLinearMapTupleType(returnBB)->getCanonicalType()); Builder.setInsertionPoint(vjp->getEntryBlock()); - auto topLevelSubcontextSize = emitMemoryLayoutSize( - Builder, original->getLocation(), pullbackTupleType); + + auto pbTupleMetatypeType = + CanMetatypeType::get(pullbackTupleType, MetatypeRepresentation::Thick); + auto pbTupleMetatypeSILType = + SILType::getPrimitiveObjectType(pbTupleMetatypeType); + auto pbTupleMetatype = + Builder.createMetatype(original->getLocation(), pbTupleMetatypeSILType); + // Create an context. pullbackContextValue = Builder.createBuiltin( original->getLocation(), - getASTContext().getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffCreateLinearMapContext)), - SILType::getNativeObjectType(getASTContext()), - SubstitutionMap(), {topLevelSubcontextSize}); + getASTContext().getIdentifier(getBuiltinName( + BuiltinValueKind::AutoDiffCreateLinearMapContextWithType)), + SILType::getNativeObjectType(getASTContext()), SubstitutionMap(), + {pbTupleMetatype}); borrowedPullbackContextValue = Builder.createBeginBorrow( original->getLocation(), pullbackContextValue); LLVM_DEBUG(getADDebugStream() @@ -148,8 +156,8 @@ class VJPCloner::Implementation final return builtinAutoDiffAllocateSubcontextGenericSignature; auto &ctx = getASTContext(); auto *decl = cast(getBuiltinValueDecl( - ctx, ctx.getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffAllocateSubcontext)))); + ctx, ctx.getIdentifier(getBuiltinName( + BuiltinValueKind::AutoDiffAllocateSubcontextWithType)))); builtinAutoDiffAllocateSubcontextGenericSignature = decl->getGenericSignature(); assert(builtinAutoDiffAllocateSubcontextGenericSignature); @@ -1067,14 +1075,21 @@ EnumInst *VJPCloner::Implementation::buildPredecessorEnumValue( assert(enumEltType == rawPtrType); auto pbTupleType = remapASTType(pullbackInfo.getLinearMapTupleType(predBB)->getCanonicalType()); - SILValue pbTupleSize = - emitMemoryLayoutSize(Builder, loc, pbTupleType); + + auto pbTupleMetatypeType = + CanMetatypeType::get(pbTupleType, MetatypeRepresentation::Thick); + auto pbTupleMetatypeSILType = + SILType::getPrimitiveObjectType(pbTupleMetatypeType); + auto pbTupleMetatype = + Builder.createMetatype(original->getLocation(), pbTupleMetatypeSILType); + auto rawBufferValue = builder.createBuiltin( loc, - getASTContext().getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffAllocateSubcontext)), + getASTContext().getIdentifier(getBuiltinName( + BuiltinValueKind::AutoDiffAllocateSubcontextWithType)), rawPtrType, SubstitutionMap(), - {borrowedPullbackContextValue, pbTupleSize}); + {borrowedPullbackContextValue, pbTupleMetatype}); + auto typedBufferValue = builder.createPointerToAddress( loc, rawBufferValue, pbTupleVal->getType().getAddressType(), diff --git a/lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp b/lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp index 01069fe3e2774..20359097fcf3d 100644 --- a/lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp +++ b/lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp @@ -146,7 +146,7 @@ static bool isBarrier(SILInstruction *inst) { case BuiltinValueKind::COWBufferForReading: case BuiltinValueKind::GetCurrentAsyncTask: case BuiltinValueKind::GetCurrentExecutor: - case BuiltinValueKind::AutoDiffCreateLinearMapContext: + case BuiltinValueKind::AutoDiffCreateLinearMapContextWithType: case BuiltinValueKind::EndAsyncLet: case BuiltinValueKind::EndAsyncLetLifetime: case BuiltinValueKind::CreateTaskGroup: @@ -199,7 +199,7 @@ static bool isBarrier(SILInstruction *inst) { case BuiltinValueKind::ResumeThrowingContinuationReturning: case BuiltinValueKind::ResumeThrowingContinuationThrowing: case BuiltinValueKind::AutoDiffProjectTopLevelSubcontext: - case BuiltinValueKind::AutoDiffAllocateSubcontext: + case BuiltinValueKind::AutoDiffAllocateSubcontextWithType: case BuiltinValueKind::AddressOfBorrowOpaque: case BuiltinValueKind::UnprotectedAddressOfBorrowOpaque: return true; diff --git a/stdlib/public/runtime/AutoDiffSupport.cpp b/stdlib/public/runtime/AutoDiffSupport.cpp index 3873e6165a7f4..372bcdac9447f 100644 --- a/stdlib/public/runtime/AutoDiffSupport.cpp +++ b/stdlib/public/runtime/AutoDiffSupport.cpp @@ -13,7 +13,7 @@ #include "AutoDiffSupport.h" #include "swift/ABI/Metadata.h" #include "swift/Runtime/HeapObject.h" - +#include "llvm/ADT/SmallVector.h" #include using namespace swift; @@ -47,6 +47,13 @@ AutoDiffLinearMapContext::AutoDiffLinearMapContext() : HeapObject(&linearMapContextHeapMetadata) { } +AutoDiffLinearMapContext::AutoDiffLinearMapContext( + const Metadata *topLevelLinearMapContextMetadata) + : HeapObject(&linearMapContextHeapMetadata) { + allocatedContextObjects.push_back(AllocatedContextObjectRecord{ + topLevelLinearMapContextMetadata, projectTopLevelSubcontext()}); +} + void *AutoDiffLinearMapContext::projectTopLevelSubcontext() const { auto offset = alignTo( sizeof(AutoDiffLinearMapContext), alignof(AutoDiffLinearMapContext)); @@ -58,6 +65,16 @@ void *AutoDiffLinearMapContext::allocate(size_t size) { return allocator.Allocate(size, alignof(AutoDiffLinearMapContext)); } +void *AutoDiffLinearMapContext::allocateSubcontext( + const Metadata *contextObjectMetadata) { + auto size = contextObjectMetadata->vw_size(); + auto align = contextObjectMetadata->vw_alignment(); + auto *contextObjectPtr = allocator.Allocate(size, align); + allocatedContextObjects.push_back( + AllocatedContextObjectRecord{contextObjectMetadata, contextObjectPtr}); + return contextObjectPtr; +} + AutoDiffLinearMapContext *swift::swift_autoDiffCreateLinearMapContext( size_t topLevelLinearMapStructSize) { auto allocationSize = alignTo( @@ -68,11 +85,31 @@ AutoDiffLinearMapContext *swift::swift_autoDiffCreateLinearMapContext( } void *swift::swift_autoDiffProjectTopLevelSubcontext( - AutoDiffLinearMapContext *allocator) { - return allocator->projectTopLevelSubcontext(); + AutoDiffLinearMapContext *linearMapContext) { + return static_cast(linearMapContext->projectTopLevelSubcontext()); } void *swift::swift_autoDiffAllocateSubcontext( AutoDiffLinearMapContext *allocator, size_t size) { return allocator->allocate(size); } + +AutoDiffLinearMapContext *swift::swift_autoDiffCreateLinearMapContextWithType( + const Metadata *topLevelLinearMapContextMetadata) { + assert(topLevelLinearMapContextMetadata->getValueWitnesses() != nullptr); + auto topLevelLinearMapContextSize = + topLevelLinearMapContextMetadata->vw_size(); + auto allocationSize = alignTo(sizeof(AutoDiffLinearMapContext), + alignof(AutoDiffLinearMapContext)) + + topLevelLinearMapContextSize; + auto *buffer = (AutoDiffLinearMapContext *)malloc(allocationSize); + return ::new (buffer) + AutoDiffLinearMapContext(topLevelLinearMapContextMetadata); +} + +void *swift::swift_autoDiffAllocateSubcontextWithType( + AutoDiffLinearMapContext *linearMapContext, + const Metadata *linearMapSubcontextMetadata) { + assert(linearMapSubcontextMetadata->getValueWitnesses() != nullptr); + return linearMapContext->allocateSubcontext(linearMapSubcontextMetadata); +} diff --git a/stdlib/public/runtime/AutoDiffSupport.h b/stdlib/public/runtime/AutoDiffSupport.h index 4fb63d82e062b..b3135ef794628 100644 --- a/stdlib/public/runtime/AutoDiffSupport.h +++ b/stdlib/public/runtime/AutoDiffSupport.h @@ -14,31 +14,95 @@ #define SWIFT_RUNTIME_AUTODIFF_SUPPORT_H #include "swift/Runtime/HeapObject.h" +#include "swift/ABI/Metadata.h" #include "swift/Runtime/Config.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Support/Allocator.h" namespace swift { - /// A data structure responsible for efficiently allocating closure contexts for /// linear maps such as pullbacks, including recursive branching trace enum /// case payloads. class AutoDiffLinearMapContext : public HeapObject { + /// A simple wrapper around a context object allocated by the + /// `AutoDiffLinearMapContext` type. This type knows all the "physical" + /// properties and behavior of the allocated context object by way of + /// storing the allocated type's `TypeMetadata`. It uses this information + /// to ensure that the allocated context object is destroyed/deinitialized + /// properly, upon its own destruction. + class [[nodiscard]] AllocatedContextObjectRecord final { + const Metadata *contextObjectMetadata; + OpaqueValue *contextObjectPtr; + + public: + AllocatedContextObjectRecord(const Metadata *contextObjectMetadata, + OpaqueValue *contextObjectPtr) + : contextObjectMetadata(contextObjectMetadata), + contextObjectPtr(contextObjectPtr) {} + + AllocatedContextObjectRecord(const Metadata *contextObjectMetadata, + void *contextObjectPtr) + : AllocatedContextObjectRecord( + contextObjectMetadata, + static_cast(contextObjectPtr)) {} + + ~AllocatedContextObjectRecord() { + if (contextObjectMetadata != nullptr && contextObjectPtr != nullptr) { + contextObjectMetadata->vw_destroy(contextObjectPtr); + } + } + + AllocatedContextObjectRecord(const AllocatedContextObjectRecord &) = delete; + + AllocatedContextObjectRecord( + AllocatedContextObjectRecord &&other) noexcept { + this->contextObjectMetadata = other.contextObjectMetadata; + this->contextObjectPtr = other.contextObjectPtr; + other.contextObjectMetadata = nullptr; + other.contextObjectPtr = nullptr; + } + + size_t size() const { return contextObjectMetadata->vw_size(); } + + size_t align() const { return contextObjectMetadata->vw_alignment(); } + }; + private: /// The underlying allocator. // TODO: Use a custom allocator so that the initial slab can be // tail-allocated. llvm::BumpPtrAllocator allocator; + /// Storage for `AllocatedContextObjectRecord`s, corresponding to the + /// subcontext allocations performed by the type. + llvm::SmallVector allocatedContextObjects; + public: - /// Creates a linear map context. + /// DEPRECATED - Use overloaded constructor taking a `const Metadata *` + /// parameter instead. This constructor might be removed as it leads to memory + /// leaks. AutoDiffLinearMapContext(); + + AutoDiffLinearMapContext(const Metadata *topLevelLinearMapContextMetadata); + /// Returns the address of the tail-allocated top-level subcontext. void *projectTopLevelSubcontext() const; + /// Allocates memory for a new subcontext. + /// + /// DEPRECATED - Use `allocateSubcontext` instead. This + /// method might be removed as it leads to memory leaks. void *allocate(size_t size); + + /// Allocates memory for a new subcontext. + void *allocateSubcontext(const Metadata *contextObjectMetadata); }; /// Creates a linear map context with a tail-allocated top-level subcontext. +/// +/// DEPRECATED - Use `swift_autoDiffCreateLinearMapContextWithType` instead. +/// This builtin might be removed as it leads to memory leaks. SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) AutoDiffLinearMapContext *swift_autoDiffCreateLinearMapContext( size_t topLevelSubcontextSize); @@ -48,9 +112,21 @@ SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) void *swift_autoDiffProjectTopLevelSubcontext(AutoDiffLinearMapContext *); /// Allocates memory for a new subcontext. +/// +/// DEPRECATED - Use `swift_autoDiffAllocateSubcontextWithType` instead. This +/// builtin might be removed as it leads to memory leaks. SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) void *swift_autoDiffAllocateSubcontext(AutoDiffLinearMapContext *, size_t size); -} +/// Creates a linear map context with a tail-allocated top-level subcontext. +SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) + AutoDiffLinearMapContext *swift_autoDiffCreateLinearMapContextWithType( + const Metadata *topLevelLinearMapContextMetadata); +/// Allocates memory for a new subcontext. +SWIFT_RUNTIME_EXPORT + SWIFT_CC(swift) void *swift_autoDiffAllocateSubcontextWithType( + AutoDiffLinearMapContext *, + const Metadata *linearMapSubcontextMetadata); +} // namespace swift #endif /* SWIFT_RUNTIME_AUTODIFF_SUPPORT_H */ diff --git a/test/AutoDiff/IRGen/runtime.swift b/test/AutoDiff/IRGen/runtime.swift index 09c848fafcc04..8a753356212a8 100644 --- a/test/AutoDiff/IRGen/runtime.swift +++ b/test/AutoDiff/IRGen/runtime.swift @@ -3,22 +3,17 @@ import Swift import _Differentiation -struct ExamplePullbackStruct { - var pb0: (T.TangentVector) -> T.TangentVector -} - -@_silgen_name("test_context_builtins") -func test_context_builtins() { - let pbStruct = ExamplePullbackStruct(pb0: { $0 }) - let context = Builtin.autoDiffCreateLinearMapContext(Builtin.sizeof(type(of: pbStruct))) +@_silgen_name("test_context_builtins_with_type") +func test_context_builtins_with_type(t: T) { + let context = Builtin.autoDiffCreateLinearMapContextWithType(T.self) let topLevelSubctxAddr = Builtin.autoDiffProjectTopLevelSubcontext(context) - UnsafeMutableRawPointer(topLevelSubctxAddr).storeBytes(of: pbStruct, as: type(of: pbStruct)) - let newBuffer = Builtin.autoDiffAllocateSubcontext(context, Builtin.sizeof(type(of: pbStruct))) - UnsafeMutableRawPointer(newBuffer).storeBytes(of: pbStruct, as: type(of: pbStruct)) + UnsafeMutableRawPointer(topLevelSubctxAddr).storeBytes(of: t, as: T.self) + let newBuffer = Builtin.autoDiffAllocateSubcontextWithType(context, T.self) + UnsafeMutableRawPointer(newBuffer).storeBytes(of: t, as: T.self) } -// CHECK-LABEL: define{{.*}}@test_context_builtins() +// CHECK-LABEL: define{{.*}}@test_context_builtins_with_type(ptr noalias nocapture %0, ptr %T) // CHECK: entry: -// CHECK: [[CTX:%.*]] = call swiftcc ptr @swift_autoDiffCreateLinearMapContext({{i[0-9]+}} {{.*}}) +// CHECK: [[CTX:%.*]] = call swiftcc ptr @swift_autoDiffCreateLinearMapContextWithType(ptr %T) // CHECK: call swiftcc ptr @swift_autoDiffProjectTopLevelSubcontext(ptr [[CTX]]) -// CHECK: [[BUF:%.*]] = call swiftcc ptr @swift_autoDiffAllocateSubcontext(ptr [[CTX]], {{i[0-9]+}} {{.*}}) +// CHECK: [[BUF:%.*]] = call swiftcc ptr @swift_autoDiffAllocateSubcontextWithType(ptr [[CTX]], ptr %T) diff --git a/test/AutoDiff/SILGen/autodiff_builtins.swift b/test/AutoDiff/SILGen/autodiff_builtins.swift index 903ba227fbf71..b369deec7809c 100644 --- a/test/AutoDiff/SILGen/autodiff_builtins.swift +++ b/test/AutoDiff/SILGen/autodiff_builtins.swift @@ -84,24 +84,20 @@ func applyDerivative_f1_vjp(t0: T) -> (T // CHECK: copy_addr [take] [[D_RESULT_BUFFER_0_FOR_LOAD]] to [init] [[ORIG_RESULT_OUT_PARAM]] // CHECK: return [[PULLBACK]] -struct ExamplePullbackStruct { - var pb0: (T.TangentVector) -> T.TangentVector -} -@_silgen_name("test_context_builtins") -func test_context_builtins() { - let pbStruct = ExamplePullbackStruct(pb0: { $0 }) - let context = Builtin.autoDiffCreateLinearMapContext(Builtin.sizeof(type(of: pbStruct))) +@_silgen_name("test_context_builtins_with_type") +func test_context_builtins_with_type(t: T) { + let context = Builtin.autoDiffCreateLinearMapContextWithType(T.self) let topLevelSubctxAddr = Builtin.autoDiffProjectTopLevelSubcontext(context) - UnsafeMutableRawPointer(topLevelSubctxAddr).storeBytes(of: pbStruct, as: type(of: pbStruct)) - let newBuffer = Builtin.autoDiffAllocateSubcontext(context, Builtin.sizeof(type(of: pbStruct))) - UnsafeMutableRawPointer(newBuffer).storeBytes(of: pbStruct, as: type(of: pbStruct)) + UnsafeMutableRawPointer(topLevelSubctxAddr).storeBytes(of: t, as: T.self) + let newBuffer = Builtin.autoDiffAllocateSubcontextWithType(context, T.self) + UnsafeMutableRawPointer(newBuffer).storeBytes(of: t, as: T.self) } -// CHECK-LABEL: sil{{.*}}@test_context_builtins -// CHECK: bb0: -// CHECK: [[CTX:%.*]] = builtin "autoDiffCreateLinearMapContext"({{%.*}} : $Builtin.Word) : $Builtin.NativeObject -// CHECK: [[BORROWED_CTX:%.*]] = begin_borrow [lexical] [[CTX]] : $Builtin.NativeObject -// CHECK: [[BUF:%.*]] = builtin "autoDiffProjectTopLevelSubcontext"([[BORROWED_CTX]] : $Builtin.NativeObject) : $Builtin.RawPointer -// CHECK: [[BUF:%.*]] = builtin "autoDiffAllocateSubcontext"([[BORROWED_CTX]] : $Builtin.NativeObject, {{.*}} : $Builtin.Word) : $Builtin.RawPointer +// CHECK-LABEL: sil{{.*}}@test_context_builtins_with_type : $@convention(thin) (@in_guaranteed T) -> () { +// CHECK: bb0({{%.*}} : $*T): +// CHECK: [[CTX:%.*]] = builtin "autoDiffCreateLinearMapContextWithType"({{%.*}} : $@thick T.Type) : $Builtin.NativeObject // users: {{.*}} +// CHECK: [[BORROWED_CTX:%.*]] = begin_borrow [lexical] [[CTX]] : $Builtin.NativeObject // users: {{.*}} +// CHECK: [[BUF:%.*]] = builtin "autoDiffProjectTopLevelSubcontext"([[BORROWED_CTX]] : $Builtin.NativeObject) : $Builtin.RawPointer // users: {{.*}} +// CHECK: [[BUF:%.*]] = builtin "autoDiffAllocateSubcontextWithType"([[BORROWED_CTX]] : $Builtin.NativeObject, {{.*}} : $@thick T.Type) : $Builtin.RawPointer // users: {{.*}} // CHECK: destroy_value [[CTX]] diff --git a/test/AutoDiff/stdlib/callee_differential_not_leaked_in_func_with_loops.swift b/test/AutoDiff/stdlib/callee_differential_not_leaked_in_func_with_loops.swift new file mode 100644 index 0000000000000..8de10d1d02d2b --- /dev/null +++ b/test/AutoDiff/stdlib/callee_differential_not_leaked_in_func_with_loops.swift @@ -0,0 +1,60 @@ +// RUN: %target-run-simple-swift +// REQUIRES: executable_test + +import _Differentiation +import StdlibUnittest + +// When the original function contains loops, we allocate a context object +// on the heap. This context object may store non-trivial objects, such as closures, +// that need to be freed explicitly, at program exit. This test verifies that the +// autodiff runtime destroys and deallocates any such objects. + +extension LifetimeTracked: AdditiveArithmetic { + public static var zero: LifetimeTracked { fatalError() } + public static func + (lhs: LifetimeTracked, rhs: LifetimeTracked) -> LifetimeTracked {fatalError()} + public static func - (lhs: LifetimeTracked, rhs: LifetimeTracked) -> LifetimeTracked {fatalError()} +} + +extension LifetimeTracked: Differentiable { + public typealias TangentVector = LifetimeTracked + public func move(by: LifetimeTracked) {fatalError()} +} + +extension LifetimeTracked { + // The original differentiable callee. + func callee(_: Float) -> Float { 42 } + + // The callee differential (pullback in this case), that is + // captured in the context object allocated on the heap in the + // presence of loops. + // + // If the autodiff runtime does not free this callee differential + // properly, the `LifetimeTracked` instance that it captures will + // also not be freed and we will have a detectable memory leak. + @derivative(of: callee, wrt: (self, f)) + func calleeDifferential(f: Float) -> (value: Float, pullback: (Float) -> (LifetimeTracked, Float)) { + return ( + value: f, + pullback: { x in (self, x) } + ) + } +} + +@differentiable(reverse) +func f(ltti: LifetimeTracked) -> Float { + for _ in 0..<1 { + } + return ltti.callee(0xDEADBEEF) +} + +var Tests = TestSuite("CalleeDifferentialLeakTest") + +Tests.test("dontLeakCalleeDifferential") { + do { + let ltti = LifetimeTracked(0xDEADBEEF) + let _ = valueWithPullback(at: ltti, of: f) + } + expectEqual(0, LifetimeTracked.instances) +} + +runAllTests() \ No newline at end of file