From c25b3b43a2640f730abb0cff408a7b7782fca39b Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:52:24 -0700 Subject: [PATCH 01/21] [Runtime] Restructure BytecodeLayouts --- include/swift/ABI/Metadata.h | 14 +- stdlib/public/runtime/BytecodeLayouts.cpp | 824 +++++++++++++++++- stdlib/public/runtime/BytecodeLayouts.h | 25 + .../layout_string_witnesses_types.swift | 5 + 4 files changed, 847 insertions(+), 21 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index 38511c67533ca..9b2cc2b04d014 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -323,13 +323,13 @@ struct TargetMetadata { } const uint8_t *getLayoutString() const { - assert(hasLayoutString()); - if (isAnyClass()) { - return asFullMetadata( - reinterpret_cast *>( - this)) - ->layoutString; - } + //assert(hasLayoutString()); + // if (isAnyClass()) { + // return asFullMetadata( + // reinterpret_cast *>( + // this)) + // ->layoutString; + // } return asFullMetadata(this)->layoutString; } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index ba1a87219b7b5..645a07f241e12 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -91,7 +91,7 @@ static void existential_destroy(OpaqueValue* object) { } template -inline static bool handleNextRefCount(const Metadata *metadata, +inline bool handleNextRefCount(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, Params... params) { uint64_t skip = reader.readBytes(); @@ -139,7 +139,7 @@ inline static bool handleNextRefCount(const Metadata *metadata, } template -inline static void handleRefCounts(const Metadata *metadata, +inline void handleRefCounts(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, Params... params) { if (N == 0) { @@ -155,7 +155,7 @@ inline static void handleRefCounts(const Metadata *metadata, } template -inline static void handleRefCounts(const Metadata *metadata, Params... params) { +inline void handleRefCounts(const Metadata *metadata, Params... params) { LayoutStringReader reader{metadata->getLayoutString(), layoutStringHeaderSize}; uintptr_t addrOffset = 0; @@ -359,10 +359,16 @@ static void handleMultiPayloadEnumGeneric(const Metadata *metadata, addrOffset += enumSize; } +void swift_release_masked(void* ptr) { + HeapObject *object = (HeapObject*)(((uintptr_t)ptr) & ~_swift_abi_SwiftSpareBitsMask); + if (object != nullptr) + object->refCounts.decrementAndMaybeDeinit(1); +} + const DestroyFuncAndMask destroyTable[] = { {(DestrFn)&skipDestroy, false}, {(DestrFn)&swift_errorRelease, true}, - {(DestrFn)&swift_release, true}, + {(DestrFn)&swift_release_masked, true}, {(DestrFn)&swift_unownedRelease, true}, {(DestrFn)&swift_weakDestroy, false}, {(DestrFn)&swift_unknownObjectRelease, true}, @@ -437,9 +443,789 @@ struct DestroyHandler { } }; +static void handleRefCountsDestroy(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr); + +static void endDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + return; +} + +static void errorDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + SwiftError *error = *(SwiftError**)(addr + addrOffset); + swift_errorRelease(error); +} + +static void nativeStrongDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + swift_release(object); + // if (object != nullptr) + // object->refCounts.decrementAndMaybeDeinit(1); +} + +static void unownedDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + swift_unownedRelease(object); +} + +static void weakDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + swift_weakDestroy((WeakReference *)(addr + addrOffset)); +} + +static void unknownDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + void *object = *(void**)(addr + addrOffset); + swift_unknownObjectRelease(object); +} + +static void unknownUnownedDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + UnownedReference *object = (UnownedReference*)(addr + addrOffset); + swift_unknownObjectUnownedDestroy(object); +} + +static void unknownWeakDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + swift_unknownObjectWeakDestroy((WeakReference *)(addr + addrOffset)); +} + +static void bridgeDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); +} + +static void singlePayloadEnumSimpleDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (extraTagBytesPattern) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = + readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); + if (tagBytes) { + xiTagBytesPattern = 0; + } + } + + if (xiTagBytesPattern) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + uint64_t tagBytes = + readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + if (tagBytes >= xiTagValues) { + return; + } + } + + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag == 0) { + reader.skip(sizeof(size_t) * 2); + } else { + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag != 0) { + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumGenericDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (extraTagBytesPattern) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); + + if (tagBytes) { + xiType = nullptr; + } + } + + if (xiType) { + auto tag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset), + numEmptyCases); + if (tag == 0) { + return; + } + } + + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void multiPayloadEnumFNDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(numPayloads, refCountBytes, enumSize); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + +static void multiPayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(numPayloads, refCountBytes, enumSize); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + + +static void multiPayloadEnumGenericDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader &reader) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + auto tagBytesOffset = enumSize - tagBytes; + + auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + +#if SWIFT_OBJC_INTEROP +static void blockDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + _Block_release((void *)(addr + addrOffset)); +} +#endif + +static void metatypeDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *type = reader.readBytes(); + type->vw_destroy((OpaqueValue *)(addr + addrOffset)); +} + +static void existentialDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); + auto* type = getExistentialTypeMetadata(object); + if (type->getValueWitnesses()->isValueInline()) { + type->vw_destroy(object); + } else { + swift_release(*(HeapObject**)object); + } +} + +static void resilientDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *type = getResilientTypeMetadata(metadata, reader); + type->vw_destroy((OpaqueValue *)(addr + addrOffset)); +} + +typedef void (*DestrFnBranchless)(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr); + +static const DestrFnBranchless destroyTableBranchless[] = { + &endDestroyBranchless, + &errorDestroyBranchless, + &nativeStrongDestroyBranchless, + &unownedDestroyBranchless, + &weakDestroyBranchless, + &unknownDestroyBranchless, + &unknownUnownedDestroyBranchless, + &unknownWeakDestroyBranchless, + &bridgeDestroyBranchless, +#if SWIFT_OBJC_INTEROP + &blockDestroyBranchless, + &unknownDestroyBranchless, +#else + nullptr, + nullptr, +#endif + nullptr, // Custom + &metatypeDestroyBranchless, + nullptr, // Generic + &existentialDestroyBranchless, + &resilientDestroyBranchless, + &singlePayloadEnumSimpleDestroyBranchless, + &singlePayloadEnumFNDestroyBranchless, + &singlePayloadEnumFNResolvedDestroyBranchless, + &singlePayloadEnumGenericDestroyBranchless, + &multiPayloadEnumFNDestroyBranchless, + &multiPayloadEnumFNResolvedDestroyBranchless, + &multiPayloadEnumGenericDestroyBranchless, +}; + +static void handleRefCountsDestroy(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + uint64_t tag = 0; + do { + tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + + destroyTableBranchless[tag](metadata, reader, addrOffset, addr); + } while (tag != 0); +} + extern "C" void swift_generic_destroy(swift::OpaqueValue *address, const Metadata *metadata) { - handleRefCounts<0, DestroyHandler>(metadata, (uint8_t *)address); + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsDestroy(metadata, reader, addrOffset, (uint8_t *)address); +} + +static void handleRefCountsInitWithCopy(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); + +static void endRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + return; +} + +static void errorRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_errorRetain(*(SwiftError**)(dest + addrOffset)); +} + +static void nativeStrongRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_retain(*(HeapObject **)(dest + addrOffset)); +} + +static void unownedRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset))); + swift_unownedRetain(object); +} + +static void weakCopyInitBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_weakCopyInit((WeakReference *)(dest + addrOffset), (WeakReference *)(src + addrOffset)); +} + +static void unknownRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + void *object = *(void**)(dest + addrOffset); + swift_unknownObjectRetain(object); +} + +static void unknownUnownedCopyInitBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + UnownedReference *objectDest = (UnownedReference*)(dest + addrOffset); + UnownedReference *objectSrc = (UnownedReference*)(src + addrOffset); + swift_unknownObjectUnownedCopyInit(objectDest, objectSrc); +} + +static void unknownWeakCopyInitBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_unknownObjectWeakCopyInit((WeakReference *)(dest + addrOffset), + (WeakReference *)(src + addrOffset)); +} + +static void bridgeRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_bridgeObjectRetain(*(void **)(dest + addrOffset)); +} + +static void singlePayloadEnumSimpleRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (extraTagBytesPattern) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = + readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + if (tagBytes) { + xiTagBytesPattern = 0; + } + } + + if (xiTagBytesPattern) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + uint64_t tagBytes = + readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + if (tagBytes >= xiTagValues) { + return; + } + } + + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (enumTag == 0) { + reader.skip(sizeof(size_t) * 2); + } else { + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumFNResolvedRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (enumTag != 0) { + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumGenericRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (extraTagBytesPattern) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + + if (tagBytes) { + xiType = nullptr; + } + } + + if (xiType) { + auto tag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), + numEmptyCases); + if (tag == 0) { + return; + } + } + + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void multiPayloadEnumFNRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(numPayloads, refCountBytes, enumSize); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + +static void multiPayloadEnumFNResolvedRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(numPayloads, refCountBytes, enumSize); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + + +static void multiPayloadEnumGenericRetainBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader &reader) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + auto tagBytesOffset = enumSize - tagBytes; + + auto enumTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); + + if (enumTag < numPayloads) { + size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); + + LayoutStringReader nestedReader = reader; + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + uintptr_t nestedAddrOffset = addrOffset; + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + } + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + addrOffset += enumSize; + }); +} + +#if SWIFT_OBJC_INTEROP +static void blockCopyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + *(void**)dest = _Block_copy(*(void**)(src + addrOffset)); +} +#endif + +static void metatypeInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *type = reader.readBytes(); + type->vw_initializeWithCopy((OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); +} + +static void existentialInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + type->vw_initializeBufferWithCopyOfBuffer((ValueBuffer*)(dest + addrOffset), + (ValueBuffer*)(src + addrOffset)); +} + +static void resilientInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *type = getResilientTypeMetadata(metadata, reader); + type->vw_initializeWithCopy((OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); +} + +typedef void (*InitWithCopyFnBranchless)(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); + +static const InitWithCopyFnBranchless initWithCopyTableBranchless[] = { + &endRetainBranchless, + &errorRetainBranchless, + &nativeStrongRetainBranchless, + &unownedRetainBranchless, + &weakCopyInitBranchless, + &unknownRetainBranchless, + &unknownUnownedCopyInitBranchless, + &unknownWeakCopyInitBranchless, + &bridgeRetainBranchless, +#if SWIFT_OBJC_INTEROP + &blockCopyBranchless, + &unknownRetainBranchless, +#else + nullptr, + nullptr, +#endif + nullptr, // Custom + &metatypeInitWithCopyBranchless, + nullptr, // Generic + &existentialInitWithCopyBranchless, + &resilientInitWithCopyBranchless, + &singlePayloadEnumSimpleRetainBranchless, + &singlePayloadEnumFNRetainBranchless, + &singlePayloadEnumFNResolvedRetainBranchless, + &singlePayloadEnumGenericRetainBranchless, + &multiPayloadEnumFNRetainBranchless, + &multiPayloadEnumFNResolvedRetainBranchless, + &multiPayloadEnumGenericRetainBranchless, +}; + +static void handleRefCountsInitWithCopy(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uint64_t tag = 0; + do { + tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + + initWithCopyTableBranchless[tag](metadata, reader, addrOffset, dest, src); + } while (tag != 0); +} + +extern "C" swift::OpaqueValue * +swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, + const Metadata *metadata) { + if (dest == src) + llvm_unreachable("WOOT"); + size_t size = metadata->vw_size(); + memcpy(dest, src, size); + + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsInitWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); + + return dest; } struct RetainFuncAndMask { @@ -464,10 +1250,20 @@ void* existential_initializeWithCopy(OpaqueValue* dest, OpaqueValue* src) { (ValueBuffer*)src); } +void* swift_retain_inline(HeapObject* object) { + if (object != nullptr) { + // Return the result of increment() to make the eventual call to + // incrementSlow a tail call, which avoids pushing a stack frame on the fast + // path on ARM64. + return object->refCounts.increment(1); + } + return object; +} + const RetainFuncAndMask retainTable[] = { {(void*)&skipRetain, true}, {(void*)&swift_errorRetain, true}, - {(void*)&swift_retain, true}, + {(void*)&swift_retain_inline, true}, {(void*)&swift_unownedRetain, true}, {(void*)&swift_weakCopyInit, false}, {(void*)&swift_unknownObjectRetain, true}, @@ -546,16 +1342,16 @@ struct CopyHandler { } }; -extern "C" swift::OpaqueValue * -swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, - const Metadata *metadata) { - size_t size = metadata->vw_size(); - memcpy(dest, src, size); +// extern "C" swift::OpaqueValue * +// swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, +// const Metadata *metadata) { +// size_t size = metadata->vw_size(); +// memcpy(dest, src, size); - handleRefCounts<0, CopyHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); +// handleRefCounts<0, CopyHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); - return dest; -} +// return dest; +// } struct TakeHandler { static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, diff --git a/stdlib/public/runtime/BytecodeLayouts.h b/stdlib/public/runtime/BytecodeLayouts.h index e3747e3edbeef..a3d9e14cd1799 100644 --- a/stdlib/public/runtime/BytecodeLayouts.h +++ b/stdlib/public/runtime/BytecodeLayouts.h @@ -70,6 +70,31 @@ struct LayoutStringReader { return returnVal; } + template + inline void readBytes(T&... result) { + uintptr_t additionalOffset = 0; + ([&] { + memcpy(&result, layoutStr + offset + additionalOffset, sizeof(T)); + additionalOffset += sizeof(T); + }(), ...); + offset += additionalOffset; + } + + template + inline T modify(F &&f) { + LayoutStringReader readerCopy = *this; + T res = f(readerCopy); + offset = readerCopy.offset; + return res; + } + + template + inline void modify(F &&f) { + LayoutStringReader readerCopy = *this; + f(readerCopy); + offset = readerCopy.offset; + } + template inline T peekBytes(size_t peekOffset = 0) const { T returnVal; diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index c21a467dd3931..224db34e51b1a 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -535,6 +535,11 @@ public func testAssign(_ ptr: UnsafeMutablePointer, from x: T) { ptr.pointee = x } +@inline(never) +public func testAssign(_ ptr: UnsafeMutablePointer, from x: UnsafeMutablePointer) { + ptr.assign(from: x, count: 1) +} + @inline(never) public func testInit(_ ptr: UnsafeMutablePointer, to x: T) { ptr.initialize(to: x) From ec3a2dc3704fe913c087f215fcaf7dbd636046c9 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:52:39 -0700 Subject: [PATCH 02/21] [Runtime] Templatize shared functionality in BytecodeLayouts --- stdlib/public/runtime/BytecodeLayouts.cpp | 296 +++------------------- 1 file changed, 39 insertions(+), 257 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 645a07f241e12..292a20ae77df4 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -72,24 +72,6 @@ static const Metadata *getResilientTypeMetadata(const Metadata *metadata, return fn(metadata->getGenericArgs()); } -typedef void (*DestrFn)(void*); - -struct DestroyFuncAndMask { - DestrFn fn; - bool isIndirect; -}; - -static void skipDestroy(void* ignore) { } - -static void existential_destroy(OpaqueValue* object) { - auto* metadata = getExistentialTypeMetadata(object); - if (metadata->getValueWitnesses()->isValueInline()) { - metadata->vw_destroy(object); - } else { - swift_release(*(HeapObject**)object); - } -} - template inline bool handleNextRefCount(const Metadata *metadata, LayoutStringReader &reader, @@ -359,90 +341,6 @@ static void handleMultiPayloadEnumGeneric(const Metadata *metadata, addrOffset += enumSize; } -void swift_release_masked(void* ptr) { - HeapObject *object = (HeapObject*)(((uintptr_t)ptr) & ~_swift_abi_SwiftSpareBitsMask); - if (object != nullptr) - object->refCounts.decrementAndMaybeDeinit(1); -} - -const DestroyFuncAndMask destroyTable[] = { - {(DestrFn)&skipDestroy, false}, - {(DestrFn)&swift_errorRelease, true}, - {(DestrFn)&swift_release_masked, true}, - {(DestrFn)&swift_unownedRelease, true}, - {(DestrFn)&swift_weakDestroy, false}, - {(DestrFn)&swift_unknownObjectRelease, true}, - {(DestrFn)&swift_unknownObjectUnownedDestroy, false}, - {(DestrFn)&swift_unknownObjectWeakDestroy, false}, - {(DestrFn)&swift_bridgeObjectRelease, true}, -#if SWIFT_OBJC_INTEROP - {(DestrFn)&_Block_release, true}, - {(DestrFn)&swift_unknownObjectRelease, true}, -#else - {nullptr, true}, - {nullptr, true}, -#endif - // TODO: how to handle Custom? - {nullptr, true}, - {nullptr, true}, - {nullptr, true}, - {(DestrFn)&existential_destroy, false}, -}; - -struct DestroyHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *addr) { - type->vw_destroy((OpaqueValue *)(addr + addrOffset)); - } - - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumSimple(reader, addr, addrOffset); - } - - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumFN(reader, resolved, addr, addrOffset); - } - - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumGeneric(reader, addr, addrOffset); - } - - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, addr); - } - - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleMultiPayloadEnumGeneric(metadata, reader, - addrOffset, addr); - } - - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *addr) { - const auto &destroyFunc = destroyTable[static_cast(tag)]; - if (SWIFT_LIKELY(destroyFunc.isIndirect)) { - destroyFunc.fn( - (void *)((*(uintptr_t *)(addr + addrOffset)))); - } else { - destroyFunc.fn(((void *)(addr + addrOffset))); - } - } -}; - static void handleRefCountsDestroy(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, @@ -469,8 +367,6 @@ static void nativeStrongDestroyBranchless(const Metadata *metadata, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); swift_release(object); - // if (object != nullptr) - // object->refCounts.decrementAndMaybeDeinit(1); } static void unownedDestroyBranchless(const Metadata *metadata, @@ -518,10 +414,12 @@ static void bridgeDestroyBranchless(const Metadata *metadata, swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); } -static void singlePayloadEnumSimpleDestroyBranchless(const Metadata *metadata, +template +static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T... param) { reader.modify([&](LayoutStringReader &reader) { uint64_t byteCountsAndOffset; size_t payloadSize; @@ -561,10 +459,12 @@ static void singlePayloadEnumSimpleDestroyBranchless(const Metadata *metadata, }); } -static void singlePayloadEnumFNDestroyBranchless(const Metadata *metadata, +template +static void singlePayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -582,10 +482,12 @@ static void singlePayloadEnumFNDestroyBranchless(const Metadata *metadata, }); } -static void singlePayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata, +template +static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag; size_t refCountBytes; @@ -601,10 +503,12 @@ static void singlePayloadEnumFNResolvedDestroyBranchless(const Metadata *metadat }); } -static void singlePayloadEnumGenericDestroyBranchless(const Metadata *metadata, +template +static void singlePayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { auto tagBytesAndOffset = reader.readBytes(); auto payloadSize = reader.readBytes(); @@ -640,10 +544,12 @@ static void singlePayloadEnumGenericDestroyBranchless(const Metadata *metadata, }); } -static void multiPayloadEnumFNDestroyBranchless(const Metadata *metadata, +template +static void multiPayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -660,7 +566,7 @@ static void multiPayloadEnumFNDestroyBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -668,10 +574,12 @@ static void multiPayloadEnumFNDestroyBranchless(const Metadata *metadata, }); } -static void multiPayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata, +template +static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = reader.readBytes(); @@ -688,7 +596,7 @@ static void multiPayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -696,11 +604,12 @@ static void multiPayloadEnumFNResolvedDestroyBranchless(const Metadata *metadata }); } - -static void multiPayloadEnumGenericDestroyBranchless(const Metadata *metadata, +template +static void multiPayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + T ...param) { reader.modify([&](LayoutStringReader &reader) { size_t tagBytes; size_t numPayloads; @@ -718,7 +627,7 @@ static void multiPayloadEnumGenericDestroyBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, addr); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -769,7 +678,7 @@ typedef void (*DestrFnBranchless)(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr); -static const DestrFnBranchless destroyTableBranchless[] = { +const DestrFnBranchless destroyTableBranchless[] = { &endDestroyBranchless, &errorDestroyBranchless, &nativeStrongDestroyBranchless, @@ -791,13 +700,13 @@ static const DestrFnBranchless destroyTableBranchless[] = { nullptr, // Generic &existentialDestroyBranchless, &resilientDestroyBranchless, - &singlePayloadEnumSimpleDestroyBranchless, - &singlePayloadEnumFNDestroyBranchless, - &singlePayloadEnumFNResolvedDestroyBranchless, - &singlePayloadEnumGenericDestroyBranchless, - &multiPayloadEnumFNDestroyBranchless, - &multiPayloadEnumFNResolvedDestroyBranchless, - &multiPayloadEnumGenericDestroyBranchless, + &singlePayloadEnumSimpleBranchless<>, + &singlePayloadEnumFNBranchless<>, + &singlePayloadEnumFNResolvedBranchless<>, + &singlePayloadEnumGenericBranchless<>, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, }; static void handleRefCountsDestroy(const Metadata *metadata, @@ -1215,8 +1124,6 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, extern "C" swift::OpaqueValue * swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - if (dest == src) - llvm_unreachable("WOOT"); size_t size = metadata->vw_size(); memcpy(dest, src, size); @@ -1228,131 +1135,6 @@ swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, return dest; } -struct RetainFuncAndMask { - void* fn; - bool isSingle; -}; - -#if SWIFT_OBJC_INTEROP -void* Block_copyForwarder(void** dest, const void** src) { - *dest = _Block_copy(*src); - return *dest; -} -#endif - -typedef void* (*RetainFn)(void*); -typedef void* (*CopyInitFn)(void*, void*); - -void* skipRetain(void* ignore) { return nullptr; } -void* existential_initializeWithCopy(OpaqueValue* dest, OpaqueValue* src) { - auto* metadata = getExistentialTypeMetadata(src); - return metadata->vw_initializeBufferWithCopyOfBuffer((ValueBuffer*)dest, - (ValueBuffer*)src); -} - -void* swift_retain_inline(HeapObject* object) { - if (object != nullptr) { - // Return the result of increment() to make the eventual call to - // incrementSlow a tail call, which avoids pushing a stack frame on the fast - // path on ARM64. - return object->refCounts.increment(1); - } - return object; -} - -const RetainFuncAndMask retainTable[] = { - {(void*)&skipRetain, true}, - {(void*)&swift_errorRetain, true}, - {(void*)&swift_retain_inline, true}, - {(void*)&swift_unownedRetain, true}, - {(void*)&swift_weakCopyInit, false}, - {(void*)&swift_unknownObjectRetain, true}, - {(void*)&swift_unknownObjectUnownedCopyInit, false}, - {(void*)&swift_unknownObjectWeakCopyInit, false}, - {(void*)&swift_bridgeObjectRetain, true}, -#if SWIFT_OBJC_INTEROP - {(void*)&Block_copyForwarder, false}, - {(void*)&objc_retain, true}, -#else - {nullptr, true}, - {nullptr, true}, -#endif - // TODO: how to handle Custom? - {nullptr, true}, - {nullptr, true}, - {nullptr, true}, - {(void*)&existential_initializeWithCopy, false}, -}; - -struct CopyHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - type->vw_initializeWithCopy((OpaqueValue*)((uintptr_t)dest + addrOffset), - (OpaqueValue*)((uintptr_t)src + addrOffset)); - } - - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumSimple(reader, src, addrOffset); - } - - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleSinglePayloadEnumFN(reader, resolved, src, addrOffset); - } - - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumGeneric(reader, src, addrOffset); - } - - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, dest, src); - } - - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - dest, src); - } - - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - const auto &retainFunc = retainTable[static_cast(tag)]; - if (SWIFT_LIKELY(retainFunc.isSingle)) { - ((RetainFn)retainFunc.fn)(*(void**)(((uintptr_t)dest + addrOffset))); - } else { - ((CopyInitFn)retainFunc.fn)((void*)((uintptr_t)dest + addrOffset), - (void*)((uintptr_t)src + addrOffset)); - } - } -}; - -// extern "C" swift::OpaqueValue * -// swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, -// const Metadata *metadata) { -// size_t size = metadata->vw_size(); -// memcpy(dest, src, size); - -// handleRefCounts<0, CopyHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); - -// return dest; -// } - struct TakeHandler { static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, uint8_t *dest, uint8_t *src) { From 2f61e2a9c958534aeb3bd7fcc800a55cfde3ab98 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:53:14 -0700 Subject: [PATCH 03/21] [Runtime] Further abstract shared functionality in BytecodeLayouts --- stdlib/public/runtime/BytecodeLayouts.cpp | 709 +++++----------------- 1 file changed, 147 insertions(+), 562 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 292a20ae77df4..753165c39dc20 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -72,79 +72,6 @@ static const Metadata *getResilientTypeMetadata(const Metadata *metadata, return fn(metadata->getGenericArgs()); } -template -inline bool handleNextRefCount(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, Params... params) { - uint64_t skip = reader.readBytes(); - auto tag = static_cast(skip >> 56); - skip &= ~(0xffULL << 56); - addrOffset += skip; - - if (SWIFT_UNLIKELY(tag == RefCountingKind::End)) { - return false; - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::Metatype)) { - auto *type = reader.readBytes(); - Handler::handleMetatype(type, addrOffset, std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::Resilient)) { - auto *type = getResilientTypeMetadata(metadata, reader); - Handler::handleMetatype(type, addrOffset, std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::SinglePayloadEnumSimple)) { - Handler::handleSinglePayloadEnumSimple(reader, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumFN)) { - Handler::handleSinglePayloadEnumFN(reader, false, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::SinglePayloadEnumFNResolved)) { - Handler::handleSinglePayloadEnumFN(reader, true, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumGeneric)) { - Handler::handleSinglePayloadEnumGeneric(reader, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumFN)) { - Handler::handleMultiPayloadEnumFN(metadata, reader, false, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::MultiPayloadEnumFNResolved)) { - Handler::handleMultiPayloadEnumFN(metadata, reader, true, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumGeneric)) { - Handler::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - std::forward(params)...); - } else { - Handler::handleReference(tag, addrOffset, std::forward(params)...); - } - - return true; -} - -template -inline void handleRefCounts(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, Params... params) { - if (N == 0) { - while (handleNextRefCount(metadata, reader, addrOffset, - std::forward(params)...)) { - } - } else { - for (unsigned i = 0; i < N; i++) { - handleNextRefCount(metadata, reader, addrOffset, - std::forward(params)...); - } - } -} - -template -inline void handleRefCounts(const Metadata *metadata, Params... params) { - LayoutStringReader reader{metadata->getLayoutString(), - layoutStringHeaderSize}; - uintptr_t addrOffset = 0; - handleRefCounts(metadata, reader, addrOffset, - std::forward(params)...); -} - static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) { switch (byteCount) { case 1: @@ -169,187 +96,17 @@ static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) { } } -static void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uint8_t *addr, - uintptr_t &addrOffset) { - auto byteCountsAndOffset = reader.readBytes(); - auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); - auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; - auto xiTagBytesOffset = - byteCountsAndOffset & std::numeric_limits::max(); - - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto payloadSize = reader.readBytes(); - auto tagBytes = - readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); - if (tagBytes) { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - goto noPayload; - } - } else { - reader.skip(sizeof(size_t)); - } - - if (xiTagBytesPattern) { - auto zeroTagValue = reader.readBytes(); - auto xiTagValues = reader.readBytes(); - - auto xiTagBytes = 1 << (xiTagBytesPattern - 1); - uint64_t tagBytes = - readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) - - zeroTagValue; - if (tagBytes >= xiTagValues) { - reader.skip(sizeof(size_t) * 2); - return; - } - } else { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - } - -noPayload: - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - reader.skip(refCountBytes); - addrOffset += skip; -} - -typedef unsigned (*GetEnumTagFn)(const uint8_t *); - -static void handleSinglePayloadEnumFN(LayoutStringReader &reader, bool resolved, - uint8_t *addr, uintptr_t &addrOffset) { - GetEnumTagFn getEnumTag; - if (resolved) { - getEnumTag = reader.readBytes(); - } else { - getEnumTag = readRelativeFunctionPointer(reader); - } - - unsigned enumTag = getEnumTag(addr + addrOffset); - - if (enumTag == 0) { - reader.skip(sizeof(size_t) * 2); - } else { - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - reader.skip(refCountBytes); - addrOffset += skip; - } -} - -static void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uint8_t *addr, - uintptr_t &addrOffset) { - auto tagBytesAndOffset = reader.readBytes(); - auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); - auto xiTagBytesOffset = - tagBytesAndOffset & std::numeric_limits::max(); - const Metadata *xiType = nullptr; - - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto payloadSize = reader.readBytes(); - auto tagBytes = - readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); - if (tagBytes) { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - goto noPayload; - } - } else { - reader.skip(sizeof(size_t)); - } - - xiType = reader.readBytes(); - - if (xiType) { - auto numEmptyCases = reader.readBytes(); - - auto tag = xiType->vw_getEnumTagSinglePayload( - (const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset), - numEmptyCases); - if (tag == 0) { - reader.skip(sizeof(size_t) * 2); - return; - } - } else { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - } - -noPayload: - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - reader.skip(refCountBytes); - addrOffset += skip; -} - -template -static void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, bool resolved, - uintptr_t &addrOffset, uint8_t *addr, - Params... params) { - GetEnumTagFn getEnumTag; - if (resolved) { - getEnumTag = reader.readBytes(); - } else { - getEnumTag = readRelativeFunctionPointer(reader); - } - - size_t numPayloads = reader.readBytes(); - size_t refCountBytes = reader.readBytes(); - size_t enumSize = reader.readBytes(); - - unsigned enumTag = getEnumTag(addr + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCounts<0, Handler>(metadata, nestedReader, nestedAddrOffset, addr, - std::forward(params)...); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; -} - -template -static void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, uint8_t *addr, - Params... params) { - auto tagBytes = reader.readBytes(); - auto numPayloads = reader.readBytes(); - auto refCountBytes = reader.readBytes(); - auto enumSize = reader.readBytes(); - auto tagBytesOffset = enumSize - tagBytes; - - auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCounts<0, Handler>(metadata, nestedReader, nestedAddrOffset, addr, - std::forward(params)...); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; -} - static void handleRefCountsDestroy(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr); -static void endDestroyBranchless(const Metadata *metadata, +template +static void handleEnd(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, - uint8_t *addr) { + uint8_t *addr, + Params ...params) { return; } @@ -414,12 +171,12 @@ static void bridgeDestroyBranchless(const Metadata *metadata, swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); } -template +template static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T... param) { + Params... params) { reader.modify([&](LayoutStringReader &reader) { uint64_t byteCountsAndOffset; size_t payloadSize; @@ -435,7 +192,7 @@ static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, auto xiTagBytesOffset = byteCountsAndOffset & std::numeric_limits::max(); - if (extraTagBytesPattern) { + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { auto extraTagBytes = 1 << (extraTagBytesPattern - 1); auto tagBytes = readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); @@ -444,7 +201,7 @@ static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, } } - if (xiTagBytesPattern) { + if (SWIFT_LIKELY(xiTagBytesPattern)) { auto xiTagBytes = 1 << (xiTagBytesPattern - 1); uint64_t tagBytes = readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) - @@ -459,12 +216,14 @@ static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, }); } -template +typedef unsigned (*GetEnumTagFn)(const uint8_t *); + +template static void singlePayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -482,12 +241,12 @@ static void singlePayloadEnumFNBranchless(const Metadata *metadata, }); } -template +template static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag; size_t refCountBytes; @@ -503,12 +262,12 @@ static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, }); } -template +template static void singlePayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { auto tagBytesAndOffset = reader.readBytes(); auto payloadSize = reader.readBytes(); @@ -544,12 +303,12 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, }); } -template +template static void multiPayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -566,7 +325,7 @@ static void multiPayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -574,12 +333,12 @@ static void multiPayloadEnumFNBranchless(const Metadata *metadata, }); } -template +template static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { GetEnumTagFn getEnumTag = reader.readBytes(); @@ -596,7 +355,7 @@ static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -604,12 +363,12 @@ static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, }); } -template +template static void multiPayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *addr, - T ...param) { + Params ...params) { reader.modify([&](LayoutStringReader &reader) { size_t tagBytes; size_t numPayloads; @@ -627,7 +386,7 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader nestedReader = reader; nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(param)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); } reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); @@ -642,6 +401,14 @@ static void blockDestroyBranchless(const Metadata *metadata, uint8_t *addr) { _Block_release((void *)(addr + addrOffset)); } + +static void objcStrongDestroyBranchless(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + objc_object *object = (objc_object*)(*(uintptr_t *)(addr + addrOffset)); + objc_release(object); +} #endif static void metatypeDestroyBranchless(const Metadata *metadata, @@ -679,7 +446,7 @@ typedef void (*DestrFnBranchless)(const Metadata *metadata, uint8_t *addr); const DestrFnBranchless destroyTableBranchless[] = { - &endDestroyBranchless, + &handleEnd, &errorDestroyBranchless, &nativeStrongDestroyBranchless, &unownedDestroyBranchless, @@ -690,7 +457,7 @@ const DestrFnBranchless destroyTableBranchless[] = { &bridgeDestroyBranchless, #if SWIFT_OBJC_INTEROP &blockDestroyBranchless, - &unknownDestroyBranchless, + &objcStrongDestroyBranchless, #else nullptr, nullptr, @@ -737,14 +504,6 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, uint8_t *dest, uint8_t *src); -static void endRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - return; -} - static void errorRetainBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, @@ -814,228 +573,22 @@ static void bridgeRetainBranchless(const Metadata *metadata, swift_bridgeObjectRetain(*(void **)(dest + addrOffset)); } -static void singlePayloadEnumSimpleRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - uint64_t byteCountsAndOffset; - size_t payloadSize; - uint64_t zeroTagValue; - size_t xiTagValues; - size_t refCountBytes; - size_t skip; - - reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); - - auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); - auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; - auto xiTagBytesOffset = - byteCountsAndOffset & std::numeric_limits::max(); - - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto tagBytes = - readTagBytes(src + addrOffset + payloadSize, extraTagBytes); - if (tagBytes) { - xiTagBytesPattern = 0; - } - } - - if (xiTagBytesPattern) { - auto xiTagBytes = 1 << (xiTagBytesPattern - 1); - uint64_t tagBytes = - readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - - zeroTagValue; - if (tagBytes >= xiTagValues) { - return; - } - } - - reader.skip(refCountBytes); - addrOffset += skip; - }); -} - -static void singlePayloadEnumFNRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); - - unsigned enumTag = getEnumTag(src + addrOffset); - - if (enumTag == 0) { - reader.skip(sizeof(size_t) * 2); - } else { - size_t refCountBytes; - size_t skip; - reader.readBytes(refCountBytes, skip); - reader.skip(refCountBytes); - addrOffset += skip; - } - }); -} - -static void singlePayloadEnumFNResolvedRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag; - size_t refCountBytes; - size_t skip; - reader.readBytes(getEnumTag, refCountBytes, skip); - - unsigned enumTag = getEnumTag(src + addrOffset); - - if (enumTag != 0) { - reader.skip(refCountBytes); - addrOffset += skip; - } - }); -} - -static void singlePayloadEnumGenericRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - auto tagBytesAndOffset = reader.readBytes(); - auto payloadSize = reader.readBytes(); - auto *xiType = reader.readBytes(); - auto numEmptyCases = reader.readBytes(); - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - - auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); - auto xiTagBytesOffset = - tagBytesAndOffset & std::numeric_limits::max(); - - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto tagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); - - if (tagBytes) { - xiType = nullptr; - } - } - - if (xiType) { - auto tag = xiType->vw_getEnumTagSinglePayload( - (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), - numEmptyCases); - if (tag == 0) { - return; - } - } - - reader.skip(refCountBytes); - addrOffset += skip; - }); -} - -static void multiPayloadEnumFNRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); - - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; - reader.readBytes(numPayloads, refCountBytes, enumSize); - - unsigned enumTag = getEnumTag(src + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; - }); -} - -static void multiPayloadEnumFNResolvedRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = reader.readBytes(); - - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; - reader.readBytes(numPayloads, refCountBytes, enumSize); - - unsigned enumTag = getEnumTag(src + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; - }); -} - - -static void multiPayloadEnumGenericRetainBranchless(const Metadata *metadata, +#if SWIFT_OBJC_INTEROP +static void blockCopyBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - reader.modify([&](LayoutStringReader &reader) { - size_t tagBytes; - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; - reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); - - auto tagBytesOffset = enumSize - tagBytes; - - auto enumTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); - } - - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; - }); + *(void**)dest = _Block_copy(*(void**)(src + addrOffset)); } -#if SWIFT_OBJC_INTEROP -static void blockCopyBranchless(const Metadata *metadata, +static void objcStrongRetainBranchless(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - *(void**)dest = _Block_copy(*(void**)(src + addrOffset)); + objc_object *object = (objc_object*)(*(uintptr_t *)(src + addrOffset)); + objc_retain(object); } #endif @@ -1069,14 +622,14 @@ static void resilientInitWithCopyBranchless(const Metadata *metadata, (OpaqueValue *)(src + addrOffset)); } -typedef void (*InitWithCopyFnBranchless)(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src); +typedef void (*InitFn)(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); -static const InitWithCopyFnBranchless initWithCopyTableBranchless[] = { - &endRetainBranchless, +static const InitFn initWithCopyTable[] = { + &handleEnd, &errorRetainBranchless, &nativeStrongRetainBranchless, &unownedRetainBranchless, @@ -1087,7 +640,7 @@ static const InitWithCopyFnBranchless initWithCopyTableBranchless[] = { &bridgeRetainBranchless, #if SWIFT_OBJC_INTEROP &blockCopyBranchless, - &unknownRetainBranchless, + &objcStrongRetainBranchless, #else nullptr, nullptr, @@ -1097,13 +650,13 @@ static const InitWithCopyFnBranchless initWithCopyTableBranchless[] = { nullptr, // Generic &existentialInitWithCopyBranchless, &resilientInitWithCopyBranchless, - &singlePayloadEnumSimpleRetainBranchless, - &singlePayloadEnumFNRetainBranchless, - &singlePayloadEnumFNResolvedRetainBranchless, - &singlePayloadEnumGenericRetainBranchless, - &multiPayloadEnumFNRetainBranchless, - &multiPayloadEnumFNResolvedRetainBranchless, - &multiPayloadEnumGenericRetainBranchless, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, }; static void handleRefCountsInitWithCopy(const Metadata *metadata, @@ -1117,7 +670,7 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, addrOffset += (tag & ~(0xFFULL << 56)); tag >>= 56; - initWithCopyTableBranchless[tag](metadata, reader, addrOffset, dest, src); + initWithCopyTable[tag](metadata, reader, addrOffset, dest, src); } while (tag != 0); } @@ -1135,73 +688,101 @@ swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, return dest; } -struct TakeHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue*)((uintptr_t)dest + addrOffset), - (OpaqueValue*)((uintptr_t)src + addrOffset)); - } - } - - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumSimple(reader, src, addrOffset); - } +static void handleRefCountsInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleSinglePayloadEnumFN(reader, resolved, src, addrOffset); - } +static void unknownWeakInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + swift_unknownObjectWeakTakeInit((WeakReference *)(dest + addrOffset), + (WeakReference *)(src + addrOffset)); +} - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumGeneric(reader, src, addrOffset); +static inline void metatypeInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *type = reader.readBytes(); + if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { + type->vw_initializeWithTake( + (OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); } +} - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, dest, src); +static void existentialInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { + type->vw_initializeWithTake((OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); } +} - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - dest, src); +static inline void resilientInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *type = getResilientTypeMetadata(metadata, reader); + if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { + type->vw_initializeWithTake( + (OpaqueValue *)(dest + addrOffset), + (OpaqueValue *)(src + addrOffset)); } +} - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - if (tag == RefCountingKind::UnknownWeak) { - swift_unknownObjectWeakTakeInit( - (WeakReference*)((uintptr_t)dest + addrOffset), - (WeakReference*)((uintptr_t)src + addrOffset)); - } else if (tag == RefCountingKind::Existential) { - auto *type = getExistentialTypeMetadata( - (OpaqueValue*)((uintptr_t)src + addrOffset)); - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue *)((uintptr_t)dest + addrOffset), - (OpaqueValue *)((uintptr_t)src + addrOffset)); - } - } - } +static const InitFn initWithTakeTable[] = { + &handleEnd, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + &unknownWeakInitWithTake, + &bridgeRetainBranchless, + nullptr, + nullptr, + nullptr, // Custom + &metatypeInitWithTake, + nullptr, // Generic + &existentialInitWithTake, + &resilientInitWithTake, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, }; +static void handleRefCountsInitWithTake(const Metadata *metadata, + LayoutStringReader &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uint64_t tag = 0; + do { + tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + + if (auto handler = initWithTakeTable[tag]) + handler(metadata, reader, addrOffset, dest, src); + } while (tag != 0); +} + extern "C" swift::OpaqueValue * swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { @@ -1213,7 +794,11 @@ swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, return dest; } - handleRefCounts<0, TakeHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + + handleRefCountsInitWithTake(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); return dest; } From 4a395cb8d3af426810a526980071f0f661ca92ce Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:53:44 -0700 Subject: [PATCH 04/21] [Runtime+IRGen] Some fixes and optimizations for layout strings --- include/swift/ABI/Metadata.h | 16 +++--- lib/IRGen/GenMeta.cpp | 15 +++-- stdlib/public/runtime/BytecodeLayouts.cpp | 12 ++-- stdlib/public/runtime/Metadata.cpp | 69 +++++++++++------------ stdlib/public/runtime/MetadataImpl.h | 5 ++ 5 files changed, 64 insertions(+), 53 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index 9b2cc2b04d014..3eaff50606d03 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -323,13 +323,13 @@ struct TargetMetadata { } const uint8_t *getLayoutString() const { - //assert(hasLayoutString()); - // if (isAnyClass()) { - // return asFullMetadata( - // reinterpret_cast *>( - // this)) - // ->layoutString; - // } + assert(hasLayoutString()); + if (isAnyClass()) { + return asFullMetadata( + reinterpret_cast *>( + this)) + ->layoutString; + } return asFullMetadata(this)->layoutString; } @@ -3778,7 +3778,7 @@ class swift_ptrauth_struct_context_descriptor(TypeContextDescriptor) } bool hasLayoutString() const { - return getTypeContextDescriptorFlags().hasLayoutString(); + return !hasForeignMetadataInitialization() && getTypeContextDescriptorFlags().hasLayoutString(); } /// Given that this type has foreign metadata initialization, return the diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index d8cab7583483d..8cc25d227c5a8 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -2580,7 +2580,8 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, auto genericSig = lowered.getNominalOrBoundGenericNominal()->getGenericSignature(); - hasLayoutString = !!typeLayoutEntry->layoutString(IGM, genericSig); + hasLayoutString = !ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal) && + !!typeLayoutEntry->layoutString(IGM, genericSig); if (!hasLayoutString && IGM.Context.LangOpts.hasFeature( @@ -4647,7 +4648,9 @@ namespace { return false; } - return !!getLayoutString(); + auto &ti = IGM.getTypeInfo(getLoweredType()); + + return !ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal) && !!getLayoutString(); } ConstantReference emitValueWitnessTable(bool relativeReference) { @@ -5163,10 +5166,10 @@ namespace { if (IGM.Context.LangOpts.hasFeature(Feature::LayoutStringValueWitnessesInstantiation) && IGM.getOptions().EnableLayoutStringValueWitnessesInstantiation) { - return !!getLayoutString() || needsSingletonMetadataInitialization(IGM, Target); + return (!!getLayoutString() || needsSingletonMetadataInitialization(IGM, Target)) && !needsForeignMetadataCompletionFunction(IGM, Target); } - return !!getLayoutString(); + return !!getLayoutString() && !needsForeignMetadataCompletionFunction(IGM, Target); } llvm::Constant *emitNominalTypeDescriptor() { @@ -6247,6 +6250,10 @@ namespace { return Target->getDeclaredType()->getCanonicalType(); } + bool hasLayoutString() { + return false; + } + void createMetadataCompletionFunction() { llvm_unreachable("foreign structs never require completion"); } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 753165c39dc20..54590686703b5 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -380,7 +380,7 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); - if (enumTag < numPayloads) { + if (SWIFT_LIKELY(enumTag < numPayloads)) { size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); LayoutStringReader nestedReader = reader; @@ -517,7 +517,8 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_retain(*(HeapObject **)(dest + addrOffset)); + HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + swift_retain(object); } static void unownedRetainBranchless(const Metadata *metadata, @@ -525,7 +526,7 @@ static void unownedRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset))); + HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); swift_unownedRetain(object); } @@ -1244,13 +1245,12 @@ void swift::swift_resolve_resilientAccessors(uint8_t *layoutStr, reader.skip(sizeof(size_t)); size_t casesBeginOffset = layoutStrOffset + reader.offset + - layoutStringHeaderSize + (numCases * sizeof(size_t)); + auto fieldCasesBeginOffset = fieldLayoutStr + (numCases * sizeof(size_t)) + reader.offset; for (size_t j = 0; j < numCases; j++) { size_t caseOffset = reader.readBytes(); - const uint8_t *caseLayoutString = fieldLayoutStr + reader.offset + - (numCases * sizeof(size_t)) + + const uint8_t *caseLayoutString = fieldCasesBeginOffset + caseOffset; swift_resolve_resilientAccessors(layoutStr, casesBeginOffset + caseOffset, diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index b39417bd967a7..3f1f7e0f480f3 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2783,10 +2783,6 @@ void swift::swift_initStructMetadataWithLayoutString( size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { if (type->vw_size() == 0 || type->getValueWitnesses()->isPOD()) { return 0; - } else if (type->hasLayoutString()) { - size_t offset = sizeof(uint64_t); - return LayoutStringReader{type->getLayoutString(), offset} - .readBytes(); } else if (auto *tuple = dyn_cast(type)) { size_t res = 0; for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { @@ -2803,6 +2799,10 @@ size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { } } return sizeof(uint64_t); + } else if (type->hasLayoutString()) { + size_t offset = sizeof(uint64_t); + return LayoutStringReader{type->getLayoutString(), offset} + .readBytes(); } else if (type->isAnyExistentialType()) { return sizeof(uint64_t); } else { @@ -2825,37 +2825,6 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, // No need to handle PODs previousFieldOffset = offset + fieldType->vw_size(); fullOffset += fieldType->vw_size(); - } else if (fieldType->hasLayoutString()) { - LayoutStringReader reader{fieldType->getLayoutString(), 0}; - const auto fieldFlags = reader.readBytes(); - const auto fieldRefCountBytes = reader.readBytes(); - if (fieldRefCountBytes > 0) { - flags |= fieldFlags; - memcpy(writer.layoutStr + writer.offset, - reader.layoutStr + layoutStringHeaderSize, fieldRefCountBytes); - - if (fieldFlags & LayoutStringFlags::HasRelativePointers) { - swift_resolve_resilientAccessors( - writer.layoutStr, writer.offset, - reader.layoutStr + layoutStringHeaderSize, fieldType); - } - - if (offset) { - LayoutStringReader tagReader {writer.layoutStr, writer.offset}; - auto writerOffsetCopy = writer.offset; - auto firstTagAndOffset = tagReader.readBytes(); - firstTagAndOffset += offset; - writer.writeBytes(firstTagAndOffset); - writer.offset = writerOffsetCopy; - } - - reader.offset = layoutStringHeaderSize + fieldRefCountBytes; - previousFieldOffset = reader.readBytes(); - writer.skip(fieldRefCountBytes); - } else { - previousFieldOffset += fieldType->vw_size(); - } - fullOffset += fieldType->vw_size(); } else if (auto *tuple = dyn_cast(fieldType)) { for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { _swift_addRefCountStringForMetatype(writer, flags, @@ -2890,6 +2859,36 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, writer.writeBytes(((uint64_t)tag << 56) | offset); previousFieldOffset = fieldType->vw_size(); fullOffset += previousFieldOffset; + } else if (fieldType->hasLayoutString()) { + LayoutStringReader reader{fieldType->getLayoutString(), 0}; + const auto fieldFlags = reader.readBytes(); + const auto fieldRefCountBytes = reader.readBytes(); + if (fieldRefCountBytes > 0) { + flags |= fieldFlags; + memcpy(writer.layoutStr + writer.offset, + reader.layoutStr + layoutStringHeaderSize, fieldRefCountBytes); + + if (fieldFlags & LayoutStringFlags::HasRelativePointers) { + swift_resolve_resilientAccessors(writer.layoutStr, writer.offset, + reader.layoutStr + layoutStringHeaderSize, fieldType); + } + + if (offset) { + LayoutStringReader tagReader {writer.layoutStr, writer.offset}; + auto writerOffsetCopy = writer.offset; + auto firstTagAndOffset = tagReader.readBytes(); + firstTagAndOffset += offset; + writer.writeBytes(firstTagAndOffset); + writer.offset = writerOffsetCopy; + } + + reader.offset = layoutStringHeaderSize + fieldRefCountBytes; + previousFieldOffset = reader.readBytes(); + writer.skip(fieldRefCountBytes); + } else { + previousFieldOffset += fieldType->vw_size(); + } + fullOffset += fieldType->vw_size(); } else if (fieldType->isAnyExistentialType()) { auto *existential = dyn_cast(fieldType); assert(existential); diff --git a/stdlib/public/runtime/MetadataImpl.h b/stdlib/public/runtime/MetadataImpl.h index 1365e24d6f783..727d1f60be3b8 100644 --- a/stdlib/public/runtime/MetadataImpl.h +++ b/stdlib/public/runtime/MetadataImpl.h @@ -94,6 +94,7 @@ struct NativeBox { static constexpr size_t isPOD = std::is_pod::value; static constexpr bool isBitwiseTakable = isPOD; static constexpr unsigned numExtraInhabitants = 0; + static constexpr bool hasLayoutString = false; static void destroy(T *value) { value->T::~T(); @@ -142,6 +143,7 @@ template struct RetainableBoxBase { #else static constexpr bool isAtomic = true; #endif + static constexpr bool hasLayoutString = false; static void destroy(T *addr) { Impl::release(*addr); @@ -254,6 +256,7 @@ struct WeakRetainableBoxBase { static constexpr bool isPOD = false; static constexpr bool isBitwiseTakable = false; static constexpr unsigned numExtraInhabitants = 0; + static constexpr bool hasLayoutString = false; // The implementation must provide implementations of: // static void destroy(T *); @@ -777,6 +780,8 @@ struct ValueWitnesses .withPOD(isPOD) .withBitwiseTakable(isBitwiseTakable); + static constexpr bool hasLayoutString = Box::hasLayoutString; + static void destroy(OpaqueValue *value, const Metadata *self) { return Box::destroy((typename Box::type*) value); } From 31de67c71370e50c793b2d0d86f6370353bfb5dd Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:54:14 -0700 Subject: [PATCH 05/21] [IRGen+Runtime] Further fixes for layout strings --- include/swift/ABI/Metadata.h | 16 +++++----- lib/IRGen/GenMeta.cpp | 24 +++++++------- stdlib/public/runtime/BytecodeLayouts.cpp | 38 +++++++++++++---------- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index 3eaff50606d03..ca9096a132574 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -323,13 +323,13 @@ struct TargetMetadata { } const uint8_t *getLayoutString() const { - assert(hasLayoutString()); - if (isAnyClass()) { - return asFullMetadata( - reinterpret_cast *>( - this)) - ->layoutString; - } + // assert(hasLayoutString()); + // if (isAnyClass()) { + // return asFullMetadata( + // reinterpret_cast *>( + // this)) + // ->layoutString; + // } return asFullMetadata(this)->layoutString; } @@ -3778,7 +3778,7 @@ class swift_ptrauth_struct_context_descriptor(TypeContextDescriptor) } bool hasLayoutString() const { - return !hasForeignMetadataInitialization() && getTypeContextDescriptorFlags().hasLayoutString(); + return getTypeContextDescriptorFlags().hasLayoutString(); } /// Given that this type has foreign metadata initialization, return the diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index 8cc25d227c5a8..3d161cef6345d 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -2580,9 +2580,10 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, auto genericSig = lowered.getNominalOrBoundGenericNominal()->getGenericSignature(); - hasLayoutString = !ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal) && - !!typeLayoutEntry->layoutString(IGM, genericSig); + hasLayoutString = !!typeLayoutEntry->layoutString(IGM, genericSig); + } + if (auto sd = dyn_cast(type)) { if (!hasLayoutString && IGM.Context.LangOpts.hasFeature( Feature::LayoutStringValueWitnessesInstantiation) && @@ -2591,9 +2592,6 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, needsSingletonMetadataInitialization(IGM, type) || (type->isGenericContext() && !isa(ti)); } - } - - if (auto sd = dyn_cast(type)) { StructContextDescriptorBuilder(IGM, sd, requireMetadata, hasLayoutString).emit(); } else if (auto ed = dyn_cast(type)) { @@ -4650,7 +4648,7 @@ namespace { auto &ti = IGM.getTypeInfo(getLoweredType()); - return !ti.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal) && !!getLayoutString(); + return !!getLayoutString(); } ConstantReference emitValueWitnessTable(bool relativeReference) { @@ -5166,10 +5164,10 @@ namespace { if (IGM.Context.LangOpts.hasFeature(Feature::LayoutStringValueWitnessesInstantiation) && IGM.getOptions().EnableLayoutStringValueWitnessesInstantiation) { - return (!!getLayoutString() || needsSingletonMetadataInitialization(IGM, Target)) && !needsForeignMetadataCompletionFunction(IGM, Target); + return !!getLayoutString() || needsSingletonMetadataInitialization(IGM, Target); } - return !!getLayoutString() && !needsForeignMetadataCompletionFunction(IGM, Target); + return !!getLayoutString(); } llvm::Constant *emitNominalTypeDescriptor() { @@ -5679,7 +5677,7 @@ namespace { llvm::Constant *emitNominalTypeDescriptor() { auto descriptor = EnumContextDescriptorBuilder( - IGM, Target, RequireMetadata, hasLayoutString()) + IGM, Target, RequireMetadata, !!getLayoutString()) .emit(); return descriptor; } @@ -5728,7 +5726,7 @@ namespace { } bool canBeConstant() { - return !HasUnfilledPayloadSize && !hasInstantiatedLayoutString(); + return !HasUnfilledPayloadSize;// && !hasInstantiatedLayoutString(); } }; @@ -6250,9 +6248,9 @@ namespace { return Target->getDeclaredType()->getCanonicalType(); } - bool hasLayoutString() { - return false; - } + // bool hasLayoutString() { + // return false; + // } void createMetadataCompletionFunction() { llvm_unreachable("foreign structs never require completion"); diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 54590686703b5..2f20957a350c8 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -369,29 +369,33 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t enumTag; + uintptr_t nestedAddrOffset; + LayoutStringReader nestedReader; reader.modify([&](LayoutStringReader &reader) { - size_t tagBytes; - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; auto tagBytesOffset = enumSize - tagBytes; - auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); - - if (SWIFT_LIKELY(enumTag < numPayloads)) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); - } - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); + addrOffset += enumSize; }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + } } #if SWIFT_OBJC_INTEROP @@ -704,7 +708,7 @@ static void unknownWeakInitWithTake(const Metadata *metadata, (WeakReference *)(src + addrOffset)); } -static inline void metatypeInitWithTake(const Metadata *metadata, +static void metatypeInitWithTake(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *dest, @@ -729,7 +733,7 @@ static void existentialInitWithTake(const Metadata *metadata, } } -static inline void resilientInitWithTake(const Metadata *metadata, +static void resilientInitWithTake(const Metadata *metadata, LayoutStringReader &reader, uintptr_t &addrOffset, uint8_t *dest, From 27b1764d131331bd8c53e73b941e70972e01e594 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:54:35 -0700 Subject: [PATCH 06/21] [Runtime+IRGen] Fix layout string flag in type layout and add array functions --- lib/IRGen/GenMeta.cpp | 14 +- lib/IRGen/GenStruct.cpp | 8 +- stdlib/public/runtime/Array.cpp | 9 + stdlib/public/runtime/BytecodeLayouts.cpp | 256 ++++++++++++---------- stdlib/public/runtime/BytecodeLayouts.h | 65 ++++++ 5 files changed, 227 insertions(+), 125 deletions(-) diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index 3d161cef6345d..b4b4198116c65 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -2581,17 +2581,17 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, auto genericSig = lowered.getNominalOrBoundGenericNominal()->getGenericSignature(); hasLayoutString = !!typeLayoutEntry->layoutString(IGM, genericSig); - } - if (auto sd = dyn_cast(type)) { if (!hasLayoutString && IGM.Context.LangOpts.hasFeature( Feature::LayoutStringValueWitnessesInstantiation) && IGM.getOptions().EnableLayoutStringValueWitnessesInstantiation) { - hasLayoutString |= requiresForeignTypeMetadata(type) || - needsSingletonMetadataInitialization(IGM, type) || + hasLayoutString |= needsSingletonMetadataInitialization(IGM, type) || (type->isGenericContext() && !isa(ti)); } + } + + if (auto sd = dyn_cast(type)) { StructContextDescriptorBuilder(IGM, sd, requireMetadata, hasLayoutString).emit(); } else if (auto ed = dyn_cast(type)) { @@ -4646,8 +4646,6 @@ namespace { return false; } - auto &ti = IGM.getTypeInfo(getLoweredType()); - return !!getLayoutString(); } @@ -6248,10 +6246,6 @@ namespace { return Target->getDeclaredType()->getCanonicalType(); } - // bool hasLayoutString() { - // return false; - // } - void createMetadataCompletionFunction() { llvm_unreachable("foreign structs never require completion"); } diff --git a/lib/IRGen/GenStruct.cpp b/lib/IRGen/GenStruct.cpp index b11c960bdce61..cc2828d841a3a 100644 --- a/lib/IRGen/GenStruct.cpp +++ b/lib/IRGen/GenStruct.cpp @@ -919,10 +919,10 @@ namespace { field.getTypeInfo().buildTypeLayoutEntry(IGM, fieldTy, useStructLayouts)); } - if (fields.size() == 1 && isFixedSize() && - getBestKnownAlignment() == *fields[0]->fixedAlignment(IGM)) { - return fields[0]; - } + // if (fields.size() == 1 && isFixedSize() && + // getBestKnownAlignment() == *fields[0]->fixedAlignment(IGM)) { + // return fields[0]; + // } return IGM.typeLayoutCache.getOrCreateAlignedGroupEntry( fields, T, getBestKnownAlignment().getValue(), *this); diff --git a/stdlib/public/runtime/Array.cpp b/stdlib/public/runtime/Array.cpp index ec9b5de9fac96..84e57199468fb 100644 --- a/stdlib/public/runtime/Array.cpp +++ b/stdlib/public/runtime/Array.cpp @@ -22,6 +22,7 @@ // //===----------------------------------------------------------------------===// +#include "BytecodeLayouts.h" #include "swift/Runtime/Config.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Metadata.h" @@ -124,6 +125,10 @@ static void array_copy_operation(OpaqueValue *dest, OpaqueValue *src, assert(copyKind == ArrayCopy::BackToFront); assert(count != 0); + if (self->hasLayoutString() && destOp == ArrayDest::Init && srcOp == ArraySource::Copy) { + return swift_generic_arrayInitWithCopy(dest, src, count, stride, self); + } + auto copy = get_witness_function(wtable); size_t i = count; do { @@ -202,6 +207,10 @@ void swift_arrayDestroy(OpaqueValue *begin, size_t count, const Metadata *self) return; auto stride = wtable->getStride(); + if (self->hasLayoutString()) { + return swift_generic_arrayDestroy(begin, count, stride, self); + } + for (size_t i = 0; i < count; ++i) { auto offset = i * stride; auto *obj = reinterpret_cast((char *)begin + offset); diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 2f20957a350c8..fb5959edfea8b 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -44,13 +44,13 @@ static Metadata *getExistentialTypeMetadata(OpaqueValue *object) { return reinterpret_cast(object)[NumWords_ValueBuffer]; } -template -static const FnTy readRelativeFunctionPointer(LayoutStringReader &reader) { +template +static const FnTy readRelativeFunctionPointer(Reader &reader) { static_assert(std::is_pointer::value); - auto absolute = reader.layoutStr + reader.offset; + auto absolute = reader.getAbsolute(); auto relativeOffset = - (uintptr_t)(intptr_t)(int32_t)reader.readBytes(); + (uintptr_t)(intptr_t)(int32_t)reader.template readBytes(); FnTy fn; #if SWIFT_PTRAUTH @@ -66,8 +66,9 @@ static const FnTy readRelativeFunctionPointer(LayoutStringReader &reader) { typedef Metadata *(*MetadataAccessor)(const Metadata *const *); +template static const Metadata *getResilientTypeMetadata(const Metadata *metadata, - LayoutStringReader &reader) { + Reader &reader) { auto fn = readRelativeFunctionPointer(reader); return fn(metadata->getGenericArgs()); } @@ -97,13 +98,13 @@ static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) { } static void handleRefCountsDestroy(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr); template static void handleEnd(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { @@ -111,7 +112,7 @@ static void handleEnd(const Metadata *metadata, } static void errorDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { SwiftError *error = *(SwiftError**)(addr + addrOffset); @@ -119,7 +120,7 @@ static void errorDestroyBranchless(const Metadata *metadata, } static void nativeStrongDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); @@ -127,7 +128,7 @@ static void nativeStrongDestroyBranchless(const Metadata *metadata, } static void unownedDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); @@ -135,14 +136,14 @@ static void unownedDestroyBranchless(const Metadata *metadata, } static void weakDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { swift_weakDestroy((WeakReference *)(addr + addrOffset)); } static void unknownDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { void *object = *(void**)(addr + addrOffset); @@ -150,7 +151,7 @@ static void unknownDestroyBranchless(const Metadata *metadata, } static void unknownUnownedDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { UnownedReference *object = (UnownedReference*)(addr + addrOffset); @@ -158,14 +159,14 @@ static void unknownUnownedDestroyBranchless(const Metadata *metadata, } static void unknownWeakDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { swift_unknownObjectWeakDestroy((WeakReference *)(addr + addrOffset)); } static void bridgeDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); @@ -173,11 +174,11 @@ static void bridgeDestroyBranchless(const Metadata *metadata, template static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params... params) { - reader.modify([&](LayoutStringReader &reader) { + reader.modify([&](LayoutStringReader1 &reader) { uint64_t byteCountsAndOffset; size_t payloadSize; uint64_t zeroTagValue; @@ -220,16 +221,16 @@ typedef unsigned (*GetEnumTagFn)(const uint8_t *); template static void singlePayloadEnumFNBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { + reader.modify([&](LayoutStringReader1 &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); unsigned enumTag = getEnumTag(addr + addrOffset); - if (enumTag == 0) { + if (SWIFT_LIKELY(enumTag == 0)) { reader.skip(sizeof(size_t) * 2); } else { size_t refCountBytes; @@ -243,11 +244,11 @@ static void singlePayloadEnumFNBranchless(const Metadata *metadata, template static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { + reader.modify([&](LayoutStringReader1 &reader) { GetEnumTagFn getEnumTag; size_t refCountBytes; size_t skip; @@ -255,7 +256,7 @@ static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, unsigned enumTag = getEnumTag(addr + addrOffset); - if (enumTag != 0) { + if (SWIFT_UNLIKELY(enumTag != 0)) { reader.skip(refCountBytes); addrOffset += skip; } @@ -264,11 +265,11 @@ static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, template static void singlePayloadEnumGenericBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { + reader.modify([&](LayoutStringReader1 &reader) { auto tagBytesAndOffset = reader.readBytes(); auto payloadSize = reader.readBytes(); auto *xiType = reader.readBytes(); @@ -280,7 +281,7 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, auto xiTagBytesOffset = tagBytesAndOffset & std::numeric_limits::max(); - if (extraTagBytesPattern) { + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { auto extraTagBytes = 1 << (extraTagBytesPattern - 1); auto tagBytes = readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); @@ -289,11 +290,11 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, } } - if (xiType) { + if (SWIFT_LIKELY(xiType)) { auto tag = xiType->vw_getEnumTagSinglePayload( (const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset), numEmptyCases); - if (tag == 0) { + if (SWIFT_LIKELY(tag == 0)) { return; } } @@ -305,67 +306,69 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, template static void multiPayloadEnumFNBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; - unsigned enumTag = getEnumTag(addr + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); - } - + enumTag = getEnumTag(addr + addrOffset); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); addrOffset += enumSize; }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + } } template static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { - reader.modify([&](LayoutStringReader &reader) { - GetEnumTagFn getEnumTag = reader.readBytes(); + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; - size_t numPayloads; - size_t refCountBytes; - size_t enumSize; + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; - unsigned enumTag = getEnumTag(addr + addrOffset); - - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); - - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); - } - + enumTag = getEnumTag(addr + addrOffset); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); addrOffset += enumSize; }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + } } template static void multiPayloadEnumGenericBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr, Params ...params) { @@ -375,17 +378,17 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, size_t enumSize; uint64_t enumTag; uintptr_t nestedAddrOffset; - LayoutStringReader nestedReader; - reader.modify([&](LayoutStringReader &reader) { + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); nestedReader = reader; nestedAddrOffset = addrOffset; auto tagBytesOffset = enumSize - tagBytes; - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); addrOffset += enumSize; }); @@ -400,14 +403,14 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, #if SWIFT_OBJC_INTEROP static void blockDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { _Block_release((void *)(addr + addrOffset)); } static void objcStrongDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { objc_object *object = (objc_object*)(*(uintptr_t *)(addr + addrOffset)); @@ -416,7 +419,7 @@ static void objcStrongDestroyBranchless(const Metadata *metadata, #endif static void metatypeDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { auto *type = reader.readBytes(); @@ -424,7 +427,7 @@ static void metatypeDestroyBranchless(const Metadata *metadata, } static void existentialDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); @@ -437,7 +440,7 @@ static void existentialDestroyBranchless(const Metadata *metadata, } static void resilientDestroyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { auto *type = getResilientTypeMetadata(metadata, reader); @@ -445,7 +448,7 @@ static void resilientDestroyBranchless(const Metadata *metadata, } typedef void (*DestrFnBranchless)(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr); @@ -481,35 +484,46 @@ const DestrFnBranchless destroyTableBranchless[] = { }; static void handleRefCountsDestroy(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - uint64_t tag = 0; - do { - tag = reader.readBytes(); + while (true) { + auto tag = reader.readBytes(); addrOffset += (tag & ~(0xFFULL << 56)); tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } destroyTableBranchless[tag](metadata, reader, addrOffset, addr); - } while (tag != 0); + } } extern "C" void swift_generic_destroy(swift::OpaqueValue *address, const Metadata *metadata) { const uint8_t *layoutStr = metadata->getLayoutString(); - LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; uintptr_t addrOffset = 0; handleRefCountsDestroy(metadata, reader, addrOffset, (uint8_t *)address); } +void swift::swift_generic_arrayDestroy(swift::OpaqueValue *address, size_t count, size_t stride, const Metadata *metadata) { + const uint8_t *layoutStr = metadata->getLayoutString(); + for (size_t i = 0; i < count; i++) { + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = i * stride; + handleRefCountsDestroy(metadata, reader, addrOffset, (uint8_t *)address); + } +} + static void handleRefCountsInitWithCopy(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src); static void errorRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -517,7 +531,7 @@ static void errorRetainBranchless(const Metadata *metadata, } static void nativeStrongRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -526,7 +540,7 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, } static void unownedRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -535,7 +549,7 @@ static void unownedRetainBranchless(const Metadata *metadata, } static void weakCopyInitBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -543,7 +557,7 @@ static void weakCopyInitBranchless(const Metadata *metadata, } static void unknownRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -552,7 +566,7 @@ static void unknownRetainBranchless(const Metadata *metadata, } static void unknownUnownedCopyInitBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -562,7 +576,7 @@ static void unknownUnownedCopyInitBranchless(const Metadata *metadata, } static void unknownWeakCopyInitBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -571,7 +585,7 @@ static void unknownWeakCopyInitBranchless(const Metadata *metadata, } static void bridgeRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -580,7 +594,7 @@ static void bridgeRetainBranchless(const Metadata *metadata, #if SWIFT_OBJC_INTEROP static void blockCopyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -588,7 +602,7 @@ static void blockCopyBranchless(const Metadata *metadata, } static void objcStrongRetainBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -598,7 +612,7 @@ static void objcStrongRetainBranchless(const Metadata *metadata, #endif static void metatypeInitWithCopyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -608,7 +622,7 @@ static void metatypeInitWithCopyBranchless(const Metadata *metadata, } static void existentialInitWithCopyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -618,7 +632,7 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, } static void resilientInitWithCopyBranchless(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -628,7 +642,7 @@ static void resilientInitWithCopyBranchless(const Metadata *metadata, } typedef void (*InitFn)(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src); @@ -665,18 +679,20 @@ static const InitFn initWithCopyTable[] = { }; static void handleRefCountsInitWithCopy(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - uint64_t tag = 0; - do { - tag = reader.readBytes(); + while (true) { + auto tag = reader.readBytes(); addrOffset += (tag & ~(0xFFULL << 56)); tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } initWithCopyTable[tag](metadata, reader, addrOffset, dest, src); - } while (tag != 0); + } } extern "C" swift::OpaqueValue * @@ -686,21 +702,37 @@ swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, memcpy(dest, src, size); const uint8_t *layoutStr = metadata->getLayoutString(); - LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; uintptr_t addrOffset = 0; handleRefCountsInitWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); return dest; } +void swift::swift_generic_arrayInitWithCopy(swift::OpaqueValue *dest, + swift::OpaqueValue *src, + size_t count, + size_t stride, + const Metadata *metadata) { + size_t size = stride * count; + memcpy(dest, src, size); + + const uint8_t *layoutStr = metadata->getLayoutString(); + for (size_t i = 0; i < count; i++) { + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = i * stride; + handleRefCountsInitWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); + } +} + static void handleRefCountsInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src); static void unknownWeakInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -709,7 +741,7 @@ static void unknownWeakInitWithTake(const Metadata *metadata, } static void metatypeInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -722,7 +754,7 @@ static void metatypeInitWithTake(const Metadata *metadata, } static void existentialInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -734,7 +766,7 @@ static void existentialInitWithTake(const Metadata *metadata, } static void resilientInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { @@ -773,19 +805,21 @@ static const InitFn initWithTakeTable[] = { }; static void handleRefCountsInitWithTake(const Metadata *metadata, - LayoutStringReader &reader, + LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - uint64_t tag = 0; - do { - tag = reader.readBytes(); + while (true) { + auto tag = reader.readBytes(); addrOffset += (tag & ~(0xFFULL << 56)); tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } if (auto handler = initWithTakeTable[tag]) handler(metadata, reader, addrOffset, dest, src); - } while (tag != 0); + } } extern "C" swift::OpaqueValue * @@ -800,7 +834,7 @@ swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, } const uint8_t *layoutStr = metadata->getLayoutString(); - LayoutStringReader reader{layoutStr, layoutStringHeaderSize}; + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; uintptr_t addrOffset = 0; handleRefCountsInitWithTake(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); @@ -973,7 +1007,7 @@ extern "C" unsigned swift_multiPayloadEnumGeneric_getEnumTag(swift::OpaqueValue *address, const Metadata *metadata) { auto addr = reinterpret_cast(address); - LayoutStringReader reader{metadata->getLayoutString(), + LayoutStringReader1 reader{metadata->getLayoutString() + layoutStringHeaderSize + sizeof(uint64_t)}; auto tagBytes = reader.readBytes(); diff --git a/stdlib/public/runtime/BytecodeLayouts.h b/stdlib/public/runtime/BytecodeLayouts.h index a3d9e14cd1799..66519530ae2b3 100644 --- a/stdlib/public/runtime/BytecodeLayouts.h +++ b/stdlib/public/runtime/BytecodeLayouts.h @@ -58,6 +58,56 @@ enum class RefCountingKind : uint8_t { // Reserved: 0x81 - 0xFF }; +struct LayoutStringReader1 { + const uint8_t *layoutStr; + + template + inline T readBytes() { + T returnVal; + memcpy(&returnVal, layoutStr, sizeof(T)); + layoutStr += sizeof(T); + return returnVal; + } + + template + inline void readBytes(T&... result) { + uintptr_t additionalOffset = 0; + ([&] { + memcpy(&result, layoutStr + additionalOffset, sizeof(T)); + additionalOffset += sizeof(T); + }(), ...); + layoutStr += additionalOffset; + } + + template + inline T modify(F &&f) { + LayoutStringReader1 readerCopy = *this; + T res = f(readerCopy); + layoutStr = readerCopy.layoutStr; + return res; + } + + template + inline void modify(F &&f) { + LayoutStringReader1 readerCopy = *this; + f(readerCopy); + layoutStr = readerCopy.layoutStr; + } + + template + inline T peekBytes(size_t peekOffset = 0) const { + T returnVal; + memcpy(&returnVal, layoutStr + peekOffset, sizeof(T)); + return returnVal; + } + + inline void skip(size_t n) { layoutStr += n; } + + inline uintptr_t getAbsolute() { + return (uintptr_t) layoutStr; + } +}; + struct LayoutStringReader { const uint8_t *layoutStr; size_t offset; @@ -103,6 +153,10 @@ struct LayoutStringReader { } inline void skip(size_t n) { offset += n; } + + inline uintptr_t getAbsolute() { + return (uintptr_t) layoutStr + offset; + } }; struct LayoutStringWriter { @@ -180,6 +234,17 @@ void swift_resolve_resilientAccessors(uint8_t *layoutStr, const uint8_t *fieldLayoutStr, const Metadata *fieldType); +void swift_generic_arrayDestroy(swift::OpaqueValue *addr, + size_t count, + size_t stride, + const Metadata *metadata); + +void swift_generic_arrayInitWithCopy(swift::OpaqueValue *dest, + swift::OpaqueValue *src, + size_t count, + size_t stride, + const Metadata *metadata); + constexpr size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t); } // namespace swift From 30d628270b34d7c5532ab1acc730f2ab18b44c2d Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:55:02 -0700 Subject: [PATCH 07/21] [IRGen+Runtime] Make more fine grained copies in layout string runtime functions --- lib/IRGen/TypeLayout.cpp | 3 +- stdlib/public/runtime/BytecodeLayouts.cpp | 424 ++++++++++++++++++---- stdlib/public/runtime/Metadata.cpp | 12 +- 3 files changed, 365 insertions(+), 74 deletions(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index 3a84c92267d10..39af07bef177c 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -364,8 +364,7 @@ class LayoutStringBuilder { uint64_t op = (static_cast(refCounting.kind) << 56) | skip; B.addInt64(op); refCountBytes += sizeof(uint64_t); - - skip = refCounting.size; + skip = 0; break; } } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index fb5959edfea8b..57a3d74b72a97 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -116,6 +116,7 @@ static void errorDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { SwiftError *error = *(SwiftError**)(addr + addrOffset); + addrOffset += sizeof(SwiftError*); swift_errorRelease(error); } @@ -124,6 +125,7 @@ static void nativeStrongDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(HeapObject*); swift_release(object); } @@ -132,6 +134,7 @@ static void unownedDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(HeapObject*); swift_unownedRelease(object); } @@ -139,7 +142,9 @@ static void weakDestroyBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - swift_weakDestroy((WeakReference *)(addr + addrOffset)); + auto *object = (WeakReference *)(addr + addrOffset); + addrOffset += sizeof(WeakReference); + swift_weakDestroy(object); } static void unknownDestroyBranchless(const Metadata *metadata, @@ -147,6 +152,7 @@ static void unknownDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { void *object = *(void**)(addr + addrOffset); + addrOffset += sizeof(void*); swift_unknownObjectRelease(object); } @@ -155,6 +161,7 @@ static void unknownUnownedDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { UnownedReference *object = (UnownedReference*)(addr + addrOffset); + addrOffset += sizeof(UnownedReference); swift_unknownObjectUnownedDestroy(object); } @@ -162,22 +169,24 @@ static void unknownWeakDestroyBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - swift_unknownObjectWeakDestroy((WeakReference *)(addr + addrOffset)); + auto *object = (WeakReference *)(addr + addrOffset); + addrOffset += sizeof(WeakReference); + swift_unknownObjectWeakDestroy(object); } static void bridgeDestroyBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - swift_bridgeObjectRelease(*(void **)(addr + addrOffset)); + auto *object = *(void **)(addr + addrOffset); + addrOffset += sizeof(void*); + swift_bridgeObjectRelease(object); } -template static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params... params) { + uint8_t *addr) { reader.modify([&](LayoutStringReader1 &reader) { uint64_t byteCountsAndOffset; size_t payloadSize; @@ -219,12 +228,10 @@ static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, typedef unsigned (*GetEnumTagFn)(const uint8_t *); -template static void singlePayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { reader.modify([&](LayoutStringReader1 &reader) { GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); @@ -242,12 +249,10 @@ static void singlePayloadEnumFNBranchless(const Metadata *metadata, }); } -template static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { reader.modify([&](LayoutStringReader1 &reader) { GetEnumTagFn getEnumTag; size_t refCountBytes; @@ -263,12 +268,10 @@ static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, }); } -template static void singlePayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { reader.modify([&](LayoutStringReader1 &reader) { auto tagBytesAndOffset = reader.readBytes(); auto payloadSize = reader.readBytes(); @@ -304,12 +307,11 @@ static void singlePayloadEnumGenericBranchless(const Metadata *metadata, }); } -template +template static void multiPayloadEnumFNBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { size_t numPayloads; size_t refCountBytes; size_t enumSize; @@ -325,22 +327,23 @@ static void multiPayloadEnumFNBranchless(const Metadata *metadata, enumTag = getEnumTag(addr + addrOffset); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; }); if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); + } else { + addrOffset += enumSize; } } -template +template static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { size_t numPayloads; size_t refCountBytes; size_t enumSize; @@ -356,22 +359,23 @@ static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, enumTag = getEnumTag(addr + addrOffset); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; }); if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); + } else { + addrOffset += enumSize; } } -template +template static void multiPayloadEnumGenericBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, - uint8_t *addr, - Params ...params) { + uint8_t *addr) { size_t tagBytes; size_t numPayloads; size_t refCountBytes; @@ -389,15 +393,258 @@ static void multiPayloadEnumGenericBranchless(const Metadata *metadata, enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); + } else { addrOffset += enumSize; + } +} + +static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = + readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + if (tagBytes) { + xiTagBytesPattern = 0; + } + } + + if (SWIFT_LIKELY(xiTagBytesPattern)) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + uint64_t tagBytes = + readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + if (tagBytes >= xiTagValues) { + return; + } + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + reader.skip(refCountBytes); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (SWIFT_LIKELY(enumTag == 0)) { + reader.skip(sizeof(size_t) * 2); + } else { + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (SWIFT_UNLIKELY(enumTag != 0)) { + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumGenericBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + + if (tagBytes) { + xiType = nullptr; + } + } + + if (SWIFT_LIKELY(xiType)) { + auto tag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), + numEmptyCases); + if (SWIFT_LIKELY(tag == 0)) { + return; + } + } + + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +template +static void multiPayloadEnumFNBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + enumTag = getEnumTag(src + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); }); if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; + } +} +template +static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + enumTag = getEnumTag(src + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; + } +} + +template +static void multiPayloadEnumGenericBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t enumTag; + uintptr_t nestedAddrOffset; + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); - HandlerFn(metadata, nestedReader, nestedAddrOffset, addr, std::forward(params)...); + nestedReader = reader; + nestedAddrOffset = addrOffset; + auto tagBytesOffset = enumSize - tagBytes; + + enumTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; } } @@ -406,7 +653,9 @@ static void blockDestroyBranchless(const Metadata *metadata, LayoutStringReader1 &reader, uintptr_t &addrOffset, uint8_t *addr) { - _Block_release((void *)(addr + addrOffset)); + void* object = (void *)(addr + addrOffset); + addrOffset += sizeof(void*); + _Block_release(object); } static void objcStrongDestroyBranchless(const Metadata *metadata, @@ -414,6 +663,7 @@ static void objcStrongDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { objc_object *object = (objc_object*)(*(uintptr_t *)(addr + addrOffset)); + addrOffset += sizeof(objc_object*); objc_release(object); } #endif @@ -423,7 +673,9 @@ static void metatypeDestroyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *addr) { auto *type = reader.readBytes(); - type->vw_destroy((OpaqueValue *)(addr + addrOffset)); + auto *object = (OpaqueValue *)(addr + addrOffset); + addrOffset += type->vw_size(); + type->vw_destroy(object); } static void existentialDestroyBranchless(const Metadata *metadata, @@ -432,6 +684,7 @@ static void existentialDestroyBranchless(const Metadata *metadata, uint8_t *addr) { OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); auto* type = getExistentialTypeMetadata(object); + addrOffset += type->vw_size(); if (type->getValueWitnesses()->isValueInline()) { type->vw_destroy(object); } else { @@ -474,10 +727,10 @@ const DestrFnBranchless destroyTableBranchless[] = { nullptr, // Generic &existentialDestroyBranchless, &resilientDestroyBranchless, - &singlePayloadEnumSimpleBranchless<>, - &singlePayloadEnumFNBranchless<>, - &singlePayloadEnumFNResolvedBranchless<>, - &singlePayloadEnumGenericBranchless<>, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, &multiPayloadEnumFNBranchless, &multiPayloadEnumFNResolvedBranchless, &multiPayloadEnumGenericBranchless, @@ -527,7 +780,10 @@ static void errorRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_errorRetain(*(SwiftError**)(dest + addrOffset)); + memcpy(dest + addrOffset, src + addrOffset, sizeof(SwiftError*)); + auto *object = *(SwiftError**)(dest + addrOffset); + addrOffset += sizeof(object); + swift_errorRetain(object); } static void nativeStrongRetainBranchless(const Metadata *metadata, @@ -535,7 +791,9 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + memcpy(dest + addrOffset, src + addrOffset, sizeof(HeapObject*)); HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(object); swift_retain(object); } @@ -544,7 +802,9 @@ static void unownedRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + memcpy(dest + addrOffset, src + addrOffset, sizeof(HeapObject*)); HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(object); swift_unownedRetain(object); } @@ -553,7 +813,10 @@ static void weakCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_weakCopyInit((WeakReference *)(dest + addrOffset), (WeakReference *)(src + addrOffset)); + auto *destObject = (WeakReference *)(dest + addrOffset); + auto *srcObject = (WeakReference *)(src + addrOffset); + addrOffset += sizeof(WeakReference); + swift_weakCopyInit(destObject, srcObject); } static void unknownRetainBranchless(const Metadata *metadata, @@ -561,7 +824,9 @@ static void unknownRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + memcpy(dest + addrOffset, src + addrOffset, sizeof(void*)); void *object = *(void**)(dest + addrOffset); + addrOffset += sizeof(void*); swift_unknownObjectRetain(object); } @@ -572,6 +837,7 @@ static void unknownUnownedCopyInitBranchless(const Metadata *metadata, uint8_t *src) { UnownedReference *objectDest = (UnownedReference*)(dest + addrOffset); UnownedReference *objectSrc = (UnownedReference*)(src + addrOffset); + addrOffset += sizeof(UnownedReference); swift_unknownObjectUnownedCopyInit(objectDest, objectSrc); } @@ -580,8 +846,10 @@ static void unknownWeakCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_unknownObjectWeakCopyInit((WeakReference *)(dest + addrOffset), - (WeakReference *)(src + addrOffset)); + auto *destObject = (WeakReference *)(dest + addrOffset); + auto *srcObject = (WeakReference *)(src + addrOffset); + addrOffset += sizeof(WeakReference); + swift_unknownObjectWeakCopyInit(destObject, srcObject); } static void bridgeRetainBranchless(const Metadata *metadata, @@ -589,7 +857,10 @@ static void bridgeRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_bridgeObjectRetain(*(void **)(dest + addrOffset)); + memcpy(dest + addrOffset, src + addrOffset, sizeof(void*)); + auto *object = *(void **)(dest + addrOffset); + addrOffset += sizeof(void*); + swift_bridgeObjectRetain(object); } #if SWIFT_OBJC_INTEROP @@ -598,7 +869,9 @@ static void blockCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - *(void**)dest = _Block_copy(*(void**)(src + addrOffset)); + auto *copy = _Block_copy(*(void**)(src + addrOffset)); + memcpy(dest + addrOffset, ©, sizeof(void*)); + addrOffset += sizeof(void*); } static void objcStrongRetainBranchless(const Metadata *metadata, @@ -606,7 +879,9 @@ static void objcStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + memcpy(dest + addrOffset, src + addrOffset, sizeof(objc_object*)); objc_object *object = (objc_object*)(*(uintptr_t *)(src + addrOffset)); + addrOffset += sizeof(objc_object*); objc_retain(object); } #endif @@ -617,8 +892,10 @@ static void metatypeInitWithCopyBranchless(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = reader.readBytes(); - type->vw_initializeWithCopy((OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); + type->vw_initializeWithCopy(destObject, srcObject); } static void existentialInitWithCopyBranchless(const Metadata *metadata, @@ -626,9 +903,14 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); - type->vw_initializeBufferWithCopyOfBuffer((ValueBuffer*)(dest + addrOffset), - (ValueBuffer*)(src + addrOffset)); + auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *destObject = (ValueBuffer *)(dest + addrOffset); + auto *srcObject = (ValueBuffer *)(src + addrOffset); + memcpy(dest + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), + src + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), + sizeof(uintptr_t) * 2); + addrOffset += type->vw_size(); + type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); } static void resilientInitWithCopyBranchless(const Metadata *metadata, @@ -637,8 +919,10 @@ static void resilientInitWithCopyBranchless(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = getResilientTypeMetadata(metadata, reader); - type->vw_initializeWithCopy((OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); + type->vw_initializeWithCopy(destObject, srcObject); } typedef void (*InitFn)(const Metadata *metadata, @@ -685,7 +969,11 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, uint8_t *src) { while (true) { auto tag = reader.readBytes(); - addrOffset += (tag & ~(0xFFULL << 56)); + auto offset = (tag & ~(0xFFULL << 56)); + if (offset) { + memcpy(dest + addrOffset, src + addrOffset, offset); + } + addrOffset += offset; tag >>= 56; if (SWIFT_UNLIKELY(tag == 0)) { return; @@ -698,9 +986,6 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, extern "C" swift::OpaqueValue * swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - size_t size = metadata->vw_size(); - memcpy(dest, src, size); - const uint8_t *layoutStr = metadata->getLayoutString(); LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; uintptr_t addrOffset = 0; @@ -714,9 +999,6 @@ void swift::swift_generic_arrayInitWithCopy(swift::OpaqueValue *dest, size_t count, size_t stride, const Metadata *metadata) { - size_t size = stride * count; - memcpy(dest, src, size); - const uint8_t *layoutStr = metadata->getLayoutString(); for (size_t i = 0; i < count; i++) { LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; @@ -736,8 +1018,11 @@ static void unknownWeakInitWithTake(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - swift_unknownObjectWeakTakeInit((WeakReference *)(dest + addrOffset), - (WeakReference *)(src + addrOffset)); + auto *destObject = (WeakReference *)(dest + addrOffset); + auto *srcObject = (WeakReference *)(src + addrOffset); + addrOffset += sizeof(WeakReference); + + swift_unknownObjectWeakTakeInit(destObject, srcObject); } static void metatypeInitWithTake(const Metadata *metadata, @@ -746,10 +1031,11 @@ static void metatypeInitWithTake(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = reader.readBytes(); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + type->vw_initializeWithTake(destObject, srcObject); } } @@ -759,9 +1045,11 @@ static void existentialInitWithTake(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake((OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + type->vw_initializeWithTake(destObject, srcObject); } } @@ -771,10 +1059,11 @@ static void resilientInitWithTake(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = getResilientTypeMetadata(metadata, reader); + auto *destObject = (OpaqueValue *)(dest + addrOffset); + auto *srcObject = (OpaqueValue *)(src + addrOffset); + addrOffset += type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue *)(dest + addrOffset), - (OpaqueValue *)(src + addrOffset)); + type->vw_initializeWithTake(destObject, srcObject); } } @@ -817,8 +1106,11 @@ static void handleRefCountsInitWithTake(const Metadata *metadata, return; } - if (auto handler = initWithTakeTable[tag]) + if (auto handler = initWithTakeTable[tag]) { handler(metadata, reader, addrOffset, dest, src); + } else { + addrOffset += sizeof(uintptr_t); + } } } diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 3f1f7e0f480f3..015511fc4c48c 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2857,8 +2857,8 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, } writer.writeBytes(((uint64_t)tag << 56) | offset); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); } else if (fieldType->hasLayoutString()) { LayoutStringReader reader{fieldType->getLayoutString(), 0}; const auto fieldFlags = reader.readBytes(); @@ -2895,14 +2895,14 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, auto tag = existential->isClassBounded() ? RefCountingKind::Unknown : RefCountingKind::Existential; writer.writeBytes(((uint64_t)tag << 56) | offset); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; + previousFieldOffset = existential->isClassBounded() ? fieldType->vw_size() - sizeof(uintptr_t) : 0; + fullOffset += fieldType->vw_size(); } else { metadata: writer.writeBytes(((uint64_t)RefCountingKind::Metatype << 56) | offset); writer.writeBytes(fieldType); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); } } From 4f6e69b35f1f006d406b31ad5c4e7d2370038933 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:55:25 -0700 Subject: [PATCH 08/21] [Runtime] Reduce instruction count in simple layout string operations --- stdlib/public/runtime/BytecodeLayouts.cpp | 42 ++++++++++++----------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 57a3d74b72a97..615346bb3c776 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -780,9 +780,9 @@ static void errorRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(SwiftError*)); - auto *object = *(SwiftError**)(dest + addrOffset); - addrOffset += sizeof(object); + SwiftError *object = *(SwiftError **)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(SwiftError*)); + addrOffset += sizeof(SwiftError *); swift_errorRetain(object); } @@ -791,10 +791,11 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(HeapObject*)); - HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); - addrOffset += sizeof(object); - swift_retain(object); + uintptr_t object = *(uintptr_t *)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(HeapObject*)); + object &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset += sizeof(HeapObject *); + swift_retain((HeapObject *)object); } static void unownedRetainBranchless(const Metadata *metadata, @@ -802,10 +803,11 @@ static void unownedRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(HeapObject*)); - HeapObject *object = (HeapObject*)((*(uintptr_t *)(dest + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); - addrOffset += sizeof(object); - swift_unownedRetain(object); + uintptr_t object = *(uintptr_t *)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(HeapObject*)); + object &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset += sizeof(HeapObject *); + swift_unownedRetain((HeapObject *)object); } static void weakCopyInitBranchless(const Metadata *metadata, @@ -824,9 +826,9 @@ static void unknownRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(void*)); - void *object = *(void**)(dest + addrOffset); - addrOffset += sizeof(void*); + void *object = *(void **)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(void*)); + addrOffset += sizeof(void *); swift_unknownObjectRetain(object); } @@ -857,8 +859,8 @@ static void bridgeRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(void*)); - auto *object = *(void **)(dest + addrOffset); + void *object = *(void **)(src + addrOffset); + memcpy(dest + addrOffset, &object, sizeof(void*)); addrOffset += sizeof(void*); swift_bridgeObjectRetain(object); } @@ -879,8 +881,8 @@ static void objcStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - memcpy(dest + addrOffset, src + addrOffset, sizeof(objc_object*)); objc_object *object = (objc_object*)(*(uintptr_t *)(src + addrOffset)); + memcpy(dest + addrOffset, &object, sizeof(objc_object*)); addrOffset += sizeof(objc_object*); objc_retain(object); } @@ -904,11 +906,11 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, uint8_t *dest, uint8_t *src) { auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *witness = *(void**)(src + addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); + memcpy(dest + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); + memcpy(dest + addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); auto *destObject = (ValueBuffer *)(dest + addrOffset); auto *srcObject = (ValueBuffer *)(src + addrOffset); - memcpy(dest + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), - src + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), - sizeof(uintptr_t) * 2); addrOffset += type->vw_size(); type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); } From 8ab845fdca870d56680f9ba08df3dd0f45d68768 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:55:51 -0700 Subject: [PATCH 09/21] [Runtime+IRGen] Fix offsets of existentials in layout strings --- lib/IRGen/TypeLayout.cpp | 10 +- stdlib/public/runtime/BytecodeLayouts.cpp | 143 ++++++++++-------- stdlib/public/runtime/Metadata.cpp | 2 +- .../layout_string_witnesses_types.swift | 41 +++++ .../layout_string_witnesses_dynamic.swift | 34 +++++ 5 files changed, 168 insertions(+), 62 deletions(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index 39af07bef177c..e788838212b27 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -360,13 +360,21 @@ class LayoutStringBuilder { break; } - default: { + case RefCountingKind::Existential: { uint64_t op = (static_cast(refCounting.kind) << 56) | skip; B.addInt64(op); refCountBytes += sizeof(uint64_t); skip = 0; break; } + + default: { + uint64_t op = (static_cast(refCounting.kind) << 56) | skip; + B.addInt64(op); + refCountBytes += sizeof(uint64_t); + skip = refCounting.size - IGM.getPointerSize().getValue(); + break; + } } } } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 615346bb3c776..c33d1aaf61ce9 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -684,7 +684,7 @@ static void existentialDestroyBranchless(const Metadata *metadata, uint8_t *addr) { OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); auto* type = getExistentialTypeMetadata(object); - addrOffset += type->vw_size(); + addrOffset += sizeof(uintptr_t) * (NumWords_ValueBuffer + 2); if (type->getValueWitnesses()->isValueInline()) { type->vw_destroy(object); } else { @@ -780,9 +780,10 @@ static void errorRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - SwiftError *object = *(SwiftError **)(src + addrOffset); + uintptr_t _addrOffset = addrOffset; + SwiftError *object = *(SwiftError **)(src + _addrOffset); memcpy(dest + addrOffset, &object, sizeof(SwiftError*)); - addrOffset += sizeof(SwiftError *); + addrOffset = _addrOffset + sizeof(SwiftError *); swift_errorRetain(object); } @@ -791,10 +792,11 @@ static void nativeStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - uintptr_t object = *(uintptr_t *)(src + addrOffset); - memcpy(dest + addrOffset, &object, sizeof(HeapObject*)); + uintptr_t _addrOffset = addrOffset; + uintptr_t object = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(HeapObject*)); object &= ~_swift_abi_SwiftSpareBitsMask; - addrOffset += sizeof(HeapObject *); + addrOffset = _addrOffset + sizeof(HeapObject *); swift_retain((HeapObject *)object); } @@ -803,10 +805,11 @@ static void unownedRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - uintptr_t object = *(uintptr_t *)(src + addrOffset); - memcpy(dest + addrOffset, &object, sizeof(HeapObject*)); + uintptr_t _addrOffset = addrOffset; + uintptr_t object = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(HeapObject*)); object &= ~_swift_abi_SwiftSpareBitsMask; - addrOffset += sizeof(HeapObject *); + addrOffset = _addrOffset + sizeof(HeapObject *); swift_unownedRetain((HeapObject *)object); } @@ -815,9 +818,10 @@ static void weakCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto *destObject = (WeakReference *)(dest + addrOffset); - auto *srcObject = (WeakReference *)(src + addrOffset); - addrOffset += sizeof(WeakReference); + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); swift_weakCopyInit(destObject, srcObject); } @@ -826,9 +830,10 @@ static void unknownRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - void *object = *(void **)(src + addrOffset); - memcpy(dest + addrOffset, &object, sizeof(void*)); - addrOffset += sizeof(void *); + uintptr_t _addrOffset = addrOffset; + void *object = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void *); swift_unknownObjectRetain(object); } @@ -837,9 +842,10 @@ static void unknownUnownedCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - UnownedReference *objectDest = (UnownedReference*)(dest + addrOffset); - UnownedReference *objectSrc = (UnownedReference*)(src + addrOffset); - addrOffset += sizeof(UnownedReference); + uintptr_t _addrOffset = addrOffset; + UnownedReference *objectDest = (UnownedReference*)(dest + _addrOffset); + UnownedReference *objectSrc = (UnownedReference*)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(UnownedReference); swift_unknownObjectUnownedCopyInit(objectDest, objectSrc); } @@ -848,9 +854,10 @@ static void unknownWeakCopyInitBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto *destObject = (WeakReference *)(dest + addrOffset); - auto *srcObject = (WeakReference *)(src + addrOffset); - addrOffset += sizeof(WeakReference); + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); swift_unknownObjectWeakCopyInit(destObject, srcObject); } @@ -859,9 +866,10 @@ static void bridgeRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - void *object = *(void **)(src + addrOffset); - memcpy(dest + addrOffset, &object, sizeof(void*)); - addrOffset += sizeof(void*); + uintptr_t _addrOffset = addrOffset; + void *object = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); swift_bridgeObjectRetain(object); } @@ -871,9 +879,10 @@ static void blockCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto *copy = _Block_copy(*(void**)(src + addrOffset)); - memcpy(dest + addrOffset, ©, sizeof(void*)); - addrOffset += sizeof(void*); + uintptr_t _addrOffset = addrOffset; + auto *copy = _Block_copy(*(void**)(src + _addrOffset)); + memcpy(dest + _addrOffset, ©, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); } static void objcStrongRetainBranchless(const Metadata *metadata, @@ -881,9 +890,10 @@ static void objcStrongRetainBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - objc_object *object = (objc_object*)(*(uintptr_t *)(src + addrOffset)); - memcpy(dest + addrOffset, &object, sizeof(objc_object*)); - addrOffset += sizeof(objc_object*); + uintptr_t _addrOffset = addrOffset; + objc_object *object = (objc_object*)(*(uintptr_t *)(src + _addrOffset)); + memcpy(dest + _addrOffset, &object, sizeof(objc_object*)); + addrOffset = _addrOffset + sizeof(objc_object*); objc_retain(object); } #endif @@ -893,10 +903,11 @@ static void metatypeInitWithCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto *type = reader.readBytes(); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); type->vw_initializeWithCopy(destObject, srcObject); } @@ -905,13 +916,14 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { - auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); - auto *witness = *(void**)(src + addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); - memcpy(dest + addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); - memcpy(dest + addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); - auto *destObject = (ValueBuffer *)(dest + addrOffset); - auto *srcObject = (ValueBuffer *)(src + addrOffset); - addrOffset += type->vw_size(); + uintptr_t _addrOffset = addrOffset; + auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); + auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); + memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); + memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); + auto *destObject = (ValueBuffer *)(dest + _addrOffset); + auto *srcObject = (ValueBuffer *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); } @@ -920,10 +932,11 @@ static void resilientInitWithCopyBranchless(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto *type = getResilientTypeMetadata(metadata, reader); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); type->vw_initializeWithCopy(destObject, srcObject); } @@ -970,12 +983,13 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, uint8_t *dest, uint8_t *src) { while (true) { + uintptr_t _addrOffset = addrOffset; auto tag = reader.readBytes(); auto offset = (tag & ~(0xFFULL << 56)); if (offset) { - memcpy(dest + addrOffset, src + addrOffset, offset); + memcpy(dest + _addrOffset, src + _addrOffset, offset); } - addrOffset += offset; + addrOffset = _addrOffset + offset; tag >>= 56; if (SWIFT_UNLIKELY(tag == 0)) { return; @@ -1032,10 +1046,11 @@ static void metatypeInitWithTake(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto *type = reader.readBytes(); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); } @@ -1046,10 +1061,11 @@ static void existentialInitWithTake(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); } @@ -1060,10 +1076,11 @@ static void resilientInitWithTake(const Metadata *metadata, uintptr_t &addrOffset, uint8_t *dest, uint8_t *src) { + uintptr_t _addrOffset = addrOffset; auto *type = getResilientTypeMetadata(metadata, reader); - auto *destObject = (OpaqueValue *)(dest + addrOffset); - auto *srcObject = (OpaqueValue *)(src + addrOffset); - addrOffset += type->vw_size(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); } @@ -1101,17 +1118,25 @@ static void handleRefCountsInitWithTake(const Metadata *metadata, uint8_t *dest, uint8_t *src) { while (true) { + uintptr_t _addrOffset = addrOffset; auto tag = reader.readBytes(); - addrOffset += (tag & ~(0xFFULL << 56)); + auto offset = (tag & ~(0xFFULL << 56)); + if (offset) { + memcpy(dest + _addrOffset, src + _addrOffset, offset); + } + _addrOffset += offset; tag >>= 56; if (SWIFT_UNLIKELY(tag == 0)) { + addrOffset = _addrOffset; return; } if (auto handler = initWithTakeTable[tag]) { + addrOffset = _addrOffset; handler(metadata, reader, addrOffset, dest, src); } else { - addrOffset += sizeof(uintptr_t); + memcpy(dest + _addrOffset, src + _addrOffset, sizeof(uintptr_t)); + addrOffset = _addrOffset + sizeof(uintptr_t); } } } @@ -1119,11 +1144,9 @@ static void handleRefCountsInitWithTake(const Metadata *metadata, extern "C" swift::OpaqueValue * swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - size_t size = metadata->vw_size(); - - memcpy(dest, src, size); - if (SWIFT_LIKELY(metadata->getValueWitnesses()->isBitwiseTakable())) { + size_t size = metadata->vw_size(); + memcpy(dest, src, size); return dest; } diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 015511fc4c48c..559f41acb9f99 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2743,7 +2743,7 @@ void swift::swift_initStructMetadataWithLayoutString( } fullOffset += fieldType->size; - previousFieldOffset = fieldType->size; + previousFieldOffset = fieldType->size - sizeof(uintptr_t); continue; } diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index 224db34e51b1a..4b2334b244959 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -223,6 +223,47 @@ public struct Recursive { } } +public protocol Foo: AnyObject {} + +public enum Repro { + public struct R1 { + let x: R2? + let y: Int + weak var z: Foo? + let w: [NSNotification.Name] + + public init(x: R2?, + y: Int, + z: Foo?, + w: [NSNotification.Name]) { + self.x = x + self.y = y + self.z = z + self.w = w + } + } + + public struct R2 { + let x: NSNotification.Name? + let y: Bool + let z: E1 + + public init(x: NSNotification.Name?, + y: Bool, + z: E1) { + self.x = x + self.y = y + self.z = z + } + } + + public enum E1: Int { + case x + case y + case z + } +} + #if os(macOS) import Foundation diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index af95e974e8cdf..6a1db82293aa7 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -24,6 +24,40 @@ class TestClass { } } +class XX: Foo { + deinit { + print("XX deinitialized!") + } +} + +func testRepro() { + let ptr = allocateInternalGenericPtr(of: Repro.R1.self) + + do { + let x = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: XX(), w: []) + testGenericInit(ptr, to: x) + } + + do { + let y = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: XX(), w: []) + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: XX deinitialized! + testGenericAssign(ptr, from: y) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: XX deinitialized! + testGenericDestroy(ptr, of: Repro.R1.self) + + ptr.deallocate() +} + +testRepro() + func testGeneric() { let ptr = allocateInternalGenericPtr(of: TestClass.self) From 7c0203dd550db14a7ff34d08d3f069a4250dfb1b Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:56:19 -0700 Subject: [PATCH 10/21] [Runtime] Combine copy and destroy in assignWithCopy --- stdlib/public/runtime/BytecodeLayouts.cpp | 643 +++++++++++++++++- .../layout_string_witnesses_dynamic.swift | 10 +- 2 files changed, 641 insertions(+), 12 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index c33d1aaf61ce9..90a7136c44bd0 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -986,7 +986,7 @@ static void handleRefCountsInitWithCopy(const Metadata *metadata, uintptr_t _addrOffset = addrOffset; auto tag = reader.readBytes(); auto offset = (tag & ~(0xFFULL << 56)); - if (offset) { + if (SWIFT_UNLIKELY(offset)) { memcpy(dest + _addrOffset, src + _addrOffset, offset); } addrOffset = _addrOffset + offset; @@ -1051,9 +1051,8 @@ static void metatypeInitWithTake(const Metadata *metadata, auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); addrOffset = _addrOffset + type->vw_size(); - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake(destObject, srcObject); - } + + type->vw_initializeWithTake(destObject, srcObject); } static void existentialInitWithTake(const Metadata *metadata, @@ -1063,8 +1062,11 @@ static void existentialInitWithTake(const Metadata *metadata, uint8_t *src) { uintptr_t _addrOffset = addrOffset; auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); + memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); + memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); @@ -1081,9 +1083,7 @@ static void resilientInitWithTake(const Metadata *metadata, auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); addrOffset = _addrOffset + type->vw_size(); - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake(destObject, srcObject); - } + type->vw_initializeWithTake(destObject, srcObject); } static const InitFn initWithTakeTable[] = { @@ -1121,7 +1121,7 @@ static void handleRefCountsInitWithTake(const Metadata *metadata, uintptr_t _addrOffset = addrOffset; auto tag = reader.readBytes(); auto offset = (tag & ~(0xFFULL << 56)); - if (offset) { + if (SWIFT_UNLIKELY(offset)) { memcpy(dest + _addrOffset, src + _addrOffset, offset); } _addrOffset += offset; @@ -1159,11 +1159,634 @@ swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, return dest; } +static void errorAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + SwiftError *destObject = *(SwiftError **)(dest + _addrOffset); + SwiftError *srcObject = *(SwiftError **)(src + _addrOffset); + memcpy(dest + addrOffset, &srcObject, sizeof(SwiftError*)); + addrOffset = _addrOffset + sizeof(SwiftError *); + swift_errorRelease(destObject); + swift_errorRetain(srcObject); +} + +static void nativeStrongAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset); + uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(HeapObject*)); + srcObject &= ~_swift_abi_SwiftSpareBitsMask; + destObject &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset = _addrOffset + sizeof(HeapObject *); + swift_release((HeapObject *)destObject); + swift_retain((HeapObject *)srcObject); +} + +static void unownedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset); + uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(HeapObject*)); + destObject &= ~_swift_abi_SwiftSpareBitsMask; + srcObject &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset = _addrOffset + sizeof(HeapObject *); + swift_unownedRelease((HeapObject *)destObject); + swift_unownedRetain((HeapObject *)srcObject); +} + +static void weakAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); + swift_weakCopyAssign(destObject, srcObject); +} + +static void unknownAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + void *destObject = *(void **)(dest + _addrOffset); + void *srcObject = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void *); + swift_unknownObjectRelease(destObject); + swift_unknownObjectRetain(srcObject); +} + +static void unknownUnownedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + UnownedReference *objectDest = (UnownedReference*)(dest + _addrOffset); + UnownedReference *objectSrc = (UnownedReference*)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(UnownedReference); + swift_unknownObjectUnownedCopyAssign(objectDest, objectSrc); +} + +static void unknownWeakAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); + swift_unknownObjectWeakCopyAssign(destObject, srcObject); +} + +static void bridgeAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + void *destObject = *(void **)(dest + _addrOffset); + void *srcObject = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); + swift_bridgeObjectRelease(destObject); + swift_bridgeObjectRetain(srcObject); +} + +#if SWIFT_OBJC_INTEROP +static void blockAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + _Block_release(*(void **)(dest + _addrOffset)); + auto *copy = _Block_copy(*(void **)(src + _addrOffset)); + memcpy(dest + _addrOffset, ©, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); +} + +static void objcStrongAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + objc_object *destObject = (objc_object*)(*(uintptr_t *)(dest + _addrOffset)); + objc_object *srcObject = (objc_object*)(*(uintptr_t *)(src + _addrOffset)); + memcpy(dest + _addrOffset, &srcObject, sizeof(objc_object*)); + addrOffset = _addrOffset + sizeof(objc_object*); + objc_release(destObject); + objc_retain(srcObject); +} +#endif + +static void metatypeAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = reader.readBytes(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_assignWithCopy(destObject, srcObject); +} + +static void existentialAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); + auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); + memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); + memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); + if (type->getValueWitnesses()->isValueInline()) { + type->vw_assignWithCopy(destObject, srcObject); + } else { + swift_release(*(HeapObject**)destObject); + memcpy(destObject, srcObject, sizeof(uintptr_t)); + swift_retain(*(HeapObject**)srcObject); + } +} + +static void resilientAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getResilientTypeMetadata(metadata, reader); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_assignWithCopy(destObject, srcObject); +} + +static void handleSingleRefCountDestroy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } + destroyTableBranchless[tag](metadata, reader, addrOffset, addr); +} + +static void handleSingleRefCountInitWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } + initWithCopyTable[tag](metadata, reader, addrOffset, dest, src); +} + +static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t srcTagBytes = 0; + uint64_t destTagBytes = 0; + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + + srcTagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + destTagBytes = readTagBytes(dest + addrOffset + payloadSize, extraTagBytes); + } + + if (SWIFT_LIKELY(xiTagBytesPattern)) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + srcTagBytes = srcTagBytes ? 0 : + readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + destTagBytes = destTagBytes ? 0 : + readTagBytes(dest + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + } + + if (srcTagBytes >= xiTagValues && destTagBytes >= xiTagValues) { + return; + } else if (destTagBytes >= xiTagValues) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (srcTagBytes >= xiTagValues) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + + unsigned srcTag = getEnumTag(src + addrOffset); + unsigned destTag = getEnumTag(dest + addrOffset); + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned srcTag = getEnumTag(src + addrOffset); + unsigned destTag = getEnumTag(dest + addrOffset); + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumGenericAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t srcTag = 0; + uint64_t destTag = 0; + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + srcTag = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + destTag = readTagBytes(dest + addrOffset + payloadSize, extraTagBytes); + } + + if (SWIFT_LIKELY(xiType)) { + if (!srcTag) { + srcTag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), + numEmptyCases); + } + + if (!destTag) { + destTag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(dest + addrOffset + xiTagBytesOffset), + numEmptyCases); + } + } + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void multiPayloadEnumFNAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned srcTag; + unsigned destTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + srcTag = getEnumTag(src + addrOffset); + destTag = getEnumTag(dest + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); + } + + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} + +static void multiPayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned srcTag; + unsigned destTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + srcTag = getEnumTag(src + addrOffset); + destTag = getEnumTag(dest + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); + } + + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} + +static void multiPayloadEnumGenericAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t srcTag; + uint64_t destTag; + uintptr_t nestedAddrOffset; + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + nestedReader = reader; + nestedAddrOffset = addrOffset; + auto tagBytesOffset = enumSize - tagBytes; + + srcTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); + destTag = readTagBytes(dest + addrOffset + tagBytesOffset, tagBytes); + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); + } + + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} + +static const InitFn assignWithCopyTable[] = { + &handleEnd, + &errorAssignWithCopy, + &nativeStrongAssignWithCopy, + &unownedAssignWithCopy, + &weakAssignWithCopy, + &unknownAssignWithCopy, + &unknownUnownedAssignWithCopy, + &unknownWeakAssignWithCopy, + &bridgeAssignWithCopy, +#if SWIFT_OBJC_INTEROP + &blockAssignWithCopy, + &objcStrongAssignWithCopy, +#else + nullptr, + nullptr, +#endif + nullptr, // Custom + &metatypeAssignWithCopy, + nullptr, // Generic + &existentialAssignWithCopy, + &resilientAssignWithCopy, + &singlePayloadEnumSimpleAssignWithCopy, + &singlePayloadEnumFNAssignWithCopy, + &singlePayloadEnumFNResolvedAssignWithCopy, + &singlePayloadEnumGenericAssignWithCopy, + &multiPayloadEnumFNAssignWithCopy, + &multiPayloadEnumFNResolvedAssignWithCopy, + &multiPayloadEnumGenericAssignWithCopy, +}; + +static void handleRefCountsAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + while (true) { + uintptr_t _addrOffset = addrOffset; + auto tag = reader.readBytes(); + auto offset = (tag & ~(0xFFULL << 56)); + if (offset) { + memcpy(dest + _addrOffset, src + _addrOffset, offset); + } + addrOffset = _addrOffset + offset; + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } + + assignWithCopyTable[tag](metadata, reader, addrOffset, dest, src); + } +} + extern "C" swift::OpaqueValue * swift_generic_assignWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - swift_generic_destroy(dest, metadata); - return swift_generic_initWithCopy(dest, src, metadata); + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsAssignWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); + + return dest; } extern "C" swift::OpaqueValue * diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index 6a1db82293aa7..2d9dd91754703 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -33,18 +33,23 @@ class XX: Foo { func testRepro() { let ptr = allocateInternalGenericPtr(of: Repro.R1.self) + var xx: XX? = XX() + do { - let x = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: XX(), w: []) + let x = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: xx!, w: []) testGenericInit(ptr, to: x) } + var yy: XX? = XX() + do { - let y = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: XX(), w: []) + let y = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: yy!, w: []) // CHECK: Before deinit print("Before deinit") // CHECK-NEXT: XX deinitialized! testGenericAssign(ptr, from: y) + xx = nil } // CHECK-NEXT: Before deinit @@ -52,6 +57,7 @@ func testRepro() { // CHECK-NEXT: XX deinitialized! testGenericDestroy(ptr, of: Repro.R1.self) + yy = nil ptr.deallocate() } From 8b3aaf54236bccf110bc35c9e7c8b3e3d65e8936 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:56:42 -0700 Subject: [PATCH 11/21] [Runtime] Compute end before looping over single payload ref count --- stdlib/public/runtime/BytecodeLayouts.cpp | 24 +++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 90a7136c44bd0..694f335af9f6b 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -1414,12 +1414,14 @@ static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, if (srcTagBytes >= xiTagValues && destTagBytes >= xiTagValues) { return; } else if (destTagBytes >= xiTagValues) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; } else if (srcTagBytes >= xiTagValues) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); } } else { @@ -1448,12 +1450,14 @@ static void singlePayloadEnumFNAssignWithCopy(const Metadata *metadata, if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { return; } else if (srcTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; } else if (destTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); } } else { @@ -1482,12 +1486,14 @@ static void singlePayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { return; } else if (srcTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; } else if (destTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); } } else { @@ -1541,12 +1547,14 @@ static void singlePayloadEnumGenericAssignWithCopy(const Metadata *metadata, if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { return; } else if (srcTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; } else if (destTag == 0) { - while (reader.layoutStr < (reader.layoutStr + refCountBytes)) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); } } else { From 56048ac19fea6ce8c55b1cebaab94c0367dbceac Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:57:06 -0700 Subject: [PATCH 12/21] [Runtime+IRGen] Fix existential offset for multiple protocol witnesses --- lib/IRGen/TypeLayout.cpp | 2 +- stdlib/public/runtime/BytecodeLayouts.cpp | 17 ++------- .../layout_string_witnesses_types.swift | 15 ++++++++ .../layout_string_witnesses_static.swift | 37 +++++++++++++++++++ 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index e788838212b27..06c5934464d62 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -364,7 +364,7 @@ class LayoutStringBuilder { uint64_t op = (static_cast(refCounting.kind) << 56) | skip; B.addInt64(op); refCountBytes += sizeof(uint64_t); - skip = 0; + skip = refCounting.size - getFixedBufferSize(IGM).getValue(); break; } diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 694f335af9f6b..9edb27a78f0a3 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -684,7 +684,7 @@ static void existentialDestroyBranchless(const Metadata *metadata, uint8_t *addr) { OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); auto* type = getExistentialTypeMetadata(object); - addrOffset += sizeof(uintptr_t) * (NumWords_ValueBuffer + 2); + addrOffset += sizeof(uintptr_t) * NumWords_ValueBuffer; if (type->getValueWitnesses()->isValueInline()) { type->vw_destroy(object); } else { @@ -918,12 +918,9 @@ static void existentialInitWithCopyBranchless(const Metadata *metadata, uint8_t *src) { uintptr_t _addrOffset = addrOffset; auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); - auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); - memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); - memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); auto *destObject = (ValueBuffer *)(dest + _addrOffset); auto *srcObject = (ValueBuffer *)(src + _addrOffset); - addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); } @@ -1062,12 +1059,9 @@ static void existentialInitWithTake(const Metadata *metadata, uint8_t *src) { uintptr_t _addrOffset = addrOffset; auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); - auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); - memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); - memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); - addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { type->vw_initializeWithTake(destObject, srcObject); } @@ -1317,12 +1311,9 @@ static void existentialAssignWithCopy(const Metadata *metadata, uint8_t *src) { uintptr_t _addrOffset = addrOffset; auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); - auto *witness = *(void**)(src + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t))); - memcpy(dest + _addrOffset + (NumWords_ValueBuffer * sizeof(uintptr_t)), &type, sizeof(uintptr_t)); - memcpy(dest + _addrOffset + ((NumWords_ValueBuffer + 1) * sizeof(uintptr_t)), &witness, sizeof(uintptr_t)); auto *destObject = (OpaqueValue *)(dest + _addrOffset); auto *srcObject = (OpaqueValue *)(src + _addrOffset); - addrOffset = _addrOffset + (sizeof(uintptr_t) * (2 + NumWords_ValueBuffer)); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); if (type->getValueWitnesses()->isValueInline()) { type->vw_assignWithCopy(destObject, srcObject); } else { diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index 4b2334b244959..9190d06cc3d37 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -264,6 +264,21 @@ public enum Repro { } } +public protocol A {} +public protocol B {} +public protocol C {} + +public struct MultiProtocolExistentialWrapper { + let x: Int = 0 + let y: any (A&B&C) + let z: AnyObject + + public init(y: any (A&B&C), z: AnyObject) { + self.y = y + self.z = z + } +} + #if os(macOS) import Foundation diff --git a/test/Interpreter/layout_string_witnesses_static.swift b/test/Interpreter/layout_string_witnesses_static.swift index 861c53ab87a96..738c21e47f114 100644 --- a/test/Interpreter/layout_string_witnesses_static.swift +++ b/test/Interpreter/layout_string_witnesses_static.swift @@ -293,6 +293,43 @@ func testExistentialStructBox() { testExistentialStructBox() +class ClassWithABC: A, B, C { + deinit { + print("ClassWithABC deinitialized!") + } +} + +func testMultiProtocolExistential() { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + + do { + let x = ClassWithABC() + testInit(ptr, to: MultiProtocolExistentialWrapper(y: x, z: SimpleClass(x: 23))) + } + + do { + let y = ClassWithABC() + + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: ClassWithABC deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testAssign(ptr, from: MultiProtocolExistentialWrapper(y: y, z: SimpleClass(x: 32))) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: ClassWithABC deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testDestroy(ptr) + + ptr.deallocate() +} + +testMultiProtocolExistential() + class ClassWithSomeClassProtocol: SomeClassProtocol { deinit { print("ClassWithSomeClassProtocol deinitialized!") From 7e3f56de242e8e63ae4bc56880709ea1b6eab761 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:57:33 -0700 Subject: [PATCH 13/21] [Runtime] Fix generic existentials in layout strings --- stdlib/public/runtime/Metadata.cpp | 2 +- .../layout_string_witnesses_types.swift | 19 +++++++--- .../layout_string_witnesses_dynamic.swift | 36 ++++++++++++++++--- .../layout_string_witnesses_static.swift | 31 ++++++++++++++++ 4 files changed, 79 insertions(+), 9 deletions(-) diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 559f41acb9f99..e7fff9ccb92a6 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2895,7 +2895,7 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, auto tag = existential->isClassBounded() ? RefCountingKind::Unknown : RefCountingKind::Existential; writer.writeBytes(((uint64_t)tag << 56) | offset); - previousFieldOffset = existential->isClassBounded() ? fieldType->vw_size() - sizeof(uintptr_t) : 0; + previousFieldOffset = fieldType->vw_size() - (existential->isClassBounded() ? sizeof(uintptr_t) : (NumWords_ValueBuffer * sizeof(uintptr_t))); fullOffset += fieldType->vw_size(); } else { metadata: diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index 9190d06cc3d37..a0fc4f63144f8 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -279,6 +279,17 @@ public struct MultiProtocolExistentialWrapper { } } +public struct AnyWrapper { + let x: Int = 0 + let y: Any + let z: AnyObject + + public init(y: Any, z: AnyObject) { + self.y = y + self.z = z + } +} + #if os(macOS) import Foundation @@ -343,8 +354,8 @@ public struct NestedWrapper { } struct InternalGeneric { - let x: T let y: Int + let x: T } public enum SinglePayloadSimpleClassEnum { @@ -615,19 +626,19 @@ public func allocateInternalGenericPtr(of tpe: T.Type) -> UnsafeMutableRawPoi @inline(never) public func testGenericAssign(_ ptr: __owned UnsafeMutableRawPointer, from x: T) { let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) - let x = InternalGeneric(x: x, y: 23) + let x = InternalGeneric(y: 23, x: x) testAssign(ptr, from: x) } @inline(never) public func testGenericInit(_ ptr: __owned UnsafeMutableRawPointer, to x: T) { let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) - let x = InternalGeneric(x: x, y: 23) + let x = InternalGeneric(y: 23, x: x) testInit(ptr, to: x) } @inline(never) public func testGenericDestroy(_ ptr: __owned UnsafeMutableRawPointer, of tpe: T.Type) { - let ptr = ptr.assumingMemoryBound(to: tpe) + let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) testDestroy(ptr) } diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index 2d9dd91754703..932b19612f79d 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -4,7 +4,7 @@ // RUN: %target-codesign %t/%target-library-name(layout_string_witnesses_types) // RUN: %target-swift-frontend -enable-experimental-feature LayoutStringValueWitnesses -enable-experimental-feature LayoutStringValueWitnessesInstantiation -enable-layout-string-value-witnesses -enable-layout-string-value-witnesses-instantiation -enable-library-evolution -enable-autolinking-runtime-compatibility-bytecode-layouts -emit-module -emit-module-path=%t/layout_string_witnesses_types_resilient.swiftmodule %S/Inputs/layout_string_witnesses_types_resilient.swift // RUN: %target-build-swift -g -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnesses -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnessesInstantiation -Xfrontend -enable-layout-string-value-witnesses -Xfrontend -enable-layout-string-value-witnesses-instantiation -Xfrontend -enable-library-evolution -c -parse-as-library -o %t/layout_string_witnesses_types_resilient.o %S/Inputs/layout_string_witnesses_types_resilient.swift -// RUN: %target-build-swift -g -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnesses -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnessesInstantiation -Xfrontend -enable-layout-string-value-witnesses -Xfrontend -enable-layout-string-value-witnesses-instantiation -Xfrontend -enable-type-layout -parse-stdlib -module-name layout_string_witnesses_dynamic -llayout_string_witnesses_types -L%t %t/layout_string_witnesses_types_resilient.o -I %t -o %t/main %s %target-rpath(%t) +// RUN: %target-build-swift -g -parse-stdlib -module-name layout_string_witnesses_dynamic -llayout_string_witnesses_types -L%t %t/layout_string_witnesses_types_resilient.o -I %t -o %t/main %s %target-rpath(%t) // RUN: %target-codesign %t/main // RUN: %target-run %t/main %t/%target-library-name(layout_string_witnesses_types) | %FileCheck %s --check-prefix=CHECK -check-prefix=CHECK-%target-os @@ -92,7 +92,35 @@ func testGeneric() { testGeneric() -func testPrespecializedStructAnyObject() { +func testGenericAny() { + let ptr = allocateInternalGenericPtr(of: Any.self) + + do { + let x: Any = TestClass() + testGenericInit(ptr, to: x as Any) + } + + do { + let y: Any = TestClass() + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + testGenericAssign(ptr, from: y as Any) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + testGenericDestroy(ptr, of: Any.self) + + ptr.deallocate() +} + +testGenericAny() + +func testPrespecializedAnyObject() { let ptr = UnsafeMutablePointer>.allocate(capacity: 1) do { @@ -120,7 +148,7 @@ func testPrespecializedStructAnyObject() { ptr.deallocate() } -testPrespecializedStructAnyObject() +testPrespecializedAnyObject() func testPrespecializedStructSimpleClass() { let ptr = UnsafeMutablePointer>.allocate(capacity: 1) @@ -866,7 +894,7 @@ func testResilientSinglePayloadEnumComplexTag() { testResilientSinglePayloadEnumComplexTag() func testResilientMultiPayloadEnumTag() { - let x = switch getResilientMultiPayloadEnumEmpty0() { + let x = switch getResilientMultiPayloadEnumEmpty0(AnyObject.self) { case .nonEmpty0: 0 case .nonEmpty1: 1 case .empty0: 2 diff --git a/test/Interpreter/layout_string_witnesses_static.swift b/test/Interpreter/layout_string_witnesses_static.swift index 738c21e47f114..8ecab346e4ea1 100644 --- a/test/Interpreter/layout_string_witnesses_static.swift +++ b/test/Interpreter/layout_string_witnesses_static.swift @@ -293,6 +293,37 @@ func testExistentialStructBox() { testExistentialStructBox() +func testAnyWrapper() { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + + do { + let x = TestClass() + testInit(ptr, to: AnyWrapper(y: x, z: SimpleClass(x: 23))) + } + + do { + let y = TestClass() + + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testAssign(ptr, from: AnyWrapper(y: y, z: SimpleClass(x: 32))) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testDestroy(ptr) + + ptr.deallocate() +} + +testAnyWrapper() + class ClassWithABC: A, B, C { deinit { print("ClassWithABC deinitialized!") From 734e9b8ed4cf0ed7b0d86a877407ce26bf6d8d89 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:57:57 -0700 Subject: [PATCH 14/21] [IRGen] Make enum metadata non-const for instantiated layout strings --- lib/IRGen/GenMeta.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index b4b4198116c65..bda693170b7de 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -5675,7 +5675,7 @@ namespace { llvm::Constant *emitNominalTypeDescriptor() { auto descriptor = EnumContextDescriptorBuilder( - IGM, Target, RequireMetadata, !!getLayoutString()) + IGM, Target, RequireMetadata, hasLayoutString()) .emit(); return descriptor; } @@ -5724,7 +5724,7 @@ namespace { } bool canBeConstant() { - return !HasUnfilledPayloadSize;// && !hasInstantiatedLayoutString(); + return !HasUnfilledPayloadSize && !hasInstantiatedLayoutString(); } }; From fd967fd9e3817e51dad8bd3d61484ad259cdccbb Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:58:25 -0700 Subject: [PATCH 15/21] [Runtime] Properly handle boxed references in layout string instantiation --- stdlib/public/runtime/Metadata.cpp | 51 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index e7fff9ccb92a6..724b11bfb6d5c 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2781,7 +2781,8 @@ void swift::swift_initStructMetadataWithLayoutString( } size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { - if (type->vw_size() == 0 || type->getValueWitnesses()->isPOD()) { + auto *vwt = type->getValueWitnesses(); + if (type->vw_size() == 0 || vwt->isPOD()) { return 0; } else if (auto *tuple = dyn_cast(type)) { size_t res = 0; @@ -2789,14 +2790,13 @@ size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { res += _swift_refCountBytesForMetatype(tuple->getElement(i).Type); } return res; + } else if (vwt == &VALUE_WITNESS_SYM(Bo) || + vwt == &VALUE_WITNESS_SYM(BO) || + vwt == &VALUE_WITNESS_SYM(Bb)) { + return sizeof(uint64_t); } else if (auto *cls = type->getClassObject()) { if (cls->isTypeMetadata()) { - auto *vwt = cls->getValueWitnesses(); - if (vwt != &VALUE_WITNESS_SYM(Bo) && - vwt != &VALUE_WITNESS_SYM(BO) && - vwt != &VALUE_WITNESS_SYM(Bb)) { - goto metadata; - } + goto metadata; } return sizeof(uint64_t); } else if (type->hasLayoutString()) { @@ -2819,9 +2819,10 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, size_t unalignedOffset = fullOffset; fullOffset = roundUpToAlignMask(fullOffset, fieldType->vw_alignment() - 1); size_t offset = fullOffset - unalignedOffset + previousFieldOffset; + auto *vwt = fieldType->getValueWitnesses(); if (fieldType->vw_size() == 0) { return; - } else if (fieldType->getValueWitnesses()->isPOD()) { + } else if (vwt->isPOD()) { // No need to handle PODs previousFieldOffset = offset + fieldType->vw_size(); fullOffset += fieldType->vw_size(); @@ -2831,6 +2832,25 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, tuple->getElement(i).Type, fullOffset, previousFieldOffset); } + } else if (vwt == &VALUE_WITNESS_SYM(Bo)) { + auto tag = RefCountingKind::NativeStrong; + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); + } else if (vwt == &VALUE_WITNESS_SYM(BO)) { + #if SWIFT_OBJC_INTEROP + auto tag = RefCountingKind::ObjC; + #else + auto tag = RefCountingKind::Unknown; + #endif + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); + } else if (vwt == &VALUE_WITNESS_SYM(Bb)) { + auto tag = RefCountingKind::Bridge; + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); } else if (auto *cls = fieldType->getClassObject()) { RefCountingKind tag; if (!cls->isTypeMetadata()) { @@ -2840,20 +2860,7 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, tag = RefCountingKind::Unknown; #endif } else { - auto *vwt = cls->getValueWitnesses(); - if (vwt == &VALUE_WITNESS_SYM(Bo)) { - tag = RefCountingKind::NativeStrong; - } else if (vwt == &VALUE_WITNESS_SYM(BO)) { - #if SWIFT_OBJC_INTEROP - tag = RefCountingKind::ObjC; - #else - tag = RefCountingKind::Unknown; - #endif - } else if (vwt == &VALUE_WITNESS_SYM(Bb)) { - tag = RefCountingKind::Bridge; - } else { - goto metadata; - }; + goto metadata; } writer.writeBytes(((uint64_t)tag << 56) | offset); From 0ff7f1db232617e861e267b8b56582296019fe5d Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:58:47 -0700 Subject: [PATCH 16/21] [IRGen] Don't generate layout strings for moveonly types --- lib/IRGen/TypeLayout.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index 06c5934464d62..f7e995679a6a2 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -1741,7 +1741,7 @@ AlignedGroupEntry::layoutString(IRGenModule &IGM, bool AlignedGroupEntry::refCountString(IRGenModule &IGM, LayoutStringBuilder &B, GenericSignature genericSig) const { - if (!isFixedSize(IGM)) { + if (!isFixedSize(IGM) || ty.isMoveOnly()) { return false; } From 21f40643802419cc1706a5598471f0f952d7ea7e Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 10:59:15 -0700 Subject: [PATCH 17/21] [Runtime] Fix singlePayloadEnumSimpleAssignWithCopy --- stdlib/public/runtime/BytecodeLayouts.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index 9edb27a78f0a3..a651966e57e30 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -1404,13 +1404,13 @@ static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, if (srcTagBytes >= xiTagValues && destTagBytes >= xiTagValues) { return; - } else if (destTagBytes >= xiTagValues) { + } else if (srcTagBytes >= xiTagValues) { const uint8_t *end = (reader.layoutStr + refCountBytes); while (reader.layoutStr < end) { handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); } return; - } else if (srcTagBytes >= xiTagValues) { + } else if (destTagBytes >= xiTagValues) { const uint8_t *end = (reader.layoutStr + refCountBytes); while (reader.layoutStr < end) { handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); From a0e48a33623d92f5c177cc29a62a928096880563 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 11:02:59 -0700 Subject: [PATCH 18/21] [Runtime] Assert when trying to get layout string from class --- include/swift/ABI/Metadata.h | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index ca9096a132574..99fb5ac17e786 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -323,13 +323,9 @@ struct TargetMetadata { } const uint8_t *getLayoutString() const { - // assert(hasLayoutString()); - // if (isAnyClass()) { - // return asFullMetadata( - // reinterpret_cast *>( - // this)) - // ->layoutString; - // } + assert(hasLayoutString()); + // Classes should not have layout strings + assert(!isAnyClass()); return asFullMetadata(this)->layoutString; } From 904418f517633f409cb2d90ebd3ba393fa37f105 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Mon, 21 Aug 2023 13:16:00 -0700 Subject: [PATCH 19/21] [Test] Remove accidentally commited broken test code --- .../layout_string_witnesses_types.swift | 41 ------------------ .../layout_string_witnesses_dynamic.swift | 42 +------------------ 2 files changed, 1 insertion(+), 82 deletions(-) diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index a0fc4f63144f8..60312f3d0635b 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -223,47 +223,6 @@ public struct Recursive { } } -public protocol Foo: AnyObject {} - -public enum Repro { - public struct R1 { - let x: R2? - let y: Int - weak var z: Foo? - let w: [NSNotification.Name] - - public init(x: R2?, - y: Int, - z: Foo?, - w: [NSNotification.Name]) { - self.x = x - self.y = y - self.z = z - self.w = w - } - } - - public struct R2 { - let x: NSNotification.Name? - let y: Bool - let z: E1 - - public init(x: NSNotification.Name?, - y: Bool, - z: E1) { - self.x = x - self.y = y - self.z = z - } - } - - public enum E1: Int { - case x - case y - case z - } -} - public protocol A {} public protocol B {} public protocol C {} diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index 932b19612f79d..e7de40bb7fe5c 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -24,46 +24,6 @@ class TestClass { } } -class XX: Foo { - deinit { - print("XX deinitialized!") - } -} - -func testRepro() { - let ptr = allocateInternalGenericPtr(of: Repro.R1.self) - - var xx: XX? = XX() - - do { - let x = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: xx!, w: []) - testGenericInit(ptr, to: x) - } - - var yy: XX? = XX() - - do { - let y = Repro.R1(x: Repro.R2(x: .NSThreadWillExit, y: true, z: .x), y: 43, z: yy!, w: []) - // CHECK: Before deinit - print("Before deinit") - - // CHECK-NEXT: XX deinitialized! - testGenericAssign(ptr, from: y) - xx = nil - } - - // CHECK-NEXT: Before deinit - print("Before deinit") - - // CHECK-NEXT: XX deinitialized! - testGenericDestroy(ptr, of: Repro.R1.self) - yy = nil - - ptr.deallocate() -} - -testRepro() - func testGeneric() { let ptr = allocateInternalGenericPtr(of: TestClass.self) @@ -894,7 +854,7 @@ func testResilientSinglePayloadEnumComplexTag() { testResilientSinglePayloadEnumComplexTag() func testResilientMultiPayloadEnumTag() { - let x = switch getResilientMultiPayloadEnumEmpty0(AnyObject.self) { + let x = switch getResilientMultiPayloadEnumEmpty0() { case .nonEmpty0: 0 case .nonEmpty1: 1 case .empty0: 2 From e3a1bac8596739242921bed88b69519ce758f00c Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Thu, 24 Aug 2023 11:01:14 -0700 Subject: [PATCH 20/21] [Runtime] Use copy of addrOffset in single payload assign with copy --- stdlib/public/runtime/BytecodeLayouts.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index a651966e57e30..aac184ad30b7a 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -1412,8 +1412,9 @@ static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, return; } else if (destTagBytes >= xiTagValues) { const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; while (reader.layoutStr < end) { - handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); } } else { reader.skip(refCountBytes); @@ -1448,8 +1449,9 @@ static void singlePayloadEnumFNAssignWithCopy(const Metadata *metadata, return; } else if (destTag == 0) { const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; while (reader.layoutStr < end) { - handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); } } else { reader.skip(refCountBytes); @@ -1484,8 +1486,9 @@ static void singlePayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, return; } else if (destTag == 0) { const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; while (reader.layoutStr < end) { - handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); } } else { reader.skip(refCountBytes); @@ -1545,8 +1548,9 @@ static void singlePayloadEnumGenericAssignWithCopy(const Metadata *metadata, return; } else if (destTag == 0) { const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; while (reader.layoutStr < end) { - handleSingleRefCountDestroy(metadata, reader, addrOffset, dest); + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); } } else { reader.skip(refCountBytes); From 97b2dc722e1ee736638313a118fa263e8fba8d95 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Thu, 24 Aug 2023 11:01:55 -0700 Subject: [PATCH 21/21] [IRGen] Assign extraTagByteCount to the correct union field in addSinglePayloadEnumFN --- lib/IRGen/TypeLayout.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index f7e995679a6a2..33ff42e6eb661 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -160,7 +160,7 @@ class LayoutStringBuilder { RefCounting op; op.kind = RefCountingKind::SinglePayloadEnumFN; op.singlePayloadEnumFN.tagFn = tagFn; - op.singlePayloadEnumSimple.extraTagByteCount = extraTagByteCount; + op.singlePayloadEnumFN.extraTagByteCount = extraTagByteCount; op.singlePayloadEnumFN.payload = payload; refCountings.push_back(op); }