diff --git a/include/swift/ABI/Metadata.h b/include/swift/ABI/Metadata.h index 38511c67533ca..99fb5ac17e786 100644 --- a/include/swift/ABI/Metadata.h +++ b/include/swift/ABI/Metadata.h @@ -324,12 +324,8 @@ struct TargetMetadata { const uint8_t *getLayoutString() const { assert(hasLayoutString()); - if (isAnyClass()) { - return asFullMetadata( - reinterpret_cast *>( - this)) - ->layoutString; - } + // Classes should not have layout strings + assert(!isAnyClass()); return asFullMetadata(this)->layoutString; } diff --git a/include/swift/AST/Builtins.def b/include/swift/AST/Builtins.def index fb271e7e1ba88..286ebdb1bfd6a 100644 --- a/include/swift/AST/Builtins.def +++ b/include/swift/AST/Builtins.def @@ -985,14 +985,14 @@ BUILTIN_MISC_OPERATION_WITH_SILGEN(CreateAsyncTaskInGroup, /// is a pure value and therefore we can consider it as readnone). BUILTIN_MISC_OPERATION_WITH_SILGEN(GlobalStringTablePointer, "globalStringTablePointer", "n", Special) -// autoDiffCreateLinearMapContext: (Builtin.Word) -> Builtin.NativeObject -BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffCreateLinearMapContext, "autoDiffCreateLinearMapContext", "", Special) +// autoDiffCreateLinearMapContextWithType: (T.Type) -> Builtin.NativeObject +BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffCreateLinearMapContextWithType, "autoDiffCreateLinearMapContextWithType", "", Special) // autoDiffProjectTopLevelSubcontext: (Builtin.NativeObject) -> Builtin.RawPointer BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffProjectTopLevelSubcontext, "autoDiffProjectTopLevelSubcontext", "n", Special) -// autoDiffAllocateSubcontext: (Builtin.NativeObject, Builtin.Word) -> Builtin.RawPointer -BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffAllocateSubcontext, "autoDiffAllocateSubcontext", "", Special) +// autoDiffAllocateSubcontextWithType: (Builtin.NativeObject, T.Type) -> Builtin.RawPointer +BUILTIN_MISC_OPERATION_WITH_SILGEN(AutoDiffAllocateSubcontextWithType, "autoDiffAllocateSubcontextWithType", "", Special) /// Build a Builtin.Executor value from an "ordinary" serial executor /// reference. diff --git a/include/swift/Runtime/RuntimeFunctions.def b/include/swift/Runtime/RuntimeFunctions.def index f578452b76a89..44cde707d176f 100644 --- a/include/swift/Runtime/RuntimeFunctions.def +++ b/include/swift/Runtime/RuntimeFunctions.def @@ -2532,12 +2532,12 @@ FUNCTION(TaskGroupDestroy, EFFECT(Concurrency), UNKNOWN_MEMEFFECTS) -// AutoDiffLinearMapContext *swift_autoDiffCreateLinearMapContext(size_t); -FUNCTION(AutoDiffCreateLinearMapContext, - swift_autoDiffCreateLinearMapContext, SwiftCC, +// AutoDiffLinearMapContext *swift_autoDiffCreateLinearMapContextWithType(const Metadata *); +FUNCTION(AutoDiffCreateLinearMapContextWithType, + swift_autoDiffCreateLinearMapContextWithType, SwiftCC, DifferentiationAvailability, RETURNS(RefCountedPtrTy), - ARGS(SizeTy), + ARGS(TypeMetadataPtrTy), ATTRS(NoUnwind), EFFECT(AutoDiff), MEMEFFECTS(ArgMemOnly)) @@ -2552,12 +2552,12 @@ FUNCTION(AutoDiffProjectTopLevelSubcontext, EFFECT(AutoDiff), MEMEFFECTS(ArgMemOnly)) -// void *swift_autoDiffAllocateSubcontext(AutoDiffLinearMapContext *, size_t); -FUNCTION(AutoDiffAllocateSubcontext, - swift_autoDiffAllocateSubcontext, SwiftCC, +// void *swift_autoDiffAllocateSubcontextWithType(AutoDiffLinearMapContext *, const Metadata *); +FUNCTION(AutoDiffAllocateSubcontextWithType, + swift_autoDiffAllocateSubcontextWithType, SwiftCC, DifferentiationAvailability, RETURNS(Int8PtrTy), - ARGS(RefCountedPtrTy, SizeTy), + ARGS(RefCountedPtrTy, TypeMetadataPtrTy), ATTRS(NoUnwind), EFFECT(AutoDiff), MEMEFFECTS(ArgMemOnly)) diff --git a/lib/AST/Builtins.cpp b/lib/AST/Builtins.cpp index 1c5e8272a3f9f..cb10d233d8a69 100644 --- a/lib/AST/Builtins.cpp +++ b/lib/AST/Builtins.cpp @@ -1609,7 +1609,8 @@ static ValueDecl *getBuildComplexEqualitySerialExecutorRef(ASTContext &ctx, static ValueDecl *getAutoDiffCreateLinearMapContext(ASTContext &ctx, Identifier id) { return getBuiltinFunction( - id, {BuiltinIntegerType::getWordType(ctx)}, ctx.TheNativeObjectType); + ctx, id, _thin, _generics(_unrestricted), + _parameters(_metatype(_typeparam(0))), _nativeObject); } static ValueDecl *getAutoDiffProjectTopLevelSubcontext(ASTContext &ctx, @@ -1621,8 +1622,8 @@ static ValueDecl *getAutoDiffProjectTopLevelSubcontext(ASTContext &ctx, static ValueDecl *getAutoDiffAllocateSubcontext(ASTContext &ctx, Identifier id) { return getBuiltinFunction( - id, {ctx.TheNativeObjectType, BuiltinIntegerType::getWordType(ctx)}, - ctx.TheRawPointerType); + ctx, id, _thin, _generics(_unrestricted), + _parameters(_nativeObject, _metatype(_typeparam(0))), _rawPointer); } static ValueDecl *getPoundAssert(ASTContext &Context, Identifier Id) { @@ -2949,13 +2950,13 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) { case BuiltinValueKind::HopToActor: return getHopToActor(Context, Id); - case BuiltinValueKind::AutoDiffCreateLinearMapContext: + case BuiltinValueKind::AutoDiffCreateLinearMapContextWithType: return getAutoDiffCreateLinearMapContext(Context, Id); case BuiltinValueKind::AutoDiffProjectTopLevelSubcontext: return getAutoDiffProjectTopLevelSubcontext(Context, Id); - case BuiltinValueKind::AutoDiffAllocateSubcontext: + case BuiltinValueKind::AutoDiffAllocateSubcontextWithType: return getAutoDiffAllocateSubcontext(Context, Id); } diff --git a/lib/IRGen/GenBuiltin.cpp b/lib/IRGen/GenBuiltin.cpp index c697557a38684..5dfaebc1d0318 100644 --- a/lib/IRGen/GenBuiltin.cpp +++ b/lib/IRGen/GenBuiltin.cpp @@ -1307,9 +1307,10 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin, return; } - if (Builtin.ID == BuiltinValueKind::AutoDiffCreateLinearMapContext) { - auto topLevelSubcontextSize = args.claimNext(); - out.add(emitAutoDiffCreateLinearMapContext(IGF, topLevelSubcontextSize) + if (Builtin.ID == BuiltinValueKind::AutoDiffCreateLinearMapContextWithType) { + auto topLevelSubcontextMetaType = args.claimNext(); + out.add(emitAutoDiffCreateLinearMapContextWithType( + IGF, topLevelSubcontextMetaType) .getAddress()); return; } @@ -1322,12 +1323,13 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin, return; } - if (Builtin.ID == BuiltinValueKind::AutoDiffAllocateSubcontext) { + if (Builtin.ID == BuiltinValueKind::AutoDiffAllocateSubcontextWithType) { Address allocatorAddr(args.claimNext(), IGF.IGM.RefCountedStructTy, IGF.IGM.getPointerAlignment()); - auto size = args.claimNext(); - out.add( - emitAutoDiffAllocateSubcontext(IGF, allocatorAddr, size).getAddress()); + auto subcontextMetatype = args.claimNext(); + out.add(emitAutoDiffAllocateSubcontextWithType(IGF, allocatorAddr, + subcontextMetatype) + .getAddress()); return; } diff --git a/lib/IRGen/GenCall.cpp b/lib/IRGen/GenCall.cpp index f39779bdf6be8..efb91c1b917e5 100644 --- a/lib/IRGen/GenCall.cpp +++ b/lib/IRGen/GenCall.cpp @@ -5483,11 +5483,13 @@ IRGenFunction::getFunctionPointerForResumeIntrinsic(llvm::Value *resume) { return fnPtr; } -Address irgen::emitAutoDiffCreateLinearMapContext( - IRGenFunction &IGF, llvm::Value *topLevelSubcontextSize) { +Address irgen::emitAutoDiffCreateLinearMapContextWithType( + IRGenFunction &IGF, llvm::Value *topLevelSubcontextMetatype) { + topLevelSubcontextMetatype = IGF.Builder.CreateBitCast( + topLevelSubcontextMetatype, IGF.IGM.TypeMetadataPtrTy); auto *call = IGF.Builder.CreateCall( - IGF.IGM.getAutoDiffCreateLinearMapContextFunctionPointer(), - {topLevelSubcontextSize}); + IGF.IGM.getAutoDiffCreateLinearMapContextWithTypeFunctionPointer(), + {topLevelSubcontextMetatype}); call->setDoesNotThrow(); call->setCallingConv(IGF.IGM.SwiftCC); return Address(call, IGF.IGM.RefCountedStructTy, @@ -5504,11 +5506,13 @@ Address irgen::emitAutoDiffProjectTopLevelSubcontext( return Address(call, IGF.IGM.Int8Ty, IGF.IGM.getPointerAlignment()); } -Address irgen::emitAutoDiffAllocateSubcontext( - IRGenFunction &IGF, Address context, llvm::Value *size) { +Address irgen::emitAutoDiffAllocateSubcontextWithType( + IRGenFunction &IGF, Address context, llvm::Value *subcontextMetatype) { + subcontextMetatype = + IGF.Builder.CreateBitCast(subcontextMetatype, IGF.IGM.TypeMetadataPtrTy); auto *call = IGF.Builder.CreateCall( - IGF.IGM.getAutoDiffAllocateSubcontextFunctionPointer(), - {context.getAddress(), size}); + IGF.IGM.getAutoDiffAllocateSubcontextWithTypeFunctionPointer(), + {context.getAddress(), subcontextMetatype}); call->setDoesNotThrow(); call->setCallingConv(IGF.IGM.SwiftCC); return Address(call, IGF.IGM.Int8Ty, IGF.IGM.getPointerAlignment()); diff --git a/lib/IRGen/GenCall.h b/lib/IRGen/GenCall.h index c64e97dc8a264..70879ce628d05 100644 --- a/lib/IRGen/GenCall.h +++ b/lib/IRGen/GenCall.h @@ -261,12 +261,15 @@ namespace irgen { CanSILFunctionType fnType, Explosion &result, Explosion &error); - Address emitAutoDiffCreateLinearMapContext( - IRGenFunction &IGF, llvm::Value *topLevelSubcontextSize); + Address emitAutoDiffCreateLinearMapContextWithType( + IRGenFunction &IGF, llvm::Value *topLevelSubcontextMetatype); + Address emitAutoDiffProjectTopLevelSubcontext( IRGenFunction &IGF, Address context); - Address emitAutoDiffAllocateSubcontext( - IRGenFunction &IGF, Address context, llvm::Value *size); + + Address + emitAutoDiffAllocateSubcontextWithType(IRGenFunction &IGF, Address context, + llvm::Value *subcontextMetatype); FunctionPointer getFunctionPointerForDispatchCall(IRGenModule &IGM, const FunctionPointer &fn); diff --git a/lib/IRGen/GenMeta.cpp b/lib/IRGen/GenMeta.cpp index effcafefee127..510dfc84d00f5 100644 --- a/lib/IRGen/GenMeta.cpp +++ b/lib/IRGen/GenMeta.cpp @@ -2586,8 +2586,7 @@ void irgen::emitLazyTypeContextDescriptor(IRGenModule &IGM, IGM.Context.LangOpts.hasFeature( Feature::LayoutStringValueWitnessesInstantiation) && IGM.getOptions().EnableLayoutStringValueWitnessesInstantiation) { - hasLayoutString |= requiresForeignTypeMetadata(type) || - needsSingletonMetadataInitialization(IGM, type) || + hasLayoutString |= needsSingletonMetadataInitialization(IGM, type) || (type->isGenericContext() && !isa(ti)); } } diff --git a/lib/IRGen/GenStruct.cpp b/lib/IRGen/GenStruct.cpp index b11c960bdce61..cc2828d841a3a 100644 --- a/lib/IRGen/GenStruct.cpp +++ b/lib/IRGen/GenStruct.cpp @@ -919,10 +919,10 @@ namespace { field.getTypeInfo().buildTypeLayoutEntry(IGM, fieldTy, useStructLayouts)); } - if (fields.size() == 1 && isFixedSize() && - getBestKnownAlignment() == *fields[0]->fixedAlignment(IGM)) { - return fields[0]; - } + // if (fields.size() == 1 && isFixedSize() && + // getBestKnownAlignment() == *fields[0]->fixedAlignment(IGM)) { + // return fields[0]; + // } return IGM.typeLayoutCache.getOrCreateAlignedGroupEntry( fields, T, getBestKnownAlignment().getValue(), *this); diff --git a/lib/IRGen/TypeLayout.cpp b/lib/IRGen/TypeLayout.cpp index 8f9829aa9baa2..c124e72cea4b7 100644 --- a/lib/IRGen/TypeLayout.cpp +++ b/lib/IRGen/TypeLayout.cpp @@ -160,7 +160,7 @@ class LayoutStringBuilder { RefCounting op; op.kind = RefCountingKind::SinglePayloadEnumFN; op.singlePayloadEnumFN.tagFn = tagFn; - op.singlePayloadEnumSimple.extraTagByteCount = extraTagByteCount; + op.singlePayloadEnumFN.extraTagByteCount = extraTagByteCount; op.singlePayloadEnumFN.payload = payload; refCountings.push_back(op); } @@ -360,12 +360,19 @@ class LayoutStringBuilder { break; } - default: { + case RefCountingKind::Existential: { uint64_t op = (static_cast(refCounting.kind) << 56) | skip; B.addInt64(op); refCountBytes += sizeof(uint64_t); + skip = refCounting.size - getFixedBufferSize(IGM).getValue(); + break; + } - skip = refCounting.size; + default: { + uint64_t op = (static_cast(refCounting.kind) << 56) | skip; + B.addInt64(op); + refCountBytes += sizeof(uint64_t); + skip = refCounting.size - IGM.getPointerSize().getValue(); break; } } @@ -1734,7 +1741,7 @@ AlignedGroupEntry::layoutString(IRGenModule &IGM, bool AlignedGroupEntry::refCountString(IRGenModule &IGM, LayoutStringBuilder &B, GenericSignature genericSig) const { - if (!isFixedSize(IGM)) { + if (!isFixedSize(IGM) || ty.isMoveOnly()) { return false; } diff --git a/lib/SIL/IR/OperandOwnership.cpp b/lib/SIL/IR/OperandOwnership.cpp index 825ea48e9e85b..b61fa8dfc6e51 100644 --- a/lib/SIL/IR/OperandOwnership.cpp +++ b/lib/SIL/IR/OperandOwnership.cpp @@ -944,7 +944,7 @@ BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, InitializeDistributedRemoteActor) BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, InitializeNonDefaultDistributedActor) -BUILTIN_OPERAND_OWNERSHIP(PointerEscape, AutoDiffAllocateSubcontext) +BUILTIN_OPERAND_OWNERSHIP(PointerEscape, AutoDiffAllocateSubcontextWithType) BUILTIN_OPERAND_OWNERSHIP(PointerEscape, AutoDiffProjectTopLevelSubcontext) // FIXME: ConvertTaskToJob is documented as taking NativePointer. It's operand's @@ -956,8 +956,7 @@ BUILTIN_OPERAND_OWNERSHIP(BitwiseEscape, BuildComplexEqualitySerialExecutorRef) BUILTIN_OPERAND_OWNERSHIP(BitwiseEscape, BuildDefaultActorExecutorRef) BUILTIN_OPERAND_OWNERSHIP(BitwiseEscape, BuildMainActorExecutorRef) -BUILTIN_OPERAND_OWNERSHIP(TrivialUse, AutoDiffCreateLinearMapContext) - +BUILTIN_OPERAND_OWNERSHIP(TrivialUse, AutoDiffCreateLinearMapContextWithType) #undef BUILTIN_OPERAND_OWNERSHIP #define SHOULD_NEVER_VISIT_BUILTIN(ID) \ diff --git a/lib/SIL/IR/ValueOwnership.cpp b/lib/SIL/IR/ValueOwnership.cpp index 104348cf8d862..572b158e6ca29 100644 --- a/lib/SIL/IR/ValueOwnership.cpp +++ b/lib/SIL/IR/ValueOwnership.cpp @@ -572,9 +572,9 @@ CONSTANT_OWNERSHIP_BUILTIN(None, InitializeDefaultActor) CONSTANT_OWNERSHIP_BUILTIN(None, DestroyDefaultActor) CONSTANT_OWNERSHIP_BUILTIN(None, InitializeDistributedRemoteActor) CONSTANT_OWNERSHIP_BUILTIN(None, InitializeNonDefaultDistributedActor) -CONSTANT_OWNERSHIP_BUILTIN(Owned, AutoDiffCreateLinearMapContext) +CONSTANT_OWNERSHIP_BUILTIN(Owned, AutoDiffCreateLinearMapContextWithType) CONSTANT_OWNERSHIP_BUILTIN(None, AutoDiffProjectTopLevelSubcontext) -CONSTANT_OWNERSHIP_BUILTIN(None, AutoDiffAllocateSubcontext) +CONSTANT_OWNERSHIP_BUILTIN(None, AutoDiffAllocateSubcontextWithType) CONSTANT_OWNERSHIP_BUILTIN(None, GetCurrentExecutor) CONSTANT_OWNERSHIP_BUILTIN(None, ResumeNonThrowingContinuationReturning) CONSTANT_OWNERSHIP_BUILTIN(None, ResumeThrowingContinuationReturning) diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index 6cf1a89f14f3a..b672f69550c50 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -2567,8 +2567,8 @@ static void visitBuiltinAddress(BuiltinInst *builtin, case BuiltinValueKind::CancelAsyncTask: case BuiltinValueKind::CreateAsyncTask: case BuiltinValueKind::CreateAsyncTaskInGroup: - case BuiltinValueKind::AutoDiffCreateLinearMapContext: - case BuiltinValueKind::AutoDiffAllocateSubcontext: + case BuiltinValueKind::AutoDiffCreateLinearMapContextWithType: + case BuiltinValueKind::AutoDiffAllocateSubcontextWithType: case BuiltinValueKind::InitializeDefaultActor: case BuiltinValueKind::InitializeDistributedRemoteActor: case BuiltinValueKind::InitializeNonDefaultDistributedActor: diff --git a/lib/SILGen/SILGenBuiltin.cpp b/lib/SILGen/SILGenBuiltin.cpp index 2dbcde3be080a..d907e73ef0e90 100644 --- a/lib/SILGen/SILGenBuiltin.cpp +++ b/lib/SILGen/SILGenBuiltin.cpp @@ -1708,16 +1708,15 @@ static ManagedValue emitBuiltinHopToActor(SILGenFunction &SGF, SILLocation loc, return ManagedValue::forObjectRValueWithoutOwnership(SGF.emitEmptyTuple(loc)); } -static ManagedValue emitBuiltinAutoDiffCreateLinearMapContext( +static ManagedValue emitBuiltinAutoDiffCreateLinearMapContextWithType( SILGenFunction &SGF, SILLocation loc, SubstitutionMap subs, ArrayRef args, SGFContext C) { ASTContext &ctx = SGF.getASTContext(); auto *builtinApply = SGF.B.createBuiltin( loc, - ctx.getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffCreateLinearMapContext)), - SILType::getNativeObjectType(ctx), - subs, + ctx.getIdentifier(getBuiltinName( + BuiltinValueKind::AutoDiffCreateLinearMapContextWithType)), + SILType::getNativeObjectType(ctx), subs, /*args*/ {args[0].getValue()}); return SGF.emitManagedRValueWithCleanup(builtinApply); } @@ -1736,16 +1735,15 @@ static ManagedValue emitBuiltinAutoDiffProjectTopLevelSubcontext( return ManagedValue::forObjectRValueWithoutOwnership(builtinApply); } -static ManagedValue emitBuiltinAutoDiffAllocateSubcontext( +static ManagedValue emitBuiltinAutoDiffAllocateSubcontextWithType( SILGenFunction &SGF, SILLocation loc, SubstitutionMap subs, ArrayRef args, SGFContext C) { ASTContext &ctx = SGF.getASTContext(); auto *builtinApply = SGF.B.createBuiltin( loc, ctx.getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffAllocateSubcontext)), - SILType::getRawPointerType(ctx), - subs, + getBuiltinName(BuiltinValueKind::AutoDiffAllocateSubcontextWithType)), + SILType::getRawPointerType(ctx), subs, /*args*/ {args[0].borrow(SGF, loc).getValue(), args[1].getValue()}); return ManagedValue::forObjectRValueWithoutOwnership(builtinApply); } diff --git a/lib/SILOptimizer/Differentiation/VJPCloner.cpp b/lib/SILOptimizer/Differentiation/VJPCloner.cpp index 7cb715b278553..50098dda2c173 100644 --- a/lib/SILOptimizer/Differentiation/VJPCloner.cpp +++ b/lib/SILOptimizer/Differentiation/VJPCloner.cpp @@ -17,6 +17,8 @@ #define DEBUG_TYPE "differentiation" +#include "swift/AST/Types.h" + #include "swift/SILOptimizer/Differentiation/VJPCloner.h" #include "swift/SILOptimizer/Analysis/DifferentiableActivityAnalysis.h" #include "swift/SILOptimizer/Differentiation/ADContext.h" @@ -118,15 +120,21 @@ class VJPCloner::Implementation final auto pullbackTupleType = remapASTType(pullbackInfo.getLinearMapTupleType(returnBB)->getCanonicalType()); Builder.setInsertionPoint(vjp->getEntryBlock()); - auto topLevelSubcontextSize = emitMemoryLayoutSize( - Builder, original->getLocation(), pullbackTupleType); + + auto pbTupleMetatypeType = + CanMetatypeType::get(pullbackTupleType, MetatypeRepresentation::Thick); + auto pbTupleMetatypeSILType = + SILType::getPrimitiveObjectType(pbTupleMetatypeType); + auto pbTupleMetatype = + Builder.createMetatype(original->getLocation(), pbTupleMetatypeSILType); + // Create an context. pullbackContextValue = Builder.createBuiltin( original->getLocation(), - getASTContext().getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffCreateLinearMapContext)), - SILType::getNativeObjectType(getASTContext()), - SubstitutionMap(), {topLevelSubcontextSize}); + getASTContext().getIdentifier(getBuiltinName( + BuiltinValueKind::AutoDiffCreateLinearMapContextWithType)), + SILType::getNativeObjectType(getASTContext()), SubstitutionMap(), + {pbTupleMetatype}); borrowedPullbackContextValue = Builder.createBeginBorrow( original->getLocation(), pullbackContextValue); LLVM_DEBUG(getADDebugStream() @@ -148,8 +156,8 @@ class VJPCloner::Implementation final return builtinAutoDiffAllocateSubcontextGenericSignature; auto &ctx = getASTContext(); auto *decl = cast(getBuiltinValueDecl( - ctx, ctx.getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffAllocateSubcontext)))); + ctx, ctx.getIdentifier(getBuiltinName( + BuiltinValueKind::AutoDiffAllocateSubcontextWithType)))); builtinAutoDiffAllocateSubcontextGenericSignature = decl->getGenericSignature(); assert(builtinAutoDiffAllocateSubcontextGenericSignature); @@ -1067,14 +1075,21 @@ EnumInst *VJPCloner::Implementation::buildPredecessorEnumValue( assert(enumEltType == rawPtrType); auto pbTupleType = remapASTType(pullbackInfo.getLinearMapTupleType(predBB)->getCanonicalType()); - SILValue pbTupleSize = - emitMemoryLayoutSize(Builder, loc, pbTupleType); + + auto pbTupleMetatypeType = + CanMetatypeType::get(pbTupleType, MetatypeRepresentation::Thick); + auto pbTupleMetatypeSILType = + SILType::getPrimitiveObjectType(pbTupleMetatypeType); + auto pbTupleMetatype = + Builder.createMetatype(original->getLocation(), pbTupleMetatypeSILType); + auto rawBufferValue = builder.createBuiltin( loc, - getASTContext().getIdentifier( - getBuiltinName(BuiltinValueKind::AutoDiffAllocateSubcontext)), + getASTContext().getIdentifier(getBuiltinName( + BuiltinValueKind::AutoDiffAllocateSubcontextWithType)), rawPtrType, SubstitutionMap(), - {borrowedPullbackContextValue, pbTupleSize}); + {borrowedPullbackContextValue, pbTupleMetatype}); + auto typedBufferValue = builder.createPointerToAddress( loc, rawBufferValue, pbTupleVal->getType().getAddressType(), diff --git a/lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp b/lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp index 01069fe3e2774..20359097fcf3d 100644 --- a/lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp +++ b/lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp @@ -146,7 +146,7 @@ static bool isBarrier(SILInstruction *inst) { case BuiltinValueKind::COWBufferForReading: case BuiltinValueKind::GetCurrentAsyncTask: case BuiltinValueKind::GetCurrentExecutor: - case BuiltinValueKind::AutoDiffCreateLinearMapContext: + case BuiltinValueKind::AutoDiffCreateLinearMapContextWithType: case BuiltinValueKind::EndAsyncLet: case BuiltinValueKind::EndAsyncLetLifetime: case BuiltinValueKind::CreateTaskGroup: @@ -199,7 +199,7 @@ static bool isBarrier(SILInstruction *inst) { case BuiltinValueKind::ResumeThrowingContinuationReturning: case BuiltinValueKind::ResumeThrowingContinuationThrowing: case BuiltinValueKind::AutoDiffProjectTopLevelSubcontext: - case BuiltinValueKind::AutoDiffAllocateSubcontext: + case BuiltinValueKind::AutoDiffAllocateSubcontextWithType: case BuiltinValueKind::AddressOfBorrowOpaque: case BuiltinValueKind::UnprotectedAddressOfBorrowOpaque: return true; diff --git a/stdlib/public/core/SetAnyHashableExtensions.swift b/stdlib/public/core/SetAnyHashableExtensions.swift index bff5639cc9ae7..bab822cf0dfe3 100644 --- a/stdlib/public/core/SetAnyHashableExtensions.swift +++ b/stdlib/public/core/SetAnyHashableExtensions.swift @@ -16,6 +16,7 @@ extension Set where Element == AnyHashable { @inlinable + @discardableResult public mutating func insert( _ newMember: __owned ConcreteElement ) -> (inserted: Bool, memberAfterInsert: ConcreteElement) { diff --git a/stdlib/public/runtime/Array.cpp b/stdlib/public/runtime/Array.cpp index ec9b5de9fac96..84e57199468fb 100644 --- a/stdlib/public/runtime/Array.cpp +++ b/stdlib/public/runtime/Array.cpp @@ -22,6 +22,7 @@ // //===----------------------------------------------------------------------===// +#include "BytecodeLayouts.h" #include "swift/Runtime/Config.h" #include "swift/Runtime/HeapObject.h" #include "swift/Runtime/Metadata.h" @@ -124,6 +125,10 @@ static void array_copy_operation(OpaqueValue *dest, OpaqueValue *src, assert(copyKind == ArrayCopy::BackToFront); assert(count != 0); + if (self->hasLayoutString() && destOp == ArrayDest::Init && srcOp == ArraySource::Copy) { + return swift_generic_arrayInitWithCopy(dest, src, count, stride, self); + } + auto copy = get_witness_function(wtable); size_t i = count; do { @@ -202,6 +207,10 @@ void swift_arrayDestroy(OpaqueValue *begin, size_t count, const Metadata *self) return; auto stride = wtable->getStride(); + if (self->hasLayoutString()) { + return swift_generic_arrayDestroy(begin, count, stride, self); + } + for (size_t i = 0; i < count; ++i) { auto offset = i * stride; auto *obj = reinterpret_cast((char *)begin + offset); diff --git a/stdlib/public/runtime/AutoDiffSupport.cpp b/stdlib/public/runtime/AutoDiffSupport.cpp index 3873e6165a7f4..372bcdac9447f 100644 --- a/stdlib/public/runtime/AutoDiffSupport.cpp +++ b/stdlib/public/runtime/AutoDiffSupport.cpp @@ -13,7 +13,7 @@ #include "AutoDiffSupport.h" #include "swift/ABI/Metadata.h" #include "swift/Runtime/HeapObject.h" - +#include "llvm/ADT/SmallVector.h" #include using namespace swift; @@ -47,6 +47,13 @@ AutoDiffLinearMapContext::AutoDiffLinearMapContext() : HeapObject(&linearMapContextHeapMetadata) { } +AutoDiffLinearMapContext::AutoDiffLinearMapContext( + const Metadata *topLevelLinearMapContextMetadata) + : HeapObject(&linearMapContextHeapMetadata) { + allocatedContextObjects.push_back(AllocatedContextObjectRecord{ + topLevelLinearMapContextMetadata, projectTopLevelSubcontext()}); +} + void *AutoDiffLinearMapContext::projectTopLevelSubcontext() const { auto offset = alignTo( sizeof(AutoDiffLinearMapContext), alignof(AutoDiffLinearMapContext)); @@ -58,6 +65,16 @@ void *AutoDiffLinearMapContext::allocate(size_t size) { return allocator.Allocate(size, alignof(AutoDiffLinearMapContext)); } +void *AutoDiffLinearMapContext::allocateSubcontext( + const Metadata *contextObjectMetadata) { + auto size = contextObjectMetadata->vw_size(); + auto align = contextObjectMetadata->vw_alignment(); + auto *contextObjectPtr = allocator.Allocate(size, align); + allocatedContextObjects.push_back( + AllocatedContextObjectRecord{contextObjectMetadata, contextObjectPtr}); + return contextObjectPtr; +} + AutoDiffLinearMapContext *swift::swift_autoDiffCreateLinearMapContext( size_t topLevelLinearMapStructSize) { auto allocationSize = alignTo( @@ -68,11 +85,31 @@ AutoDiffLinearMapContext *swift::swift_autoDiffCreateLinearMapContext( } void *swift::swift_autoDiffProjectTopLevelSubcontext( - AutoDiffLinearMapContext *allocator) { - return allocator->projectTopLevelSubcontext(); + AutoDiffLinearMapContext *linearMapContext) { + return static_cast(linearMapContext->projectTopLevelSubcontext()); } void *swift::swift_autoDiffAllocateSubcontext( AutoDiffLinearMapContext *allocator, size_t size) { return allocator->allocate(size); } + +AutoDiffLinearMapContext *swift::swift_autoDiffCreateLinearMapContextWithType( + const Metadata *topLevelLinearMapContextMetadata) { + assert(topLevelLinearMapContextMetadata->getValueWitnesses() != nullptr); + auto topLevelLinearMapContextSize = + topLevelLinearMapContextMetadata->vw_size(); + auto allocationSize = alignTo(sizeof(AutoDiffLinearMapContext), + alignof(AutoDiffLinearMapContext)) + + topLevelLinearMapContextSize; + auto *buffer = (AutoDiffLinearMapContext *)malloc(allocationSize); + return ::new (buffer) + AutoDiffLinearMapContext(topLevelLinearMapContextMetadata); +} + +void *swift::swift_autoDiffAllocateSubcontextWithType( + AutoDiffLinearMapContext *linearMapContext, + const Metadata *linearMapSubcontextMetadata) { + assert(linearMapSubcontextMetadata->getValueWitnesses() != nullptr); + return linearMapContext->allocateSubcontext(linearMapSubcontextMetadata); +} diff --git a/stdlib/public/runtime/AutoDiffSupport.h b/stdlib/public/runtime/AutoDiffSupport.h index 4fb63d82e062b..b3135ef794628 100644 --- a/stdlib/public/runtime/AutoDiffSupport.h +++ b/stdlib/public/runtime/AutoDiffSupport.h @@ -14,31 +14,95 @@ #define SWIFT_RUNTIME_AUTODIFF_SUPPORT_H #include "swift/Runtime/HeapObject.h" +#include "swift/ABI/Metadata.h" #include "swift/Runtime/Config.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/Support/Allocator.h" namespace swift { - /// A data structure responsible for efficiently allocating closure contexts for /// linear maps such as pullbacks, including recursive branching trace enum /// case payloads. class AutoDiffLinearMapContext : public HeapObject { + /// A simple wrapper around a context object allocated by the + /// `AutoDiffLinearMapContext` type. This type knows all the "physical" + /// properties and behavior of the allocated context object by way of + /// storing the allocated type's `TypeMetadata`. It uses this information + /// to ensure that the allocated context object is destroyed/deinitialized + /// properly, upon its own destruction. + class [[nodiscard]] AllocatedContextObjectRecord final { + const Metadata *contextObjectMetadata; + OpaqueValue *contextObjectPtr; + + public: + AllocatedContextObjectRecord(const Metadata *contextObjectMetadata, + OpaqueValue *contextObjectPtr) + : contextObjectMetadata(contextObjectMetadata), + contextObjectPtr(contextObjectPtr) {} + + AllocatedContextObjectRecord(const Metadata *contextObjectMetadata, + void *contextObjectPtr) + : AllocatedContextObjectRecord( + contextObjectMetadata, + static_cast(contextObjectPtr)) {} + + ~AllocatedContextObjectRecord() { + if (contextObjectMetadata != nullptr && contextObjectPtr != nullptr) { + contextObjectMetadata->vw_destroy(contextObjectPtr); + } + } + + AllocatedContextObjectRecord(const AllocatedContextObjectRecord &) = delete; + + AllocatedContextObjectRecord( + AllocatedContextObjectRecord &&other) noexcept { + this->contextObjectMetadata = other.contextObjectMetadata; + this->contextObjectPtr = other.contextObjectPtr; + other.contextObjectMetadata = nullptr; + other.contextObjectPtr = nullptr; + } + + size_t size() const { return contextObjectMetadata->vw_size(); } + + size_t align() const { return contextObjectMetadata->vw_alignment(); } + }; + private: /// The underlying allocator. // TODO: Use a custom allocator so that the initial slab can be // tail-allocated. llvm::BumpPtrAllocator allocator; + /// Storage for `AllocatedContextObjectRecord`s, corresponding to the + /// subcontext allocations performed by the type. + llvm::SmallVector allocatedContextObjects; + public: - /// Creates a linear map context. + /// DEPRECATED - Use overloaded constructor taking a `const Metadata *` + /// parameter instead. This constructor might be removed as it leads to memory + /// leaks. AutoDiffLinearMapContext(); + + AutoDiffLinearMapContext(const Metadata *topLevelLinearMapContextMetadata); + /// Returns the address of the tail-allocated top-level subcontext. void *projectTopLevelSubcontext() const; + /// Allocates memory for a new subcontext. + /// + /// DEPRECATED - Use `allocateSubcontext` instead. This + /// method might be removed as it leads to memory leaks. void *allocate(size_t size); + + /// Allocates memory for a new subcontext. + void *allocateSubcontext(const Metadata *contextObjectMetadata); }; /// Creates a linear map context with a tail-allocated top-level subcontext. +/// +/// DEPRECATED - Use `swift_autoDiffCreateLinearMapContextWithType` instead. +/// This builtin might be removed as it leads to memory leaks. SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) AutoDiffLinearMapContext *swift_autoDiffCreateLinearMapContext( size_t topLevelSubcontextSize); @@ -48,9 +112,21 @@ SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) void *swift_autoDiffProjectTopLevelSubcontext(AutoDiffLinearMapContext *); /// Allocates memory for a new subcontext. +/// +/// DEPRECATED - Use `swift_autoDiffAllocateSubcontextWithType` instead. This +/// builtin might be removed as it leads to memory leaks. SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) void *swift_autoDiffAllocateSubcontext(AutoDiffLinearMapContext *, size_t size); -} +/// Creates a linear map context with a tail-allocated top-level subcontext. +SWIFT_RUNTIME_EXPORT SWIFT_CC(swift) + AutoDiffLinearMapContext *swift_autoDiffCreateLinearMapContextWithType( + const Metadata *topLevelLinearMapContextMetadata); +/// Allocates memory for a new subcontext. +SWIFT_RUNTIME_EXPORT + SWIFT_CC(swift) void *swift_autoDiffAllocateSubcontextWithType( + AutoDiffLinearMapContext *, + const Metadata *linearMapSubcontextMetadata); +} // namespace swift #endif /* SWIFT_RUNTIME_AUTODIFF_SUPPORT_H */ diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index ba1a87219b7b5..aac184ad30b7a 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -44,13 +44,13 @@ static Metadata *getExistentialTypeMetadata(OpaqueValue *object) { return reinterpret_cast(object)[NumWords_ValueBuffer]; } -template -static const FnTy readRelativeFunctionPointer(LayoutStringReader &reader) { +template +static const FnTy readRelativeFunctionPointer(Reader &reader) { static_assert(std::is_pointer::value); - auto absolute = reader.layoutStr + reader.offset; + auto absolute = reader.getAbsolute(); auto relativeOffset = - (uintptr_t)(intptr_t)(int32_t)reader.readBytes(); + (uintptr_t)(intptr_t)(int32_t)reader.template readBytes(); FnTy fn; #if SWIFT_PTRAUTH @@ -66,103 +66,13 @@ static const FnTy readRelativeFunctionPointer(LayoutStringReader &reader) { typedef Metadata *(*MetadataAccessor)(const Metadata *const *); +template static const Metadata *getResilientTypeMetadata(const Metadata *metadata, - LayoutStringReader &reader) { + Reader &reader) { auto fn = readRelativeFunctionPointer(reader); return fn(metadata->getGenericArgs()); } -typedef void (*DestrFn)(void*); - -struct DestroyFuncAndMask { - DestrFn fn; - bool isIndirect; -}; - -static void skipDestroy(void* ignore) { } - -static void existential_destroy(OpaqueValue* object) { - auto* metadata = getExistentialTypeMetadata(object); - if (metadata->getValueWitnesses()->isValueInline()) { - metadata->vw_destroy(object); - } else { - swift_release(*(HeapObject**)object); - } -} - -template -inline static bool handleNextRefCount(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, Params... params) { - uint64_t skip = reader.readBytes(); - auto tag = static_cast(skip >> 56); - skip &= ~(0xffULL << 56); - addrOffset += skip; - - if (SWIFT_UNLIKELY(tag == RefCountingKind::End)) { - return false; - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::Metatype)) { - auto *type = reader.readBytes(); - Handler::handleMetatype(type, addrOffset, std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::Resilient)) { - auto *type = getResilientTypeMetadata(metadata, reader); - Handler::handleMetatype(type, addrOffset, std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::SinglePayloadEnumSimple)) { - Handler::handleSinglePayloadEnumSimple(reader, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumFN)) { - Handler::handleSinglePayloadEnumFN(reader, false, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::SinglePayloadEnumFNResolved)) { - Handler::handleSinglePayloadEnumFN(reader, true, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::SinglePayloadEnumGeneric)) { - Handler::handleSinglePayloadEnumGeneric(reader, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumFN)) { - Handler::handleMultiPayloadEnumFN(metadata, reader, false, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == - RefCountingKind::MultiPayloadEnumFNResolved)) { - Handler::handleMultiPayloadEnumFN(metadata, reader, true, addrOffset, - std::forward(params)...); - } else if (SWIFT_UNLIKELY(tag == RefCountingKind::MultiPayloadEnumGeneric)) { - Handler::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - std::forward(params)...); - } else { - Handler::handleReference(tag, addrOffset, std::forward(params)...); - } - - return true; -} - -template -inline static void handleRefCounts(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, Params... params) { - if (N == 0) { - while (handleNextRefCount(metadata, reader, addrOffset, - std::forward(params)...)) { - } - } else { - for (unsigned i = 0; i < N; i++) { - handleNextRefCount(metadata, reader, addrOffset, - std::forward(params)...); - } - } -} - -template -inline static void handleRefCounts(const Metadata *metadata, Params... params) { - LayoutStringReader reader{metadata->getLayoutString(), - layoutStringHeaderSize}; - uintptr_t addrOffset = 0; - handleRefCounts(metadata, reader, addrOffset, - std::forward(params)...); -} - static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) { switch (byteCount) { case 1: @@ -187,464 +97,1699 @@ static uint64_t readTagBytes(const uint8_t *addr, uint8_t byteCount) { } } -static void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uint8_t *addr, - uintptr_t &addrOffset) { - auto byteCountsAndOffset = reader.readBytes(); - auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); - auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; - auto xiTagBytesOffset = - byteCountsAndOffset & std::numeric_limits::max(); +static void handleRefCountsDestroy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr); + +template +static void handleEnd(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr, + Params ...params) { + return; +} - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto payloadSize = reader.readBytes(); - auto tagBytes = - readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); - if (tagBytes) { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - goto noPayload; - } - } else { - reader.skip(sizeof(size_t)); - } +static void errorDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + SwiftError *error = *(SwiftError**)(addr + addrOffset); + addrOffset += sizeof(SwiftError*); + swift_errorRelease(error); +} - if (xiTagBytesPattern) { - auto zeroTagValue = reader.readBytes(); - auto xiTagValues = reader.readBytes(); +static void nativeStrongDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(HeapObject*); + swift_release(object); +} - auto xiTagBytes = 1 << (xiTagBytesPattern - 1); - uint64_t tagBytes = - readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) - - zeroTagValue; - if (tagBytes >= xiTagValues) { - reader.skip(sizeof(size_t) * 2); - return; +static void unownedDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + HeapObject *object = (HeapObject*)((*(uintptr_t *)(addr + addrOffset)) & ~_swift_abi_SwiftSpareBitsMask); + addrOffset += sizeof(HeapObject*); + swift_unownedRelease(object); +} + +static void weakDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *object = (WeakReference *)(addr + addrOffset); + addrOffset += sizeof(WeakReference); + swift_weakDestroy(object); +} + +static void unknownDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + void *object = *(void**)(addr + addrOffset); + addrOffset += sizeof(void*); + swift_unknownObjectRelease(object); +} + +static void unknownUnownedDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + UnownedReference *object = (UnownedReference*)(addr + addrOffset); + addrOffset += sizeof(UnownedReference); + swift_unknownObjectUnownedDestroy(object); +} + +static void unknownWeakDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *object = (WeakReference *)(addr + addrOffset); + addrOffset += sizeof(WeakReference); + swift_unknownObjectWeakDestroy(object); +} + +static void bridgeDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *object = *(void **)(addr + addrOffset); + addrOffset += sizeof(void*); + swift_bridgeObjectRelease(object); +} + +static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = + readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); + if (tagBytes) { + xiTagBytesPattern = 0; + } + } + + if (SWIFT_LIKELY(xiTagBytesPattern)) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + uint64_t tagBytes = + readTagBytes(addr + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + if (tagBytes >= xiTagValues) { + return; + } } - } else { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - } -noPayload: - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - reader.skip(refCountBytes); - addrOffset += skip; + reader.skip(refCountBytes); + addrOffset += skip; + }); } typedef unsigned (*GetEnumTagFn)(const uint8_t *); -static void handleSinglePayloadEnumFN(LayoutStringReader &reader, bool resolved, - uint8_t *addr, uintptr_t &addrOffset) { - GetEnumTagFn getEnumTag; - if (resolved) { - getEnumTag = reader.readBytes(); - } else { - getEnumTag = readRelativeFunctionPointer(reader); - } +static void singlePayloadEnumFNBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); - unsigned enumTag = getEnumTag(addr + addrOffset); + unsigned enumTag = getEnumTag(addr + addrOffset); - if (enumTag == 0) { - reader.skip(sizeof(size_t) * 2); - } else { + if (SWIFT_LIKELY(enumTag == 0)) { + reader.skip(sizeof(size_t) * 2); + } else { + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned enumTag = getEnumTag(addr + addrOffset); + + if (SWIFT_UNLIKELY(enumTag != 0)) { + reader.skip(refCountBytes); + addrOffset += skip; + } + }); +} + +static void singlePayloadEnumGenericBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + reader.modify([&](LayoutStringReader1 &reader) { + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); auto refCountBytes = reader.readBytes(); auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); + + if (tagBytes) { + xiType = nullptr; + } + } + + if (SWIFT_LIKELY(xiType)) { + auto tag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset), + numEmptyCases); + if (SWIFT_LIKELY(tag == 0)) { + return; + } + } + reader.skip(refCountBytes); addrOffset += skip; + }); +} + +template +static void multiPayloadEnumFNBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + enumTag = getEnumTag(addr + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); + } else { + addrOffset += enumSize; } } -static void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uint8_t *addr, - uintptr_t &addrOffset) { - auto tagBytesAndOffset = reader.readBytes(); - auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); - auto xiTagBytesOffset = - tagBytesAndOffset & std::numeric_limits::max(); - const Metadata *xiType = nullptr; +template +static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + enumTag = getEnumTag(addr + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); + } else { + addrOffset += enumSize; + } +} - if (extraTagBytesPattern) { - auto extraTagBytes = 1 << (extraTagBytesPattern - 1); - auto payloadSize = reader.readBytes(); - auto tagBytes = - readTagBytes(addr + addrOffset + payloadSize, extraTagBytes); - if (tagBytes) { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - goto noPayload; - } +template +static void multiPayloadEnumGenericBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t enumTag; + uintptr_t nestedAddrOffset; + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + nestedReader = reader; + nestedAddrOffset = addrOffset; + auto tagBytesOffset = enumSize - tagBytes; + + enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, addr); } else { - reader.skip(sizeof(size_t)); + addrOffset += enumSize; } +} - xiType = reader.readBytes(); +static void singlePayloadEnumSimpleBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = + readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + if (tagBytes) { + xiTagBytesPattern = 0; + } + } - if (xiType) { - auto numEmptyCases = reader.readBytes(); + if (SWIFT_LIKELY(xiTagBytesPattern)) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + uint64_t tagBytes = + readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + if (tagBytes >= xiTagValues) { + return; + } + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + reader.skip(refCountBytes); + addrOffset += skip; + }); +} - auto tag = xiType->vw_getEnumTagSinglePayload( - (const OpaqueValue *)(addr + addrOffset + xiTagBytesOffset), - numEmptyCases); - if (tag == 0) { +static void singlePayloadEnumFNBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + + unsigned enumTag = getEnumTag(src + addrOffset); + + if (SWIFT_LIKELY(enumTag == 0)) { reader.skip(sizeof(size_t) * 2); - return; + } else { + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; } - } else { - reader.skip(sizeof(uint64_t) + sizeof(size_t)); - } + }); +} -noPayload: - auto refCountBytes = reader.readBytes(); - auto skip = reader.readBytes(); - reader.skip(refCountBytes); - addrOffset += skip; -} - -template -static void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, bool resolved, - uintptr_t &addrOffset, uint8_t *addr, - Params... params) { - GetEnumTagFn getEnumTag; - if (resolved) { - getEnumTag = reader.readBytes(); - } else { - getEnumTag = readRelativeFunctionPointer(reader); - } +static void singlePayloadEnumFNResolvedBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); - size_t numPayloads = reader.readBytes(); - size_t refCountBytes = reader.readBytes(); - size_t enumSize = reader.readBytes(); + unsigned enumTag = getEnumTag(src + addrOffset); - unsigned enumTag = getEnumTag(addr + addrOffset); + if (SWIFT_UNLIKELY(enumTag != 0)) { + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + } + }); +} - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); +static void singlePayloadEnumGenericBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + auto tagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + + if (tagBytes) { + xiType = nullptr; + } + } - LayoutStringReader nestedReader = reader; + if (SWIFT_LIKELY(xiType)) { + auto tag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), + numEmptyCases); + if (SWIFT_LIKELY(tag == 0)) { + return; + } + } + + reader.skip(refCountBytes); + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +template +static void multiPayloadEnumFNBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + enumTag = getEnumTag(src + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCounts<0, Handler>(metadata, nestedReader, nestedAddrOffset, addr, - std::forward(params)...); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; } +} - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; +template +static void multiPayloadEnumFNResolvedBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned enumTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + enumTag = getEnumTag(src + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; + } } -template -static void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, uint8_t *addr, - Params... params) { - auto tagBytes = reader.readBytes(); - auto numPayloads = reader.readBytes(); - auto refCountBytes = reader.readBytes(); - auto enumSize = reader.readBytes(); - auto tagBytesOffset = enumSize - tagBytes; +template +static void multiPayloadEnumGenericBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t enumTag; + uintptr_t nestedAddrOffset; + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + nestedReader = reader; + nestedAddrOffset = addrOffset; + auto tagBytesOffset = enumSize - tagBytes; + + enumTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(enumTag < numPayloads)) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(enumTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + HandlerFn(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + } else { + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; + } +} - auto enumTag = readTagBytes(addr + addrOffset + tagBytesOffset, tagBytes); +#if SWIFT_OBJC_INTEROP +static void blockDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + void* object = (void *)(addr + addrOffset); + addrOffset += sizeof(void*); + _Block_release(object); +} - if (enumTag < numPayloads) { - size_t refCountOffset = reader.peekBytes(enumTag * sizeof(size_t)); +static void objcStrongDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + objc_object *object = (objc_object*)(*(uintptr_t *)(addr + addrOffset)); + addrOffset += sizeof(objc_object*); + objc_release(object); +} +#endif - LayoutStringReader nestedReader = reader; - nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); - uintptr_t nestedAddrOffset = addrOffset; - handleRefCounts<0, Handler>(metadata, nestedReader, nestedAddrOffset, addr, - std::forward(params)...); +static void metatypeDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *type = reader.readBytes(); + auto *object = (OpaqueValue *)(addr + addrOffset); + addrOffset += type->vw_size(); + type->vw_destroy(object); +} + +static void existentialDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + OpaqueValue *object = (OpaqueValue *)(addr + addrOffset); + auto* type = getExistentialTypeMetadata(object); + addrOffset += sizeof(uintptr_t) * NumWords_ValueBuffer; + if (type->getValueWitnesses()->isValueInline()) { + type->vw_destroy(object); + } else { + swift_release(*(HeapObject**)object); } +} - reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); - addrOffset += enumSize; +static void resilientDestroyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto *type = getResilientTypeMetadata(metadata, reader); + type->vw_destroy((OpaqueValue *)(addr + addrOffset)); } -const DestroyFuncAndMask destroyTable[] = { - {(DestrFn)&skipDestroy, false}, - {(DestrFn)&swift_errorRelease, true}, - {(DestrFn)&swift_release, true}, - {(DestrFn)&swift_unownedRelease, true}, - {(DestrFn)&swift_weakDestroy, false}, - {(DestrFn)&swift_unknownObjectRelease, true}, - {(DestrFn)&swift_unknownObjectUnownedDestroy, false}, - {(DestrFn)&swift_unknownObjectWeakDestroy, false}, - {(DestrFn)&swift_bridgeObjectRelease, true}, +typedef void (*DestrFnBranchless)(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr); + +const DestrFnBranchless destroyTableBranchless[] = { + &handleEnd, + &errorDestroyBranchless, + &nativeStrongDestroyBranchless, + &unownedDestroyBranchless, + &weakDestroyBranchless, + &unknownDestroyBranchless, + &unknownUnownedDestroyBranchless, + &unknownWeakDestroyBranchless, + &bridgeDestroyBranchless, #if SWIFT_OBJC_INTEROP - {(DestrFn)&_Block_release, true}, - {(DestrFn)&swift_unknownObjectRelease, true}, + &blockDestroyBranchless, + &objcStrongDestroyBranchless, #else - {nullptr, true}, - {nullptr, true}, + nullptr, + nullptr, #endif - // TODO: how to handle Custom? - {nullptr, true}, - {nullptr, true}, - {nullptr, true}, - {(DestrFn)&existential_destroy, false}, + nullptr, // Custom + &metatypeDestroyBranchless, + nullptr, // Generic + &existentialDestroyBranchless, + &resilientDestroyBranchless, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, }; -struct DestroyHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *addr) { - type->vw_destroy((OpaqueValue *)(addr + addrOffset)); - } +static void handleRefCountsDestroy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + while (true) { + auto tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumSimple(reader, addr, addrOffset); + destroyTableBranchless[tag](metadata, reader, addrOffset, addr); } +} - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumFN(reader, resolved, addr, addrOffset); - } +extern "C" void +swift_generic_destroy(swift::OpaqueValue *address, const Metadata *metadata) { + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsDestroy(metadata, reader, addrOffset, (uint8_t *)address); +} - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleSinglePayloadEnumGeneric(reader, addr, addrOffset); +void swift::swift_generic_arrayDestroy(swift::OpaqueValue *address, size_t count, size_t stride, const Metadata *metadata) { + const uint8_t *layoutStr = metadata->getLayoutString(); + for (size_t i = 0; i < count; i++) { + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = i * stride; + handleRefCountsDestroy(metadata, reader, addrOffset, (uint8_t *)address); } +} - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, addr); - } +static void handleRefCountsInitWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); + +static void errorRetainBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + SwiftError *object = *(SwiftError **)(src + _addrOffset); + memcpy(dest + addrOffset, &object, sizeof(SwiftError*)); + addrOffset = _addrOffset + sizeof(SwiftError *); + swift_errorRetain(object); +} - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *addr) { - ::handleMultiPayloadEnumGeneric(metadata, reader, - addrOffset, addr); - } +static void nativeStrongRetainBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + uintptr_t object = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(HeapObject*)); + object &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset = _addrOffset + sizeof(HeapObject *); + swift_retain((HeapObject *)object); +} - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *addr) { - const auto &destroyFunc = destroyTable[static_cast(tag)]; - if (SWIFT_LIKELY(destroyFunc.isIndirect)) { - destroyFunc.fn( - (void *)((*(uintptr_t *)(addr + addrOffset)))); - } else { - destroyFunc.fn(((void *)(addr + addrOffset))); - } - } -}; +static void unownedRetainBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + uintptr_t object = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(HeapObject*)); + object &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset = _addrOffset + sizeof(HeapObject *); + swift_unownedRetain((HeapObject *)object); +} -extern "C" void -swift_generic_destroy(swift::OpaqueValue *address, const Metadata *metadata) { - handleRefCounts<0, DestroyHandler>(metadata, (uint8_t *)address); +static void weakCopyInitBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); + swift_weakCopyInit(destObject, srcObject); } -struct RetainFuncAndMask { - void* fn; - bool isSingle; -}; +static void unknownRetainBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + void *object = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void *); + swift_unknownObjectRetain(object); +} + +static void unknownUnownedCopyInitBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + UnownedReference *objectDest = (UnownedReference*)(dest + _addrOffset); + UnownedReference *objectSrc = (UnownedReference*)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(UnownedReference); + swift_unknownObjectUnownedCopyInit(objectDest, objectSrc); +} + +static void unknownWeakCopyInitBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); + swift_unknownObjectWeakCopyInit(destObject, srcObject); +} + +static void bridgeRetainBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + void *object = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &object, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); + swift_bridgeObjectRetain(object); +} #if SWIFT_OBJC_INTEROP -void* Block_copyForwarder(void** dest, const void** src) { - *dest = _Block_copy(*src); - return *dest; +static void blockCopyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *copy = _Block_copy(*(void**)(src + _addrOffset)); + memcpy(dest + _addrOffset, ©, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); +} + +static void objcStrongRetainBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + objc_object *object = (objc_object*)(*(uintptr_t *)(src + _addrOffset)); + memcpy(dest + _addrOffset, &object, sizeof(objc_object*)); + addrOffset = _addrOffset + sizeof(objc_object*); + objc_retain(object); } #endif -typedef void* (*RetainFn)(void*); -typedef void* (*CopyInitFn)(void*, void*); - -void* skipRetain(void* ignore) { return nullptr; } -void* existential_initializeWithCopy(OpaqueValue* dest, OpaqueValue* src) { - auto* metadata = getExistentialTypeMetadata(src); - return metadata->vw_initializeBufferWithCopyOfBuffer((ValueBuffer*)dest, - (ValueBuffer*)src); -} - -const RetainFuncAndMask retainTable[] = { - {(void*)&skipRetain, true}, - {(void*)&swift_errorRetain, true}, - {(void*)&swift_retain, true}, - {(void*)&swift_unownedRetain, true}, - {(void*)&swift_weakCopyInit, false}, - {(void*)&swift_unknownObjectRetain, true}, - {(void*)&swift_unknownObjectUnownedCopyInit, false}, - {(void*)&swift_unknownObjectWeakCopyInit, false}, - {(void*)&swift_bridgeObjectRetain, true}, +static void metatypeInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = reader.readBytes(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_initializeWithCopy(destObject, srcObject); +} + +static void existentialInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); + auto *destObject = (ValueBuffer *)(dest + _addrOffset); + auto *srcObject = (ValueBuffer *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); + type->vw_initializeBufferWithCopyOfBuffer(destObject, srcObject); +} + +static void resilientInitWithCopyBranchless(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getResilientTypeMetadata(metadata, reader); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_initializeWithCopy(destObject, srcObject); +} + +typedef void (*InitFn)(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); + +static const InitFn initWithCopyTable[] = { + &handleEnd, + &errorRetainBranchless, + &nativeStrongRetainBranchless, + &unownedRetainBranchless, + &weakCopyInitBranchless, + &unknownRetainBranchless, + &unknownUnownedCopyInitBranchless, + &unknownWeakCopyInitBranchless, + &bridgeRetainBranchless, #if SWIFT_OBJC_INTEROP - {(void*)&Block_copyForwarder, false}, - {(void*)&objc_retain, true}, + &blockCopyBranchless, + &objcStrongRetainBranchless, #else - {nullptr, true}, - {nullptr, true}, + nullptr, + nullptr, #endif - // TODO: how to handle Custom? - {nullptr, true}, - {nullptr, true}, - {nullptr, true}, - {(void*)&existential_initializeWithCopy, false}, + nullptr, // Custom + &metatypeInitWithCopyBranchless, + nullptr, // Generic + &existentialInitWithCopyBranchless, + &resilientInitWithCopyBranchless, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, }; -struct CopyHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - type->vw_initializeWithCopy((OpaqueValue*)((uintptr_t)dest + addrOffset), - (OpaqueValue*)((uintptr_t)src + addrOffset)); - } +static void handleRefCountsInitWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + while (true) { + uintptr_t _addrOffset = addrOffset; + auto tag = reader.readBytes(); + auto offset = (tag & ~(0xFFULL << 56)); + if (SWIFT_UNLIKELY(offset)) { + memcpy(dest + _addrOffset, src + _addrOffset, offset); + } + addrOffset = _addrOffset + offset; + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumSimple(reader, src, addrOffset); + initWithCopyTable[tag](metadata, reader, addrOffset, dest, src); } +} - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleSinglePayloadEnumFN(reader, resolved, src, addrOffset); - } +extern "C" swift::OpaqueValue * +swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, + const Metadata *metadata) { + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsInitWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumGeneric(reader, src, addrOffset); - } + return dest; +} - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, dest, src); +void swift::swift_generic_arrayInitWithCopy(swift::OpaqueValue *dest, + swift::OpaqueValue *src, + size_t count, + size_t stride, + const Metadata *metadata) { + const uint8_t *layoutStr = metadata->getLayoutString(); + for (size_t i = 0; i < count; i++) { + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = i * stride; + handleRefCountsInitWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); } +} - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - dest, src); +static void handleRefCountsInitWithTake(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src); + +static void unknownWeakInitWithTake(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto *destObject = (WeakReference *)(dest + addrOffset); + auto *srcObject = (WeakReference *)(src + addrOffset); + addrOffset += sizeof(WeakReference); + + swift_unknownObjectWeakTakeInit(destObject, srcObject); +} + +static void metatypeInitWithTake(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = reader.readBytes(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + + type->vw_initializeWithTake(destObject, srcObject); +} + +static void existentialInitWithTake(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto* type = getExistentialTypeMetadata((OpaqueValue*)(src + addrOffset)); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); + if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { + type->vw_initializeWithTake(destObject, srcObject); } +} - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - const auto &retainFunc = retainTable[static_cast(tag)]; - if (SWIFT_LIKELY(retainFunc.isSingle)) { - ((RetainFn)retainFunc.fn)(*(void**)(((uintptr_t)dest + addrOffset))); +static void resilientInitWithTake(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getResilientTypeMetadata(metadata, reader); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_initializeWithTake(destObject, srcObject); +} + +static const InitFn initWithTakeTable[] = { + &handleEnd, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + &unknownWeakInitWithTake, + &bridgeRetainBranchless, + nullptr, + nullptr, + nullptr, // Custom + &metatypeInitWithTake, + nullptr, // Generic + &existentialInitWithTake, + &resilientInitWithTake, + &singlePayloadEnumSimpleBranchless, + &singlePayloadEnumFNBranchless, + &singlePayloadEnumFNResolvedBranchless, + &singlePayloadEnumGenericBranchless, + &multiPayloadEnumFNBranchless, + &multiPayloadEnumFNResolvedBranchless, + &multiPayloadEnumGenericBranchless, +}; + +static void handleRefCountsInitWithTake(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + while (true) { + uintptr_t _addrOffset = addrOffset; + auto tag = reader.readBytes(); + auto offset = (tag & ~(0xFFULL << 56)); + if (SWIFT_UNLIKELY(offset)) { + memcpy(dest + _addrOffset, src + _addrOffset, offset); + } + _addrOffset += offset; + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + addrOffset = _addrOffset; + return; + } + + if (auto handler = initWithTakeTable[tag]) { + addrOffset = _addrOffset; + handler(metadata, reader, addrOffset, dest, src); } else { - ((CopyInitFn)retainFunc.fn)((void*)((uintptr_t)dest + addrOffset), - (void*)((uintptr_t)src + addrOffset)); + memcpy(dest + _addrOffset, src + _addrOffset, sizeof(uintptr_t)); + addrOffset = _addrOffset + sizeof(uintptr_t); } } -}; +} extern "C" swift::OpaqueValue * -swift_generic_initWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, +swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - size_t size = metadata->vw_size(); - memcpy(dest, src, size); + if (SWIFT_LIKELY(metadata->getValueWitnesses()->isBitwiseTakable())) { + size_t size = metadata->vw_size(); + memcpy(dest, src, size); + return dest; + } + + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = 0; - handleRefCounts<0, CopyHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); + handleRefCountsInitWithTake(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); return dest; } -struct TakeHandler { - static inline void handleMetatype(const Metadata *type, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue*)((uintptr_t)dest + addrOffset), - (OpaqueValue*)((uintptr_t)src + addrOffset)); - } - } +static void errorAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + SwiftError *destObject = *(SwiftError **)(dest + _addrOffset); + SwiftError *srcObject = *(SwiftError **)(src + _addrOffset); + memcpy(dest + addrOffset, &srcObject, sizeof(SwiftError*)); + addrOffset = _addrOffset + sizeof(SwiftError *); + swift_errorRelease(destObject); + swift_errorRetain(srcObject); +} - static inline void handleSinglePayloadEnumSimple(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumSimple(reader, src, addrOffset); - } +static void nativeStrongAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset); + uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(HeapObject*)); + srcObject &= ~_swift_abi_SwiftSpareBitsMask; + destObject &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset = _addrOffset + sizeof(HeapObject *); + swift_release((HeapObject *)destObject); + swift_retain((HeapObject *)srcObject); +} - static inline void handleSinglePayloadEnumFN(LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleSinglePayloadEnumFN(reader, resolved, src, addrOffset); - } +static void unownedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + uintptr_t destObject = *(uintptr_t *)(dest + _addrOffset); + uintptr_t srcObject = *(uintptr_t *)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(HeapObject*)); + destObject &= ~_swift_abi_SwiftSpareBitsMask; + srcObject &= ~_swift_abi_SwiftSpareBitsMask; + addrOffset = _addrOffset + sizeof(HeapObject *); + swift_unownedRelease((HeapObject *)destObject); + swift_unownedRetain((HeapObject *)srcObject); +} + +static void weakAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); + swift_weakCopyAssign(destObject, srcObject); +} + +static void unknownAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + void *destObject = *(void **)(dest + _addrOffset); + void *srcObject = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void *); + swift_unknownObjectRelease(destObject); + swift_unknownObjectRetain(srcObject); +} + +static void unknownUnownedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + UnownedReference *objectDest = (UnownedReference*)(dest + _addrOffset); + UnownedReference *objectSrc = (UnownedReference*)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(UnownedReference); + swift_unknownObjectUnownedCopyAssign(objectDest, objectSrc); +} + +static void unknownWeakAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *destObject = (WeakReference *)(dest + _addrOffset); + auto *srcObject = (WeakReference *)(src + _addrOffset); + addrOffset = _addrOffset + sizeof(WeakReference); + swift_unknownObjectWeakCopyAssign(destObject, srcObject); +} + +static void bridgeAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + void *destObject = *(void **)(dest + _addrOffset); + void *srcObject = *(void **)(src + _addrOffset); + memcpy(dest + _addrOffset, &srcObject, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); + swift_bridgeObjectRelease(destObject); + swift_bridgeObjectRetain(srcObject); +} + +#if SWIFT_OBJC_INTEROP +static void blockAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + _Block_release(*(void **)(dest + _addrOffset)); + auto *copy = _Block_copy(*(void **)(src + _addrOffset)); + memcpy(dest + _addrOffset, ©, sizeof(void*)); + addrOffset = _addrOffset + sizeof(void*); +} - static inline void handleSinglePayloadEnumGeneric(LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleSinglePayloadEnumGeneric(reader, src, addrOffset); +static void objcStrongAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + objc_object *destObject = (objc_object*)(*(uintptr_t *)(dest + _addrOffset)); + objc_object *srcObject = (objc_object*)(*(uintptr_t *)(src + _addrOffset)); + memcpy(dest + _addrOffset, &srcObject, sizeof(objc_object*)); + addrOffset = _addrOffset + sizeof(objc_object*); + objc_release(destObject); + objc_retain(srcObject); +} +#endif + +static void metatypeAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = reader.readBytes(); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_assignWithCopy(destObject, srcObject); +} + +static void existentialAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getExistentialTypeMetadata((OpaqueValue*)(src + _addrOffset)); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + (sizeof(uintptr_t) * NumWords_ValueBuffer); + if (type->getValueWitnesses()->isValueInline()) { + type->vw_assignWithCopy(destObject, srcObject); + } else { + swift_release(*(HeapObject**)destObject); + memcpy(destObject, srcObject, sizeof(uintptr_t)); + swift_retain(*(HeapObject**)srcObject); } +} + +static void resilientAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + uintptr_t _addrOffset = addrOffset; + auto *type = getResilientTypeMetadata(metadata, reader); + auto *destObject = (OpaqueValue *)(dest + _addrOffset); + auto *srcObject = (OpaqueValue *)(src + _addrOffset); + addrOffset = _addrOffset + type->vw_size(); + type->vw_assignWithCopy(destObject, srcObject); +} - static inline void handleMultiPayloadEnumFN(const Metadata *metadata, - LayoutStringReader &reader, - bool resolved, - uintptr_t &addrOffset, - uint8_t *dest, uint8_t *src) { - ::handleMultiPayloadEnumFN(metadata, reader, resolved, - addrOffset, dest, src); +static void handleSingleRefCountDestroy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *addr) { + auto tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; } + destroyTableBranchless[tag](metadata, reader, addrOffset, addr); +} - static inline void handleMultiPayloadEnumGeneric(const Metadata *metadata, - LayoutStringReader &reader, - uintptr_t &addrOffset, - uint8_t *dest, - uint8_t *src) { - ::handleMultiPayloadEnumGeneric(metadata, reader, addrOffset, - dest, src); +static void handleSingleRefCountInitWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + auto tag = reader.readBytes(); + addrOffset += (tag & ~(0xFFULL << 56)); + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; } + initWithCopyTable[tag](metadata, reader, addrOffset, dest, src); +} - static inline void handleReference(RefCountingKind tag, uintptr_t addrOffset, - uint8_t *dest, uint8_t *src) { - if (tag == RefCountingKind::UnknownWeak) { - swift_unknownObjectWeakTakeInit( - (WeakReference*)((uintptr_t)dest + addrOffset), - (WeakReference*)((uintptr_t)src + addrOffset)); - } else if (tag == RefCountingKind::Existential) { - auto *type = getExistentialTypeMetadata( - (OpaqueValue*)((uintptr_t)src + addrOffset)); - if (SWIFT_UNLIKELY(!type->getValueWitnesses()->isBitwiseTakable())) { - type->vw_initializeWithTake( - (OpaqueValue *)((uintptr_t)dest + addrOffset), - (OpaqueValue *)((uintptr_t)src + addrOffset)); +static void singlePayloadEnumSimpleAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t srcTagBytes = 0; + uint64_t destTagBytes = 0; + uint64_t byteCountsAndOffset; + size_t payloadSize; + uint64_t zeroTagValue; + size_t xiTagValues; + size_t refCountBytes; + size_t skip; + + reader.readBytes(byteCountsAndOffset, payloadSize, zeroTagValue, xiTagValues, refCountBytes, skip); + + auto extraTagBytesPattern = (uint8_t)(byteCountsAndOffset >> 62); + auto xiTagBytesPattern = ((uint8_t)(byteCountsAndOffset >> 59)) & 0x7; + auto xiTagBytesOffset = + byteCountsAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + + srcTagBytes = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + destTagBytes = readTagBytes(dest + addrOffset + payloadSize, extraTagBytes); + } + + if (SWIFT_LIKELY(xiTagBytesPattern)) { + auto xiTagBytes = 1 << (xiTagBytesPattern - 1); + srcTagBytes = srcTagBytes ? 0 : + readTagBytes(src + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + destTagBytes = destTagBytes ? 0 : + readTagBytes(dest + addrOffset + xiTagBytesOffset, xiTagBytes) - + zeroTagValue; + } + + if (srcTagBytes >= xiTagValues && destTagBytes >= xiTagValues) { + return; + } else if (srcTagBytes >= xiTagValues) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTagBytes >= xiTagValues) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; + while (reader.layoutStr < end) { + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + size_t refCountBytes; + size_t skip; + reader.readBytes(refCountBytes, skip); + + unsigned srcTag = getEnumTag(src + addrOffset); + unsigned destTag = getEnumTag(dest + addrOffset); + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; + while (reader.layoutStr < end) { + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag; + size_t refCountBytes; + size_t skip; + reader.readBytes(getEnumTag, refCountBytes, skip); + + unsigned srcTag = getEnumTag(src + addrOffset); + unsigned destTag = getEnumTag(dest + addrOffset); + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; + while (reader.layoutStr < end) { + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void singlePayloadEnumGenericAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + reader.modify([&](LayoutStringReader1 &reader) { + uint64_t srcTag = 0; + uint64_t destTag = 0; + auto tagBytesAndOffset = reader.readBytes(); + auto payloadSize = reader.readBytes(); + auto *xiType = reader.readBytes(); + auto numEmptyCases = reader.readBytes(); + auto refCountBytes = reader.readBytes(); + auto skip = reader.readBytes(); + + auto extraTagBytesPattern = (uint8_t)(tagBytesAndOffset >> 62); + auto xiTagBytesOffset = + tagBytesAndOffset & std::numeric_limits::max(); + + if (SWIFT_UNLIKELY(extraTagBytesPattern)) { + auto extraTagBytes = 1 << (extraTagBytesPattern - 1); + srcTag = readTagBytes(src + addrOffset + payloadSize, extraTagBytes); + destTag = readTagBytes(dest + addrOffset + payloadSize, extraTagBytes); + } + + if (SWIFT_LIKELY(xiType)) { + if (!srcTag) { + srcTag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(src + addrOffset + xiTagBytesOffset), + numEmptyCases); + } + + if (!destTag) { + destTag = xiType->vw_getEnumTagSinglePayload( + (const OpaqueValue *)(dest + addrOffset + xiTagBytesOffset), + numEmptyCases); } } + + if (SWIFT_UNLIKELY(srcTag == 0 && destTag == 0)) { + return; + } else if (srcTag == 0) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + while (reader.layoutStr < end) { + handleSingleRefCountInitWithCopy(metadata, reader, addrOffset, dest, src); + } + return; + } else if (destTag == 0) { + const uint8_t *end = (reader.layoutStr + refCountBytes); + auto nestedAddrOffset = addrOffset; + while (reader.layoutStr < end) { + handleSingleRefCountDestroy(metadata, reader, nestedAddrOffset, dest); + } + } else { + reader.skip(refCountBytes); + } + + memcpy(dest + addrOffset, src + addrOffset, skip); + addrOffset += skip; + }); +} + +static void multiPayloadEnumFNAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned srcTag; + unsigned destTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = readRelativeFunctionPointer(reader); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + srcTag = getEnumTag(src + addrOffset); + destTag = getEnumTag(dest + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); } -}; -extern "C" swift::OpaqueValue * -swift_generic_initWithTake(swift::OpaqueValue *dest, swift::OpaqueValue *src, - const Metadata *metadata) { - size_t size = metadata->vw_size(); + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} - memcpy(dest, src, size); +static void multiPayloadEnumFNResolvedAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + LayoutStringReader1 nestedReader; + uintptr_t nestedAddrOffset; + unsigned srcTag; + unsigned destTag; + + reader.modify([&](LayoutStringReader1 &reader) { + GetEnumTagFn getEnumTag = reader.readBytes(); + reader.readBytes(numPayloads, refCountBytes, enumSize); + nestedReader = reader; + nestedAddrOffset = addrOffset; + + srcTag = getEnumTag(src + addrOffset); + destTag = getEnumTag(dest + addrOffset); + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); + } - if (SWIFT_LIKELY(metadata->getValueWitnesses()->isBitwiseTakable())) { - return dest; + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} + +static void multiPayloadEnumGenericAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + size_t tagBytes; + size_t numPayloads; + size_t refCountBytes; + size_t enumSize; + uint64_t srcTag; + uint64_t destTag; + uintptr_t nestedAddrOffset; + LayoutStringReader1 nestedReader; + reader.modify([&](LayoutStringReader1 &reader) { + reader.readBytes(tagBytes, numPayloads, refCountBytes, enumSize); + + nestedReader = reader; + nestedAddrOffset = addrOffset; + auto tagBytesOffset = enumSize - tagBytes; + + srcTag = readTagBytes(src + addrOffset + tagBytesOffset, tagBytes); + destTag = readTagBytes(dest + addrOffset + tagBytesOffset, tagBytes); + + reader.skip(refCountBytes + (numPayloads * sizeof(size_t))); + }); + + if (SWIFT_LIKELY(srcTag < numPayloads && destTag < numPayloads)) { + addrOffset += enumSize; + size_t srcRefCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + size_t destRefCountOffset = nestedReader.peekBytes(destTag * sizeof(size_t)); + LayoutStringReader1 nestedReaderDest = nestedReader; + nestedReader.skip((numPayloads * sizeof(size_t)) + srcRefCountOffset); + nestedReaderDest.skip((numPayloads * sizeof(size_t)) + destRefCountOffset); + auto nestedAddrOffsetDest = nestedAddrOffset; + handleRefCountsDestroy(metadata, nestedReaderDest, nestedAddrOffsetDest, dest); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (destTag > numPayloads) { + addrOffset += enumSize; + size_t refCountOffset = nestedReader.peekBytes(srcTag * sizeof(size_t)); + nestedReader.skip((numPayloads * sizeof(size_t)) + refCountOffset); + handleRefCountsInitWithCopy(metadata, nestedReader, nestedAddrOffset, dest, src); + auto trailingBytes = addrOffset - nestedAddrOffset; + if (trailingBytes) + memcpy(dest + nestedAddrOffset, src + nestedAddrOffset, trailingBytes); + return; + } else if (srcTag > numPayloads) { + handleRefCountsDestroy(metadata, nestedReader, nestedAddrOffset, dest); } - handleRefCounts<0, TakeHandler>(metadata, (uint8_t *)dest, (uint8_t *)src); + memcpy(dest + addrOffset, src + addrOffset, enumSize); + addrOffset += enumSize; +} + +static const InitFn assignWithCopyTable[] = { + &handleEnd, + &errorAssignWithCopy, + &nativeStrongAssignWithCopy, + &unownedAssignWithCopy, + &weakAssignWithCopy, + &unknownAssignWithCopy, + &unknownUnownedAssignWithCopy, + &unknownWeakAssignWithCopy, + &bridgeAssignWithCopy, +#if SWIFT_OBJC_INTEROP + &blockAssignWithCopy, + &objcStrongAssignWithCopy, +#else + nullptr, + nullptr, +#endif + nullptr, // Custom + &metatypeAssignWithCopy, + nullptr, // Generic + &existentialAssignWithCopy, + &resilientAssignWithCopy, + &singlePayloadEnumSimpleAssignWithCopy, + &singlePayloadEnumFNAssignWithCopy, + &singlePayloadEnumFNResolvedAssignWithCopy, + &singlePayloadEnumGenericAssignWithCopy, + &multiPayloadEnumFNAssignWithCopy, + &multiPayloadEnumFNResolvedAssignWithCopy, + &multiPayloadEnumGenericAssignWithCopy, +}; - return dest; +static void handleRefCountsAssignWithCopy(const Metadata *metadata, + LayoutStringReader1 &reader, + uintptr_t &addrOffset, + uint8_t *dest, + uint8_t *src) { + while (true) { + uintptr_t _addrOffset = addrOffset; + auto tag = reader.readBytes(); + auto offset = (tag & ~(0xFFULL << 56)); + if (offset) { + memcpy(dest + _addrOffset, src + _addrOffset, offset); + } + addrOffset = _addrOffset + offset; + tag >>= 56; + if (SWIFT_UNLIKELY(tag == 0)) { + return; + } + + assignWithCopyTable[tag](metadata, reader, addrOffset, dest, src); + } } extern "C" swift::OpaqueValue * swift_generic_assignWithCopy(swift::OpaqueValue *dest, swift::OpaqueValue *src, const Metadata *metadata) { - swift_generic_destroy(dest, metadata); - return swift_generic_initWithCopy(dest, src, metadata); + const uint8_t *layoutStr = metadata->getLayoutString(); + LayoutStringReader1 reader{layoutStr + layoutStringHeaderSize}; + uintptr_t addrOffset = 0; + handleRefCountsAssignWithCopy(metadata, reader, addrOffset, (uint8_t *)dest, (uint8_t *)src); + + return dest; } extern "C" swift::OpaqueValue * @@ -805,7 +1950,7 @@ extern "C" unsigned swift_multiPayloadEnumGeneric_getEnumTag(swift::OpaqueValue *address, const Metadata *metadata) { auto addr = reinterpret_cast(address); - LayoutStringReader reader{metadata->getLayoutString(), + LayoutStringReader1 reader{metadata->getLayoutString() + layoutStringHeaderSize + sizeof(uint64_t)}; auto tagBytes = reader.readBytes(); @@ -1081,13 +2226,12 @@ void swift::swift_resolve_resilientAccessors(uint8_t *layoutStr, reader.skip(sizeof(size_t)); size_t casesBeginOffset = layoutStrOffset + reader.offset + - layoutStringHeaderSize + (numCases * sizeof(size_t)); + auto fieldCasesBeginOffset = fieldLayoutStr + (numCases * sizeof(size_t)) + reader.offset; for (size_t j = 0; j < numCases; j++) { size_t caseOffset = reader.readBytes(); - const uint8_t *caseLayoutString = fieldLayoutStr + reader.offset + - (numCases * sizeof(size_t)) + + const uint8_t *caseLayoutString = fieldCasesBeginOffset + caseOffset; swift_resolve_resilientAccessors(layoutStr, casesBeginOffset + caseOffset, diff --git a/stdlib/public/runtime/BytecodeLayouts.h b/stdlib/public/runtime/BytecodeLayouts.h index e3747e3edbeef..66519530ae2b3 100644 --- a/stdlib/public/runtime/BytecodeLayouts.h +++ b/stdlib/public/runtime/BytecodeLayouts.h @@ -58,6 +58,56 @@ enum class RefCountingKind : uint8_t { // Reserved: 0x81 - 0xFF }; +struct LayoutStringReader1 { + const uint8_t *layoutStr; + + template + inline T readBytes() { + T returnVal; + memcpy(&returnVal, layoutStr, sizeof(T)); + layoutStr += sizeof(T); + return returnVal; + } + + template + inline void readBytes(T&... result) { + uintptr_t additionalOffset = 0; + ([&] { + memcpy(&result, layoutStr + additionalOffset, sizeof(T)); + additionalOffset += sizeof(T); + }(), ...); + layoutStr += additionalOffset; + } + + template + inline T modify(F &&f) { + LayoutStringReader1 readerCopy = *this; + T res = f(readerCopy); + layoutStr = readerCopy.layoutStr; + return res; + } + + template + inline void modify(F &&f) { + LayoutStringReader1 readerCopy = *this; + f(readerCopy); + layoutStr = readerCopy.layoutStr; + } + + template + inline T peekBytes(size_t peekOffset = 0) const { + T returnVal; + memcpy(&returnVal, layoutStr + peekOffset, sizeof(T)); + return returnVal; + } + + inline void skip(size_t n) { layoutStr += n; } + + inline uintptr_t getAbsolute() { + return (uintptr_t) layoutStr; + } +}; + struct LayoutStringReader { const uint8_t *layoutStr; size_t offset; @@ -70,6 +120,31 @@ struct LayoutStringReader { return returnVal; } + template + inline void readBytes(T&... result) { + uintptr_t additionalOffset = 0; + ([&] { + memcpy(&result, layoutStr + offset + additionalOffset, sizeof(T)); + additionalOffset += sizeof(T); + }(), ...); + offset += additionalOffset; + } + + template + inline T modify(F &&f) { + LayoutStringReader readerCopy = *this; + T res = f(readerCopy); + offset = readerCopy.offset; + return res; + } + + template + inline void modify(F &&f) { + LayoutStringReader readerCopy = *this; + f(readerCopy); + offset = readerCopy.offset; + } + template inline T peekBytes(size_t peekOffset = 0) const { T returnVal; @@ -78,6 +153,10 @@ struct LayoutStringReader { } inline void skip(size_t n) { offset += n; } + + inline uintptr_t getAbsolute() { + return (uintptr_t) layoutStr + offset; + } }; struct LayoutStringWriter { @@ -155,6 +234,17 @@ void swift_resolve_resilientAccessors(uint8_t *layoutStr, const uint8_t *fieldLayoutStr, const Metadata *fieldType); +void swift_generic_arrayDestroy(swift::OpaqueValue *addr, + size_t count, + size_t stride, + const Metadata *metadata); + +void swift_generic_arrayInitWithCopy(swift::OpaqueValue *dest, + swift::OpaqueValue *src, + size_t count, + size_t stride, + const Metadata *metadata); + constexpr size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t); } // namespace swift diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index b39417bd967a7..724b11bfb6d5c 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2743,7 +2743,7 @@ void swift::swift_initStructMetadataWithLayoutString( } fullOffset += fieldType->size; - previousFieldOffset = fieldType->size; + previousFieldOffset = fieldType->size - sizeof(uintptr_t); continue; } @@ -2781,28 +2781,28 @@ void swift::swift_initStructMetadataWithLayoutString( } size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { - if (type->vw_size() == 0 || type->getValueWitnesses()->isPOD()) { + auto *vwt = type->getValueWitnesses(); + if (type->vw_size() == 0 || vwt->isPOD()) { return 0; - } else if (type->hasLayoutString()) { - size_t offset = sizeof(uint64_t); - return LayoutStringReader{type->getLayoutString(), offset} - .readBytes(); } else if (auto *tuple = dyn_cast(type)) { size_t res = 0; for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { res += _swift_refCountBytesForMetatype(tuple->getElement(i).Type); } return res; + } else if (vwt == &VALUE_WITNESS_SYM(Bo) || + vwt == &VALUE_WITNESS_SYM(BO) || + vwt == &VALUE_WITNESS_SYM(Bb)) { + return sizeof(uint64_t); } else if (auto *cls = type->getClassObject()) { if (cls->isTypeMetadata()) { - auto *vwt = cls->getValueWitnesses(); - if (vwt != &VALUE_WITNESS_SYM(Bo) && - vwt != &VALUE_WITNESS_SYM(BO) && - vwt != &VALUE_WITNESS_SYM(Bb)) { - goto metadata; - } + goto metadata; } return sizeof(uint64_t); + } else if (type->hasLayoutString()) { + size_t offset = sizeof(uint64_t); + return LayoutStringReader{type->getLayoutString(), offset} + .readBytes(); } else if (type->isAnyExistentialType()) { return sizeof(uint64_t); } else { @@ -2819,12 +2819,53 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, size_t unalignedOffset = fullOffset; fullOffset = roundUpToAlignMask(fullOffset, fieldType->vw_alignment() - 1); size_t offset = fullOffset - unalignedOffset + previousFieldOffset; + auto *vwt = fieldType->getValueWitnesses(); if (fieldType->vw_size() == 0) { return; - } else if (fieldType->getValueWitnesses()->isPOD()) { + } else if (vwt->isPOD()) { // No need to handle PODs previousFieldOffset = offset + fieldType->vw_size(); fullOffset += fieldType->vw_size(); + } else if (auto *tuple = dyn_cast(fieldType)) { + for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { + _swift_addRefCountStringForMetatype(writer, flags, + tuple->getElement(i).Type, fullOffset, + previousFieldOffset); + } + } else if (vwt == &VALUE_WITNESS_SYM(Bo)) { + auto tag = RefCountingKind::NativeStrong; + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); + } else if (vwt == &VALUE_WITNESS_SYM(BO)) { + #if SWIFT_OBJC_INTEROP + auto tag = RefCountingKind::ObjC; + #else + auto tag = RefCountingKind::Unknown; + #endif + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); + } else if (vwt == &VALUE_WITNESS_SYM(Bb)) { + auto tag = RefCountingKind::Bridge; + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); + } else if (auto *cls = fieldType->getClassObject()) { + RefCountingKind tag; + if (!cls->isTypeMetadata()) { + #if SWIFT_OBJC_INTEROP + tag = RefCountingKind::ObjC; + #else + tag = RefCountingKind::Unknown; + #endif + } else { + goto metadata; + } + + writer.writeBytes(((uint64_t)tag << 56) | offset); + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); } else if (fieldType->hasLayoutString()) { LayoutStringReader reader{fieldType->getLayoutString(), 0}; const auto fieldFlags = reader.readBytes(); @@ -2835,9 +2876,8 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, reader.layoutStr + layoutStringHeaderSize, fieldRefCountBytes); if (fieldFlags & LayoutStringFlags::HasRelativePointers) { - swift_resolve_resilientAccessors( - writer.layoutStr, writer.offset, - reader.layoutStr + layoutStringHeaderSize, fieldType); + swift_resolve_resilientAccessors(writer.layoutStr, writer.offset, + reader.layoutStr + layoutStringHeaderSize, fieldType); } if (offset) { @@ -2856,54 +2896,20 @@ void swift::_swift_addRefCountStringForMetatype(LayoutStringWriter &writer, previousFieldOffset += fieldType->vw_size(); } fullOffset += fieldType->vw_size(); - } else if (auto *tuple = dyn_cast(fieldType)) { - for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { - _swift_addRefCountStringForMetatype(writer, flags, - tuple->getElement(i).Type, fullOffset, - previousFieldOffset); - } - } else if (auto *cls = fieldType->getClassObject()) { - RefCountingKind tag; - if (!cls->isTypeMetadata()) { - #if SWIFT_OBJC_INTEROP - tag = RefCountingKind::ObjC; - #else - tag = RefCountingKind::Unknown; - #endif - } else { - auto *vwt = cls->getValueWitnesses(); - if (vwt == &VALUE_WITNESS_SYM(Bo)) { - tag = RefCountingKind::NativeStrong; - } else if (vwt == &VALUE_WITNESS_SYM(BO)) { - #if SWIFT_OBJC_INTEROP - tag = RefCountingKind::ObjC; - #else - tag = RefCountingKind::Unknown; - #endif - } else if (vwt == &VALUE_WITNESS_SYM(Bb)) { - tag = RefCountingKind::Bridge; - } else { - goto metadata; - }; - } - - writer.writeBytes(((uint64_t)tag << 56) | offset); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; } else if (fieldType->isAnyExistentialType()) { auto *existential = dyn_cast(fieldType); assert(existential); auto tag = existential->isClassBounded() ? RefCountingKind::Unknown : RefCountingKind::Existential; writer.writeBytes(((uint64_t)tag << 56) | offset); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; + previousFieldOffset = fieldType->vw_size() - (existential->isClassBounded() ? sizeof(uintptr_t) : (NumWords_ValueBuffer * sizeof(uintptr_t))); + fullOffset += fieldType->vw_size(); } else { metadata: writer.writeBytes(((uint64_t)RefCountingKind::Metatype << 56) | offset); writer.writeBytes(fieldType); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; + previousFieldOffset = 0; + fullOffset += fieldType->vw_size(); } } diff --git a/stdlib/public/runtime/MetadataImpl.h b/stdlib/public/runtime/MetadataImpl.h index 1365e24d6f783..727d1f60be3b8 100644 --- a/stdlib/public/runtime/MetadataImpl.h +++ b/stdlib/public/runtime/MetadataImpl.h @@ -94,6 +94,7 @@ struct NativeBox { static constexpr size_t isPOD = std::is_pod::value; static constexpr bool isBitwiseTakable = isPOD; static constexpr unsigned numExtraInhabitants = 0; + static constexpr bool hasLayoutString = false; static void destroy(T *value) { value->T::~T(); @@ -142,6 +143,7 @@ template struct RetainableBoxBase { #else static constexpr bool isAtomic = true; #endif + static constexpr bool hasLayoutString = false; static void destroy(T *addr) { Impl::release(*addr); @@ -254,6 +256,7 @@ struct WeakRetainableBoxBase { static constexpr bool isPOD = false; static constexpr bool isBitwiseTakable = false; static constexpr unsigned numExtraInhabitants = 0; + static constexpr bool hasLayoutString = false; // The implementation must provide implementations of: // static void destroy(T *); @@ -777,6 +780,8 @@ struct ValueWitnesses .withPOD(isPOD) .withBitwiseTakable(isBitwiseTakable); + static constexpr bool hasLayoutString = Box::hasLayoutString; + static void destroy(OpaqueValue *value, const Metadata *self) { return Box::destroy((typename Box::type*) value); } diff --git a/test/AutoDiff/IRGen/runtime.swift b/test/AutoDiff/IRGen/runtime.swift index 09c848fafcc04..8a753356212a8 100644 --- a/test/AutoDiff/IRGen/runtime.swift +++ b/test/AutoDiff/IRGen/runtime.swift @@ -3,22 +3,17 @@ import Swift import _Differentiation -struct ExamplePullbackStruct { - var pb0: (T.TangentVector) -> T.TangentVector -} - -@_silgen_name("test_context_builtins") -func test_context_builtins() { - let pbStruct = ExamplePullbackStruct(pb0: { $0 }) - let context = Builtin.autoDiffCreateLinearMapContext(Builtin.sizeof(type(of: pbStruct))) +@_silgen_name("test_context_builtins_with_type") +func test_context_builtins_with_type(t: T) { + let context = Builtin.autoDiffCreateLinearMapContextWithType(T.self) let topLevelSubctxAddr = Builtin.autoDiffProjectTopLevelSubcontext(context) - UnsafeMutableRawPointer(topLevelSubctxAddr).storeBytes(of: pbStruct, as: type(of: pbStruct)) - let newBuffer = Builtin.autoDiffAllocateSubcontext(context, Builtin.sizeof(type(of: pbStruct))) - UnsafeMutableRawPointer(newBuffer).storeBytes(of: pbStruct, as: type(of: pbStruct)) + UnsafeMutableRawPointer(topLevelSubctxAddr).storeBytes(of: t, as: T.self) + let newBuffer = Builtin.autoDiffAllocateSubcontextWithType(context, T.self) + UnsafeMutableRawPointer(newBuffer).storeBytes(of: t, as: T.self) } -// CHECK-LABEL: define{{.*}}@test_context_builtins() +// CHECK-LABEL: define{{.*}}@test_context_builtins_with_type(ptr noalias nocapture %0, ptr %T) // CHECK: entry: -// CHECK: [[CTX:%.*]] = call swiftcc ptr @swift_autoDiffCreateLinearMapContext({{i[0-9]+}} {{.*}}) +// CHECK: [[CTX:%.*]] = call swiftcc ptr @swift_autoDiffCreateLinearMapContextWithType(ptr %T) // CHECK: call swiftcc ptr @swift_autoDiffProjectTopLevelSubcontext(ptr [[CTX]]) -// CHECK: [[BUF:%.*]] = call swiftcc ptr @swift_autoDiffAllocateSubcontext(ptr [[CTX]], {{i[0-9]+}} {{.*}}) +// CHECK: [[BUF:%.*]] = call swiftcc ptr @swift_autoDiffAllocateSubcontextWithType(ptr [[CTX]], ptr %T) diff --git a/test/AutoDiff/SILGen/autodiff_builtins.swift b/test/AutoDiff/SILGen/autodiff_builtins.swift index 903ba227fbf71..b369deec7809c 100644 --- a/test/AutoDiff/SILGen/autodiff_builtins.swift +++ b/test/AutoDiff/SILGen/autodiff_builtins.swift @@ -84,24 +84,20 @@ func applyDerivative_f1_vjp(t0: T) -> (T // CHECK: copy_addr [take] [[D_RESULT_BUFFER_0_FOR_LOAD]] to [init] [[ORIG_RESULT_OUT_PARAM]] // CHECK: return [[PULLBACK]] -struct ExamplePullbackStruct { - var pb0: (T.TangentVector) -> T.TangentVector -} -@_silgen_name("test_context_builtins") -func test_context_builtins() { - let pbStruct = ExamplePullbackStruct(pb0: { $0 }) - let context = Builtin.autoDiffCreateLinearMapContext(Builtin.sizeof(type(of: pbStruct))) +@_silgen_name("test_context_builtins_with_type") +func test_context_builtins_with_type(t: T) { + let context = Builtin.autoDiffCreateLinearMapContextWithType(T.self) let topLevelSubctxAddr = Builtin.autoDiffProjectTopLevelSubcontext(context) - UnsafeMutableRawPointer(topLevelSubctxAddr).storeBytes(of: pbStruct, as: type(of: pbStruct)) - let newBuffer = Builtin.autoDiffAllocateSubcontext(context, Builtin.sizeof(type(of: pbStruct))) - UnsafeMutableRawPointer(newBuffer).storeBytes(of: pbStruct, as: type(of: pbStruct)) + UnsafeMutableRawPointer(topLevelSubctxAddr).storeBytes(of: t, as: T.self) + let newBuffer = Builtin.autoDiffAllocateSubcontextWithType(context, T.self) + UnsafeMutableRawPointer(newBuffer).storeBytes(of: t, as: T.self) } -// CHECK-LABEL: sil{{.*}}@test_context_builtins -// CHECK: bb0: -// CHECK: [[CTX:%.*]] = builtin "autoDiffCreateLinearMapContext"({{%.*}} : $Builtin.Word) : $Builtin.NativeObject -// CHECK: [[BORROWED_CTX:%.*]] = begin_borrow [lexical] [[CTX]] : $Builtin.NativeObject -// CHECK: [[BUF:%.*]] = builtin "autoDiffProjectTopLevelSubcontext"([[BORROWED_CTX]] : $Builtin.NativeObject) : $Builtin.RawPointer -// CHECK: [[BUF:%.*]] = builtin "autoDiffAllocateSubcontext"([[BORROWED_CTX]] : $Builtin.NativeObject, {{.*}} : $Builtin.Word) : $Builtin.RawPointer +// CHECK-LABEL: sil{{.*}}@test_context_builtins_with_type : $@convention(thin) (@in_guaranteed T) -> () { +// CHECK: bb0({{%.*}} : $*T): +// CHECK: [[CTX:%.*]] = builtin "autoDiffCreateLinearMapContextWithType"({{%.*}} : $@thick T.Type) : $Builtin.NativeObject // users: {{.*}} +// CHECK: [[BORROWED_CTX:%.*]] = begin_borrow [lexical] [[CTX]] : $Builtin.NativeObject // users: {{.*}} +// CHECK: [[BUF:%.*]] = builtin "autoDiffProjectTopLevelSubcontext"([[BORROWED_CTX]] : $Builtin.NativeObject) : $Builtin.RawPointer // users: {{.*}} +// CHECK: [[BUF:%.*]] = builtin "autoDiffAllocateSubcontextWithType"([[BORROWED_CTX]] : $Builtin.NativeObject, {{.*}} : $@thick T.Type) : $Builtin.RawPointer // users: {{.*}} // CHECK: destroy_value [[CTX]] diff --git a/test/AutoDiff/stdlib/callee_differential_not_leaked_in_func_with_loops.swift b/test/AutoDiff/stdlib/callee_differential_not_leaked_in_func_with_loops.swift new file mode 100644 index 0000000000000..8de10d1d02d2b --- /dev/null +++ b/test/AutoDiff/stdlib/callee_differential_not_leaked_in_func_with_loops.swift @@ -0,0 +1,60 @@ +// RUN: %target-run-simple-swift +// REQUIRES: executable_test + +import _Differentiation +import StdlibUnittest + +// When the original function contains loops, we allocate a context object +// on the heap. This context object may store non-trivial objects, such as closures, +// that need to be freed explicitly, at program exit. This test verifies that the +// autodiff runtime destroys and deallocates any such objects. + +extension LifetimeTracked: AdditiveArithmetic { + public static var zero: LifetimeTracked { fatalError() } + public static func + (lhs: LifetimeTracked, rhs: LifetimeTracked) -> LifetimeTracked {fatalError()} + public static func - (lhs: LifetimeTracked, rhs: LifetimeTracked) -> LifetimeTracked {fatalError()} +} + +extension LifetimeTracked: Differentiable { + public typealias TangentVector = LifetimeTracked + public func move(by: LifetimeTracked) {fatalError()} +} + +extension LifetimeTracked { + // The original differentiable callee. + func callee(_: Float) -> Float { 42 } + + // The callee differential (pullback in this case), that is + // captured in the context object allocated on the heap in the + // presence of loops. + // + // If the autodiff runtime does not free this callee differential + // properly, the `LifetimeTracked` instance that it captures will + // also not be freed and we will have a detectable memory leak. + @derivative(of: callee, wrt: (self, f)) + func calleeDifferential(f: Float) -> (value: Float, pullback: (Float) -> (LifetimeTracked, Float)) { + return ( + value: f, + pullback: { x in (self, x) } + ) + } +} + +@differentiable(reverse) +func f(ltti: LifetimeTracked) -> Float { + for _ in 0..<1 { + } + return ltti.callee(0xDEADBEEF) +} + +var Tests = TestSuite("CalleeDifferentialLeakTest") + +Tests.test("dontLeakCalleeDifferential") { + do { + let ltti = LifetimeTracked(0xDEADBEEF) + let _ = valueWithPullback(at: ltti, of: f) + } + expectEqual(0, LifetimeTracked.instances) +} + +runAllTests() \ No newline at end of file diff --git a/test/IDE/complete_type_relation_any_some.swift b/test/IDE/complete_type_relation_any_some.swift new file mode 100644 index 0000000000000..c14ed956f1154 --- /dev/null +++ b/test/IDE/complete_type_relation_any_some.swift @@ -0,0 +1,23 @@ +// RUN: %empty-directory(%t) +// RUN: %swift-ide-test -batch-code-completion -source-filename %s -filecheck %raw-FileCheck --completion-output-dir %t + +protocol Shape {} + +struct Square : Shape {} + +struct Test { + let protocolType: Shape + let structType: Square + + func testAny() -> any Shape { + return self.#^WITH_ANY_CONTEXTUAL_TYPE^# +// WITH_ANY_CONTEXTUAL_TYPE-DAG: Decl[InstanceVar]/CurrNominal/TypeRelation[Convertible]: protocolType[#any Shape#]; +// WITH_ANY_CONTEXTUAL_TYPE-DAG: Decl[InstanceVar]/CurrNominal/TypeRelation[Convertible]: structType[#Square#]; + } + + func testSome() -> some Shape { + return self.#^WITH_SOME_CONTEXTUAL_TYPE^# +// WITH_SOME_CONTEXTUAL_TYPE-DAG: Decl[InstanceVar]/CurrNominal: protocolType[#any Shape#]; +// WITH_SOME_CONTEXTUAL_TYPE-DAG: Decl[InstanceVar]/CurrNominal/TypeRelation[Convertible]: structType[#Square#]; + } +} diff --git a/test/IRGen/tuple_conformances.swift b/test/IRGen/tuple_conformances.swift index bcb9075da4849..a85fb8c4a53de 100644 --- a/test/IRGen/tuple_conformances.swift +++ b/test/IRGen/tuple_conformances.swift @@ -1,5 +1,8 @@ // RUN: %target-swift-frontend -emit-ir -parse-stdlib -primary-file %s -enable-experimental-feature TupleConformances -parse-as-library | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + import Swift protocol P { diff --git a/test/Interpreter/Inputs/layout_string_witnesses_types.swift b/test/Interpreter/Inputs/layout_string_witnesses_types.swift index c21a467dd3931..60312f3d0635b 100644 --- a/test/Interpreter/Inputs/layout_string_witnesses_types.swift +++ b/test/Interpreter/Inputs/layout_string_witnesses_types.swift @@ -223,6 +223,32 @@ public struct Recursive { } } +public protocol A {} +public protocol B {} +public protocol C {} + +public struct MultiProtocolExistentialWrapper { + let x: Int = 0 + let y: any (A&B&C) + let z: AnyObject + + public init(y: any (A&B&C), z: AnyObject) { + self.y = y + self.z = z + } +} + +public struct AnyWrapper { + let x: Int = 0 + let y: Any + let z: AnyObject + + public init(y: Any, z: AnyObject) { + self.y = y + self.z = z + } +} + #if os(macOS) import Foundation @@ -287,8 +313,8 @@ public struct NestedWrapper { } struct InternalGeneric { - let x: T let y: Int + let x: T } public enum SinglePayloadSimpleClassEnum { @@ -535,6 +561,11 @@ public func testAssign(_ ptr: UnsafeMutablePointer, from x: T) { ptr.pointee = x } +@inline(never) +public func testAssign(_ ptr: UnsafeMutablePointer, from x: UnsafeMutablePointer) { + ptr.assign(from: x, count: 1) +} + @inline(never) public func testInit(_ ptr: UnsafeMutablePointer, to x: T) { ptr.initialize(to: x) @@ -554,19 +585,19 @@ public func allocateInternalGenericPtr(of tpe: T.Type) -> UnsafeMutableRawPoi @inline(never) public func testGenericAssign(_ ptr: __owned UnsafeMutableRawPointer, from x: T) { let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) - let x = InternalGeneric(x: x, y: 23) + let x = InternalGeneric(y: 23, x: x) testAssign(ptr, from: x) } @inline(never) public func testGenericInit(_ ptr: __owned UnsafeMutableRawPointer, to x: T) { let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) - let x = InternalGeneric(x: x, y: 23) + let x = InternalGeneric(y: 23, x: x) testInit(ptr, to: x) } @inline(never) public func testGenericDestroy(_ ptr: __owned UnsafeMutableRawPointer, of tpe: T.Type) { - let ptr = ptr.assumingMemoryBound(to: tpe) + let ptr = ptr.assumingMemoryBound(to: InternalGeneric.self) testDestroy(ptr) } diff --git a/test/Interpreter/layout_string_witnesses_dynamic.swift b/test/Interpreter/layout_string_witnesses_dynamic.swift index af95e974e8cdf..e7de40bb7fe5c 100644 --- a/test/Interpreter/layout_string_witnesses_dynamic.swift +++ b/test/Interpreter/layout_string_witnesses_dynamic.swift @@ -4,7 +4,7 @@ // RUN: %target-codesign %t/%target-library-name(layout_string_witnesses_types) // RUN: %target-swift-frontend -enable-experimental-feature LayoutStringValueWitnesses -enable-experimental-feature LayoutStringValueWitnessesInstantiation -enable-layout-string-value-witnesses -enable-layout-string-value-witnesses-instantiation -enable-library-evolution -enable-autolinking-runtime-compatibility-bytecode-layouts -emit-module -emit-module-path=%t/layout_string_witnesses_types_resilient.swiftmodule %S/Inputs/layout_string_witnesses_types_resilient.swift // RUN: %target-build-swift -g -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnesses -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnessesInstantiation -Xfrontend -enable-layout-string-value-witnesses -Xfrontend -enable-layout-string-value-witnesses-instantiation -Xfrontend -enable-library-evolution -c -parse-as-library -o %t/layout_string_witnesses_types_resilient.o %S/Inputs/layout_string_witnesses_types_resilient.swift -// RUN: %target-build-swift -g -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnesses -Xfrontend -enable-experimental-feature -Xfrontend LayoutStringValueWitnessesInstantiation -Xfrontend -enable-layout-string-value-witnesses -Xfrontend -enable-layout-string-value-witnesses-instantiation -Xfrontend -enable-type-layout -parse-stdlib -module-name layout_string_witnesses_dynamic -llayout_string_witnesses_types -L%t %t/layout_string_witnesses_types_resilient.o -I %t -o %t/main %s %target-rpath(%t) +// RUN: %target-build-swift -g -parse-stdlib -module-name layout_string_witnesses_dynamic -llayout_string_witnesses_types -L%t %t/layout_string_witnesses_types_resilient.o -I %t -o %t/main %s %target-rpath(%t) // RUN: %target-codesign %t/main // RUN: %target-run %t/main %t/%target-library-name(layout_string_witnesses_types) | %FileCheck %s --check-prefix=CHECK -check-prefix=CHECK-%target-os @@ -52,7 +52,35 @@ func testGeneric() { testGeneric() -func testPrespecializedStructAnyObject() { +func testGenericAny() { + let ptr = allocateInternalGenericPtr(of: Any.self) + + do { + let x: Any = TestClass() + testGenericInit(ptr, to: x as Any) + } + + do { + let y: Any = TestClass() + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + testGenericAssign(ptr, from: y as Any) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + testGenericDestroy(ptr, of: Any.self) + + ptr.deallocate() +} + +testGenericAny() + +func testPrespecializedAnyObject() { let ptr = UnsafeMutablePointer>.allocate(capacity: 1) do { @@ -80,7 +108,7 @@ func testPrespecializedStructAnyObject() { ptr.deallocate() } -testPrespecializedStructAnyObject() +testPrespecializedAnyObject() func testPrespecializedStructSimpleClass() { let ptr = UnsafeMutablePointer>.allocate(capacity: 1) diff --git a/test/Interpreter/layout_string_witnesses_static.swift b/test/Interpreter/layout_string_witnesses_static.swift index 861c53ab87a96..8ecab346e4ea1 100644 --- a/test/Interpreter/layout_string_witnesses_static.swift +++ b/test/Interpreter/layout_string_witnesses_static.swift @@ -293,6 +293,74 @@ func testExistentialStructBox() { testExistentialStructBox() +func testAnyWrapper() { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + + do { + let x = TestClass() + testInit(ptr, to: AnyWrapper(y: x, z: SimpleClass(x: 23))) + } + + do { + let y = TestClass() + + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testAssign(ptr, from: AnyWrapper(y: y, z: SimpleClass(x: 32))) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: TestClass deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testDestroy(ptr) + + ptr.deallocate() +} + +testAnyWrapper() + +class ClassWithABC: A, B, C { + deinit { + print("ClassWithABC deinitialized!") + } +} + +func testMultiProtocolExistential() { + let ptr = UnsafeMutablePointer.allocate(capacity: 1) + + do { + let x = ClassWithABC() + testInit(ptr, to: MultiProtocolExistentialWrapper(y: x, z: SimpleClass(x: 23))) + } + + do { + let y = ClassWithABC() + + // CHECK: Before deinit + print("Before deinit") + + // CHECK-NEXT: ClassWithABC deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testAssign(ptr, from: MultiProtocolExistentialWrapper(y: y, z: SimpleClass(x: 32))) + } + + // CHECK-NEXT: Before deinit + print("Before deinit") + + // CHECK-NEXT: ClassWithABC deinitialized! + // CHECK-NEXT: SimpleClass deinitialized! + testDestroy(ptr) + + ptr.deallocate() +} + +testMultiProtocolExistential() + class ClassWithSomeClassProtocol: SomeClassProtocol { deinit { print("ClassWithSomeClassProtocol deinitialized!") diff --git a/test/Interpreter/lazy/deferred_syntax_errors.swift b/test/Interpreter/lazy/deferred_syntax_errors.swift index 6788805237b04..5e112c69994b8 100644 --- a/test/Interpreter/lazy/deferred_syntax_errors.swift +++ b/test/Interpreter/lazy/deferred_syntax_errors.swift @@ -1,6 +1,9 @@ // REQUIRES: OS=macosx // RUN: %target-jit-run %s -enable-experimental-feature LazyImmediate | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + // Tests that parsing of function bodies is deferred // to runtime when the interpreter is invoked using // experimental lazy compilation. Syntax errors in diff --git a/test/Interpreter/lazy/deferred_type_errors.swift b/test/Interpreter/lazy/deferred_type_errors.swift index 8b68308b09282..b905f932264dd 100644 --- a/test/Interpreter/lazy/deferred_type_errors.swift +++ b/test/Interpreter/lazy/deferred_type_errors.swift @@ -1,6 +1,9 @@ // REQUIRES: OS=macosx // RUN: %target-jit-run %s -enable-experimental-feature LazyImmediate | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + // Tests that type checking of function bodies is // deferred to runtime when the interpreter is invoked // using experimental lazy compilation. Type errors in diff --git a/test/Interpreter/lazy/globals.swift b/test/Interpreter/lazy/globals.swift index e4b2b9381ae30..17da6a282d42c 100644 --- a/test/Interpreter/lazy/globals.swift +++ b/test/Interpreter/lazy/globals.swift @@ -1,6 +1,9 @@ // REQUIRES: OS=macosx // RUN: %target-jit-run %s -enable-experimental-feature LazyImmediate | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + // Tests that piecewise compilation works with global variables let x = 1 diff --git a/test/Interpreter/lazy/linkage_promotion.swift b/test/Interpreter/lazy/linkage_promotion.swift index 72d4f3d5c095f..1dbfcc4134633 100644 --- a/test/Interpreter/lazy/linkage_promotion.swift +++ b/test/Interpreter/lazy/linkage_promotion.swift @@ -2,6 +2,9 @@ // REQUIRES: swift_interpreter // RUN: %target-jit-run %s -enable-experimental-feature LazyImmediate | %FileCheck %s +// -enable-experimental-feature requires an asserts build +// REQUIRES: asserts + // Tests that the linkage of private symbols is // promoted to hidden external, allowing // single-function compilation of non-public symbols. diff --git a/test/lit.site.cfg.in b/test/lit.site.cfg.in index ff2857bd1f5ff..8544f2a22bcea 100644 --- a/test/lit.site.cfg.in +++ b/test/lit.site.cfg.in @@ -13,6 +13,7 @@ import os import platform import sys +import lit.util config.cmake = "@CMAKE_COMMAND@" config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@" @@ -170,6 +171,6 @@ if '@SWIFT_SWIFT_PARSER@' == 'TRUE': # Let the main config do the real work. if config.test_exec_root is None: - config.test_exec_root = os.path.dirname(os.path.realpath(__file__)) + config.test_exec_root = os.path.dirname(lit.util.abs_path_preserve_drive(__file__)) lit_config.load_config( config, os.path.join(config.swift_src_root, "test", "lit.cfg"))