diff --git a/SwiftCompilerSources/Sources/Optimizer/Analysis/AliasAnalysis.swift b/SwiftCompilerSources/Sources/Optimizer/Analysis/AliasAnalysis.swift index 94a3553ad60f8..b238b39087af9 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Analysis/AliasAnalysis.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Analysis/AliasAnalysis.swift @@ -253,7 +253,7 @@ struct AliasAnalysis { case let storeBorrow as StoreBorrowInst: return memLoc.mayAlias(with: storeBorrow.destination, self) ? .init(write: true) : .noEffects - case let mdi as MarkDependenceInst: + case let mdi as MarkDependenceInstruction: if mdi.base.type.isAddress && memLoc.mayAlias(with: mdi.base, self) { return .init(read: true) } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceDiagnostics.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceDiagnostics.swift index b1e9e0bd11d58..79c33934a86f2 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceDiagnostics.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceDiagnostics.swift @@ -123,7 +123,7 @@ private func analyze(dependence: LifetimeDependence, _ context: FunctionPassCont // Check each lifetime-dependent use via a def-use visitor var walker = DiagnoseDependenceWalker(diagnostics, context) defer { walker.deinitialize() } - let result = walker.walkDown(root: dependence.dependentValue) + let result = walker.walkDown(dependence: dependence) // The walk may abort without a diagnostic error. assert(!error || result == .abortWalk) return result == .continueWalk @@ -354,7 +354,7 @@ private struct LifetimeVariable { } /// Walk up an address into which a dependent value has been stored. If any address in the use-def chain is a -/// mark_dependence, follow the depenedence base rather than the forwarded value. If any of the dependence bases in +/// mark_dependence, follow the dependence base rather than the forwarded value. If any of the dependence bases in /// within the current scope is with (either local checkInoutResult), then storing a value into that address is /// nonescaping. /// diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceInsertion.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceInsertion.swift index 0895d4e1abfb5..a79cc2f2859fd 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceInsertion.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceInsertion.swift @@ -286,26 +286,21 @@ private func insertMarkDependencies(value: Value, initializer: Instruction?, _ context: FunctionPassContext) { var currentValue = value for base in bases { - let markDep = builder.createMarkDependence( - value: currentValue, base: base, kind: .Unresolved) - if value.type.isAddress { // Address dependencies cannot be represented as SSA values, so it does not make sense to replace any uses of the // dependent address. - // - // TODO: insert a separate mark_dependence_addr instruction with no return value and do not update currentValue. - } else { - // TODO: implement non-inout parameter dependencies. This assumes that currentValue is the apply immediately - // preceeding the mark_dependence. - let uses = currentValue.uses.lazy.filter { - if $0.isScopeEndingUse { - return false - } - let inst = $0.instruction - return inst != markDep && inst != initializer && !(inst is Deallocation) + _ = builder.createMarkDependenceAddr(value: currentValue, base: base, kind: .Unresolved) + continue + } + let markDep = builder.createMarkDependence(value: currentValue, base: base, kind: .Unresolved) + let uses = currentValue.uses.lazy.filter { + if $0.isScopeEndingUse { + return false } - uses.replaceAll(with: markDep, context) + let inst = $0.instruction + return inst != markDep && inst != initializer && !(inst is Deallocation) } + uses.replaceAll(with: markDep, context) currentValue = markDep } } diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceScopeFixup.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceScopeFixup.swift index 29eff8cb331c7..99bd76386f9ba 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceScopeFixup.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/LifetimeDependenceScopeFixup.swift @@ -112,7 +112,7 @@ let lifetimeDependenceScopeFixupPass = FunctionPass( let localReachabilityCache = LocalVariableReachabilityCache() for instruction in function.instructions { - guard let markDep = instruction as? MarkDependenceInst else { + guard let markDep = instruction as? MarkDependenceInstruction else { continue } guard let innerLifetimeDep = LifetimeDependence(markDep, context) else { @@ -129,46 +129,71 @@ let lifetimeDependenceScopeFixupPass = FunctionPass( } } -private extension MarkDependenceInst { +private extension MarkDependenceInstruction { /// Rewrite the mark_dependence base operand to ignore inner borrow scopes (begin_borrow, load_borrow). /// /// Note: this could be done as a general simplification, e.g. after inlining. But currently this is only relevant for /// diagnostics. func rewriteSkippingBorrow(scope: LifetimeDependence.Scope, _ context: FunctionPassContext) -> LifetimeDependence { guard let newScope = scope.ignoreBorrowScope(context) else { - return LifetimeDependence(scope: scope, dependentValue: self) + return LifetimeDependence(scope: scope, markDep: self)! } let newBase = newScope.parentValue if newBase != self.baseOperand.value { self.baseOperand.set(to: newBase, context) } - return LifetimeDependence(scope: newScope, dependentValue: self) + return LifetimeDependence(scope: newScope, markDep: self)! } - /// Rewrite the mark_dependence base operand, setting it to a function argument. - /// - /// To handle more than one function argument, new mark_dependence instructions will be chained. - /// This is called when the dependent value is returned by the function and the dependence base is in the caller. func redirectFunctionReturn(to args: SingleInlineArray, _ context: FunctionPassContext) { - var updatedMarkDep: MarkDependenceInst? + var updatedMarkDep: MarkDependenceInstruction? for arg in args { guard let currentMarkDep = updatedMarkDep else { self.baseOperand.set(to: arg, context) updatedMarkDep = self continue } - let newMarkDep = Builder(after: currentMarkDep, location: currentMarkDep.location, context) - .createMarkDependence(value: currentMarkDep, base: arg, kind: .Unresolved) - let uses = currentMarkDep.uses.lazy.filter { - let inst = $0.instruction - return inst != newMarkDep + switch currentMarkDep { + case let mdi as MarkDependenceInst: + updatedMarkDep = mdi.redirectFunctionReturnForward(to: arg, input: mdi, context) + case let mdi as MarkDependenceAddrInst: + updatedMarkDep = mdi.redirectFunctionReturnAddress(to: arg, context) + default: + fatalError("unexpected MarkDependenceInstruction") } - uses.replaceAll(with: newMarkDep, context) - updatedMarkDep = newMarkDep } } } +private extension MarkDependenceInst { + /// Rewrite the mark_dependence base operand, setting it to a function argument. + /// + /// This is called when the dependent value is returned by the function and the dependence base is in the caller. + func redirectFunctionReturnForward(to arg: FunctionArgument, input: MarkDependenceInst, + _ context: FunctionPassContext) -> MarkDependenceInst { + // To handle more than one function argument, new mark_dependence instructions will be chained. + let newMarkDep = Builder(after: input, location: input.location, context) + .createMarkDependence(value: input, base: arg, kind: .Unresolved) + let uses = input.uses.lazy.filter { + let inst = $0.instruction + return inst != newMarkDep + } + uses.replaceAll(with: newMarkDep, context) + return newMarkDep + } +} + +private extension MarkDependenceAddrInst { + /// Rewrite the mark_dependence_addr base operand, setting it to a function argument. + /// + /// This is called when the dependent value is returned by the function and the dependence base is in the caller. + func redirectFunctionReturnAddress(to arg: FunctionArgument, _ context: FunctionPassContext) + -> MarkDependenceAddrInst { + return Builder(after: self, location: self.location, context) + .createMarkDependenceAddr(value: self.address, base: arg, kind: .Unresolved) + } +} + /// Transitively extend nested scopes that enclose the dependence base. /// /// If the parent function returns the dependent value, then this returns the function arguments that represent the @@ -211,8 +236,8 @@ private func extendScopes(dependence: LifetimeDependence, var dependsOnArgs = SingleInlineArray() for scopeExtension in scopeExtensions { var scopeExtension = scopeExtension - guard var useRange = computeDependentUseRange(of: dependence.dependentValue, within: &scopeExtension, - localReachabilityCache, context) else { + guard var useRange = computeDependentUseRange(of: dependence, within: &scopeExtension, localReachabilityCache, + context) else { continue } @@ -463,12 +488,12 @@ extension ScopeExtension { } } -/// Return an InstructionRange covering all the dependent uses of 'value'. -private func computeDependentUseRange(of value: Value, within scopeExtension: inout ScopeExtension, +/// Return an InstructionRange covering all the dependent uses of 'dependence'. +private func computeDependentUseRange(of dependence: LifetimeDependence, within scopeExtension: inout ScopeExtension, _ localReachabilityCache: LocalVariableReachabilityCache, _ context: FunctionPassContext) -> InstructionRange? { - + let function = dependence.function guard var ownershipRange = scopeExtension.computeRange(localReachabilityCache, context) else { return nil } @@ -476,7 +501,6 @@ private func computeDependentUseRange(of value: Value, within scopeExtension: in // The innermost scope that must be extended must dominate all uses. var useRange = InstructionRange(begin: scopeExtension.innerScope.extendableBegin!.instruction, context) - let function = value.parentFunction var walker = LifetimeDependentUseWalker(function, localReachabilityCache, context) { // Do not extend the useRange past the ownershipRange. let dependentInst = $0.instruction @@ -487,7 +511,7 @@ private func computeDependentUseRange(of value: Value, within scopeExtension: in } defer {walker.deinitialize()} - _ = walker.walkDown(root: value) + _ = walker.walkDown(dependence: dependence) log("Scope fixup for dependent uses:\n\(useRange)") @@ -495,9 +519,12 @@ private func computeDependentUseRange(of value: Value, within scopeExtension: in // Lifetime dependenent uses may not be dominated by the access. The dependent value may be used by a phi or stored // into a memory location. The access may be conditional relative to such uses. If any use was not dominated, then - // `useRange` will include the function entry. + // `useRange` will include the function entry. There is not way to directly check + // useRange.isValid. useRange.blockRange.isValid is not a strong enough check because it will always succeed when + // useRange.begin == entryBlock even if a use if above useRange.begin. let firstInst = function.entryBlock.instructions.first! if firstInst != useRange.begin, useRange.contains(firstInst) { + useRange.deinitialize() return nil } return useRange diff --git a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift index 02b0e8371d430..ae70c9eb22787 100644 --- a/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift +++ b/SwiftCompilerSources/Sources/Optimizer/FunctionPasses/ObjectOutliner.swift @@ -305,7 +305,8 @@ private func isValidUseOfObject(_ use: Operand) -> Bool { is DeallocStackRefInst, is StrongRetainInst, is StrongReleaseInst, - is FixLifetimeInst: + is FixLifetimeInst, + is MarkDependenceAddrInst: return true case let mdi as MarkDependenceInst: diff --git a/SwiftCompilerSources/Sources/Optimizer/TestPasses/MemBehaviorDumper.swift b/SwiftCompilerSources/Sources/Optimizer/TestPasses/MemBehaviorDumper.swift index fd05530b805d7..d7b352454d9fa 100644 --- a/SwiftCompilerSources/Sources/Optimizer/TestPasses/MemBehaviorDumper.swift +++ b/SwiftCompilerSources/Sources/Optimizer/TestPasses/MemBehaviorDumper.swift @@ -75,6 +75,7 @@ private extension Instruction { is BuiltinInst, is StoreBorrowInst, is MarkDependenceInst, + is MarkDependenceAddrInst, is DebugValueInst: return true default: diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/AddressUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/AddressUtils.swift index c1f4afbb48cdf..40bd631f5d0b2 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/AddressUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/AddressUtils.swift @@ -56,22 +56,26 @@ protocol AddressUseVisitor { -> WalkResult /// A loaded address use propagates the value at the address. - mutating func loadedAddressUse(of operand: Operand, into value: Value) + mutating func loadedAddressUse(of operand: Operand, intoValue value: Value) -> WalkResult /// A loaded address use propagates the value at the address to the /// destination address operand. - mutating func loadedAddressUse(of operand: Operand, into address: Operand) + mutating func loadedAddressUse(of operand: Operand, intoAddress address: Operand) -> WalkResult /// Yielding an address may modify the value at the yield, but not past the yield. The yielded value may escape, but /// only if its type is escapable. mutating func yieldedAddressUse(of operand: Operand) -> WalkResult - /// A non-address owned `value` whose ownership depends on the in-memory - /// value at `address`, such as `mark_dependence %value on %address`. - mutating func dependentAddressUse(of operand: Operand, into value: Value) - -> WalkResult + /// A forwarded `value` whose ownership depends on the in-memory value at the address `operand`. + /// `%value` may be an address type, but only uses of this address value are dependent. + /// e.g. `%value = mark_dependence %_ on %operand` + mutating func dependentAddressUse(of operand: Operand, dependentValue value: Value) -> WalkResult + + /// A dependent `address` whose lifetime depends on the in-memory value at `address`. + /// e.g. `mark_dependence_addr %address on %operand` + mutating func dependentAddressUse(of operand: Operand, dependentAddress address: Value) -> WalkResult /// A pointer escape may propagate the address beyond the current instruction. mutating func escapingAddressUse(of operand: Operand) -> WalkResult @@ -91,34 +95,11 @@ extension AddressUseVisitor { case is EndAccessInst, is EndApplyInst, is AbortApplyInst, is EndBorrowInst: return scopeEndingAddressUse(of: operand) - case let markDep as MarkDependenceInst: - if markDep.valueOperand == operand { - return projectedAddressUse(of: operand, into: markDep) - } - assert(markDep.baseOperand == operand) - // If another address depends on the current address, - // handle it like a projection. - if markDep.type.isAddress { - return projectedAddressUse(of: operand, into: markDep) - } - switch markDep.dependenceKind { - case .Unresolved: - if LifetimeDependence(markDep, context) == nil { - break - } - fallthrough - case .NonEscaping: - // Note: This is unreachable from InteriorUseVisitor because the base address of a `mark_dependence - // [nonescaping]` must be a `begin_access`, and interior liveness does not check uses of the accessed address. - return dependentAddressUse(of: operand, into: markDep) - case .Escaping: - break - } - // A potentially escaping value depends on this address. - return escapingAddressUse(of: operand) + case is MarkDependenceInstruction: + return classifyMarkDependence(operand: operand) case let pai as PartialApplyInst where !pai.mayEscape: - return dependentAddressUse(of: operand, into: pai) + return dependentAddressUse(of: operand, dependentValue: pai) case let pai as PartialApplyInst where pai.mayEscape: return escapingAddressUse(of: operand) @@ -156,14 +137,14 @@ extension AddressUseVisitor { case is LoadInst, is LoadUnownedInst, is LoadWeakInst, is ValueMetatypeInst, is ExistentialMetatypeInst, is PackElementGetInst: let svi = operand.instruction as! SingleValueInstruction - return loadedAddressUse(of: operand, into: svi) + return loadedAddressUse(of: operand, intoValue: svi) case is YieldInst: return yieldedAddressUse(of: operand) case let sdai as SourceDestAddrInstruction where sdai.sourceOperand == operand: - return loadedAddressUse(of: operand, into: sdai.destinationOperand) + return loadedAddressUse(of: operand, intoAddress: sdai.destinationOperand) case let sdai as SourceDestAddrInstruction where sdai.destinationOperand == operand: @@ -198,6 +179,57 @@ extension AddressUseVisitor { return unknownAddressUse(of: operand) } } + + private mutating func classifyMarkDependence(operand: Operand) -> WalkResult { + switch operand.instruction { + case let markDep as MarkDependenceInst: + if markDep.valueOperand == operand { + return projectedAddressUse(of: operand, into: markDep) + } + assert(markDep.baseOperand == operand) + // If another address depends on the current address, + // handle it like a projection. + if markDep.type.isAddress { + return projectedAddressUse(of: operand, into: markDep) + } + switch markDep.dependenceKind { + case .Unresolved: + if LifetimeDependence(markDep, context) == nil { + break + } + fallthrough + case .NonEscaping: + // Note: This is unreachable from InteriorUseVisitor because the base address of a `mark_dependence + // [nonescaping]` must be a `begin_access`, and interior liveness does not check uses of the accessed address. + return dependentAddressUse(of: operand, dependentValue: markDep) + case .Escaping: + break + } + // A potentially escaping value depends on this address. + return escapingAddressUse(of: operand) + + case let markDep as MarkDependenceAddrInst: + if markDep.addressOperand == operand { + return leafAddressUse(of: operand) + } + switch markDep.dependenceKind { + case .Unresolved: + if LifetimeDependence(markDep, context) == nil { + break + } + fallthrough + case .NonEscaping: + return dependentAddressUse(of: operand, dependentAddress: markDep.address) + case .Escaping: + break + } + // A potentially escaping value depends on this address. + return escapingAddressUse(of: operand) + + default: + fatalError("Unexpected MarkDependenceInstruction") + } + } } extension AccessBase { @@ -369,12 +401,12 @@ extension AddressInitializationWalker { return convention.isIndirectIn ? .continueWalk : .abortWalk } - mutating func loadedAddressUse(of operand: Operand, into value: Value) + mutating func loadedAddressUse(of operand: Operand, intoValue value: Value) -> WalkResult { return .continueWalk } - mutating func loadedAddressUse(of operand: Operand, into address: Operand) + mutating func loadedAddressUse(of operand: Operand, intoAddress address: Operand) -> WalkResult { return .continueWalk } @@ -385,8 +417,11 @@ extension AddressInitializationWalker { return .continueWalk } - mutating func dependentAddressUse(of operand: Operand, into value: Value) - -> WalkResult { + mutating func dependentAddressUse(of operand: Operand, dependentValue value: Value) -> WalkResult { + return .continueWalk + } + + mutating func dependentAddressUse(of operand: Operand, dependentAddress address: Value) -> WalkResult { return .continueWalk } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift index 1aad4306866ef..d992481da692b 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/LifetimeDependenceUtils.swift @@ -15,8 +15,8 @@ // gatherVariableIntroducers(for:) is a use-def walk that returns the values that most closely associated with the // variable declarations that the given value holds an instance of. // -// LifetimeDependence.init models the lifetime dependence for a FunctionArgument or a MarkDependenceInst, categorizing -// the kind of dependence scope that the lifetime represents. +// LifetimeDependence.init models the lifetime dependence for a FunctionArgument or a MarkDependenceInstruction, +// categorizing the kind of dependence scope that the lifetime represents. // // LifetimeDependence.Scope.computeRange() computes the instruction range covered by a dependence scope. // @@ -133,6 +133,7 @@ struct LifetimeDependence : CustomStringConvertible { } let scope: Scope let dependentValue: Value + let markDepInst: MarkDependenceInstruction? var parentValue: Value { scope.parentValue } @@ -155,6 +156,7 @@ extension LifetimeDependence { } self.scope = Scope(base: arg, context) self.dependentValue = arg + self.markDepInst = nil } // Construct a LifetimeDependence from a return value. This only @@ -175,23 +177,40 @@ extension LifetimeDependence { assert(value.ownership == .owned, "unsafe apply result must be owned") self.scope = Scope(base: value, context) self.dependentValue = value + self.markDepInst = nil } /// Construct LifetimeDependence from mark_dependence [unresolved] or mark_dependence [nonescaping]. /// - /// For any LifetimeDependence constructed from a mark_dependence, its `dependentValue` will be the result of the - /// mark_dependence. + /// For a LifetimeDependence constructed from a mark_dependence, its `dependentValue` is the result of the + /// mark_dependence. For a mark_dependence_addr, `dependentValue` is the address operand. /// /// Returns 'nil' for unknown dependence. - init?(_ markDep: MarkDependenceInst, _ context: some Context) { + init?(_ markDep: MarkDependenceInstruction, _ context: some Context) { + self.init(scope: Scope(base: markDep.base, context), markDep: markDep) + } + + /// Construct LifetimeDependence from mark_dependence [unresolved] or mark_dependence [nonescaping] with a custom + /// Scope. + /// + /// Returns 'nil' for unknown dependence. + init?(scope: LifetimeDependence.Scope, markDep: MarkDependenceInstruction) { switch markDep.dependenceKind { case .Unresolved, .NonEscaping: - self.scope = Scope(base: markDep.base, context) - self.dependentValue = markDep + self.scope = scope + switch markDep { + case let md as MarkDependenceInst: + self.dependentValue = md + case let md as MarkDependenceAddrInst: + self.dependentValue = md.address + default: + fatalError("unexpected MarkDependenceInstruction") + } + self.markDepInst = markDep case .Escaping: return nil } - } + } /// Compute the range of the dependence scope. /// @@ -208,7 +227,7 @@ extension LifetimeDependence { } func resolve(_ context: some Context) { - if let mdi = dependentValue as? MarkDependenceInst { + if let mdi = markDepInst { mdi.resolveToNonEscaping() } } @@ -500,9 +519,9 @@ extension LifetimeDependence.Scope { /// returnedDependence(address: FunctionArgument, on: Operand) -> WalkResult /// yieldedDependence(result: Operand) -> WalkResult /// Start walking: -/// walkDown(root: Value) +/// walkDown(dependence: LifetimeDependence) /// -/// Note: this may visit values that are not dominated by `root` because of dependent phi operands. +/// Note: this may visit values that are not dominated by `dependence` because of dependent phi operands. protocol LifetimeDependenceDefUseWalker : ForwardingDefUseWalker, OwnershipUseVisitor, AddressUseVisitor { @@ -535,18 +554,15 @@ extension LifetimeDependenceDefUseWalker { // Start a forward walk. extension LifetimeDependenceDefUseWalker { - mutating func walkDown(root: Value) -> WalkResult { - if root.type.isAddress { - if let md = root as? MarkDependenceInst, !root.isEscapable { - // LifetimeDependence.dependentValue is typically a mark_dependence. If its 'value' address is a non-Escapable - // local variable, then consider all other reachable uses of that local variable to be dependent uses. Remember - // the operand to the mark_dependence as if it was a store. Diagnostics will consider this the point of variable - // initialization. - if visitStoredUses(of: md.valueOperand, into: md.value) == .abortWalk { - return .abortWalk - } - } - // The root address may also be an escapable mark_dependence that guards its address uses (unsafeAddress), or an + mutating func walkDown(dependence: LifetimeDependence) -> WalkResult { + if let mdAddr = dependence.markDepInst as? MarkDependenceAddrInst { + // All reachable uses of the dependent address are dependent uses. Treat the mark_dependence_addr base operand as + // if it was a store's address operand. Diagnostics will consider this the point of variable initialization. + return visitStoredUses(of: mdAddr.baseOperand, into: mdAddr.address) + } + let root = dependence.dependentValue + if root.type.isAddress { + // The root address may be an escapable mark_dependence that guards its address uses (unsafeAddress), or an // allocation or incoming argument. In all these cases, it is sufficient to walk down the address uses. return walkDownAddressUses(of: root) } @@ -568,7 +584,7 @@ extension LifetimeDependenceDefUseWalker { // Override ForwardingDefUseWalker. mutating func walkDown(operand: Operand) -> WalkResult { - // Initially delegate all usess to OwnershipUseVisitor. + // Initially delegate all uses to OwnershipUseVisitor. // walkDownDefault will be called for uses that forward ownership. return classify(operand: operand) } @@ -615,7 +631,7 @@ extension LifetimeDependenceDefUseWalker { // like [nonescaping] even though they are not considered OSSA // borrows until after resolution. assert(operand == mdi.baseOperand) - return dependentUse(of: operand, into: mdi) + return dependentUse(of: operand, dependentValue: mdi) case is ExistentialMetatypeInst, is FixLifetimeInst, is WitnessMethodInst, is DynamicMethodBranchInst, is ValueMetatypeInst, @@ -660,11 +676,24 @@ extension LifetimeDependenceDefUseWalker { return escapingDependence(on: operand) } - mutating func dependentUse(of operand: Operand, into value: Value) + // Handle address or non-address operands. + mutating func dependentUse(of operand: Operand, dependentValue value: Value) -> WalkResult { return walkDownUses(of: value, using: operand) } + // Handle address or non-address operands. + mutating func dependentUse(of operand: Operand, dependentAddress address: Value) + -> WalkResult { + // Consider this a leaf use in addition to the dependent address uses, which might all occur earlier. + if leafUse(of: operand) == .abortWalk { + return .abortWalk + } + // The lifetime dependence is effectively "copied into" the dependent address. Find all uses of the dependent + // address as if this were a stored use. + return visitStoredUses(of: operand, into: address) + } + mutating func borrowingUse(of operand: Operand, by borrowInst: BorrowingInstruction) -> WalkResult { @@ -720,8 +749,7 @@ extension LifetimeDependenceDefUseWalker { return visitAppliedUse(of: operand, by: apply) } - mutating func loadedAddressUse(of operand: Operand, into value: Value) - -> WalkResult { + mutating func loadedAddressUse(of operand: Operand, intoValue value: Value) -> WalkResult { // Record the load itself, in case the loaded value is Escapable. if leafUse(of: operand) == .abortWalk { return .abortWalk @@ -729,8 +757,8 @@ extension LifetimeDependenceDefUseWalker { return walkDownUses(of: value, using: operand) } - mutating func loadedAddressUse(of operand: Operand, into address: Operand) - -> WalkResult { + // copy_addr + mutating func loadedAddressUse(of operand: Operand, intoAddress address: Operand) -> WalkResult { if leafUse(of: operand) == .abortWalk { return .abortWalk } @@ -745,11 +773,15 @@ extension LifetimeDependenceDefUseWalker { } } - mutating func dependentAddressUse(of operand: Operand, into value: Value) - -> WalkResult { - walkDownUses(of: value, using: operand) + mutating func dependentAddressUse(of operand: Operand, dependentValue value: Value) -> WalkResult { + dependentUse(of: operand, dependentValue: value) } + // mark_dependence_addr + mutating func dependentAddressUse(of operand: Operand, dependentAddress address: Value) -> WalkResult { + dependentUse(of: operand, dependentAddress: address) + } + mutating func escapingAddressUse(of operand: Operand) -> WalkResult { if let mdi = operand.instruction as? MarkDependenceInst { assert(!mdi.isUnresolved && !mdi.isNonEscaping, @@ -797,13 +829,15 @@ extension LifetimeDependenceDefUseWalker { } } - // Visit stores to a local variable (alloc_box), temporary storage (alloc_stack). This handles stores of the entire - // value and stores to a tuple element. Stores to a field within another nominal value are considered lifetime - // dependence leaf uses; the type system enforces non-escapability on the aggregate value. + // Visit a dependent local variable (alloc_box), or temporary storage (alloc_stack). The depenedency is typically from + // storing a dependent value at `address`, but may be from an outright `mark_dependence_addr`. // - // If 'operand' is an address, then the "store" corresponds to initialization via an @out argument. The initial - // call to visitStoredUses will have 'operand == address' where the "stored value" is the temporary stack - // allocation for the @out parameter. + // This handles stores of the entire value and stores into a member. Storing into a member makes the entire aggregate + // a dependent value. + // + // If 'operand' is an address, then the "store" corresponds to a dependency on another in-memory value. This may + // result from `copy_addr`, `mark_dependence_addr`, or initialization of an applied `@out` argument that depends on + // another indirect parameter. private mutating func visitStoredUses(of operand: Operand, into address: Value) -> WalkResult { assert(address.type.isAddress) @@ -871,19 +905,30 @@ extension LifetimeDependenceDefUseWalker { case .load: switch localAccess.instruction! { case let load as LoadInst: - return loadedAddressUse(of: localAccess.operand!, into: load) + return loadedAddressUse(of: localAccess.operand!, intoValue: load) case let load as LoadBorrowInst: - return loadedAddressUse(of: localAccess.operand!, into: load) + return loadedAddressUse(of: localAccess.operand!, intoValue: load) case let copyAddr as SourceDestAddrInstruction: - return loadedAddressUse(of: localAccess.operand!, into: copyAddr.destinationOperand) + return loadedAddressUse(of: localAccess.operand!, intoAddress: copyAddr.destinationOperand) + default: + return .abortWalk + } + case .dependenceSource: + switch localAccess.instruction! { + case let md as MarkDependenceInst: + if md.type.isAddress { + return loadedAddressUse(of: localAccess.operand!, intoAddress: md.valueOperand) + } + return loadedAddressUse(of: localAccess.operand!, intoValue: md) + case let md as MarkDependenceAddrInst: + return loadedAddressUse(of: localAccess.operand!, intoAddress: md.addressOperand) default: return .abortWalk } - case .dependence: - // An address-forwarding mark_dependence is simply a marker that indicates the start of an in-memory - // dependent value. Typically, it has no uses. If it does have uses, then they are visited earlier by - // LocalVariableAccessWalker to record any other local accesses. - return .continueWalk + case .dependenceDest: + // Simply a marker that indicates the start of an in-memory dependent value. If this was a mark_dependence, uses + // of its forwarded address has were visited by LocalVariableAccessWalker and recorded as separate local accesses. + return .continueWalk case .store: let si = localAccess.operand!.instruction as! StoringInstruction assert(si.sourceOperand == initialValue, "the only reachable store should be the current assignment") @@ -921,13 +966,13 @@ extension LifetimeDependenceDefUseWalker { // because a mark_dependence [nonescaping] represents the // dependence. if let result = apply.singleDirectResult, !result.isEscapable { - if dependentUse(of: operand, into: result) == .abortWalk { + if dependentUse(of: operand, dependentValue: result) == .abortWalk { return .abortWalk } } for resultAddr in apply.indirectResultOperands where !resultAddr.value.isEscapable { - if visitStoredUses(of: operand, into: resultAddr.value) == .abortWalk { + if dependentUse(of: operand, dependentAddress: resultAddr.value) == .abortWalk { return .abortWalk } } @@ -1023,10 +1068,23 @@ private struct LifetimeDependenceUsePrinter : LifetimeDependenceDefUseWalker { let lifetimeDependenceUseTest = FunctionTest("lifetime_dependence_use") { function, arguments, context in let value = arguments.takeValue() + var dependence: LifetimeDependence? + switch value { + case let arg as FunctionArgument: + dependence = LifetimeDependence(arg, context) + case let md as MarkDependenceInstruction: + dependence = LifetimeDependence(md, context) + default: + break + } + guard let dependence = dependence else { + print("Invalid dependence scope: \(value)") + return + } print("LifetimeDependence uses of: \(value)") var printer = LifetimeDependenceUsePrinter(function: function, context) defer { printer.deinitialize() } - _ = printer.walkDown(root: value) + _ = printer.walkDown(dependence: dependence) } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/LocalVariableUtils.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/LocalVariableUtils.swift index 1b289170954d5..2736f60ca0531 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/LocalVariableUtils.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/LocalVariableUtils.swift @@ -36,6 +36,29 @@ private func log(_ message: @autoclosure () -> String) { // Local variables are accessed in one of these ways. // // Note: @in is only immutable up to when it is destroyed, so still requires a local live range. +// +// A .dependenceSource access creates a new dependent value that keeps this local alive. +// +// %local = alloc_stack // this local +// %md = mark_dependence %val on %local +// mark_dependence_addr %adr on %local +// +// The effect of .dependenceSource on reachability is like a load of this local. The dependent value depends on any +// value in this local that reaches this point. +// +// A .dependenceDest access is the point where another value becomes dependent on this local: +// +// %local = alloc_stack // this local +// %md = mark_dependence %local on %val +// mark_dependence_addr %local on %val +// +// The effect of .dependenceDest on reachability is like a store of this local. All uses of this local reachable from +// this point are uses of the dependence base. +// +// Note that the same mark_dependence_addr often involves two locals: +// +// mark_dependence_addr %localDest on %localSource +// struct LocalVariableAccess: CustomStringConvertible { enum Kind { case incomingArgument // @in, @inout, @inout_aliasable @@ -43,7 +66,8 @@ struct LocalVariableAccess: CustomStringConvertible { case inoutYield // indirect yield from this accessor case beginAccess // Reading or reassigning a 'var' case load // Reading a 'let'. Returning 'var' from an initializer. - case dependence // A mark_dependence after an apply with an indirect result. No effect. + case dependenceSource // A value/address depends on this local here (like a load) + case dependenceDest // This local depends on another value/address here (like a store) case store // 'var' initialization and destruction case apply // indirect arguments case escape // alloc_box captures @@ -77,7 +101,7 @@ struct LocalVariableAccess: CustomStringConvertible { case .`init`, .modify: return true } - case .load, .dependence: + case .load, .dependenceSource, .dependenceDest: return false case .incomingArgument, .outgoingArgument, .store, .inoutYield: return true @@ -116,8 +140,10 @@ struct LocalVariableAccess: CustomStringConvertible { str += "beginAccess" case .load: str += "load" - case .dependence: - str += "dependence" + case .dependenceSource: + str += "dependenceSource" + case .dependenceDest: + str += "dependenceDest" case .store: str += "store" case .apply: @@ -152,7 +178,7 @@ class LocalVariableAccessInfo: CustomStringConvertible { case .`init`, .modify: break // lazily compute full assignment } - case .load, .dependence: + case .load, .dependenceSource, .dependenceDest: self._isFullyAssigned = false case .store: if let store = localAccess.instruction as? StoringInstruction { @@ -365,7 +391,7 @@ extension LocalVariableAccessWalker : ForwardingDefUseWalker { break case let markDep as MarkDependenceInst: assert(markDep.baseOperand == operand) - visit(LocalVariableAccess(.dependence, operand)) + visit(LocalVariableAccess(.dependenceSource, operand)) default: visit(LocalVariableAccess(.escape, operand)) } @@ -381,10 +407,6 @@ extension LocalVariableAccessWalker : ForwardingDefUseWalker { extension LocalVariableAccessWalker: AddressUseVisitor { private mutating func walkDownAddressUses(address: Value) -> WalkResult { for operand in address.uses.ignoreTypeDependence { - if let md = operand.instruction as? MarkDependenceInst, operand == md.valueOperand { - // Record the forwarding mark_dependence as a fake access before continuing to walk down. - visit(LocalVariableAccess(.dependence, operand)) - } if classifyAddress(operand: operand) == .abortWalk { return .abortWalk } @@ -399,6 +421,13 @@ extension LocalVariableAccessWalker: AddressUseVisitor { // temporaries do not have access scopes, so we need to walk down any projection that may be used to initialize the // temporary. mutating func projectedAddressUse(of operand: Operand, into value: Value) -> WalkResult { + // Intercept mark_dependence destination to record an access point which can be used like a store when finding all + // uses that affect the base after the point that the dependence was marked. + if let md = value as? MarkDependenceInst { + assert(operand == md.valueOperand) + visit(LocalVariableAccess(.dependenceDest, operand)) + // walk down the forwarded address as usual... + } return walkDownAddressUses(address: value) } @@ -432,6 +461,9 @@ extension LocalVariableAccessWalker: AddressUseVisitor { is PackElementSetInst: // Handle instructions that initialize both temporaries and local variables. visit(LocalVariableAccess(.store, operand)) + case let md as MarkDependenceAddrInst: + assert(operand == md.addressOperand) + visit(LocalVariableAccess(.dependenceDest, operand)) case is DeallocStackInst: break default: @@ -452,7 +484,7 @@ extension LocalVariableAccessWalker: AddressUseVisitor { return .continueWalk } - mutating func dependentAddressUse(of operand: Operand, into value: Value) -> WalkResult { + mutating func dependentAddressUse(of operand: Operand, dependentValue value: Value) -> WalkResult { // Find all uses of partial_apply [on_stack]. if let pai = value as? PartialApplyInst, !pai.mayEscape { var walker = NonEscapingClosureDefUseWalker(context) @@ -468,12 +500,17 @@ extension LocalVariableAccessWalker: AddressUseVisitor { return .continueWalk } - mutating func loadedAddressUse(of operand: Operand, into value: Value) -> WalkResult { + mutating func dependentAddressUse(of operand: Operand, dependentAddress address: Value) -> WalkResult { + // No other dependent uses can access to memory at this address. + return .continueWalk + } + + mutating func loadedAddressUse(of operand: Operand, intoValue value: Value) -> WalkResult { visit(LocalVariableAccess(.load, operand)) return .continueWalk } - mutating func loadedAddressUse(of operand: Operand, into address: Operand) -> WalkResult { + mutating func loadedAddressUse(of operand: Operand, intoAddress address: Operand) -> WalkResult { visit(LocalVariableAccess(.load, operand)) return .continueWalk } diff --git a/SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift b/SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift index 7d327f9642de9..ec3c03b82ccaf 100644 --- a/SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift +++ b/SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift @@ -208,7 +208,8 @@ struct InteriorLivenessResult: CustomDebugStringConvertible { /// - forwardingUse(of:isInnerlifetime:) /// - interiorPointerUse(of:into:) /// - pointerEscapingUse(of:) -/// - dependentUse(of:into:) +/// - dependentUse(of:dependentValue:) +/// - dependentUse(of:dependentAddress:) /// - borrowingUse(of:by:) /// /// This only visits the first level of uses. The implementation may transitively visit forwarded, borrowed, dependent, @@ -269,7 +270,13 @@ protocol OwnershipUseVisitor { /// there are no explicit scope-ending operations. Instead /// BorrowingInstruction.scopeEndingOperands will return the final /// consumes in the dependent value's forwarding chain. - mutating func dependentUse(of operand: Operand, into value: Value) -> WalkResult + mutating func dependentUse(of operand: Operand, dependentValue value: Value) -> WalkResult + + /// A use that creates an implicit borrow scope over all reachable uses of a value stored in + /// `dependentAddress`. This could conservatively be handled has `pointerEscapingUse`, but note that accessing the + /// `dependentAddress` only keeps the original owner alive, it cannot modify the original (modifying a dependent + /// address is still just a "read" of the dependence source. + mutating func dependentUse(of operand: Operand, dependentAddress address: Value) -> WalkResult /// A use that is scoped to an inner borrow scope. mutating func borrowingUse(of operand: Operand, by borrowInst: BorrowingInstruction) -> WalkResult @@ -384,6 +391,9 @@ extension OwnershipUseVisitor { return forwardingUse(of: operand, isInnerLifetime: false) case .pointerEscape: + if let mdai = operand.instruction as? MarkDependenceAddrInst, operand == mdai.baseOperand { + return dependentUse(of: operand, dependentAddress: mdai.address) + } return pointerEscapingUse(of: operand) case .instantaneousUse, .forwardingUnowned, .unownedInstantaneousUse, .bitwiseEscape: @@ -413,6 +423,9 @@ extension OwnershipUseVisitor { if operand.instruction is ProjectBoxInst { return visitInteriorPointerUse(of: operand) } + if let mdai = operand.instruction as? MarkDependenceAddrInst, operand == mdai.baseOperand { + return dependentUse(of: operand, dependentAddress: mdai.address) + } return pointerEscapingUse(of: operand) case .instantaneousUse, .forwardingUnowned, .unownedInstantaneousUse, .bitwiseEscape, .endBorrow, .reborrow: @@ -436,12 +449,14 @@ extension OwnershipUseVisitor { switch operand.instruction { case let pai as PartialApplyInst: assert(!pai.mayEscape) - return dependentUse(of: operand, into: pai) + return dependentUse(of: operand, dependentValue: pai) case let mdi as MarkDependenceInst: + // .borrow operand ownership only applies to the base operand of a non-escaping markdep that forwards a + // non-address value. assert(operand == mdi.baseOperand && mdi.isNonEscaping) - return dependentUse(of: operand, into: mdi) + return dependentUse(of: operand, dependentValue: mdi) case let bfi as BorrowedFromInst where !bfi.borrowedPhi.isReborrow: - return dependentUse(of: operand, into: bfi) + return dependentUse(of: operand, dependentValue: bfi) default: return borrowingUse(of: operand, by: BorrowingInstruction(operand.instruction)!) @@ -586,7 +601,7 @@ extension InteriorUseWalker: OwnershipUseVisitor { } // Handle partial_apply [on_stack] and mark_dependence [nonescaping]. - mutating func dependentUse(of operand: Operand, into value: Value) -> WalkResult { + mutating func dependentUse(of operand: Operand, dependentValue value: Value) -> WalkResult { // OSSA lifetime ignores trivial types. if value.type.isTrivial(in: function) { return .continueWalk @@ -602,6 +617,12 @@ extension InteriorUseWalker: OwnershipUseVisitor { return walkDownUses(of: value) } + mutating func dependentUse(of operand: Operand, dependentAddress address: Value) -> WalkResult { + // An mutable local variable depends a value that depends on the original interior pointer. This would require data + // flow to find local uses. InteriorUseWalker only walks the SSA uses. + pointerEscapingUse(of: operand) + } + mutating func pointerEscapingUse(of operand: Operand) -> WalkResult { if useVisitor(operand) == .abortWalk { return .abortWalk @@ -699,12 +720,12 @@ extension InteriorUseWalker: AddressUseVisitor { return .continueWalk } - mutating func loadedAddressUse(of operand: Operand, into value: Value) + mutating func loadedAddressUse(of operand: Operand, intoValue value: Value) -> WalkResult { return .continueWalk } - mutating func loadedAddressUse(of operand: Operand, into address: Operand) + mutating func loadedAddressUse(of operand: Operand, intoAddress address: Operand) -> WalkResult { return .continueWalk } @@ -713,7 +734,7 @@ extension InteriorUseWalker: AddressUseVisitor { return .continueWalk } - mutating func dependentAddressUse(of operand: Operand, into value: Value) + mutating func dependentAddressUse(of operand: Operand, dependentValue value: Value) -> WalkResult { // For Escapable values, simply continue the walk. if value.mayEscape { @@ -727,6 +748,11 @@ extension InteriorUseWalker: AddressUseVisitor { return escapingAddressUse(of: operand) } + mutating func dependentAddressUse(of operand: Operand, dependentAddress address: Value) -> WalkResult { + // TODO: consider data flow that finds reachable uses of `dependentAddress`. + return escapingAddressUse(of: operand) + } + mutating func escapingAddressUse(of operand: Operand) -> WalkResult { pointerStatus.setEscaping(operand: operand) return ignoreEscape ? .continueWalk : .abortWalk diff --git a/SwiftCompilerSources/Sources/SIL/Builder.swift b/SwiftCompilerSources/Sources/SIL/Builder.swift index 2954dd363c3cf..2af0327a1e6bf 100644 --- a/SwiftCompilerSources/Sources/SIL/Builder.swift +++ b/SwiftCompilerSources/Sources/SIL/Builder.swift @@ -568,11 +568,17 @@ public struct Builder { return notifyNew(endMutation.getAs(EndCOWMutationInst.self)) } - public func createMarkDependence(value: Value, base: Value, kind: MarkDependenceInst.DependenceKind) -> MarkDependenceInst { + public func createMarkDependence(value: Value, base: Value, kind: MarkDependenceKind) -> MarkDependenceInst { let markDependence = bridged.createMarkDependence(value.bridged, base.bridged, BridgedInstruction.MarkDependenceKind(rawValue: kind.rawValue)!) return notifyNew(markDependence.getAs(MarkDependenceInst.self)) } + + public func createMarkDependenceAddr(value: Value, base: Value, kind: MarkDependenceKind) -> MarkDependenceAddrInst { + let markDependence = bridged.createMarkDependenceAddr( + value.bridged, base.bridged, BridgedInstruction.MarkDependenceKind(rawValue: kind.rawValue)!) + return notifyNew(markDependence.getAs(MarkDependenceAddrInst.self)) + } @discardableResult public func createEndAccess(beginAccess: BeginAccessInst) -> EndAccessInst { diff --git a/SwiftCompilerSources/Sources/SIL/Instruction.swift b/SwiftCompilerSources/Sources/SIL/Instruction.swift index f7a3df59e4ca2..84d38f87beaa6 100644 --- a/SwiftCompilerSources/Sources/SIL/Instruction.swift +++ b/SwiftCompilerSources/Sources/SIL/Instruction.swift @@ -1082,22 +1082,34 @@ class GetAsyncContinuationAddrInst : SingleValueInstruction, UnaryInstruction {} final public class ExtractExecutorInst : SingleValueInstruction {} -final public -class MarkDependenceInst : SingleValueInstruction { - public enum DependenceKind: Int32 { - case Unresolved = 0 - case Escaping = 1 - case NonEscaping = 2 - } +public enum MarkDependenceKind: Int32 { + case Unresolved = 0 + case Escaping = 1 + case NonEscaping = 2 +} + +public protocol MarkDependenceInstruction: Instruction { + var baseOperand: Operand { get } + var base: Value { get } + var dependenceKind: MarkDependenceKind { get } + func resolveToNonEscaping() + func settleToEscaping() +} + +extension MarkDependenceInstruction { + public var isNonEscaping: Bool { dependenceKind == .NonEscaping } + public var isUnresolved: Bool { dependenceKind == .Unresolved } +} + +final public class MarkDependenceInst : SingleValueInstruction, MarkDependenceInstruction { public var valueOperand: Operand { operands[0] } public var baseOperand: Operand { operands[1] } public var value: Value { return valueOperand.value } public var base: Value { return baseOperand.value } - public var dependenceKind: DependenceKind { - DependenceKind(rawValue: bridged.MarkDependenceInst_dependenceKind().rawValue)! + + public var dependenceKind: MarkDependenceKind { + MarkDependenceKind(rawValue: bridged.MarkDependenceInst_dependenceKind().rawValue)! } - public var isNonEscaping: Bool { dependenceKind == .NonEscaping } - public var isUnresolved: Bool { dependenceKind == .Unresolved } public func resolveToNonEscaping() { bridged.MarkDependenceInst_resolveToNonEscaping() @@ -1112,6 +1124,25 @@ class MarkDependenceInst : SingleValueInstruction { } } +final public class MarkDependenceAddrInst : Instruction, MarkDependenceInstruction { + public var addressOperand: Operand { operands[0] } + public var baseOperand: Operand { operands[1] } + public var address: Value { return addressOperand.value } + public var base: Value { return baseOperand.value } + + public var dependenceKind: MarkDependenceKind { + MarkDependenceKind(rawValue: bridged.MarkDependenceAddrInst_dependenceKind().rawValue)! + } + + public func resolveToNonEscaping() { + bridged.MarkDependenceAddrInst_resolveToNonEscaping() + } + + public func settleToEscaping() { + bridged.MarkDependenceAddrInst_settleToEscaping() + } +} + final public class RefToBridgeObjectInst : SingleValueInstruction { public var convertedOperand: Operand { operands[0] } public var maskOperand: Operand { operands[1] } diff --git a/SwiftCompilerSources/Sources/SIL/Registration.swift b/SwiftCompilerSources/Sources/SIL/Registration.swift index e0d5b5894de66..3a6b1c7c73960 100644 --- a/SwiftCompilerSources/Sources/SIL/Registration.swift +++ b/SwiftCompilerSources/Sources/SIL/Registration.swift @@ -195,6 +195,7 @@ public func registerSILClasses() { register(ObjCMetatypeToObjectInst.self) register(ValueToBridgeObjectInst.self) register(MarkDependenceInst.self) + register(MarkDependenceAddrInst.self) register(RefToBridgeObjectInst.self) register(BridgeObjectToRefInst.self) register(BridgeObjectToWordInst.self) diff --git a/SwiftCompilerSources/Sources/SIL/Utilities/WalkUtils.swift b/SwiftCompilerSources/Sources/SIL/Utilities/WalkUtils.swift index babcfbebc8b12..a5ecef2512688 100644 --- a/SwiftCompilerSources/Sources/SIL/Utilities/WalkUtils.swift +++ b/SwiftCompilerSources/Sources/SIL/Utilities/WalkUtils.swift @@ -541,6 +541,12 @@ extension AddressDefUseWalker { } else { return unmatchedPath(address: operand, path: path) } + case is MarkDependenceAddrInst: + if operand.index == 0 { + return leafUse(address: operand, path: path) + } else { + return unmatchedPath(address: operand, path: path) + } default: return leafUse(address: operand, path: path) } diff --git a/docs/SIL/Instructions.md b/docs/SIL/Instructions.md index 533b9b69ee3c1..5314f01f464f0 100644 --- a/docs/SIL/Instructions.md +++ b/docs/SIL/Instructions.md @@ -1964,39 +1964,118 @@ destroy the value, such as `release_value`, `strong_release`, ### mark_dependence ``` -sil-instruction :: 'mark_dependence' '[nonescaping]'? sil-operand 'on' sil-operand +sil-instruction :: 'mark_dependence' mark-dep-option? sil-operand 'on' sil-operand +mark-dep-option ::= '[nonescaping]' +mark-dep-option ::= '[unresolved]' %2 = mark_dependence %value : $*T on %base : $Builtin.NativeObject ``` `%base` must not be identical to `%value`. -Indicates that the validity of `%value` depends on the value of `%base`. +The value of the result depends on the value of `%base`. Operations that would destroy `%base` must not be moved before any -instructions which depend on the result of this instruction, exactly as +instructions that depend on the result of this instruction, exactly as if the address had been directly derived from that operand (e.g. using `ref_element_addr`). -The result is the forwarded value of `%value`. `%value` may be an -address, but it could be an address in a non-obvious form, such as a -Builtin.RawPointer or a struct containing the same. - -`%base` may have either object or address type. In the latter case, the -dependency is on the current value stored in the address. - -The optional `nonescaping` attribute indicates that no value derived -from `%value` escapes the lifetime of `%base`. As with escaping -`mark_dependence`, all values transitively forwarded from `%value` -must be destroyed within the lifetime of `base`. Unlike escaping -`mark_dependence`, this must be statically verifiable. Additionally, -unlike escaping `mark_dependence`, nonescaping `mark_dependence` may -produce a value of non-`Escapable` type. A non-`Escapable` -`mark_dependence` extends the lifetime of `%base` into copies of -`%value` and values transitively forwarded from those copies. If the -`mark_dependence` forwards an address, then it extends the lifetime -through loads from that address. Unlike escaping `mark_dependence`, no -value derived from `%value` may have a bitwise escape (conversion to -UnsafePointer) or pointer escape (unknown use). +The result is the forwarded value of `%value`. If `%value` is an +address, then result is also an address, and the semantics are the +same as the non-address form: the dependency is on any value derived +from the resulting address. The value could also be a +Builtin.RawPointer or a struct containing the same, in which case, +pointed-to values have a dependency if they are derived from this +instruction's result. Note that in-memory values are only dependent on +base if they are derived from this instruction's result. In this +example, the load of `%dependent_value` depends on `%base`, but the +load of `%independent_value` does not: + +``` +%dependent_address = mark_dependence %original_address on %base +%dependent_value = load [copy] %dependent_address +%independent_value = load %original_address +destroy_value %base +``` + +`%base` may have either object or address type. If it is an address, +then the dependency is on the current value stored at the address. + +The optional `nonescaping` attribute indicates that the lifetime +guarantee is statically verifiable via a def-use walk starting at this +instruction's result. No value derived from a nonescaping +`mark_dependence` may have a bitwise escape (conversion to +UnsafePointer) or pointer escape (unknown use). The `unresolved` +attribute indicates that this verification is required but has not yet +been diagnosed. + +`mark_dependence` may only have a non-`Escapable` result if it also +has a `nonescaping` or `unresolved` attribute. A non-`Escapable` +`mark_dependence` extends the lifetime of `%base` through copies of +`%value` and values transitively forwarded from those copies. If +`%value` is an address, then that includes loads from the +address. None of those values may be used by a bitwise escape +(conversion to UnsafePointer) or pointer escape (unknown use). In this +example, the apply depends on `%base` because `%value` has a +non-`Escapable` type: + +``` +%dependent_address = mark_dependence [nonescaping] %value : %*NonescapableType on %base +%dependent_value = load %dependent_address +%copied_value = copy_value %dependent_value +apply %f(%dependent_value) +destroy_value %base +``` + +### mark_dependence_addr + +``` +sil-instruction :: 'mark_dependence_addr' mark-dep-option? sil-operand 'on' sil-operand +mark-dep-option ::= '[nonescaping]' +mark-dep-option ::= '[unresolved]' + +mark_dependence_addr [nonescaping] %address : $*T on %base : $Builtin.NativeObject +``` + +The in-memory value at `%address` depends on the value of `%base`. +Operations that would destroy `%base` must not be moved before any +instructions that depend on that value, exactly as if the location at +`%address` aliases `%base` on all paths reachable from this instruction. + +In this example, the load of `%dependent_value` depends on `%base`: + +``` +mark_dependence_addr %address on %base +%dependent_value = load [copy] %address +destroy_value %base +``` + +`%base` may have either object or address type. If it is an address, +then the dependency is on the current value stored at the address. + +The optional `nonescaping` attribute indicates that the lifetime +guarantee is statically verifiable via a data flow over all paths +reachable from this instruction considering all addresses that may +alias with `%address`. No aliasing address may be used by a bitwise +escape (conversion to UnsafePointer) or pointer escape (unknown +use). The `unresolved` attribute indicates that this verification is +required but has not yet been diagnosed. + +`mark_dependence_addr` may only have a non-`Escapable` `%address` if +it also has a `nonescaping` or `unresolved` attribute. A +non-`Escapable` `mark_dependence_addr` extends the lifetime of `%base` +through values loaded from the memory location at `%address` and +through any transitively forwarded or copied values. None of those +values may be used by a bitwise escape (conversion to UnsafePointer) +or pointer escape (unknown use). In this example, the apply depends on +`%base` because `%address` has a non-`Escapable` type: + +``` +mark_dependence_addr [nonescaping] %address : %*NonescapableType on %base +%dependent_value = load %address +%copied_value = copy_value %dependent_value +apply %f(%dependent_value) +destroy_value %base +``` ### is_unique diff --git a/include/swift/SIL/AddressWalker.h b/include/swift/SIL/AddressWalker.h index df09d97fe3d7b..590e56333155a 100644 --- a/include/swift/SIL/AddressWalker.h +++ b/include/swift/SIL/AddressWalker.h @@ -305,19 +305,25 @@ TransitiveAddressWalker::walk(SILValue projectedAddress) { continue; } - if (auto *mdi = dyn_cast(user)) { + if (auto mdi = MarkDependenceInstruction(user)) { // TODO: continue walking the dependent value, which may not be an // address. See AddressUtils.swift. Until that is implemented, this must // be considered a pointer escape. - if (op->get() == mdi->getBase()) { + if (op->get() == mdi.getBase()) { recordEscape(op, AddressUseKind::Dependent); callVisitUse(op); continue; } - - // If we are the value use, look through it. - transitiveResultUses(op); - continue; + if (auto *mdi = dyn_cast(user)) { + // If we are the value use of a forwarding markdep, look through it. + transitiveResultUses(op); + continue; + } + if (auto *mdi = dyn_cast(user)) { + // The address operand is simply a leaf use. + callVisitUse(op); + continue; + } } // We were unable to recognize this user, so set AddressUseKind to unknown diff --git a/include/swift/SIL/InstWrappers.h b/include/swift/SIL/InstWrappers.h index 2d5bbcc8a7adf..d2c352e50dda5 100644 --- a/include/swift/SIL/InstWrappers.h +++ b/include/swift/SIL/InstWrappers.h @@ -293,7 +293,7 @@ class ForwardingOperation { case SILInstructionKind::DifferentiableFunctionInst: return nullptr; case SILInstructionKind::MarkDependenceInst: - return &forwardingInst->getOperandRef(MarkDependenceInst::Value); + return &forwardingInst->getOperandRef(MarkDependenceInst::Dependent); case SILInstructionKind::RefToBridgeObjectInst: return &forwardingInst->getOperandRef(RefToBridgeObjectInst::ConvertedOperand); diff --git a/include/swift/SIL/SILBridging.h b/include/swift/SIL/SILBridging.h index f1a4c23928202..884b261380bb9 100644 --- a/include/swift/SIL/SILBridging.h +++ b/include/swift/SIL/SILBridging.h @@ -777,6 +777,9 @@ struct BridgedInstruction { BRIDGED_INLINE MarkDependenceKind MarkDependenceInst_dependenceKind() const; BRIDGED_INLINE void MarkDependenceInst_resolveToNonEscaping() const; BRIDGED_INLINE void MarkDependenceInst_settleToEscaping() const; + BRIDGED_INLINE MarkDependenceKind MarkDependenceAddrInst_dependenceKind() const; + BRIDGED_INLINE void MarkDependenceAddrInst_resolveToNonEscaping() const; + BRIDGED_INLINE void MarkDependenceAddrInst_settleToEscaping() const; BRIDGED_INLINE SwiftInt BeginAccessInst_getAccessKind() const; BRIDGED_INLINE bool BeginAccessInst_isStatic() const; BRIDGED_INLINE bool BeginAccessInst_isUnsafe() const; @@ -1222,6 +1225,9 @@ struct BridgedBuilder{ bool keepUnique) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createMarkDependence( BridgedValue value, BridgedValue base, BridgedInstruction::MarkDependenceKind dependenceKind) const; + + SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createMarkDependenceAddr( + BridgedValue value, BridgedValue base, BridgedInstruction::MarkDependenceKind dependenceKind) const; SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedInstruction createEndAccess(BridgedValue value) const; diff --git a/include/swift/SIL/SILBridgingImpl.h b/include/swift/SIL/SILBridgingImpl.h index 7e4bce5442ffb..a541e556e4bf0 100644 --- a/include/swift/SIL/SILBridgingImpl.h +++ b/include/swift/SIL/SILBridgingImpl.h @@ -1356,6 +1356,18 @@ void BridgedInstruction::MarkDependenceInst_settleToEscaping() const { getAs()->settleToEscaping(); } +BridgedInstruction::MarkDependenceKind BridgedInstruction::MarkDependenceAddrInst_dependenceKind() const { + return (MarkDependenceKind)getAs()->dependenceKind(); +} + +void BridgedInstruction::MarkDependenceAddrInst_resolveToNonEscaping() const { + getAs()->resolveToNonEscaping(); +} + +void BridgedInstruction::MarkDependenceAddrInst_settleToEscaping() const { + getAs()->settleToEscaping(); +} + SwiftInt BridgedInstruction::BeginAccessInst_getAccessKind() const { return (SwiftInt)getAs()->getAccessKind(); } @@ -2378,6 +2390,12 @@ BridgedInstruction BridgedBuilder::createMarkDependence(BridgedValue value, Brid return {unbridged().createMarkDependence(regularLoc(), value.getSILValue(), base.getSILValue(), swift::MarkDependenceKind(kind))}; } +BridgedInstruction BridgedBuilder::createMarkDependenceAddr(BridgedValue value, BridgedValue base, BridgedInstruction::MarkDependenceKind kind) const { + return {unbridged().createMarkDependenceAddr( + regularLoc(), value.getSILValue(), base.getSILValue(), + swift::MarkDependenceKind(kind))}; +} + BridgedInstruction BridgedBuilder::createEndAccess(BridgedValue value) const { return {unbridged().createEndAccess(regularLoc(), value.getSILValue(), false)}; } diff --git a/include/swift/SIL/SILBuilder.h b/include/swift/SIL/SILBuilder.h index 59781fecccdd4..51ebeff1932c9 100644 --- a/include/swift/SIL/SILBuilder.h +++ b/include/swift/SIL/SILBuilder.h @@ -2338,6 +2338,14 @@ class SILBuilder { forwardingOwnershipKind, dependenceKind)); } + MarkDependenceAddrInst * + createMarkDependenceAddr(SILLocation Loc, SILValue address, SILValue base, + MarkDependenceKind dependenceKind) { + return insert(new (getModule()) MarkDependenceAddrInst( + getSILDebugLocation(Loc), address, base, + dependenceKind)); + } + IsUniqueInst *createIsUnique(SILLocation Loc, SILValue operand) { auto Int1Ty = SILType::getBuiltinIntegerType(1, getASTContext()); return insert(new (getModule()) IsUniqueInst(getSILDebugLocation(Loc), diff --git a/include/swift/SIL/SILCloner.h b/include/swift/SIL/SILCloner.h index 66d6a2c2d8490..b7519a9922098 100644 --- a/include/swift/SIL/SILCloner.h +++ b/include/swift/SIL/SILCloner.h @@ -3142,6 +3142,17 @@ void SILCloner::visitMarkDependenceInst(MarkDependenceInst *Inst) { Inst->dependenceKind())); } +template +void SILCloner:: +visitMarkDependenceAddrInst(MarkDependenceAddrInst *Inst) { + getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope())); + recordClonedInstruction( + Inst, getBuilder().createMarkDependenceAddr( + getOpLocation(Inst->getLoc()), getOpValue(Inst->getAddress()), + getOpValue(Inst->getBase()), + Inst->dependenceKind())); +} + template void SILCloner::visitStrongReleaseInst(StrongReleaseInst *Inst) { diff --git a/include/swift/SIL/SILInstruction.h b/include/swift/SIL/SILInstruction.h index 6ac404710587a..295a0767d651b 100644 --- a/include/swift/SIL/SILInstruction.h +++ b/include/swift/SIL/SILInstruction.h @@ -8750,66 +8750,28 @@ enum class MarkDependenceKind { }; static_assert(2 == SILNode::NumMarkDependenceKindBits, "Size mismatch"); -/// Indicates that the validity of the first operand ("the value") depends on -/// the value of the second operand ("the base"). Operations that would destroy -/// the base must not be moved before any instructions which depend on the -/// result of this instruction, exactly as if the address had been obviously -/// derived from that operand (e.g. using ``ref_element_addr``). The result is -/// always equal to the first operand and thus forwards ownership through the -/// first operand. This is a "regular" use of the second operand (i.e. the -/// second operand must be live at the use point). -/// -/// Example: -/// -/// %base = ... -/// %value = ... @trivial value ... -/// %value_dependent_on_base = mark_dependence %value on %base -/// ... -/// use(%value_dependent_on_base) (1) -/// ... -/// destroy_value %base (2) -/// -/// (2) can never move before (1). In English this is a way for the compiler -/// writer to say to the optimizer: 'This subset of uses of "value" (the uses of -/// result) have a dependence on "base" being alive. Do not allow for things -/// that /may/ destroy base to be moved earlier than any of these uses of -/// "value"'. -/// -/// The dependent 'value' may be marked 'nonescaping', which guarantees that the -/// lifetime dependence is statically enforceable. In this case, the compiler -/// must be able to follow all values forwarded from the dependent 'value', and -/// recognize all final (non-forwarded, non-escaping) use points. This implies -/// that `findPointerEscape` is false. A diagnostic pass checks that the -/// incoming SIL to verify that these use points are all initially within the -/// 'base' lifetime. Regular 'mark_dependence' semantics ensure that -/// optimizations cannot violate the lifetime dependence after diagnostics. -class MarkDependenceInst - : public InstructionBase { - friend SILBuilder; - +template +class MarkDependenceInstBase : public InstructionBase { FixedOperandList<2> Operands; - USE_SHARED_UINT8; + TEMPLATE_USE_SHARED_UINT8(BaseTy); - MarkDependenceInst(SILDebugLocation DebugLoc, SILValue value, SILValue base, - ValueOwnershipKind forwardingOwnershipKind, - MarkDependenceKind dependenceKind) - : InstructionBase(DebugLoc, value->getType(), forwardingOwnershipKind), - Operands{this, value, base} { - sharedUInt8().MarkDependenceInst.dependenceKind = uint8_t(dependenceKind); +protected: + template + MarkDependenceInstBase(SILDebugLocation DebugLoc, SILValue value, + SILValue base, MarkDependenceKind dependenceKind, + Rest &&...rest) + : InstructionBase(DebugLoc, std::forward(rest)...), + Operands{this, value, base} { + sharedUInt8().MarkDependenceInstBase.dependenceKind = + uint8_t(dependenceKind); } public: - enum { Value, Base }; - - SILValue getValue() const { return Operands[Value].get(); } + enum { Dependent, Base }; SILValue getBase() const { return Operands[Base].get(); } - void setValue(SILValue newVal) { - Operands[Value].set(newVal); - } void setBase(SILValue newVal) { Operands[Base].set(newVal); } @@ -8818,9 +8780,14 @@ class MarkDependenceInst MutableArrayRef getAllOperands() { return Operands.asArray(); } MarkDependenceKind dependenceKind() const { - return MarkDependenceKind(sharedUInt8().MarkDependenceInst.dependenceKind); + return MarkDependenceKind( + sharedUInt8().MarkDependenceInstBase.dependenceKind); } + /// True if the lifetime dependence is statically enforceable. If so, the + /// compiler can follow all values forwarded from the result, and recognize + /// all final (non-forwarded, non-escaping) use points. This implies that + /// `findPointerEscape` is false. bool isNonEscaping() const { return dependenceKind() == MarkDependenceKind::NonEscaping; } @@ -8833,13 +8800,41 @@ class MarkDependenceInst } void resolveToNonEscaping() { - sharedUInt8().MarkDependenceInst.dependenceKind = + sharedUInt8().MarkDependenceInstBase.dependenceKind = uint8_t(MarkDependenceKind::NonEscaping); } void settleToEscaping() { - sharedUInt8().MarkDependenceInst.dependenceKind = + sharedUInt8().MarkDependenceInstBase.dependenceKind = uint8_t(MarkDependenceKind::Escaping); + } +}; + +/// The result forwards the value of the first operand ('value') and depends on +/// the second operand ('base'). +/// +/// The 'value' and the forwarded result are both either an object type or an +/// address type. The semantics are the same in each case. +/// +/// 'base' may have either object or address type independent from the type of +/// 'value'. If 'base' is an address, then the dependency is on the current +/// value stored at the address. +class MarkDependenceInst + : public MarkDependenceInstBase { + friend SILBuilder; + + MarkDependenceInst(SILDebugLocation DebugLoc, SILValue value, SILValue base, + ValueOwnershipKind forwardingOwnershipKind, + MarkDependenceKind dependenceKind) + : MarkDependenceInstBase(DebugLoc, value, base, dependenceKind, + value->getType(), forwardingOwnershipKind) {} + +public: + SILValue getValue() const { return getAllOperands()[Dependent].get(); } + + void setValue(SILValue newVal) { + getAllOperands()[Dependent].set(newVal); } // True if the dependence is limited to the scope of an OSSA lifetime. Only @@ -8860,6 +8855,97 @@ class MarkDependenceInst llvm::function_ref visitUnknownUse); }; +/// The in-memory value at the first operand ('address') depends on the value of +/// the second operand ('base'). This is as if the location at 'address' aliases +/// 'base' on all paths reachable from this instruction. +/// +/// 'base' may have either object or address type. If 'base' is an address, then +/// the dependency is on the current value stored at the address. +class MarkDependenceAddrInst + : public MarkDependenceInstBase { + friend SILBuilder; + + MarkDependenceAddrInst(SILDebugLocation DebugLoc, SILValue value, + SILValue base, MarkDependenceKind dependenceKind) + : MarkDependenceInstBase(DebugLoc, value, base, dependenceKind) {} + +public: + SILValue getAddress() const { return getAllOperands()[Dependent].get(); } + + void setAddress(SILValue newVal) { + getAllOperands()[Dependent].set(newVal); + } +}; + +/// Shared API for MarkDependenceInst and MarkDependenceAddrInst. +class MarkDependenceInstruction { + const SILInstruction *inst = nullptr; + +public: + explicit MarkDependenceInstruction(const SILInstruction *inst) { + switch (inst->getKind()) { + case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: + this->inst = inst; + break; + default: + break; + } + } + + explicit operator bool() const { return inst != nullptr; } + + SILValue getBase() const { + if (inst) { + switch (inst->getKind()) { + case SILInstructionKind::MarkDependenceInst: + return cast(inst)->getBase(); + case SILInstructionKind::MarkDependenceAddrInst: + return cast(inst)->getBase(); + default: + break; + } + } + return SILValue(); + } + + SILValue getDependent() const { + if (inst) { + switch (inst->getKind()) { + case SILInstructionKind::MarkDependenceInst: + return cast(inst)->getValue(); + case SILInstructionKind::MarkDependenceAddrInst: + return cast(inst)->getAddress(); + default: + break; + } + } + return SILValue(); + } + + SILType getType() const { + if (auto *mdi = dyn_cast(inst)) + return mdi->getType(); + + return SILType(); + } + + bool isNonEscaping() const { + if (inst) { + switch (inst->getKind()) { + case SILInstructionKind::MarkDependenceInst: + return cast(inst)->isNonEscaping(); + case SILInstructionKind::MarkDependenceAddrInst: + return cast(inst)->isNonEscaping(); + default: + break; + } + } + return false; + } +}; + /// Promote an Objective-C block that is on the stack to the heap, or simply /// retain a block that is already on the heap. class CopyBlockInst diff --git a/include/swift/SIL/SILNode.h b/include/swift/SIL/SILNode.h index 6237311062296..091c4b5441775 100644 --- a/include/swift/SIL/SILNode.h +++ b/include/swift/SIL/SILNode.h @@ -285,8 +285,8 @@ class alignas(8) SILNode : pointerEscape : 1, fromVarDecl : 1); - SHARED_FIELD(MarkDependenceInst, uint8_t - dependenceKind : NumMarkDependenceKindBits); + SHARED_TEMPLATE2_FIELD(SILInstructionKind, typename, MarkDependenceInstBase, + uint8_t dependenceKind : NumMarkDependenceKindBits); // Do not use `_sharedUInt8_private` outside of SILNode. } _sharedUInt8_private; diff --git a/include/swift/SIL/SILNodes.def b/include/swift/SIL/SILNodes.def index d85b257279250..06b3f22ece711 100644 --- a/include/swift/SIL/SILNodes.def +++ b/include/swift/SIL/SILNodes.def @@ -441,6 +441,8 @@ ABSTRACT_VALUE_AND_INST(SingleValueInstruction, ValueBase, SILInstruction) SingleValueInstruction, None, DoesNotRelease) SINGLE_VALUE_INST(ValueToBridgeObjectInst, value_to_bridge_object, SingleValueInstruction, None, DoesNotRelease) + // MarkDependenceAddrInst has read effects for the base operand. See + // getMemoryBehavior() SINGLE_VALUE_INST(MarkDependenceInst, mark_dependence, SingleValueInstruction, None, DoesNotRelease) SINGLE_VALUE_INST(CopyBlockInst, copy_block, @@ -901,7 +903,12 @@ NON_VALUE_INST(IgnoredUseInst, ignored_use, NON_VALUE_INST(IncrementProfilerCounterInst, increment_profiler_counter, SILInstruction, MayReadWrite, DoesNotRelease) -NODE_RANGE(NonValueInstruction, UnreachableInst, IncrementProfilerCounterInst) +// MarkDependenceAddrInst has read effects for the base operand. See +// getMemoryBehavior(). +NON_VALUE_INST(MarkDependenceAddrInst, mark_dependence_addr, + SILInstruction, None, DoesNotRelease) + +NODE_RANGE(NonValueInstruction, UnreachableInst, MarkDependenceAddrInst) ABSTRACT_INST(MultipleValueInstruction, SILInstruction) MULTIPLE_VALUE_INST(BeginApplyInst, begin_apply, diff --git a/lib/IRGen/IRGenSIL.cpp b/lib/IRGen/IRGenSIL.cpp index 2a1e37df82ada..086da5d307efa 100644 --- a/lib/IRGen/IRGenSIL.cpp +++ b/lib/IRGen/IRGenSIL.cpp @@ -1427,6 +1427,7 @@ class IRGenSILFunction : llvm_unreachable("unimplemented"); } void visitMarkDependenceInst(MarkDependenceInst *i); + void visitMarkDependenceAddrInst(MarkDependenceAddrInst *i); void visitCopyBlockInst(CopyBlockInst *i); void visitCopyBlockWithoutEscapingInst(CopyBlockWithoutEscapingInst *i) { llvm_unreachable("not valid in canonical SIL"); @@ -6136,6 +6137,11 @@ void IRGenSILFunction::visitMarkDependenceInst(swift::MarkDependenceInst *i) { } } +void IRGenSILFunction:: +visitMarkDependenceAddrInst(swift::MarkDependenceAddrInst *i) { + // Dependency-marking is purely for SIL. No result. +} + void IRGenSILFunction::visitCopyBlockInst(CopyBlockInst *i) { Explosion lowered = getLoweredExplosion(i->getOperand()); llvm::Value *copied = emitBlockCopyCall(lowered.claimNext()); diff --git a/lib/SIL/IR/OperandOwnership.cpp b/lib/SIL/IR/OperandOwnership.cpp index 310544bbf63b4..fcee5da5c0314 100644 --- a/lib/SIL/IR/OperandOwnership.cpp +++ b/lib/SIL/IR/OperandOwnership.cpp @@ -664,18 +664,25 @@ OperandOwnership OperandOwnershipClassifier::visitCopyBlockWithoutEscapingInst( return OperandOwnership::UnownedInstantaneousUse; } +template +static OperandOwnership +visitMarkDependenceInstBase(MarkDependenceInstBase *mdi) { +} + OperandOwnership OperandOwnershipClassifier::visitMarkDependenceInst(MarkDependenceInst *mdi) { // If we are analyzing "the value", we forward ownership. - if (getOperandIndex() == MarkDependenceInst::Value) { + if (getOperandIndex() == MarkDependenceInst::Dependent) { return getOwnershipKind().getForwardingOperandOwnership( /*allowUnowned*/true); } if (mdi->isNonEscaping()) { - if (!mdi->getType().isAddress()) { - // This creates a "dependent value", just like on-stack partial_apply, - // which we treat like a borrow. - return OperandOwnership::Borrow; + if (auto *svi = dyn_cast(mdi)) { + if (!svi->getType().isAddress()) { + // This creates a "dependent value", just like on-stack partial_apply, + // which we treat like a borrow. + return OperandOwnership::Borrow; + } } return OperandOwnership::AnyInteriorPointer; } @@ -684,9 +691,15 @@ OperandOwnershipClassifier::visitMarkDependenceInst(MarkDependenceInst *mdi) { // lifetime. return OperandOwnership::UnownedInstantaneousUse; } - // FIXME: Add an end_dependence instruction so we can treat mark_dependence as - // a borrow of the base (mark_dependence %base -> end_dependence is analogous - // to a borrow scope). + return OperandOwnership::PointerEscape; +} + +OperandOwnership OperandOwnershipClassifier:: +visitMarkDependenceAddrInst(MarkDependenceAddrInst *mdai) { + // If we are analyzing "the value", this is a trivial use + if (getOperandIndex() == MarkDependenceInst::Dependent) { + return OperandOwnership::TrivialUse; + } return OperandOwnership::PointerEscape; } diff --git a/lib/SIL/IR/SILInstruction.cpp b/lib/SIL/IR/SILInstruction.cpp index 2cf1fa76631b5..45b09113095da 100644 --- a/lib/SIL/IR/SILInstruction.cpp +++ b/lib/SIL/IR/SILInstruction.cpp @@ -870,6 +870,10 @@ namespace { return true; } + bool visitMarkDependenceAddrInst(const MarkDependenceAddrInst *RHS) { + return true; + } + bool visitOpenExistentialRefInst(const OpenExistentialRefInst *RHS) { return true; } @@ -1092,8 +1096,8 @@ MemoryBehavior SILInstruction::getMemoryBehavior() const { llvm_unreachable("Covered switch isn't covered?!"); } - if (auto *mdi = dyn_cast(this)) { - if (mdi->getBase()->getType().isAddress()) + if (auto mdi = MarkDependenceInstruction(this)) { + if (mdi.getBase()->getType().isAddress()) return MemoryBehavior::MayRead; return MemoryBehavior::None; } diff --git a/lib/SIL/IR/SILPrinter.cpp b/lib/SIL/IR/SILPrinter.cpp index 2258b85dce3db..5645fb6b02627 100644 --- a/lib/SIL/IR/SILPrinter.cpp +++ b/lib/SIL/IR/SILPrinter.cpp @@ -2701,7 +2701,8 @@ class SILPrinter : public SILInstructionVisitor { void visitClassifyBridgeObjectInst(ClassifyBridgeObjectInst *CBOI) { *this << getIDAndType(CBOI->getOperand()); } - void visitMarkDependenceInst(MarkDependenceInst *MDI) { + template + void visitMarkDependenceInstBase(MarkDependenceInstBase *MDI) { switch (MDI->dependenceKind()) { case MarkDependenceKind::Unresolved: *this << "[unresolved] "; @@ -2712,10 +2713,17 @@ class SILPrinter : public SILInstructionVisitor { *this << "[nonescaping] "; break; } - *this << getIDAndType(MDI->getValue()) << " on " - << getIDAndType(MDI->getBase()); + *this << + getIDAndType(MDI->getOperand(MarkDependenceInstBase::Dependent)) + << " on " << getIDAndType(MDI->getBase()); + } + void visitMarkDependenceInst(MarkDependenceInst *MDI) { + visitMarkDependenceInstBase(MDI); printForwardingOwnershipKind(MDI, MDI->getValue()); } + void visitMarkDependenceAddrInst(MarkDependenceAddrInst *MDI) { + visitMarkDependenceInstBase(MDI); + } void visitCopyBlockInst(CopyBlockInst *RI) { *this << getIDAndType(RI->getOperand()); } diff --git a/lib/SIL/Parser/ParseSIL.cpp b/lib/SIL/Parser/ParseSIL.cpp index ce77bc4d9dfce..127e979f35ad9 100644 --- a/lib/SIL/Parser/ParseSIL.cpp +++ b/lib/SIL/Parser/ParseSIL.cpp @@ -3937,7 +3937,8 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B, break; } - case SILInstructionKind::MarkDependenceInst: { + case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: { std::optional dependenceKind; SILValue Base; auto parseDependenceKind = [](StringRef Str) { @@ -3955,13 +3956,22 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B, if (!dependenceKind) { dependenceKind = MarkDependenceKind::Escaping; } - ValueOwnershipKind forwardingOwnership = Val->getOwnershipKind(); - if (parseForwardingOwnershipKind(forwardingOwnership) - || parseSILDebugLocation(InstLoc, B)) - return true; + if (Opcode == SILInstructionKind::MarkDependenceInst) { + ValueOwnershipKind forwardingOwnership = Val->getOwnershipKind(); + if (parseForwardingOwnershipKind(forwardingOwnership) + || parseSILDebugLocation(InstLoc, B)) { + return true; + } + ResultVal = B.createMarkDependence(InstLoc, Val, Base, + forwardingOwnership, + dependenceKind.value()); + } else { + if (parseSILDebugLocation(InstLoc, B)) + return true; - ResultVal = B.createMarkDependence(InstLoc, Val, Base, forwardingOwnership, - dependenceKind.value()); + ResultVal = B.createMarkDependenceAddr(InstLoc, Val, Base, + dependenceKind.value()); + } break; } diff --git a/lib/SIL/Utils/InstructionUtils.cpp b/lib/SIL/Utils/InstructionUtils.cpp index 5c0c9ed1661e5..f1e002b0d4a00 100644 --- a/lib/SIL/Utils/InstructionUtils.cpp +++ b/lib/SIL/Utils/InstructionUtils.cpp @@ -558,6 +558,7 @@ RuntimeEffect swift::getRuntimeEffect(SILInstruction *inst, SILType &impactType) case SILInstructionKind::ClassifyBridgeObjectInst: case SILInstructionKind::ValueToBridgeObjectInst: case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: case SILInstructionKind::MergeIsolationRegionInst: case SILInstructionKind::MoveValueInst: case SILInstructionKind::DropDeinitInst: diff --git a/lib/SIL/Utils/MemAccessUtils.cpp b/lib/SIL/Utils/MemAccessUtils.cpp index 79ef07c6d717e..554711f2d2c4f 100644 --- a/lib/SIL/Utils/MemAccessUtils.cpp +++ b/lib/SIL/Utils/MemAccessUtils.cpp @@ -2830,6 +2830,7 @@ void swift::visitAccessedAddress(SILInstruction *I, case SILInstructionKind::ExistentialMetatypeInst: case SILInstructionKind::FixLifetimeInst: case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: case SILInstructionKind::GlobalAddrInst: case SILInstructionKind::HasSymbolInst: case SILInstructionKind::HopToExecutorInst: diff --git a/lib/SIL/Utils/MemoryLocations.cpp b/lib/SIL/Utils/MemoryLocations.cpp index 4dbbe349b21db..a36e0163d779e 100644 --- a/lib/SIL/Utils/MemoryLocations.cpp +++ b/lib/SIL/Utils/MemoryLocations.cpp @@ -339,7 +339,7 @@ bool MemoryLocations::analyzeLocationUsesRecursively(SILValue V, unsigned locIdx break; case SILInstructionKind::MarkDependenceInst: { auto *mdi = cast(user); - if (use == &mdi->getAllOperands()[MarkDependenceInst::Value]) { + if (use == &mdi->getAllOperands()[MarkDependenceInst::Dependent]) { if (!analyzeLocationUsesRecursively(mdi, locIdx, collectedVals, subLocationMap)) return false; } diff --git a/lib/SIL/Verifier/LinearLifetimeChecker.cpp b/lib/SIL/Verifier/LinearLifetimeChecker.cpp index e20fc089b8d77..8f1b35ef3575b 100644 --- a/lib/SIL/Verifier/LinearLifetimeChecker.cpp +++ b/lib/SIL/Verifier/LinearLifetimeChecker.cpp @@ -41,7 +41,7 @@ using namespace swift; /// Return true if \p operand can legally be consumed by another operand of the /// same instruction (in parallel). bool isParallelOperand(Operand *operand) { - return isa(operand->getUser()) + return isa(operand->getUser()) //!!! why??? || operand->getOperandOwnership() == OperandOwnership::Reborrow || operand->getOperandOwnership() == OperandOwnership::GuaranteedForwarding; } diff --git a/lib/SIL/Verifier/MemoryLifetimeVerifier.cpp b/lib/SIL/Verifier/MemoryLifetimeVerifier.cpp index 9630417841f25..d83fc493224d9 100644 --- a/lib/SIL/Verifier/MemoryLifetimeVerifier.cpp +++ b/lib/SIL/Verifier/MemoryLifetimeVerifier.cpp @@ -889,16 +889,18 @@ void MemoryLifetimeVerifier::checkBlock(SILBasicBlock *block, Bits &bits) { locations.clearBits(bits, opVal); break; } - case SILInstructionKind::MarkDependenceInst: { - auto *mdi = cast(&I); - if (mdi->getBase()->getType().isAddress() && + case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: { + auto mdi = MarkDependenceInstruction(&I); + if (mdi.getBase()->getType().isAddress() && // In case the mark_dependence is used for a closure it might be that the base // is "self" in an initializer and "self" is not fully initialized, yet. - !mdi->getType().isFunction()) { - requireBitsSet(bits, mdi->getBase(), &I); + (!mdi.getType() || !mdi.getType().isFunction())) { + requireBitsSet(bits, mdi.getBase(), &I); } // TODO: check that the base operand is alive during the whole lifetime - // of the value operand. + // of the value operand. This requires treating all transitive uses of + // 'mdi' as uses of 'base' (including copies for non-Escapable types). break; } default: diff --git a/lib/SIL/Verifier/SILVerifier.cpp b/lib/SIL/Verifier/SILVerifier.cpp index f92b17bdf9d26..6e990b8d309d7 100644 --- a/lib/SIL/Verifier/SILVerifier.cpp +++ b/lib/SIL/Verifier/SILVerifier.cpp @@ -711,6 +711,7 @@ struct ImmutableAddressUseVerifier { break; } case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: case SILInstructionKind::LoadBorrowInst: case SILInstructionKind::ExistentialMetatypeInst: case SILInstructionKind::ValueMetatypeInst: @@ -2193,7 +2194,7 @@ class SILVerifier : public SILVerifierBase { "mark_dependence [nonescaping] must be an owned value"); } } - + void checkPartialApplyInst(PartialApplyInst *PAI) { auto resultInfo = requireObjectType(SILFunctionType, PAI, "result of partial_apply"); diff --git a/lib/SILOptimizer/Analysis/RegionAnalysis.cpp b/lib/SILOptimizer/Analysis/RegionAnalysis.cpp index ac959e18f6733..9881890ac65c9 100644 --- a/lib/SILOptimizer/Analysis/RegionAnalysis.cpp +++ b/lib/SILOptimizer/Analysis/RegionAnalysis.cpp @@ -3003,6 +3003,7 @@ CONSTANT_TRANSLATION(UncheckedAddrCastInst, Assign) CONSTANT_TRANSLATION(UncheckedEnumDataInst, Assign) CONSTANT_TRANSLATION(UncheckedOwnershipConversionInst, Assign) CONSTANT_TRANSLATION(IndexRawPointerInst, Assign) +CONSTANT_TRANSLATION(MarkDependenceAddrInst, Assign) CONSTANT_TRANSLATION(InitExistentialMetatypeInst, Assign) CONSTANT_TRANSLATION(OpenExistentialMetatypeInst, Assign) diff --git a/lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp b/lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp index 810de799902d0..b56e7cf72e5d4 100644 --- a/lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp +++ b/lib/SILOptimizer/LoopTransforms/COWArrayOpt.cpp @@ -471,7 +471,7 @@ bool COWArrayOpt::checkSafeArrayAddressUses(UserList &AddressUsers) { continue; } - if (isa(UseInst)) { + if (MarkDependenceInstruction(UseInst)) { continue; } @@ -569,8 +569,9 @@ bool COWArrayOpt::checkSafeArrayValueUses(UserList &ArrayValueUsers) { continue; } - if (isa(UseInst)) + if (MarkDependenceInstruction(UseInst)) { continue; + } if (isa(UseInst)) continue; @@ -638,8 +639,9 @@ bool COWArrayOpt::checkSafeArrayElementUse(SILInstruction *UseInst, // // The struct_extract, unchecked_ref_cast is handled below in the // "Transitive SafeArrayElementUse" code. - if (isa(UseInst)) + if (MarkDependenceInstruction(UseInst)) { return true; + } if (isa(UseInst)) return true; diff --git a/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp b/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp index 5db55087240c3..0b12222f412b0 100644 --- a/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp +++ b/lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp @@ -303,10 +303,9 @@ void ArrayInfo::classifyUsesOfArray(SILValue arrayValue) { // above as the array would be passed indirectly. if (isFixLifetimeUseOfArray(user, arrayValue)) continue; - if (auto *MDI = dyn_cast(user)) { - if (MDI->getBase() == arrayValue) { + if (auto mdi = MarkDependenceInstruction(user)) { + if (mdi.getBase() == arrayValue) continue; - } } // Check if this is a forEach call on the array. if (TryApplyInst *forEachCall = isForEachUseOfArray(user, arrayValue)) { diff --git a/lib/SILOptimizer/Mandatory/CapturePromotion.cpp b/lib/SILOptimizer/Mandatory/CapturePromotion.cpp index 41a181dc285ae..70e6f813a123c 100644 --- a/lib/SILOptimizer/Mandatory/CapturePromotion.cpp +++ b/lib/SILOptimizer/Mandatory/CapturePromotion.cpp @@ -1217,7 +1217,7 @@ static bool findEscapeOrMutationUses(Operand *op, if (isa(parent)) return false; state.accumulatedEscapes.push_back( - &userMDI->getOperandRef(MarkDependenceInst::Value)); + &userMDI->getOperandRef(MarkDependenceInst::Dependent)); return true; } } diff --git a/lib/SILOptimizer/Mandatory/MoveOnlyBorrowToDestructureUtils.cpp b/lib/SILOptimizer/Mandatory/MoveOnlyBorrowToDestructureUtils.cpp index 7d77d62d9879b..a2300bcb41265 100644 --- a/lib/SILOptimizer/Mandatory/MoveOnlyBorrowToDestructureUtils.cpp +++ b/lib/SILOptimizer/Mandatory/MoveOnlyBorrowToDestructureUtils.cpp @@ -1551,10 +1551,10 @@ static bool gatherBorrows(SILValue rootValue, // escape. Is it legal to canonicalize ForwardingUnowned? case OperandOwnership::ForwardingUnowned: case OperandOwnership::PointerEscape: - if (auto *md = dyn_cast(use->getUser())) { + if (auto mdi = MarkDependenceInstruction(use->getUser())) { // mark_depenence uses only keep its base value alive; they do not use // the base value itself and are irrelevant for destructuring. - if (use == &md->getOperandRef(MarkDependenceInst::Base)) + if (use->get() == mdi.getBase()) continue; } return false; diff --git a/lib/SILOptimizer/Mandatory/MoveOnlyWrappedTypeEliminator.cpp b/lib/SILOptimizer/Mandatory/MoveOnlyWrappedTypeEliminator.cpp index d76d858557cb3..c386e131c03a7 100644 --- a/lib/SILOptimizer/Mandatory/MoveOnlyWrappedTypeEliminator.cpp +++ b/lib/SILOptimizer/Mandatory/MoveOnlyWrappedTypeEliminator.cpp @@ -188,6 +188,7 @@ struct SILMoveOnlyWrappedTypeEliminatorVisitor NO_UPDATE_NEEDED(DestructureStruct) NO_UPDATE_NEEDED(SelectEnum) NO_UPDATE_NEEDED(MarkDependence) + NO_UPDATE_NEEDED(MarkDependenceAddr) NO_UPDATE_NEEDED(DestroyAddr) NO_UPDATE_NEEDED(DeallocStack) NO_UPDATE_NEEDED(DeallocBox) diff --git a/lib/SILOptimizer/Mandatory/PMOMemoryUseCollector.cpp b/lib/SILOptimizer/Mandatory/PMOMemoryUseCollector.cpp index f1780359e5d73..c46ac043bbbd4 100644 --- a/lib/SILOptimizer/Mandatory/PMOMemoryUseCollector.cpp +++ b/lib/SILOptimizer/Mandatory/PMOMemoryUseCollector.cpp @@ -223,10 +223,9 @@ bool ElementUseCollector::collectContainerUses(SILValue boxValue) { return false; continue; } - if (auto *md = dyn_cast(user)) { - // Another value depends on the current in-memory value. Consider that a - // load. - if (md->getBase() == ui->get()) { + if (auto mdi = MarkDependenceInstruction(user)) { + // Another value depends on the current in-memory value. + if (mdi.getBase() == ui->get()) { Uses.emplace_back(user, PMOUseKind::DependenceBase); continue; } @@ -471,23 +470,31 @@ bool ElementUseCollector::collectUses(SILValue Pointer) { if (User->isDebugInstruction()) continue; - if (auto *md = dyn_cast(User)) { - if (md->getBase() == UI->get()) { + if (auto *mdi = dyn_cast(User)) { + if (mdi->getBase() == UI->get()) { Uses.emplace_back(User, PMOUseKind::DependenceBase); continue; } - SILValue value = md->getValue(); - assert(value == UI->get() && "missing mark_dependence use"); + assert(mdi->getValue() == UI->get() && "missing mark_dependence use"); // A mark_dependence creates a new dependent value in the same memory // location. Analogous to a load + init. Uses.emplace_back(User, PMOUseKind::Load); Uses.emplace_back(User, PMOUseKind::Initialization); - if (!collectUses(md)) + // Follow a forwarding mark_dependence. + if (!collectUses(mdi)) return false; continue; } - + if (auto *mdi = dyn_cast(User)) { + if (mdi->getBase() == UI->get()) { + Uses.emplace_back(User, PMOUseKind::DependenceBase); + continue; + } + // Ignore the address use. It can be eliminated if the allocation is + // unused. + continue; + } // Otherwise, the use is something complicated, it escapes. Uses.emplace_back(User, PMOUseKind::Escape); } diff --git a/lib/SILOptimizer/Mandatory/PredictableMemOpt.cpp b/lib/SILOptimizer/Mandatory/PredictableMemOpt.cpp index 37a4bd11b301a..20e0211c91b5a 100644 --- a/lib/SILOptimizer/Mandatory/PredictableMemOpt.cpp +++ b/lib/SILOptimizer/Mandatory/PredictableMemOpt.cpp @@ -1965,6 +1965,9 @@ class OptimizeDeadAlloc { SILValue promoteMarkDepBase(MarkDependenceInst *md, ArrayRef availableValues); + void promoteMarkDepAddrBase(MarkDependenceAddrInst *md, + ArrayRef availableValues); + /// Promote a load take cleaning up everything except for RAUWing the /// instruction with the aggregated result. The routine returns the new /// aggregated result to the caller and expects the caller to eventually RAUW @@ -2067,6 +2070,9 @@ bool OptimizeDeadAlloc::canRemoveDeadAllocation() { if (use->getOperandOwnership() == OperandOwnership::ForwardingUnowned) return false; } + // FIXME: Lifetime completion on Boundary::Liveness requires that 'src' + // has no escaping uses. Check escaping uses here and either bailout or + // request completion on Boundary::Availability. valuesNeedingLifetimeCompletion.insert(src); } } @@ -2204,9 +2210,13 @@ bool OptimizeDeadAlloc::canPromoteTake( void OptimizeDeadAlloc::removeDeadAllocation() { for (auto idxVal : llvm::enumerate(promotions.markDepBases.instructions())) { - auto *md = cast(idxVal.value()); auto vals = promotions.markDepBases.availableValues(idxVal.index()); - promoteMarkDepBase(md, vals); + if (auto *mdi = dyn_cast(idxVal.value())) { + promoteMarkDepBase(mdi, vals); + continue; + } + auto *mda = cast(idxVal.value()); + promoteMarkDepAddrBase(mda, vals); } // If our memory is trivially typed, we can just remove it without needing to @@ -2333,6 +2343,22 @@ SILValue OptimizeDeadAlloc::promoteMarkDepBase( return dependentValue; } +void OptimizeDeadAlloc::promoteMarkDepAddrBase( + MarkDependenceAddrInst *md, ArrayRef availableValues) { + + SILValue dependentAddress = md->getAddress(); + LLVM_DEBUG(llvm::dbgs() << " *** Promoting mark_dependence_addr base: " + << *md + << " To address: " << dependentAddress); + SILBuilderWithScope B(md); + for (auto &availableValue : availableValues) { + B.createMarkDependenceAddr(md->getLoc(), dependentAddress, + availableValue.getValue(), + md->dependenceKind()); + } + deleter.deleteIfDead(md); +} + SILValue OptimizeDeadAlloc::promoteLoadTake(LoadInst *li, ArrayRef availableValues) { diff --git a/lib/SILOptimizer/SILCombiner/SILCombiner.h b/lib/SILOptimizer/SILCombiner/SILCombiner.h index 4c0b0833532f4..5f5de2e0bb1bc 100644 --- a/lib/SILOptimizer/SILCombiner/SILCombiner.h +++ b/lib/SILOptimizer/SILCombiner/SILCombiner.h @@ -290,6 +290,7 @@ class SILCombiner : SILInstruction *visitAllocRefDynamicInst(AllocRefDynamicInst *ARDI); SILInstruction *visitMarkDependenceInst(MarkDependenceInst *MDI); + SILInstruction *visitMarkDependenceAddrInst(MarkDependenceAddrInst *MDI); SILInstruction *visitConvertFunctionInst(ConvertFunctionInst *CFI); SILInstruction * visitConvertEscapeToNoEscapeInst(ConvertEscapeToNoEscapeInst *Cvt); diff --git a/lib/SILOptimizer/SILCombiner/SILCombinerMiscVisitors.cpp b/lib/SILOptimizer/SILCombiner/SILCombinerMiscVisitors.cpp index c0df6073ca399..29bc111fcf678 100644 --- a/lib/SILOptimizer/SILCombiner/SILCombinerMiscVisitors.cpp +++ b/lib/SILOptimizer/SILCombiner/SILCombinerMiscVisitors.cpp @@ -1653,7 +1653,11 @@ static bool isLiteral(SILValue val) { return isa(val); } -SILInstruction *SILCombiner::visitMarkDependenceInst(MarkDependenceInst *mdi) { +template +static SILInstruction *combineMarkDependenceBaseInst( + MarkDependenceInstBase *mdi, + SILCombiner *C) { + if (!mdi->getFunction()->hasOwnership()) { // Simplify the base operand of a MarkDependenceInst to eliminate // unnecessary instructions that aren't adding value. @@ -1664,7 +1668,7 @@ SILInstruction *SILCombiner::visitMarkDependenceInst(MarkDependenceInst *mdi) { if (eiBase->hasOperand()) { mdi->setBase(eiBase->getOperand()); if (eiBase->use_empty()) { - eraseInstFromFunction(*eiBase); + C->eraseInstFromFunction(*eiBase); } return mdi; } @@ -1675,7 +1679,7 @@ SILInstruction *SILCombiner::visitMarkDependenceInst(MarkDependenceInst *mdi) { if (auto *ier = dyn_cast(mdi->getBase())) { mdi->setBase(ier->getOperand()); if (ier->use_empty()) - eraseInstFromFunction(*ier); + C->eraseInstFromFunction(*ier); return mdi; } @@ -1684,7 +1688,7 @@ SILInstruction *SILCombiner::visitMarkDependenceInst(MarkDependenceInst *mdi) { if (auto *oeri = dyn_cast(mdi->getBase())) { mdi->setBase(oeri->getOperand()); if (oeri->use_empty()) - eraseInstFromFunction(*oeri); + C->eraseInstFromFunction(*oeri); return mdi; } } @@ -1695,12 +1699,17 @@ SILInstruction *SILCombiner::visitMarkDependenceInst(MarkDependenceInst *mdi) { { SILType baseType = mdi->getBase()->getType(); if (baseType.getObjectType().isTrivial(*mdi->getFunction())) { - SILValue value = mdi->getValue(); - mdi->replaceAllUsesWith(value); - return eraseInstFromFunction(*mdi); + if (auto mdValue = dyn_cast(mdi)) { + auto &valOper = mdi->getAllOperands()[MarkDependenceInst::Dependent]; + mdValue->replaceAllUsesWith(valOper.get()); + } + return C->eraseInstFromFunction(*mdi); } } + return nullptr; +} +SILInstruction *SILCombiner::visitMarkDependenceInst(MarkDependenceInst *mdi) { if (isLiteral(mdi->getValue())) { // A literal lives forever, so no mark_dependence is needed. // This pattern can occur after StringOptimization when a utf8CString of @@ -1708,8 +1717,12 @@ SILInstruction *SILCombiner::visitMarkDependenceInst(MarkDependenceInst *mdi) { replaceInstUsesWith(*mdi, mdi->getValue()); return eraseInstFromFunction(*mdi); } + return combineMarkDependenceBaseInst(mdi, this); +} - return nullptr; +SILInstruction * +SILCombiner::visitMarkDependenceAddrInst(MarkDependenceAddrInst *mdi) { + return combineMarkDependenceBaseInst(mdi, this); } /// Returns true if reference counting and debug_value users of a global_value diff --git a/lib/SILOptimizer/Transforms/ArrayCountPropagation.cpp b/lib/SILOptimizer/Transforms/ArrayCountPropagation.cpp index c83bc6b45c9d5..6dd13d072c41a 100644 --- a/lib/SILOptimizer/Transforms/ArrayCountPropagation.cpp +++ b/lib/SILOptimizer/Transforms/ArrayCountPropagation.cpp @@ -128,8 +128,8 @@ bool ArrayAllocation::recursivelyCollectUses(ValueBase *Def) { isa(User)) continue; - if (auto *MDI = dyn_cast(User)) { - if (Def == MDI->getBase()) { + if (auto mdi = MarkDependenceInstruction(User)) { + if (Def == mdi.getBase()) { continue; } } diff --git a/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp b/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp index 0761f5637055c..ffad29fb43799 100644 --- a/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp +++ b/lib/SILOptimizer/UtilityPasses/SerializeSILPass.cpp @@ -166,6 +166,7 @@ static bool hasOpaqueArchetype(TypeExpansionContext context, case SILInstructionKind::ClassifyBridgeObjectInst: case SILInstructionKind::ValueToBridgeObjectInst: case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: case SILInstructionKind::MergeIsolationRegionInst: case SILInstructionKind::CopyBlockInst: case SILInstructionKind::CopyBlockWithoutEscapingInst: diff --git a/lib/SILOptimizer/Utils/SILInliner.cpp b/lib/SILOptimizer/Utils/SILInliner.cpp index d08230e72d6d3..7c8f8283889ce 100644 --- a/lib/SILOptimizer/Utils/SILInliner.cpp +++ b/lib/SILOptimizer/Utils/SILInliner.cpp @@ -878,6 +878,7 @@ InlineCost swift::instructionInlineCost(SILInstruction &I) { case SILInstructionKind::BeginBorrowInst: case SILInstructionKind::BorrowedFromInst: case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: case SILInstructionKind::MergeIsolationRegionInst: case SILInstructionKind::PreviousDynamicFunctionRefInst: case SILInstructionKind::DynamicFunctionRefInst: diff --git a/lib/Serialization/DeserializeSIL.cpp b/lib/Serialization/DeserializeSIL.cpp index 0e7eb3869bdd8..9b105ff264fa9 100644 --- a/lib/Serialization/DeserializeSIL.cpp +++ b/lib/Serialization/DeserializeSIL.cpp @@ -2418,6 +2418,18 @@ bool SILDeserializer::readSILInstruction(SILFunction *Fn, MarkDependenceKind(Attr)); break; } + case SILInstructionKind::MarkDependenceAddrInst: { + auto Ty = MF->getType(TyID); + auto Ty2 = MF->getType(TyID2); + ResultInst = Builder.createMarkDependenceAddr( + Loc, + getLocalValue(Builder.maybeGetFunction(), ValID, + getSILType(Ty, (SILValueCategory)TyCategory, Fn)), + getLocalValue(Builder.maybeGetFunction(), ValID2, + getSILType(Ty2, (SILValueCategory)TyCategory2, Fn)), + MarkDependenceKind(Attr)); + break; + } case SILInstructionKind::BeginDeallocRefInst: { auto Ty = MF->getType(TyID); auto Ty2 = MF->getType(TyID2); diff --git a/lib/Serialization/ModuleFormat.h b/lib/Serialization/ModuleFormat.h index 3aa7d63caea20..79d76efcf18b3 100644 --- a/lib/Serialization/ModuleFormat.h +++ b/lib/Serialization/ModuleFormat.h @@ -58,7 +58,7 @@ const uint16_t SWIFTMODULE_VERSION_MAJOR = 0; /// describe what change you made. The content of this comment isn't important; /// it just ensures a conflict if two people change the module format. /// Don't worry about adhering to the 80-column limit for this line. -const uint16_t SWIFTMODULE_VERSION_MINOR = 930; // default override table +const uint16_t SWIFTMODULE_VERSION_MINOR = 931; // mark_dependence_addr /// A standard hash seed used for all string hashes in a serialized module. /// diff --git a/lib/Serialization/SerializeSIL.cpp b/lib/Serialization/SerializeSIL.cpp index ce44a0da5933e..053f2dc15c8f0 100644 --- a/lib/Serialization/SerializeSIL.cpp +++ b/lib/Serialization/SerializeSIL.cpp @@ -1854,6 +1854,7 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) { case SILInstructionKind::DeallocPartialRefInst: case SILInstructionKind::BeginDeallocRefInst: case SILInstructionKind::MarkDependenceInst: + case SILInstructionKind::MarkDependenceAddrInst: case SILInstructionKind::IndexAddrInst: case SILInstructionKind::IndexRawPointerInst: { SILValue operand, operand2; @@ -1879,6 +1880,11 @@ void SILSerializer::writeSILInstruction(const SILInstruction &SI) { operand = MDI->getValue(); operand2 = MDI->getBase(); Attr = unsigned(MDI->dependenceKind()); + } else if (SI.getKind() == SILInstructionKind::MarkDependenceAddrInst) { + const MarkDependenceAddrInst *MDI = cast(&SI); + operand = MDI->getAddress(); + operand2 = MDI->getBase(); + Attr = unsigned(MDI->dependenceKind()); } else { const IndexAddrInst *IAI = cast(&SI); operand = IAI->getBase(); diff --git a/test/SIL/Parser/basic2.sil b/test/SIL/Parser/basic2.sil index d14e3d5e1ce5a..1a87c8280a933 100644 --- a/test/SIL/Parser/basic2.sil +++ b/test/SIL/Parser/basic2.sil @@ -313,6 +313,22 @@ bb0(%0 : @owned $Builtin.NativeObject, %1 : @owned $Builtin.NativeObject): return %9999 : $() } +// CHECK-LABEL: sil [ossa] @test_mark_dependence_addr : $@convention(thin) (@in Builtin.NativeObject, @in Builtin.NativeObject) -> () { +// CHECK: mark_dependence_addr %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject +// CHECK: mark_dependence_addr [nonescaping] %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject +// CHECK: mark_dependence_addr [unresolved] %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject +// CHECK: } // end sil function 'test_mark_dependence_addr' +sil [ossa] @test_mark_dependence_addr : $@convention(thin) (@in Builtin.NativeObject, @in Builtin.NativeObject) -> () { +bb0(%0 : $*Builtin.NativeObject, %1 : $*Builtin.NativeObject): + mark_dependence_addr %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject + mark_dependence_addr [nonescaping] %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject + mark_dependence_addr [unresolved] %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject + destroy_addr %1 : $*Builtin.NativeObject + destroy_addr %0 : $*Builtin.NativeObject + %9999 = tuple() + return %9999 : $() +} + // CHECK-LABEL: sil [ossa] @testMarkUnresolvedNonCopyableValueInst : $@convention(thin) (@guaranteed Klass) -> () { // CHECK: mark_unresolved_non_copyable_value [consumable_and_assignable] %{{[0-9]+}} : $*MoveOnlyPair // CHECK: } // end sil function 'testMarkUnresolvedNonCopyableValueInst' diff --git a/test/SIL/Serialization/basic2.sil b/test/SIL/Serialization/basic2.sil index 30991fbd570c9..37e0218e314c7 100644 --- a/test/SIL/Serialization/basic2.sil +++ b/test/SIL/Serialization/basic2.sil @@ -228,6 +228,22 @@ bb0(%0 : @owned $Builtin.NativeObject, %1 : @owned $Builtin.NativeObject): return %9999 : $() } +// CHECK-LABEL: sil [ossa] @test_mark_dependence_addr : $@convention(thin) (@in Builtin.NativeObject, @in Builtin.NativeObject) -> () { +// CHECK: mark_dependence_addr %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject +// CHECK: mark_dependence_addr [nonescaping] %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject +// CHECK: mark_dependence_addr [unresolved] %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject +// CHECK: } // end sil function 'test_mark_dependence_addr' +sil [ossa] @test_mark_dependence_addr : $@convention(thin) (@in Builtin.NativeObject, @in Builtin.NativeObject) -> () { +bb0(%0 : $*Builtin.NativeObject, %1 : $*Builtin.NativeObject): + mark_dependence_addr %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject + mark_dependence_addr [nonescaping] %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject + mark_dependence_addr [unresolved] %1 : $*Builtin.NativeObject on %0 : $*Builtin.NativeObject + destroy_addr %1 : $*Builtin.NativeObject + destroy_addr %0 : $*Builtin.NativeObject + %9999 = tuple() + return %9999 : $() +} + // CHECK-LABEL: sil [ossa] @test_moveonlywrapper_to_copyable_addr : $@convention(thin) (@in Builtin.NativeObject) -> () { // CHECK: moveonlywrapper_to_copyable_addr // CHECK: } // end sil function 'test_moveonlywrapper_to_copyable_addr' diff --git a/test/SIL/memory_lifetime.sil b/test/SIL/memory_lifetime.sil index b37770503f79d..76c02a3538db9 100644 --- a/test/SIL/memory_lifetime.sil +++ b/test/SIL/memory_lifetime.sil @@ -834,14 +834,14 @@ bb0(%0 : @owned $T, %1 : @owned $Inner): return %8 } -sil [ossa] @mark_dependence_uninit_value : $@convention(thin) (@owned T, @owned Inner) -> @owned Inner { +sil [ossa] @mark_dependence_addr : $@convention(method) (@owned T, @owned Inner) -> @owned Inner { bb0(%0 : @owned $T, %1 : @owned $Inner): %2 = alloc_stack $T store %0 to [init] %2 %4 = alloc_stack $Inner store %1 to [init] %4 - %6 = mark_dependence %4 on %2 - %7 = begin_access [read] [dynamic] %6 + mark_dependence_addr %4 on %2 + %7 = begin_access [read] [dynamic] %4 %8 = load [take] %7 end_access %7 dealloc_stack %4 @@ -864,5 +864,3 @@ bb0: %10 = tuple () return %10 } - - diff --git a/test/SIL/memory_lifetime_failures.sil b/test/SIL/memory_lifetime_failures.sil index cc2fa1af9e292..728581fef0c0e 100644 --- a/test/SIL/memory_lifetime_failures.sil +++ b/test/SIL/memory_lifetime_failures.sil @@ -826,3 +826,19 @@ bb0: } +// CHECK: SIL memory lifetime failure in @mark_dependence_addr_uninit_base: memory is not initialized, but should be +sil [ossa] @mark_dependence_addr_uninit_base : $@convention(thin) (@owned T, @owned Inner) -> @owned Inner { +bb0(%0 : @owned $T, %1 : @owned $Inner): + %2 = alloc_stack $T + store %0 to [init] %2 + %4 = alloc_stack $Inner + store %1 to [init] %4 + destroy_addr %2 + mark_dependence %4 on %2 + %7 = begin_access [read] [dynamic] %4 + %8 = load [take] %7 + end_access %7 + dealloc_stack %4 + dealloc_stack %2 + return %8 +} diff --git a/test/SILOptimizer/lifetime_dependence/dependence_insertion.sil b/test/SILOptimizer/lifetime_dependence/dependence_insertion.sil index 60e8c8082eca5..5efefb9d2661a 100644 --- a/test/SILOptimizer/lifetime_dependence/dependence_insertion.sil +++ b/test/SILOptimizer/lifetime_dependence/dependence_insertion.sil @@ -57,7 +57,7 @@ sil @useNE : $@convention(thin) (@guaranteed NE) -> () // CHECK: [[IN:%.*]] = alloc_stack $AnyObject // CHECK: [[SB:%.*]] = store_borrow [[MV]] to [[IN]] // CHECK: apply %{{.*}}([[OUT]], [[SB]]) : $@convention(thin) (@in_guaranteed AnyObject) -> @lifetime(borrow 0) @out NE -// CHECK: [[MD:%.*]] = mark_dependence [unresolved] [[OUT]] on %0 +// CHECK: mark_dependence_addr [unresolved] [[OUT]] on %0 // CHECK: end_borrow [[SB]] // CHECK: end_borrow [[TEMP]] // CHECK: [[LD:%.*]] = load [take] [[OUT]] @@ -95,7 +95,7 @@ bb0(%0 : @guaranteed $AnyObject): // CHECK: [[MD:%.*]] = mark_unresolved_non_copyable_value [consumable_and_assignable] [[ALLOC]] // CHECK: [[ACCESS:%.*]] = begin_access [modify] [unknown] %0 // CHECK: apply %{{.*}}([[MD]], [[ACCESS]]) : $@convention(thin) (@inout AnyObject) -> @lifetime(borrow 0) @out NCE -// CHECK: mark_dependence [unresolved] [[ALLOC]] on [[ACCESS]] +// CHECK: mark_dependence_addr [unresolved] [[ALLOC]] on [[ACCESS]] // CHECK: end_access [[ACCESS]] // CHECK-LABEL: } // end sil function 'testInoutSpanProp' sil [available 9999] [ossa] @testInoutSpanProp : $@convention(method) (@inout AnyObject) -> @lifetime(borrow 0) @owned NCE { diff --git a/test/SILOptimizer/mem-behavior.sil b/test/SILOptimizer/mem-behavior.sil index 0e2eab7be76c0..465013615ad5b 100644 --- a/test/SILOptimizer/mem-behavior.sil +++ b/test/SILOptimizer/mem-behavior.sil @@ -1916,3 +1916,28 @@ bb0(%0 : $*C, %1 : $*C, %2 : @owned $C): %5 = mark_dependence %1 on %0 return %4 } + +// CHECK-LABEL: @test_mark_dependence_addr +// CHECK: PAIR #0. +// CHECK-NEXT: mark_dependence_addr %1 : $*C on %2 : $C +// CHECK-NEXT: %0 = argument of bb0 : $*C +// CHECK-NEXT: r=0,w=0 +// CHECK: PAIR #1. +// CHECK-NEXT: mark_dependence_addr %1 : $*C on %2 : $C +// CHECK-NEXT: %1 = argument of bb0 : $*C +// CHECK-NEXT: r=0,w=0 +// CHECK: PAIR #2. +// CHECK-NEXT: mark_dependence_addr %1 : $*C on %0 : $*C +// CHECK-NEXT: %0 = argument of bb0 : $*C +// CHECK-NEXT: r=1,w=0 +// CHECK: PAIR #3. +// CHECK-NEXT: mark_dependence_addr %1 : $*C on %0 : $*C +// CHECK-NEXT: %1 = argument of bb0 : $*C +// CHECK-NEXT: r=0,w=0 +sil [ossa] @test_mark_dependence_addr : $@convention(thin) (@in_guaranteed C, @in_guaranteed C, @guaranteed C) -> () { +bb0(%0 : $*C, %1 : $*C, %2 : @guaranteed $C): + mark_dependence_addr %1 on %2 + mark_dependence_addr %1 on %0 + %99 = tuple () + return %99 : $() +}