[ownership] Change guaranteed args from transformation terminators to not require end_borrows.

For those who are unaware, a transformation terminator is a terminator like
switch_enum/checked_cast_br that always dominate their successor blocks. Since
they dominate their successor blocks by design and transform their input into
the args form, we can validate that they obey guaranteed ownership semantics
just like a forwarding instruction.

Beyond removing unnecessary code bloat, this also makes it significantly more
easier to optimize/work with transformation terminators when converting @owned
-> @guaranteed since we do not need to find end_borrow points when the owned
value is consumed.

<rdar://problem/59097063>

Author: gottesmm

include/swift/SIL/SILBuilder.h

@@ -802,6 +802,12 @@ class SILBuilder {
   }
 
   EndBorrowInst *createEndBorrow(SILLocation loc, SILValue borrowedValue) {
+    if (auto *arg = dyn_cast<SILPhiArgument>(borrowedValue)) {
+      if (auto *ti = arg->getSingleTerminator()) {
+        assert(!ti->isTransformationTerminator() &&
+               "Transforming terminators do not have end_borrow");
+      }
+    }
     return insert(new (getModule())
                       EndBorrowInst(getSILDebugLocation(loc), borrowedValue));
   }

include/swift/SIL/SILInstruction.h

@@ -7042,6 +7042,30 @@ class TermInst : public NonValueInstruction {
   bool isProgramTerminating() const;
 
   TermKind getTermKind() const { return TermKind(getKind()); }
+
+  /// Returns true if this is a transformation terminator.
+  bool isTransformationTerminator() const {
+    switch (getTermKind()) {
+    case TermKind::UnwindInst:
+    case TermKind::UnreachableInst:
+    case TermKind::ReturnInst:
+    case TermKind::ThrowInst:
+    case TermKind::YieldInst:
+    case TermKind::TryApplyInst:
+    case TermKind::BranchInst:
+    case TermKind::CondBranchInst:
+    case TermKind::SwitchValueInst:
+    case TermKind::SwitchEnumAddrInst:
+    case TermKind::DynamicMethodBranchInst:
+    case TermKind::CheckedCastAddrBranchInst:
+    case TermKind::CheckedCastValueBranchInst:
+      return false;
+    case TermKind::SwitchEnumInst:
+    case TermKind::CheckedCastBranchInst:
+      return true;
+    }
+    llvm_unreachable("Covered switch isn't covered.");
+  }
 };
 
 /// UnreachableInst - Position in the code which would be undefined to reach.

lib/SIL/DynamicCasts.cpp

@@ -1254,9 +1254,7 @@ void swift::emitIndirectConditionalCastWithScalar(
     case CastConsumptionKind::TakeOnSuccess:
       break;
     case CastConsumptionKind::CopyOnSuccess: {
-      SILValue originalSuccValue = succValue;
       succValue = B.emitCopyValueOperation(loc, succValue);
-      B.emitEndBorrowOperation(loc, originalSuccValue);
       B.emitEndBorrowOperation(loc, srcValue);
       break;
     }
@@ -1295,7 +1293,6 @@ void swift::emitIndirectConditionalCastWithScalar(
                                 StoreOwnershipQualifier::Init);
       break;
     case CastConsumptionKind::CopyOnSuccess:
-      B.emitEndBorrowOperation(loc, failValue);
       B.emitEndBorrowOperation(loc, srcValue);
       break;
     case CastConsumptionKind::BorrowAlways:

lib/SIL/OwnershipUtils.cpp

@@ -66,6 +66,15 @@ bool swift::isGuaranteedForwardingValueKind(SILNodeKind kind) {
 }
 
 bool swift::isGuaranteedForwardingValue(SILValue value) {
+  // If we have an argument from a transforming terminator, we can forward
+  // guaranteed.
+  if (auto *arg = dyn_cast<SILArgument>(value)) {
+    if (auto *ti = arg->getSingleTerminator()) {
+      if (ti->isTransformationTerminator()) {
+        return true;
+      }
+    }
+  }
   return isGuaranteedForwardingValueKind(
       value->getKindOfRepresentativeSILNodeInObject());
 }
@@ -173,9 +182,18 @@ bool swift::getUnderlyingBorrowIntroducingValues(
     // Otherwise if v is an ownership forwarding value, add its defining
     // instruction
     if (isGuaranteedForwardingValue(v)) {
-      auto *i = v->getDefiningInstruction();
-      assert(i);
-      llvm::transform(i->getAllOperands(), std::back_inserter(worklist),
+      if (auto *i = v->getDefiningInstruction()) {
+        llvm::transform(i->getAllOperands(), std::back_inserter(worklist),
+                        [](const Operand &op) -> SILValue { return op.get(); });
+        continue;
+      }
+
+      // Otherwise, we should have a block argument that is defined by a single
+      // predecessor terminator.
+      auto *arg = cast<SILPhiArgument>(v);
+      auto *termInst = arg->getSingleTerminator();
+      assert(termInst && termInst->isTransformationTerminator());
+      llvm::transform(termInst->getAllOperands(), std::back_inserter(worklist),
                       [](const Operand &op) -> SILValue { return op.get(); });
       continue;
     }

lib/SIL/SILOwnershipVerifier.cpp

@@ -334,9 +334,55 @@ bool SILValueOwnershipChecker::gatherUsers(
       continue;
     }
 
-    // Otherwise if we have a terminator, add any as uses any end_borrow to
-    // ensure that the subscope is completely enclsed within the super scope. We
-    // require all of our arguments to be either trivial or guaranteed.
+    // See if our forwarding terminator is a transformation terminator. If so,
+    // just add all of the uses of its successors to the worklist to visit as
+    // users.
+    if (ti->isTransformationTerminator()) {
+      for (auto &succ : ti->getSuccessors()) {
+        auto *succBlock = succ.getBB();
+
+        // If we do not have any arguments, then continue.
+        if (succBlock->args_empty())
+          continue;
+
+        // Otherwise, make sure that all arguments are trivial or guaranteed. If
+        // we fail, emit an error.
+        //
+        // TODO: We could ignore this error and emit a more specific error on
+        // the actual terminator.
+        for (auto *succArg : succBlock->getSILPhiArguments()) {
+          // *NOTE* We do not emit an error here since we want to allow for more
+          // specific errors to be found during use_verification.
+          //
+          // TODO: Add a flag that associates the terminator instruction with
+          // needing to be verified. If it isn't verified appropriately, assert
+          // when the verifier is destroyed.
+          auto succArgOwnershipKind = succArg->getOwnershipKind();
+          if (!succArgOwnershipKind.isCompatibleWith(ownershipKind)) {
+            // This is where the error would go.
+            continue;
+          }
+
+          // If we have an any value, just continue.
+          if (succArgOwnershipKind == ValueOwnershipKind::None)
+            continue;
+
+          // Otherwise add all users of this BBArg to the worklist to visit
+          // recursively.
+          llvm::copy(succArg->getUses(), std::back_inserter(users));
+        }
+      }
+      continue;
+    }
+
+    // Otherwise, we are dealing with the merging of true phis. To work with
+    // this we will eventually need to create a notion of forwarding borrow
+    // scopes.
+
+    // But until then, we validate that the argument has an end_borrow that acts
+    // as a subscope that is compeltely enclosed within the scopes of all
+    // incoming values. We require all of our arguments to be either trivial or
+    // guaranteed.
     for (auto &succ : ti->getSuccessors()) {
       auto *succBlock = succ.getBB();
 

lib/SILGen/SILGenBuilder.cpp

@@ -199,6 +199,13 @@ ManagedValue SILGenBuilder::createGuaranteedPhiArgument(SILType type) {
   return SGF.emitManagedBorrowedArgumentWithCleanup(arg);
 }
 
+ManagedValue
+SILGenBuilder::createGuaranteedTransformingTerminatorArgument(SILType type) {
+  SILPhiArgument *arg =
+      getInsertionBB()->createPhiArgument(type, ValueOwnershipKind::Guaranteed);
+  return ManagedValue::forUnmanaged(arg);
+}
+
 ManagedValue SILGenBuilder::createAllocRef(
     SILLocation loc, SILType refType, bool objc, bool canAllocOnStack,
     ArrayRef<SILType> inputElementTypes,

lib/SILGen/SILGenBuilder.h

@@ -127,6 +127,15 @@ class SILGenBuilder : public SILBuilder {
   ManagedValue createOwnedPhiArgument(SILType type);
   ManagedValue createGuaranteedPhiArgument(SILType type);
 
+  /// For arguments from terminators that are "transforming terminators". These
+  /// types of guaranteed arguments are validated as part of the operand of the
+  /// transforming terminator since transforming terminators are guaranteed to
+  /// be the only predecessor of our parent block.
+  ///
+  /// NOTE: Two examples of transforming terminators are switch_enum,
+  /// checked_cast_br.
+  ManagedValue createGuaranteedTransformingTerminatorArgument(SILType type);
+
   using SILBuilder::createMarkUninitialized;
   ManagedValue createMarkUninitialized(ValueDecl *decl, ManagedValue operand,
                                        MarkUninitializedInst::Kind muKind);

lib/SILGen/SILGenDynamicCast.cpp

@@ -185,7 +185,7 @@ namespace {
           // argument.
           ManagedValue argument;
           if (!shouldTakeOnSuccess(consumption)) {
-            argument = SGF.B.createGuaranteedPhiArgument(
+            argument = SGF.B.createGuaranteedTransformingTerminatorArgument(
                 origTargetTL.getLoweredType());
           } else {
             argument =
@@ -236,7 +236,8 @@ namespace {
         switch (consumption) {
         case CastConsumptionKind::BorrowAlways:
         case CastConsumptionKind::CopyOnSuccess:
-          SGF.B.createGuaranteedPhiArgument(operandValue.getType());
+          SGF.B.createGuaranteedTransformingTerminatorArgument(
+              operandValue.getType());
           handleFalse(None);
           break;
         case CastConsumptionKind::TakeAlways:

lib/SILGen/SILGenExpr.cpp

@@ -165,8 +165,14 @@ namespace {
 struct EndBorrowCleanup : Cleanup {
   SILValue borrowedValue;
 
-  EndBorrowCleanup(SILValue borrowedValue)
-      : borrowedValue(borrowedValue) {}
+  EndBorrowCleanup(SILValue borrowedValue) : borrowedValue(borrowedValue) {
+    if (auto *arg = dyn_cast<SILPhiArgument>(borrowedValue)) {
+      if (auto *ti = arg->getSingleTerminator()) {
+        assert(!ti->isTransformationTerminator() &&
+               "Transforming terminators do not have end_borrow");
+      }
+    }
+  }
 
   void emit(SILGenFunction &SGF, CleanupLocation l,
             ForUnwind_t forUnwind) override {

lib/SILGen/SILGenPattern.cpp

@@ -1921,7 +1921,7 @@ void PatternMatchEmission::emitEnumElementObjectDispatch(
 
       SILValue eltValue;
       if (isPlusZero) {
-        origCMV = {SGF.B.createGuaranteedPhiArgument(eltTy),
+        origCMV = {SGF.B.createGuaranteedTransformingTerminatorArgument(eltTy),
                    CastConsumptionKind::BorrowAlways};
       } else {
         origCMV = {SGF.B.createOwnedPhiArgument(eltTy),
@@ -1971,7 +1971,7 @@ void PatternMatchEmission::emitEnumElementObjectDispatch(
   if (SILBasicBlock *defaultBB = blocks.getDefaultBlock()) {
     SGF.B.setInsertionPoint(defaultBB);
     if (isPlusZero) {
-      SGF.B.createGuaranteedPhiArgument(src.getType());
+      SGF.B.createGuaranteedTransformingTerminatorArgument(src.getType());
     } else {
       SGF.B.createOwnedPhiArgument(src.getType());
     }