diff --git a/lib/SILOptimizer/Differentiation/PullbackCloner.cpp b/lib/SILOptimizer/Differentiation/PullbackCloner.cpp
index 9cfdd4cad9cce..0cdc7e4c69d7c 100644
--- a/lib/SILOptimizer/Differentiation/PullbackCloner.cpp
+++ b/lib/SILOptimizer/Differentiation/PullbackCloner.cpp
@@ -1780,8 +1780,12 @@ class PullbackCloner::Implementation final
   void visitMoveValueInst(MoveValueInst *mvi) {
     switch (getTangentValueCategory(mvi)) {
     case SILValueCategory::Address:
-      llvm::report_fatal_error("AutoDiff does not support move_value with "
-                               "SILValueCategory::Address");
+      LLVM_DEBUG(getADDebugStream() << "AutoDiff does not support move_value with "
+                 "SILValueCategory::Address");
+      getContext().emitNondifferentiabilityError(
+        mvi, getInvoker(), diag::autodiff_expression_not_differentiable_note);
+      errorOccurred = true;
+      return;
     case SILValueCategory::Object:
       visitValueOwnershipInst(mvi, /*needZeroResAdj=*/true);
     }
@@ -3121,8 +3125,21 @@ void PullbackCloner::Implementation::visitSILBasicBlock(SILBasicBlock *bb) {
         break;
       }
       }
-    } else
-      llvm::report_fatal_error("do not know how to handle this incoming bb argument");
+    } else {
+      LLVM_DEBUG(getADDebugStream() <<
+                 "do not know how to handle this incoming bb argument");
+      if (auto term = bbArg->getSingleTerminator()) {
+        getContext().emitNondifferentiabilityError(term, getInvoker(),
+          diag::autodiff_expression_not_differentiable_note);
+      } else {
+        // This will be a bit confusing, but still better than nothing.
+        getContext().emitNondifferentiabilityError(bbArg, getInvoker(),
+          diag::autodiff_expression_not_differentiable_note);
+      }
+
+      errorOccurred = true;
+      return;
+    }
   }
 
   // 3. Build the pullback successor cases for the `switch_enum`
diff --git a/test/AutoDiff/SILOptimizer/differentiation_diagnostics.swift b/test/AutoDiff/SILOptimizer/differentiation_diagnostics.swift
index 34f09959d7680..6df2c5c5300ac 100644
--- a/test/AutoDiff/SILOptimizer/differentiation_diagnostics.swift
+++ b/test/AutoDiff/SILOptimizer/differentiation_diagnostics.swift
@@ -46,6 +46,17 @@ func try_apply_rethrows(_ x: Float) -> Float {
   return x
 }
 
+// This generates `try_apply` which we do not know to handle yet, therefore
+// one should use a.differentialMap here. If / when differentiation of throwing
+// functions will be supported, we'd need to remove this diagnostics.
+// expected-error @+2 {{function is not differentiable}}
+// expected-note @+2 {{when differentiating this function definition}}
+@differentiable(reverse)
+func map_nondiff(_ a: [Float]) -> [Float] {
+  // expected-note @+1 {{expression is not differentiable}}
+  return a.map { $0 }
+}
+
 //===----------------------------------------------------------------------===//
 // Unreachable
 //===----------------------------------------------------------------------===//