diff --git a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h
index 030be328e97fd..bf9694556f901 100644
--- a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h
+++ b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h
@@ -157,7 +157,10 @@ struct MaskableOpRewritePattern : OpRewritePattern<SourceOp> {
     if (failed(newOp))
       return failure();
 
-    rewriter.replaceOp(rootOp, *newOp);
+    if (rootOp->getNumResults() == 0 || *newOp == Value())
+      rewriter.eraseOp(rootOp);
+    else
+      rewriter.replaceOp(rootOp, *newOp);
     return success();
   }