Enable DestinationPropagation by default

compiler/rustc_index/src/interval.rs

@@ -140,6 +140,49 @@ impl<I: Idx> IntervalSet<I> {
         result
     }
 
+    /// Specialized version of `insert` when we know that the inserted point is *before* any
+    /// contained.
+    pub fn prepend(&mut self, point: I) {
+        let point = point.index() as u32;
+
+        if let Some((first_start, _)) = self.map.first_mut() {
+            assert!(point < *first_start);
+            if point + 1 == *first_start {
+                *first_start = point;
+            } else {
+                self.map.insert(0, (point, point));
+            }
+        } else {
+            // If the map is empty, push is faster than insert.
+            self.map.push((point, point));
+        }
+
+        debug_assert!(
+            self.check_invariants(),
+            "wrong intervals after prepend {point:?} to {self:?}"
+        );
+    }
+
+    /// Specialized version of `insert` when we know that the inserted point is *after* any
+    /// contained.
+    pub fn append(&mut self, point: I) {
+        let point = point.index() as u32;
+
+        if let Some((_, last_end)) = self.map.last_mut()
+            && let _ = assert!(*last_end < point)
+            && point == *last_end + 1
+        {
+            *last_end = point;
+        } else {
+            self.map.push((point, point));
+        }
+
+        debug_assert!(
+            self.check_invariants(),
+            "wrong intervals after append {point:?} to {self:?}"
+        );
+    }
+
     pub fn contains(&self, needle: I) -> bool {
         let needle = needle.index() as u32;
         let Some(last) = self.map.partition_point(|r| r.0 <= needle).checked_sub(1) else {
@@ -325,6 +368,14 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         self.ensure_row(row).insert(point)
     }
 
+    pub fn prepend(&mut self, row: R, point: C) {
+        self.ensure_row(row).prepend(point)
+    }
+
+    pub fn append(&mut self, row: R, point: C) {
+        self.ensure_row(row).append(point)
+    }
+
     pub fn contains(&self, row: R, point: C) -> bool {
         self.row(row).is_some_and(|r| r.contains(point))
     }

compiler/rustc_mir_dataflow/src/points.rs

@@ -3,7 +3,7 @@ use rustc_index::interval::SparseIntervalMatrix;
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::{self, BasicBlock, Body, Location};
 
-use crate::framework::{Analysis, Results, ResultsVisitor, visit_results};
+use crate::framework::{Analysis, Direction, Results, ResultsVisitor, visit_results};
 
 /// Maps between a `Location` and a `PointIndex` (and vice versa).
 pub struct DenseLocationMap {
@@ -105,27 +105,47 @@ where
     N: Idx,
     A: Analysis<'tcx, Domain = DenseBitSet<N>>,
 {
-    let values = SparseIntervalMatrix::new(elements.num_points());
-    let mut visitor = Visitor { elements, values };
-    visit_results(
-        body,
-        body.basic_blocks.reverse_postorder().iter().copied(),
-        &mut analysis,
-        &results,
-        &mut visitor,
-    );
-    visitor.values
+    let mut values = SparseIntervalMatrix::new(elements.num_points());
+    let reachable_blocks = mir::traversal::reachable_as_bitset(body);
+    if A::Direction::IS_BACKWARD {
+        // Iterate blocks in decreasing order, to visit locations in decreasing order. This
+        // allows to use the more efficient `prepend` method to interval sets.
+        let callback = |state: &DenseBitSet<N>, location| {
+            let point = elements.point_from_location(location);
+            // Use internal iterator manually as it is much more efficient.
+            state.iter().for_each(|node| values.prepend(node, point));
+        };
+        let mut visitor = Visitor { callback };
+        visit_results(
+            body,
+            // Note the `.rev()`.
+            body.basic_blocks.indices().filter(|&bb| reachable_blocks.contains(bb)).rev(),
+            &mut analysis,
+            &results,
+            &mut visitor,
+        );
+    } else {
+        // Iterate blocks in increasing order, to visit locations in increasing order. This
+        // allows to use the more efficient `append` method to interval sets.
+        let callback = |state: &DenseBitSet<N>, location| {
+            let point = elements.point_from_location(location);
+            // Use internal iterator manually as it is much more efficient.
+            state.iter().for_each(|node| values.append(node, point));
+        };
+        let mut visitor = Visitor { callback };
+        visit_results(body, reachable_blocks.iter(), &mut analysis, &results, &mut visitor);
+    }
+    values
 }
 
-struct Visitor<'a, N: Idx> {
-    elements: &'a DenseLocationMap,
-    values: SparseIntervalMatrix<N, PointIndex>,
+struct Visitor<F> {
+    callback: F,
 }
 
-impl<'tcx, A, N> ResultsVisitor<'tcx, A> for Visitor<'_, N>
+impl<'tcx, A, F> ResultsVisitor<'tcx, A> for Visitor<F>
 where
-    A: Analysis<'tcx, Domain = DenseBitSet<N>>,
-    N: Idx,
+    A: Analysis<'tcx>,
+    F: FnMut(&A::Domain, Location),
 {
     fn visit_after_primary_statement_effect<'mir>(
         &mut self,
@@ -134,11 +154,7 @@ where
         _statement: &'mir mir::Statement<'tcx>,
         location: Location,
     ) {
-        let point = self.elements.point_from_location(location);
-        // Use internal iterator manually as it is much more efficient.
-        state.iter().for_each(|node| {
-            self.values.insert(node, point);
-        });
+        (self.callback)(state, location);
     }
 
     fn visit_after_primary_terminator_effect<'mir>(
@@ -148,10 +164,6 @@ where
         _terminator: &'mir mir::Terminator<'tcx>,
         location: Location,
     ) {
-        let point = self.elements.point_from_location(location);
-        // Use internal iterator manually as it is much more efficient.
-        state.iter().for_each(|node| {
-            self.values.insert(node, point);
-        });
+        (self.callback)(state, location);
     }
 }

compiler/rustc_mir_transform/src/dest_prop.rs

@@ -158,7 +158,7 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation {
         //  2. Despite being an overall perf improvement, this still causes a 30% regression in
         //     keccak. We can temporarily fix this by bounding function size, but in the long term
         //     we should fix this by being smarter about invalidating analysis results.
-        sess.mir_opt_level() >= 3
+        sess.mir_opt_level() >= 2
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {

compiler/rustc_mir_transform/src/lib.rs

@@ -153,7 +153,6 @@ declare_passes! {
     mod match_branches : MatchBranchSimplification;
     mod mentioned_items : MentionedItems;
     mod multiple_return_terminators : MultipleReturnTerminators;
-    mod nrvo : RenameReturnPlace;
     mod post_drop_elaboration : CheckLiveDrops;
     mod prettify : ReorderBasicBlocks, ReorderLocals;
     mod promote_consts : PromoteTemps;
@@ -708,15 +707,14 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'
             &jump_threading::JumpThreading,
             &early_otherwise_branch::EarlyOtherwiseBranch,
             &simplify_comparison_integral::SimplifyComparisonIntegral,
-            &dest_prop::DestinationPropagation,
             &o1(simplify_branches::SimplifyConstCondition::Final),
             &o1(remove_noop_landing_pads::RemoveNoopLandingPads),
             &o1(simplify::SimplifyCfg::Final),
             // After the last SimplifyCfg, because this wants one-block functions.
             &strip_debuginfo::StripDebugInfo,
             &copy_prop::CopyProp,
             &dead_store_elimination::DeadStoreElimination::Final,
-            &nrvo::RenameReturnPlace,
+            &dest_prop::DestinationPropagation,
             &simplify::SimplifyLocals::Final,
             &multiple_return_terminators::MultipleReturnTerminators,
             &large_enums::EnumSizeOpt { discrepancy: 128 },