Enable CopyProp by default, tune the impl a bit

compiler/rustc_middle/src/ty/adt.rs

@@ -406,6 +406,7 @@ impl<'tcx> AdtDef<'tcx> {
     }
 
     /// Return the index of `VariantDef` given a variant id.
+    #[inline]
     pub fn variant_index_with_id(self, vid: DefId) -> VariantIdx {
         self.variants()
             .iter_enumerated()

compiler/rustc_mir_transform/src/copy_prop.rs

@@ -22,7 +22,7 @@ pub struct CopyProp;
 
 impl<'tcx> MirPass<'tcx> for CopyProp {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        sess.mir_opt_level() >= 4
+        sess.mir_opt_level() >= 1
     }
 
     #[instrument(level = "trace", skip(self, tcx, body))]
@@ -96,7 +96,7 @@ fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> BitSet<Local> {
     fully_moved
 }
 
-/// Utility to help performing subtitution of `*pattern` by `target`.
+/// Utility to help performing substitution of `*pattern` by `target`.
 struct Replacer<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     fully_moved: BitSet<Local>,

compiler/rustc_mir_transform/src/ssa.rs

@@ -19,6 +19,33 @@ pub struct SsaLocals {
     copy_classes: IndexVec<Local, Local>,
 }
 
+/// We often encounter MIR bodies with 1 or 2 basic blocks. In those cases, it's unnecessary to
+/// actually compute dominators, we can just compare block indices because bb0 is always the first
+/// block, and in any body all other blocks are always always dominated by bb0.
+struct SmallDominators {
+    inner: Option<Dominators<BasicBlock>>,
+}
+
+trait DomExt {
+    fn dominates(self, _other: Self, dominators: &SmallDominators) -> bool;
+}
+
+impl DomExt for Location {
+    fn dominates(self, other: Location, dominators: &SmallDominators) -> bool {
+        if self.block == other.block {
+            self.statement_index <= other.statement_index
+        } else {
+            dominators.dominates(self.block, other.block)
+        }
+    }
+}
+
+impl SmallDominators {
+    fn dominates(&self, dom: BasicBlock, node: BasicBlock) -> bool {
+        if let Some(inner) = &self.inner { inner.dominates(dom, node) } else { dom < node }
+    }
+}
+
 impl SsaLocals {
     pub fn new<'tcx>(
         tcx: TyCtxt<'tcx>,
@@ -29,7 +56,9 @@ impl SsaLocals {
         let assignment_order = Vec::new();
 
         let assignments = IndexVec::from_elem(Set1::Empty, &body.local_decls);
-        let dominators = body.basic_blocks.dominators();
+        let dominators =
+            if body.basic_blocks.len() > 2 { Some(body.basic_blocks.dominators()) } else { None };
+        let dominators = SmallDominators { inner: dominators };
         let mut visitor = SsaVisitor { assignments, assignment_order, dominators };
 
         for (local, decl) in body.local_decls.iter_enumerated() {
@@ -41,8 +70,14 @@ impl SsaLocals {
             }
         }
 
-        for (bb, data) in traversal::reverse_postorder(body) {
-            visitor.visit_basic_block_data(bb, data);
+        if body.basic_blocks.len() > 2 {
+            for (bb, data) in traversal::reverse_postorder(body) {
+                visitor.visit_basic_block_data(bb, data);
+            }
+        } else {
+            for (bb, data) in body.basic_blocks.iter_enumerated() {
+                visitor.visit_basic_block_data(bb, data);
+            }
         }
 
         for var_debug_info in &body.var_debug_info {
@@ -139,7 +174,7 @@ enum LocationExtended {
 }
 
 struct SsaVisitor {
-    dominators: Dominators<BasicBlock>,
+    dominators: SmallDominators,
     assignments: IndexVec<Local, Set1<LocationExtended>>,
     assignment_order: Vec<Local>,
 }

tests/codegen/move-operands.rs

@@ -1,4 +1,4 @@
-// compile-flags: -C no-prepopulate-passes -Zmir-enable-passes=+DestinationPropagation
+// compile-flags: -C no-prepopulate-passes -Zmir-enable-passes=+DestinationPropagation,+CopyProp
 
 #![crate_type = "lib"]
 

tests/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs

@@ -21,23 +21,20 @@ pub struct U(f32, f32, f32, f32);
 // CHECK-LABEL: @build_array_s
 #[no_mangle]
 pub fn build_array_s(x: [f32; 4]) -> S<4> {
-    // CHECK: call void @llvm.memcpy.{{.+}}({{.*}}, i{{[0-9]+}} 16, i1 false)
     // CHECK: call void @llvm.memcpy.{{.+}}({{.*}}, i{{[0-9]+}} 16, i1 false)
     S::<4>(x)
 }
 
 // CHECK-LABEL: @build_array_t
 #[no_mangle]
 pub fn build_array_t(x: [f32; 4]) -> T {
-    // CHECK: call void @llvm.memcpy.{{.+}}({{.*}}, i{{[0-9]+}} 16, i1 false)
     // CHECK: call void @llvm.memcpy.{{.+}}({{.*}}, i{{[0-9]+}} 16, i1 false)
     T(x)
 }
 
 // CHECK-LABEL: @build_array_u
 #[no_mangle]
 pub fn build_array_u(x: [f32; 4]) -> U {
-    // CHECK: call void @llvm.memcpy.{{.+}}({{.*}}, i{{[0-9]+}} 16, i1 false)
     // CHECK: call void @llvm.memcpy.{{.+}}({{.*}}, i{{[0-9]+}} 16, i1 false)
     unsafe { std::mem::transmute(x) }
 }

tests/mir-opt/copy-prop/custom_move_arg.f.CopyProp.diff

@@ -0,0 +1,30 @@
+- // MIR for `f` before CopyProp
++ // MIR for `f` after CopyProp
+
+  fn f(_1: NotCopy) -> () {
+      let mut _0: ();                      // return place in scope 0 at $DIR/custom_move_arg.rs:+0:19: +0:19
+      let mut _2: NotCopy;                 // in scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL
+      let mut _3: NotCopy;                 // in scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL
+
+      bb0: {
+-         _2 = move _1;                    // scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL
+          _0 = opaque::<NotCopy>(move _1) -> bb1; // scope 0 at $DIR/custom_move_arg.rs:+3:9: +3:41
+                                           // mir::Constant
+                                           // + span: $DIR/custom_move_arg.rs:15:24: 15:30
+                                           // + literal: Const { ty: fn(NotCopy) {opaque::<NotCopy>}, val: Value(<ZST>) }
+      }
+
+      bb1: {
+-         _3 = move _2;                    // scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL
+-         _0 = opaque::<NotCopy>(_3) -> bb2; // scope 0 at $DIR/custom_move_arg.rs:+7:9: +7:35
++         _0 = opaque::<NotCopy>(_1) -> bb2; // scope 0 at $DIR/custom_move_arg.rs:+7:9: +7:35
+                                           // mir::Constant
+                                           // + span: $DIR/custom_move_arg.rs:19:24: 19:30
+                                           // + literal: Const { ty: fn(NotCopy) {opaque::<NotCopy>}, val: Value(<ZST>) }
+      }
+
+      bb2: {
+          return;                          // scope 0 at $DIR/custom_move_arg.rs:+10:9: +10:17
+      }
+  }
+

tests/mir-opt/copy-prop/custom_move_arg.rs

@@ -0,0 +1,32 @@
+// unit-test: CopyProp
+
+#![feature(custom_mir, core_intrinsics)]
+#![allow(unused_assignments)]
+extern crate core;
+use core::intrinsics::mir::*;
+
+struct NotCopy(bool);
+
+// EMIT_MIR custom_move_arg.f.CopyProp.diff
+#[custom_mir(dialect = "analysis", phase = "post-cleanup")]
+fn f(_1: NotCopy) {
+    mir!({
+        let _2 = Move(_1);
+        Call(RET, bb1, opaque(Move(_1)))
+    }
+    bb1 = {
+        let _3 = Move(_2);
+        Call(RET, bb2, opaque(_3))
+    }
+    bb2 = {
+        Return()
+    })
+}
+
+#[inline(never)]
+fn opaque<T>(_t: T) {}
+
+fn main() {
+    f(NotCopy(true));
+    println!("hi");
+}

tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.diff

@@ -10,19 +10,21 @@
       scope 2 (inlined <u8 as Add>::add) { // at $DIR/inherit_overflow.rs:7:13: 7:47
           debug self => _1;                // in scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
           debug other => _2;               // in scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
-          let mut _3: u8;                  // in scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
-          let mut _4: u8;                  // in scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
-          let mut _5: (u8, bool);          // in scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
+          let mut _3: (u8, bool);          // in scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
       }
 
       bb0: {
+          StorageLive(_1);                 // scope 0 at $DIR/inherit_overflow.rs:+3:13: +3:47
           _1 = const u8::MAX;              // scope 0 at $DIR/inherit_overflow.rs:+3:13: +3:47
+          StorageLive(_2);                 // scope 0 at $DIR/inherit_overflow.rs:+3:13: +3:47
           _2 = const 1_u8;                 // scope 0 at $DIR/inherit_overflow.rs:+3:13: +3:47
-          _5 = CheckedAdd(const u8::MAX, const 1_u8); // scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
-          assert(!move (_5.1: bool), "attempt to compute `{} + {}`, which would overflow", const u8::MAX, const 1_u8) -> bb1; // scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
+          _3 = CheckedAdd(const u8::MAX, const 1_u8); // scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
+          assert(!move (_3.1: bool), "attempt to compute `{} + {}`, which would overflow", const u8::MAX, const 1_u8) -> bb1; // scope 2 at $SRC_DIR/core/src/ops/arith.rs:LL:COL
       }
 
       bb1: {
+          StorageDead(_2);                 // scope 0 at $DIR/inherit_overflow.rs:+3:13: +3:47
+          StorageDead(_1);                 // scope 0 at $DIR/inherit_overflow.rs:+3:13: +3:47
           return;                          // scope 0 at $DIR/inherit_overflow.rs:+4:2: +4:2
       }
   }

tests/mir-opt/inline/inline_generator.main.Inline.diff

@@ -7,29 +7,27 @@
       let mut _2: std::pin::Pin<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>; // in scope 0 at $DIR/inline_generator.rs:+1:14: +1:32
       let mut _3: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 0 at $DIR/inline_generator.rs:+1:23: +1:31
       let mut _4: [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 0 at $DIR/inline_generator.rs:+1:28: +1:31
-+     let mut _7: bool;                    // in scope 0 at $DIR/inline_generator.rs:+1:33: +1:46
++     let mut _5: bool;                    // in scope 0 at $DIR/inline_generator.rs:+1:33: +1:46
       scope 1 {
           debug _r => _1;                  // in scope 1 at $DIR/inline_generator.rs:+1:9: +1:11
       }
 +     scope 2 (inlined g) {                // at $DIR/inline_generator.rs:9:28: 9:31
 +     }
 +     scope 3 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new) { // at $DIR/inline_generator.rs:9:14: 9:32
 +         debug pointer => _3;             // in scope 3 at $SRC_DIR/core/src/pin.rs:LL:COL
-+         let mut _5: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 3 at $SRC_DIR/core/src/pin.rs:LL:COL
 +         scope 4 {
 +             scope 5 (inlined Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>::new_unchecked) { // at $SRC_DIR/core/src/pin.rs:LL:COL
-+                 debug pointer => _5;     // in scope 5 at $SRC_DIR/core/src/pin.rs:LL:COL
-+                 let mut _6: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 5 at $SRC_DIR/core/src/pin.rs:LL:COL
++                 debug pointer => _3;     // in scope 5 at $SRC_DIR/core/src/pin.rs:LL:COL
 +             }
 +         }
 +     }
 +     scope 6 (inlined g::{closure#0}) {   // at $DIR/inline_generator.rs:9:33: 9:46
-+         debug a => _7;                   // in scope 6 at $DIR/inline_generator.rs:15:6: 15:7
-+         let mut _8: i32;                 // in scope 6 at $DIR/inline_generator.rs:15:17: 15:39
-+         let mut _9: u32;                 // in scope 6 at $DIR/inline_generator.rs:15:5: 15:41
++         debug a => _5;                   // in scope 6 at $DIR/inline_generator.rs:15:6: 15:7
++         let mut _6: i32;                 // in scope 6 at $DIR/inline_generator.rs:15:17: 15:39
++         let mut _7: u32;                 // in scope 6 at $DIR/inline_generator.rs:15:5: 15:41
++         let mut _8: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 6 at $DIR/inline_generator.rs:15:5: 15:41
++         let mut _9: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 6 at $DIR/inline_generator.rs:15:5: 15:41
 +         let mut _10: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 6 at $DIR/inline_generator.rs:15:5: 15:41
-+         let mut _11: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 6 at $DIR/inline_generator.rs:15:5: 15:41
-+         let mut _12: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]; // in scope 6 at $DIR/inline_generator.rs:15:5: 15:41
 +     }
 
       bb0: {
@@ -64,28 +62,22 @@
 -     }
 - 
 -     bb2: {
-+         StorageLive(_5);                 // scope 4 at $SRC_DIR/core/src/pin.rs:LL:COL
-+         _5 = move _3;                    // scope 4 at $SRC_DIR/core/src/pin.rs:LL:COL
-+         StorageLive(_6);                 // scope 5 at $SRC_DIR/core/src/pin.rs:LL:COL
-+         _6 = move _5;                    // scope 5 at $SRC_DIR/core/src/pin.rs:LL:COL
-+         _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]> { pointer: move _6 }; // scope 5 at $SRC_DIR/core/src/pin.rs:LL:COL
-+         StorageDead(_6);                 // scope 5 at $SRC_DIR/core/src/pin.rs:LL:COL
-+         StorageDead(_5);                 // scope 4 at $SRC_DIR/core/src/pin.rs:LL:COL
++         _2 = Pin::<&mut [generator@$DIR/inline_generator.rs:15:5: 15:8]> { pointer: move _3 }; // scope 5 at $SRC_DIR/core/src/pin.rs:LL:COL
           StorageDead(_3);                 // scope 0 at $DIR/inline_generator.rs:+1:31: +1:32
 -         _1 = <[generator@$DIR/inline_generator.rs:15:5: 15:8] as Generator<bool>>::resume(move _2, const false) -> [return: bb3, unwind: bb4]; // scope 0 at $DIR/inline_generator.rs:+1:14: +1:46
 -                                          // mir::Constant
 -                                          // + span: $DIR/inline_generator.rs:9:33: 9:39
 -                                          // + literal: Const { ty: for<'a> fn(Pin<&'a mut [generator@$DIR/inline_generator.rs:15:5: 15:8]>, bool) -> GeneratorState<<[generator@$DIR/inline_generator.rs:15:5: 15:8] as Generator<bool>>::Yield, <[generator@$DIR/inline_generator.rs:15:5: 15:8] as Generator<bool>>::Return> {<[generator@$DIR/inline_generator.rs:15:5: 15:8] as Generator<bool>>::resume}, val: Value(<ZST>) }
-+         StorageLive(_7);                 // scope 0 at $DIR/inline_generator.rs:+1:33: +1:46
-+         _7 = const false;                // scope 0 at $DIR/inline_generator.rs:+1:33: +1:46
-+         _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]); // scope 6 at $DIR/inline_generator.rs:15:5: 15:41
-+         _9 = discriminant((*_10));       // scope 6 at $DIR/inline_generator.rs:15:5: 15:41
-+         switchInt(move _9) -> [0: bb3, 1: bb8, 3: bb7, otherwise: bb9]; // scope 6 at $DIR/inline_generator.rs:15:5: 15:41
++         StorageLive(_5);                 // scope 0 at $DIR/inline_generator.rs:+1:33: +1:46
++         _5 = const false;                // scope 0 at $DIR/inline_generator.rs:+1:33: +1:46
++         _8 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]); // scope 6 at $DIR/inline_generator.rs:15:5: 15:41
++         _7 = discriminant((*_8));        // scope 6 at $DIR/inline_generator.rs:15:5: 15:41
++         switchInt(move _7) -> [0: bb3, 1: bb8, 3: bb7, otherwise: bb9]; // scope 6 at $DIR/inline_generator.rs:15:5: 15:41
       }
 
 -     bb3: {
 +     bb1: {
-+         StorageDead(_7);                 // scope 0 at $DIR/inline_generator.rs:+1:33: +1:46
++         StorageDead(_5);                 // scope 0 at $DIR/inline_generator.rs:+1:33: +1:46
           StorageDead(_2);                 // scope 0 at $DIR/inline_generator.rs:+1:45: +1:46
           StorageDead(_4);                 // scope 0 at $DIR/inline_generator.rs:+1:46: +1:47
           _0 = const ();                   // scope 0 at $DIR/inline_generator.rs:+0:11: +2:2
@@ -99,33 +91,33 @@
 +     }
 + 
 +     bb3: {
-+         StorageLive(_8);                 // scope 6 at $DIR/inline_generator.rs:15:17: 15:39
-+         switchInt(_7) -> [0: bb5, otherwise: bb4]; // scope 6 at $DIR/inline_generator.rs:15:20: 15:21
++         StorageLive(_6);                 // scope 6 at $DIR/inline_generator.rs:15:17: 15:39
++         switchInt(_5) -> [0: bb5, otherwise: bb4]; // scope 6 at $DIR/inline_generator.rs:15:20: 15:21
 +     }
 + 
 +     bb4: {
-+         _8 = const 7_i32;                // scope 6 at $DIR/inline_generator.rs:15:24: 15:25
++         _6 = const 7_i32;                // scope 6 at $DIR/inline_generator.rs:15:24: 15:25
 +         goto -> bb6;                     // scope 6 at $DIR/inline_generator.rs:15:17: 15:39
 +     }
 + 
 +     bb5: {
-+         _8 = const 13_i32;               // scope 6 at $DIR/inline_generator.rs:15:35: 15:37
++         _6 = const 13_i32;               // scope 6 at $DIR/inline_generator.rs:15:35: 15:37
 +         goto -> bb6;                     // scope 6 at $DIR/inline_generator.rs:15:17: 15:39
 +     }
 + 
 +     bb6: {
-+         _1 = GeneratorState::<i32, bool>::Yielded(move _8); // scope 6 at $DIR/inline_generator.rs:15:11: 15:39
-+         _11 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]); // scope 6 at $DIR/inline_generator.rs:15:11: 15:39
-+         discriminant((*_11)) = 3;        // scope 6 at $DIR/inline_generator.rs:15:11: 15:39
++         _1 = GeneratorState::<i32, bool>::Yielded(move _6); // scope 6 at $DIR/inline_generator.rs:15:11: 15:39
++         _9 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]); // scope 6 at $DIR/inline_generator.rs:15:11: 15:39
++         discriminant((*_9)) = 3;         // scope 6 at $DIR/inline_generator.rs:15:11: 15:39
 +         goto -> bb1;                     // scope 0 at $DIR/inline_generator.rs:15:11: 15:39
 +     }
 + 
 +     bb7: {
-+         StorageLive(_8);                 // scope 6 at $DIR/inline_generator.rs:15:5: 15:41
-+         StorageDead(_8);                 // scope 6 at $DIR/inline_generator.rs:15:38: 15:39
-+         _1 = GeneratorState::<i32, bool>::Complete(_7); // scope 6 at $DIR/inline_generator.rs:15:41: 15:41
-+         _12 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]); // scope 6 at $DIR/inline_generator.rs:15:41: 15:41
-+         discriminant((*_12)) = 1;        // scope 6 at $DIR/inline_generator.rs:15:41: 15:41
++         StorageLive(_6);                 // scope 6 at $DIR/inline_generator.rs:15:5: 15:41
++         StorageDead(_6);                 // scope 6 at $DIR/inline_generator.rs:15:38: 15:39
++         _1 = GeneratorState::<i32, bool>::Complete(_5); // scope 6 at $DIR/inline_generator.rs:15:41: 15:41
++         _10 = deref_copy (_2.0: &mut [generator@$DIR/inline_generator.rs:15:5: 15:8]); // scope 6 at $DIR/inline_generator.rs:15:41: 15:41
++         discriminant((*_10)) = 1;        // scope 6 at $DIR/inline_generator.rs:15:41: 15:41
 +         goto -> bb1;                     // scope 0 at $DIR/inline_generator.rs:15:41: 15:41
 +     }
 +