Auto merge of #156858 - cjgillot:post-mono-mir-opts, r=<try>

Add post-mono MIR optimizations

Author: bors

Cargo.lock

@@ -3814,6 +3814,7 @@ dependencies = [
  "rustc_macros",
  "rustc_metadata",
  "rustc_middle",
+ "rustc_mir_transform",
  "rustc_serialize",
  "rustc_session",
  "rustc_span",
@@ -4459,7 +4460,6 @@ dependencies = [
 name = "rustc_monomorphize"
 version = "0.0.0"
 dependencies = [
- "rustc_abi",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_hir",

compiler/rustc_attr_parsing/src/attributes/prototype.rs

@@ -93,6 +93,7 @@ fn parse_dialect(
         sym::analysis => MirDialect::Analysis,
         sym::built => MirDialect::Built,
         sym::runtime => MirDialect::Runtime,
+        sym::mono => MirDialect::Mono,
 
         _ => {
             cx.adcx().expected_specific_argument(span, &[sym::analysis, sym::built, sym::runtime]);
@@ -136,43 +137,28 @@ fn check_custom_mir(
     failed: &mut bool,
 ) {
     let attr_span = cx.attr_span;
+    let Some((phase, phase_span)) = phase else { return };
     let Some((dialect, dialect_span)) = dialect else {
-        if let Some((_, phase_span)) = phase {
+        *failed = true;
+        cx.emit_err(session_diagnostics::CustomMirPhaseRequiresDialect { attr_span, phase_span });
+        return;
+    };
+
+    match (dialect, phase) {
+        (MirDialect::Built, _)
+        | (MirDialect::Analysis, MirPhase::Optimized)
+        | (MirDialect::Mono, MirPhase::PostCleanup) => {
             *failed = true;
-            cx.emit_err(session_diagnostics::CustomMirPhaseRequiresDialect {
+            cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase {
+                dialect,
+                phase,
                 attr_span,
+                dialect_span,
                 phase_span,
             });
         }
-        return;
-    };
-
-    match dialect {
-        MirDialect::Analysis => {
-            if let Some((MirPhase::Optimized, phase_span)) = phase {
-                *failed = true;
-                cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase {
-                    dialect,
-                    phase: MirPhase::Optimized,
-                    attr_span,
-                    dialect_span,
-                    phase_span,
-                });
-            }
-        }
-
-        MirDialect::Built => {
-            if let Some((phase, phase_span)) = phase {
-                *failed = true;
-                cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase {
-                    dialect,
-                    phase,
-                    attr_span,
-                    dialect_span,
-                    phase_span,
-                });
-            }
-        }
-        MirDialect::Runtime => {}
+        (MirDialect::Analysis, MirPhase::Initial | MirPhase::PostCleanup)
+        | (MirDialect::Runtime, MirPhase::Initial | MirPhase::PostCleanup | MirPhase::Optimized)
+        | (MirDialect::Mono, MirPhase::Initial | MirPhase::Optimized) => {}
     }
 }

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -263,7 +263,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
         .mir
         .args_iter()
         .map(|local| {
-            let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty);
+            let arg_ty = fx.mir.local_decls[local].ty;
 
             // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482
             if Some(local) == fx.mir.spread_arg {
@@ -352,7 +352,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
     }
 
     for local in fx.mir.vars_and_temps_iter() {
-        let ty = fx.monomorphize(fx.mir.local_decls[local].ty);
+        let ty = fx.mir.local_decls[local].ty;
         let layout = fx.layout_of(ty);
 
         let is_ssa = ssa_analyzed[local].is_ssa(fx, ty);
@@ -461,9 +461,9 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     };
 
     let extra_args = &args[fn_sig.inputs().skip_binder().len()..];
-    let extra_args = fx.tcx.mk_type_list_from_iter(
-        extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.node.ty(fx.mir, fx.tcx))),
-    );
+    let extra_args = fx
+        .tcx
+        .mk_type_list_from_iter(extra_args.iter().map(|op_arg| op_arg.node.ty(fx.mir, fx.tcx)));
     let fn_abi = if let Some(instance) = instance {
         FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args)
     } else {

compiler/rustc_codegen_cranelift/src/base.rs

@@ -44,7 +44,7 @@ pub(crate) fn codegen_fn<'tcx>(
     let symbol_name = tcx.symbol_name(instance).name.to_string();
     let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name);
 
-    let mir = tcx.instance_mir(instance.def);
+    let mir = rustc_mir_transform::build_codegen_mir(tcx, instance);
     let _mir_guard = crate::PrintOnPanic(|| {
         let mut buf = Vec::new();
         with_no_trimmed_paths!({
@@ -282,10 +282,8 @@ fn verify_func(tcx: TyCtxt<'_>, writer: &crate::pretty_clif::CommentWriter, func
 }
 
 fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
-    let arg_uninhabited = fx
-        .mir
-        .args_iter()
-        .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).is_uninhabited());
+    let arg_uninhabited =
+        fx.mir.args_iter().any(|arg| fx.layout_of(fx.mir.local_decls[arg].ty).is_uninhabited());
     if arg_uninhabited {
         fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
         fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
@@ -297,19 +295,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
         .generic_activity("codegen prelude")
         .run(|| crate::abi::codegen_fn_prelude(fx, start_block));
 
-    let reachable_blocks = traversal::mono_reachable_as_bitset(fx.mir, fx.tcx, fx.instance);
-
     for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() {
         let block = fx.get_block(bb);
         fx.bcx.switch_to_block(block);
 
-        if !reachable_blocks.contains(bb) {
-            // We want to skip this block, because it's not reachable. But we still create
-            // the block so terminators in other blocks can reference it.
-            fx.bcx.ins().trap(TrapCode::user(1 /* unreachable */).unwrap());
-            continue;
-        }
-
         if bb_data.is_cleanup {
             if cfg!(not(feature = "unwinding")) {
                 continue;
@@ -698,8 +687,8 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                     ref operand,
                     to_ty,
                 ) => {
-                    let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx));
-                    let to_layout = fx.layout_of(fx.monomorphize(to_ty));
+                    let from_ty = operand.ty(&fx.mir.local_decls, fx.tcx);
+                    let to_layout = fx.layout_of(to_ty);
                     match *from_ty.kind() {
                         ty::FnDef(def_id, args) => {
                             let func_ref = fx.get_function_ref(
@@ -722,7 +711,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                     ref operand,
                     to_ty,
                 ) => {
-                    let to_layout = fx.layout_of(fx.monomorphize(to_ty));
+                    let to_layout = fx.layout_of(to_ty);
                     let operand = codegen_operand(fx, operand);
                     lval.write_cvalue(fx, operand.cast_pointer_to(to_layout));
                 }
@@ -752,7 +741,6 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                 ) => {
                     let operand = codegen_operand(fx, operand);
                     let from_ty = operand.layout().ty;
-                    let to_ty = fx.monomorphize(to_ty);
 
                     fn is_wide_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
                         ty.builtin_deref(true).is_some_and(|pointee_ty| {
@@ -824,8 +812,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                 }
                 Rvalue::Repeat(ref operand, times) => {
                     let operand = codegen_operand(fx, operand);
-                    let times = fx
-                        .monomorphize(times)
+                    let times = times
                         .try_to_target_usize(fx.tcx)
                         .expect("expected monomorphic const in codegen");
                     if operand.layout().size.bytes() == 0 {
@@ -862,7 +849,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                     if matches!(**kind, AggregateKind::RawPtr(..)) =>
                 {
                     let ty = to_place_and_rval.1.ty(&fx.mir.local_decls, fx.tcx);
-                    let layout = fx.layout_of(fx.monomorphize(ty));
+                    let layout = fx.layout_of(ty);
                     let [data, meta] = &*operands.raw else {
                         bug!("RawPtr fields: {operands:?}");
                     };
@@ -957,8 +944,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
 fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value {
     match *place.layout().ty.kind() {
         ty::Array(_elem_ty, len) => {
-            let len = fx
-                .monomorphize(len)
+            let len = len
                 .try_to_target_usize(fx.tcx)
                 .expect("expected monomorphic const in codegen") as i64;
             fx.bcx.ins().iconst(fx.pointer_type, len)
@@ -981,7 +967,7 @@ pub(crate) fn codegen_place<'tcx>(
             }
             PlaceElem::OpaqueCast(ty) => bug!("encountered OpaqueCast({ty}) in codegen"),
             PlaceElem::UnwrapUnsafeBinder(ty) => {
-                cplace = cplace.place_transmute_type(fx, fx.monomorphize(ty));
+                cplace = cplace.place_transmute_type(fx, ty);
             }
             PlaceElem::Field(field, _ty) => {
                 cplace = cplace.place_field(fx, field);

compiler/rustc_codegen_cranelift/src/common.rs

@@ -2,7 +2,6 @@ use cranelift_codegen::isa::TargetFrontendConfig;
 use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
 use rustc_abi::{Float, Integer, Primitive};
 use rustc_index::IndexVec;
-use rustc_middle::ty::TypeFoldable;
 use rustc_middle::ty::layout::{
     self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
 };
@@ -342,17 +341,6 @@ impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> {
 }
 
 impl<'tcx> FunctionCx<'_, '_, 'tcx> {
-    pub(crate) fn monomorphize<T>(&self, value: T) -> T
-    where
-        T: TypeFoldable<TyCtxt<'tcx>> + Copy,
-    {
-        self.instance.instantiate_mir_and_normalize_erasing_regions(
-            self.tcx,
-            ty::TypingEnv::fully_monomorphized(),
-            ty::EarlyBinder::bind(value),
-        )
-    }
-
     pub(crate) fn clif_type(&self, ty: Ty<'tcx>) -> Option<Type> {
         clif_type_from_ty(self.tcx, ty)
     }

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -78,7 +78,7 @@ pub(crate) fn eval_mir_constant<'tcx>(
     fx: &FunctionCx<'_, '_, 'tcx>,
     constant: &ConstOperand<'tcx>,
 ) -> (ConstValue, Ty<'tcx>) {
-    let cv = fx.monomorphize(constant.const_);
+    let cv = constant.const_;
     // This cannot fail because we checked all required_consts in advance.
     let val = cv
         .eval(fx.tcx, ty::TypingEnv::fully_monomorphized(), constant.span)

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -111,7 +111,7 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
                         .span_err(span, "asm! and global_asm! sym operands are not yet supported");
                 }
 
-                let const_ = fx.monomorphize(value.const_);
+                let const_ = value.const_;
                 if let ty::FnDef(def_id, args) = *const_.ty().kind() {
                     let instance = ty::Instance::resolve_for_fn_ptr(
                         fx.tcx,

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -180,7 +180,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
 
             // Make sure this is actually a SIMD vector.
-            let idx_ty = fx.monomorphize(idx.node.ty(fx.mir, fx.tcx));
+            let idx_ty = idx.node.ty(fx.mir, fx.tcx);
             if !idx_ty.is_simd()
                 || !matches!(idx_ty.simd_size_and_type(fx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
             {

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -23,6 +23,7 @@ extern crate rustc_hir;
 extern crate rustc_incremental;
 extern crate rustc_index;
 extern crate rustc_log;
+extern crate rustc_mir_transform;
 extern crate rustc_session;
 extern crate rustc_span;
 extern crate rustc_symbol_mangling;

compiler/rustc_codegen_ssa/Cargo.toml

@@ -27,6 +27,7 @@ rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_middle = { path = "../rustc_middle" }
+rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }