Make build_codegen_mir return a Cow.

Author: cjgillot

Cargo.lock

@@ -3835,7 +3835,6 @@ dependencies = [
  "rustc_macros",
  "rustc_metadata",
  "rustc_middle",
- "rustc_mir_transform",
  "rustc_serialize",
  "rustc_session",
  "rustc_span",

compiler/rustc_codegen_cranelift/src/base.rs

@@ -8,6 +8,7 @@ use rustc_ast::InlineAsmOptions;
 use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization;
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_index::IndexVec;
+use rustc_middle::mono::{InstantiationMode, MonoItem};
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv as _};
@@ -45,6 +46,10 @@ pub(crate) fn codegen_fn<'tcx>(
     let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name);
 
     let mir = tcx.build_codegen_mir(instance);
+    let mir = match MonoItem::Fn(instance).instantiation_mode(tcx) {
+        InstantiationMode::LocalCopy => &*mir.borrow(),
+        InstantiationMode::GloballyShared { .. } => &*mir.steal(),
+    };
     let _mir_guard = crate::PrintOnPanic(|| {
         let mut buf = Vec::new();
         with_no_trimmed_paths!({

compiler/rustc_codegen_cranelift/src/common.rs

@@ -266,7 +266,7 @@ pub(crate) fn create_wrapper_function(
     module.define_function(wrapper_func_id, &mut ctx).unwrap();
 }
 
-pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
+pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
     pub(crate) module: &'m mut dyn Module,
     pub(crate) debug_context: Option<&'clif mut DebugContext>,
     pub(crate) tcx: TyCtxt<'tcx>,
@@ -278,7 +278,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) cgu_name: Symbol,
     pub(crate) instance: Instance<'tcx>,
     pub(crate) symbol_name: String,
-    pub(crate) mir: &'tcx Body<'tcx>,
+    pub(crate) mir: &'m Body<'tcx>,
     pub(crate) fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
 
     pub(crate) bcx: FunctionBuilder<'clif>,

compiler/rustc_codegen_ssa/Cargo.toml

@@ -27,7 +27,6 @@ rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_middle = { path = "../rustc_middle" }
-rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -15,7 +15,7 @@ use super::FunctionCx;
 use crate::traits::*;
 
 pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
-    fx: &FunctionCx<'a, 'tcx, Bx>,
+    fx: &FunctionCx<'_, 'a, 'tcx, Bx>,
     traversal_order: &[mir::BasicBlock],
 ) -> DenseBitSet<mir::Local> {
     let mir = fx.mir;
@@ -65,13 +65,13 @@ enum LocalKind {
     SSA(DefLocation),
 }
 
-struct LocalAnalyzer<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> {
-    fx: &'a FunctionCx<'b, 'tcx, Bx>,
+struct LocalAnalyzer<'a, 'mir, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> {
+    fx: &'a FunctionCx<'mir, 'b, 'tcx, Bx>,
     dominators: &'a Dominators<mir::BasicBlock>,
     locals: IndexVec<mir::Local, LocalKind>,
 }
 
-impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> {
+impl<'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'_, '_, 'b, 'tcx, Bx> {
     fn define(&mut self, local: mir::Local, location: DefLocation) {
         let fx = self.fx;
         let kind = &mut self.locals[local];
@@ -166,7 +166,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
     }
 }
 
-impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer<'a, 'b, 'tcx, Bx> {
+impl<'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer<'_, '_, 'b, 'tcx, Bx> {
     fn visit_assign(
         &mut self,
         place: &mir::Place<'tcx>,

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -46,17 +46,17 @@ enum CallKind {
 
 /// Used by `FunctionCx::codegen_terminator` for emitting common patterns
 /// e.g., creating a basic block, calling a function, etc.
-struct TerminatorCodegenHelper<'tcx> {
+struct TerminatorCodegenHelper<'mir, 'tcx> {
     bb: mir::BasicBlock,
-    terminator: &'tcx mir::Terminator<'tcx>,
+    terminator: &'mir mir::Terminator<'tcx>,
 }
 
-impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
+impl<'a, 'tcx> TerminatorCodegenHelper<'_, 'tcx> {
     /// Returns the appropriate `Funclet` for the current funclet, if on MSVC,
     /// either already previously cached, or newly created, by `landing_pad_for`.
     fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &'b mut FunctionCx<'_, 'a, 'tcx, Bx>,
     ) -> Option<&'b Bx::Funclet> {
         let cleanup_kinds = fx.cleanup_kinds.as_ref()?;
         let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?;
@@ -82,7 +82,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// stuff in it or next to it.
     fn llbb_with_cleanup<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> Bx::BasicBlock {
         let (needs_landing_pad, is_cleanupret) = self.llbb_characteristics(fx, target);
@@ -106,7 +106,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 
     fn llbb_characteristics<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> (bool, bool) {
         if let Some(ref cleanup_kinds) = fx.cleanup_kinds {
@@ -131,7 +131,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 
     fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>,
         bx: &mut Bx,
         target: mir::BasicBlock,
         mergeable_succ: bool,
@@ -160,7 +160,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// return destination `destination` and the unwind action `unwind`.
     fn do_call<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>,
         bx: &mut Bx,
         fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
         fn_ptr: Bx::Value,
@@ -302,7 +302,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     /// Generates inline assembly with optional `destination` and `unwind`.
     fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>,
         bx: &mut Bx,
         template: &[InlineAsmTemplatePiece],
         operands: &[InlineAsmOperandRef<'tcx, Bx>],
@@ -369,9 +369,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 }
 
 /// Codegen implementations for some terminator variants.
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> {
     /// Generates code for a `Resume` terminator.
-    fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx) {
+    fn codegen_resume_terminator(
+        &mut self,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
+        bx: &mut Bx,
+    ) {
         if let Some(funclet) = helper.funclet(self) {
             bx.cleanup_ret(funclet, None);
         } else {
@@ -388,7 +392,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_switchint_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         discr: &mir::Operand<'tcx>,
         targets: &SwitchTargets,
@@ -600,7 +604,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     #[tracing::instrument(level = "trace", skip(self, helper, bx))]
     fn codegen_drop_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         source_info: &mir::SourceInfo,
         location: mir::Place<'tcx>,
@@ -698,7 +702,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_assert_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         cond: &mir::Operand<'tcx>,
@@ -799,7 +803,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_terminate_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         reason: UnwindTerminateReason,
@@ -830,7 +834,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     /// Returns `Some` if this is indeed a panic intrinsic and codegen is done.
     fn codegen_panic_intrinsic(
         &mut self,
-        helper: &TerminatorCodegenHelper<'tcx>,
+        helper: &TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         intrinsic: ty::IntrinsicDef,
         instance: Instance<'tcx>,
@@ -897,7 +901,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_call_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         terminator: &mir::Terminator<'tcx>,
         func: &mir::Operand<'tcx>,
@@ -1399,7 +1403,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     fn codegen_asm_terminator(
         &mut self,
-        helper: TerminatorCodegenHelper<'tcx>,
+        helper: TerminatorCodegenHelper<'_, 'tcx>,
         bx: &mut Bx,
         asm_macro: InlineAsmMacro,
         terminator: &mir::Terminator<'tcx>,
@@ -1524,7 +1528,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         &mut self,
         bx: &mut Bx,
         bb: mir::BasicBlock,
-        terminator: &'tcx mir::Terminator<'tcx>,
+        terminator: &mir::Terminator<'tcx>,
     ) -> MergingSucc {
         debug!("codegen_terminator: {:?}", terminator);
 

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -9,7 +9,7 @@ use crate::errors;
 use crate::mir::operand::OperandRef;
 use crate::traits::*;
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> {
     pub(crate) fn eval_mir_constant_to_operand(
         &self,
         bx: &mut Bx,

compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs

@@ -4,7 +4,7 @@ use rustc_middle::mir::coverage::CoverageKind;
 use super::FunctionCx;
 use crate::traits::*;
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> {
     pub(crate) fn codegen_coverage(&self, bx: &mut Bx, kind: &CoverageKind, scope: SourceScope) {
         // Determine the instance that coverage data was originally generated for.
         let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) {

compiler/rustc_codegen_ssa/src/mir/debuginfo.rs

@@ -214,7 +214,7 @@ fn calculate_debuginfo_offset<
     DebugInfoOffset { direct_offset, indirect_offsets, result: place }
 }
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> {
     pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) {
         bx.set_span(source_info.span);
         if let Some(dbg_loc) = self.dbg_loc(source_info) {

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -52,7 +52,7 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     bx.memset(dst, val, size, align, flags);
 }
 
-impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> {
     /// In the `Fallback` case, returns the instance that should be called instead.
     pub fn codegen_intrinsic_call(
         &mut self,