Re-add call site inlining attributes

Author: saethlin

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -5,7 +5,7 @@ use rustc_hir::find_attr;
 use rustc_middle::middle::codegen_fn_attrs::{
     CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, SanitizerFnAttrs, TargetFeature,
 };
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::{self, Instance, TyCtxt};
 use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet};
 use rustc_span::sym;
 use rustc_symbol_mangling::mangle_internal_symbol;
@@ -42,22 +42,31 @@ pub(crate) fn remove_string_attr_from_llfn(llfn: &Value, name: &str) {
 
 /// Get LLVM attribute for the provided inline heuristic.
 #[inline]
-fn inline_attr<'ll>(
+pub(crate) fn inline_attr<'tcx, 'll>(
     cx: &SimpleCx<'ll>,
-    sess: &Session,
-    inline: InlineAttr,
+    tcx: TyCtxt<'tcx>,
+    instance: Instance<'tcx>,
+    codegen_fn_attrs: &CodegenFnAttrs,
 ) -> Option<&'ll Attribute> {
-    if !sess.opts.unstable_opts.inline_llvm {
+    if !tcx.sess.opts.unstable_opts.inline_llvm {
         // disable LLVM inlining
         return Some(AttributeKind::NoInline.create_attr(cx.llcx));
     }
+
+    // `optnone` requires `noinline`
+    let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) {
+        (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never,
+        (InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint,
+        (inline, _) => inline,
+    };
+
     match inline {
         InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
         InlineAttr::Always | InlineAttr::Force { .. } => {
             Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
         }
         InlineAttr::Never => {
-            if sess.target.arch != Arch::AmdGpu {
+            if tcx.sess.target.arch != Arch::AmdGpu {
                 Some(AttributeKind::NoInline.create_attr(cx.llcx))
             } else {
                 None
@@ -412,14 +421,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
     }
 
     if let Some(instance) = instance {
-        // `optnone` requires `noinline`
-        let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) {
-            (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never,
-            (InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint,
-            (inline, _) => inline,
-        };
-
-        to_add.extend(inline_attr(cx, sess, inline));
+        to_add.extend(inline_attr(cx, tcx, instance, codegen_fn_attrs));
     }
 
     if sess.must_emit_unwind_tables() {

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1419,6 +1419,21 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             )
         };
 
+        if let Some(callee_instance) = callee_instance {
+            // Attributes on the function definition being called
+            let callee_attrs = self.cx.tcx.codegen_fn_attrs(callee_instance.def_id());
+
+            if let Some(inlining_rule) =
+                attributes::inline_attr(&self.cx, self.cx.tcx, callee_instance, callee_attrs)
+            {
+                attributes::apply_to_callsite(
+                    call,
+                    llvm::AttributePlace::Function,
+                    &[inlining_rule],
+                );
+            }
+        }
+
         if let Some(fn_abi) = fn_abi {
             fn_abi.apply_attrs_callsite(self, call);
         }

tests/codegen-llvm/call-site-inline-attributes.rs

@@ -0,0 +1,40 @@
+//@ compile-flags: -O -Zinline-mir=no -Cno-prepopulate-passes -Zmerge-functions=disabled
+
+#![crate_type = "lib"]
+
+// This test checks that we add inlinehint for #[inline], noinline for #[inline(never)], and
+// alwaysinline for #[inline(always)] to call sites.
+
+#[unsafe(no_mangle)]
+fn calls_something_noinline() {
+    // CHECK-LABEL @calls_something_noinline
+    // CHECK: call void @{{.*}}noinline_fn() #[[NOINLINE:[0-9]+]]
+    noinline_fn();
+}
+
+#[inline(never)]
+fn noinline_fn() {}
+
+#[unsafe(no_mangle)]
+fn calls_something_inline() {
+    // CHECK-LABEL @calls_something_inlinehint
+    // CHECK: call void @{{.*}}inlinehint_fn() #[[INLINEHINT:[0-9]+]]
+    inlinehint_fn();
+}
+
+#[inline]
+fn inlinehint_fn() {}
+
+#[unsafe(no_mangle)]
+fn calls_something_alwaysinline() {
+    // CHECK-LABEL @calls_something_alwaysinline
+    // CHECK: call void @{{.*}}alwaysinline_fn() #[[ALWAYSINLINE:[0-9]+]]
+    alwaysinline_fn();
+}
+
+#[inline(always)]
+fn alwaysinline_fn() {}
+
+//CHECK: attributes #[[NOINLINE]] = {{{.*}} noinline {{.*}}}
+//CHECK: attributes #[[INLINEHINT]] = {{{.*}} inlinehint {{.*}}}
+//CHECK: attributes #[[ALWAYSINLINE]] = {{{.*}} alwaysinline {{.*}}}