Add a noinline attribute at the callsite when we bail on adding alwaysinline

Author: Jamesbarford

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1434,20 +1434,31 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
             // Only propagate `#[inline(always)]` to the callsite when there is
             // an attribute and the caller and callee are compatible for
-            // inlining here.
+            // inlining here. Otherwise we explicitly emit a `noinline` to
+            // ensure that the function will not get inlined through an LLVM
+            // pass.
             if attributes::has_inline_always_callsite_attribute(
                 self.cx.tcx,
                 callee_attrs,
                 callee_instance,
-            ) && self.tcx.is_call_inline_able_at_callsite(
-                &callee_attrs.target_features,
-                &caller_attrs.target_features,
             ) {
-                attributes::apply_to_callsite(
-                    call,
-                    llvm::AttributePlace::Function,
-                    &[AttributeKind::AlwaysInline.create_attr(self.cx.llcx)],
-                );
+                if self.tcx.is_call_inline_able_at_callsite(
+                    &callee_attrs.target_features,
+                    &caller_attrs.target_features,
+                ) {
+                    attributes::apply_to_callsite(
+                        call,
+                        llvm::AttributePlace::Function,
+                        &[AttributeKind::AlwaysInline.create_attr(self.cx.llcx)],
+                    );
+                } else {
+                    // Ensure the function call will not be inlined.
+                    attributes::apply_to_callsite(
+                        call,
+                        llvm::AttributePlace::Function,
+                        &[AttributeKind::NoInline.create_attr(self.cx.llcx)],
+                    );
+                }
             }
         }
 

tests/codegen-llvm/inline-always-callsite-noinline-cmpxchg16b.rs

@@ -0,0 +1,28 @@
+//@ compile-flags: --crate-type=lib --target x86_64-unknown-linux-gnu -O -Zinline-mir=no -C no-prepopulate-passes
+//@ needs-llvm-components: x86
+//@ only-x86_64
+//@ ignore-backends: gcc
+
+#![feature(core_intrinsics, target_feature_inline_always)]
+#![allow(incomplete_features)]
+
+use std::intrinsics::{AtomicOrdering, atomic_load};
+
+#[inline(always)]
+#[target_feature(enable = "cmpxchg16b")]
+#[unsafe(no_mangle)]
+pub fn load(x: *const u128) -> u128 {
+    unsafe { atomic_load::<u128, { AtomicOrdering::Relaxed }>(x) }
+}
+
+#[unsafe(no_mangle)]
+// CHECK-LABEL: define{{.*}} @load_core(
+pub fn load_core(x: *const u128) -> u128 {
+    // `cmpxchg16b` is not enabled on the caller, so the ineligible
+    // `#[inline(always)]` callee must be marked `noinline` at the callsite.
+    //
+    // CHECK: %_0 = {{(tail )?}}call{{.*}} @load(ptr{{.*}} %x) [[CALL_ATTRS:#[0-9]+]]
+    unsafe { load(x) }
+}
+
+// CHECK: attributes [[CALL_ATTRS]] = { {{.*}}noinline{{.*}} }