Rollup merge of rust-lang#157983 - folkertdev:tailcc-lift-same-signature-restriction, r=WaffleLapkin

Lift the same-signature restriction for `extern "tail"`

tracking issue: rust-lang#157427

The `extern "tail"` calling convention uses callee cleanup (i.e. the callee restores the stack, not the caller). Hence, the same-signature restriction that is normally required to codegen tail calls does not apply.

We need the ABI to deviate from `extern "Rust"` to make sure indirect arguments are passed by stack offset, not via a pointer into the caller's stack frame (the value would potentially be overwritten). For standard tail calls we work around this problem by storing the value in the caller's caller, but for `extern "tail"` that doesn't work reliably because the signatures can be different.

I'm not sure about unsized arguments. That feature seems really broken, so I'm not sure how much work we should put into trying to do something reasonable there. Fundamentally I don't think we can support unsized arguments in `extern "tail"` calls.

Also we can't really promote using this yet due to `tailcc` being a bit broken on LLVM 22 on x86_64.

r? WaffleLapkin
cc @bjorn3

Author: JonathanBrouwer

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1317,7 +1317,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     }
                     LocalRef::Operand(arg) => {
                         let Ref(place_value) = arg.val else {
-                            bug!("only `Ref` should use `PassMode::Indirect`");
+                            bug!(
+                                "only `Ref` should use `PassMode::Indirect`, but got {:?}",
+                                arg.val
+                            );
                         };
                         bx.typed_place_copy(place_value, tmp.val, fn_abi.args[i].layout);
                         op.val = arg.val;

compiler/rustc_hir_typeck/src/check.rs

@@ -90,7 +90,7 @@ pub(super) fn check_fn<'a, 'tcx>(
         }
 
         // Check that argument is Sized.
-        if !params_can_be_unsized {
+        if !params_can_be_unsized || fn_sig.abi() == rustc_abi::ExternAbi::RustTail {
             fcx.require_type_is_sized(
                 param_ty,
                 param.ty_span,

compiler/rustc_mir_build/src/check_tail_calls.rs

@@ -147,7 +147,9 @@ impl<'tcx> TailCallCkVisitor<'_, 'tcx> {
         // ```
         // we should think what is the expected behavior here.
         // (we should probably just accept this by revealing opaques?)
-        if caller_sig.inputs_and_output != callee_sig.inputs_and_output {
+        if caller_sig.inputs_and_output != callee_sig.inputs_and_output
+            && !matches!(callee_sig.abi(), ExternAbi::RustTail)
+        {
             let caller_ty = self.tcx.type_of(self.caller_def_id).skip_binder();
 
             self.report_signature_mismatch(
@@ -189,6 +191,12 @@ impl<'tcx> TailCallCkVisitor<'_, 'tcx> {
         if callee_sig.c_variadic() {
             self.report_c_variadic_callee(expr.span);
         }
+
+        for &arg_ty in callee_sig.inputs() {
+            if !arg_ty.is_sized(self.tcx, self.typing_env) {
+                self.report_unsized_argument(expr.span, arg_ty);
+            }
+        }
     }
 
     /// Returns true if the caller function needs a location argument
@@ -417,6 +425,17 @@ impl<'tcx> TailCallCkVisitor<'_, 'tcx> {
 
         self.found_errors = Err(err);
     }
+
+    fn report_unsized_argument(&mut self, sp: Span, arg_ty: Ty<'tcx>) {
+        let err = self
+            .tcx
+            .dcx()
+            .struct_span_err(sp, format!("unsized arguments cannot be used in a tail call"))
+            .with_note(format!("unsized argument of type `{arg_ty}`"))
+            .emit();
+
+        self.found_errors = Err(err);
+    }
 }
 
 impl<'a, 'tcx> Visitor<'a, 'tcx> for TailCallCkVisitor<'a, 'tcx> {

compiler/rustc_target/src/callconv/mod.rs

@@ -827,6 +827,9 @@ impl<'a, Ty> FnAbi<'a, Ty> {
                             ArgAttribute::default()
                         };
                         arg.cast_to_with_attrs(Reg { kind: RegKind::Integer, size }, attr.into());
+                    } else if self.conv == CanonAbi::RustTail {
+                        assert!(arg.layout.is_sized(), "extern \"tail\" arguments must be sized");
+                        arg.pass_by_stack_offset(None);
                     }
                 }
 

compiler/rustc_ty_utils/src/abi.rs

@@ -458,8 +458,10 @@ fn fn_abi_sanity_check<'tcx>(
                 // omitted entirely in the calling convention.
                 assert!(arg.is_ignore());
             }
-            if let PassMode::Indirect { on_stack, .. } = arg.mode {
-                assert!(!on_stack, "rust abi shouldn't use on_stack");
+            if let PassMode::Indirect { on_stack, .. } = arg.mode
+                && spec_abi != ExternAbi::RustTail
+            {
+                assert!(!on_stack, "rustic abi {spec_abi:?} shouldn't use on_stack");
             }
         } else if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
             assert_matches!(

tests/ui/explicit-tail-calls/no-unsized-arguments.aarch64.stderr

@@ -0,0 +1,31 @@
+error[E0277]: the size for values of type `[u8]` cannot be known at compilation time
+  --> $DIR/no-unsized-arguments.rs:40:42
+   |
+LL |     extern "tail" fn unsized_argument(x: [u8]) -> u8 {
+   |                                          ^^^^ doesn't have a size known at compile-time
+   |
+   = help: the trait `Sized` is not implemented for `[u8]`
+help: function arguments must have a statically known size, borrowed slices always have a known size
+   |
+LL |     extern "tail" fn unsized_argument(x: &[u8]) -> u8 {
+   |                                          +
+
+error: unsized arguments cannot be used in a tail call
+  --> $DIR/no-unsized-arguments.rs:33:5
+   |
+LL |     become unsized_argument(b);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: unsized argument of type `[u8]`
+
+error: unsized arguments cannot be used in a tail call
+  --> $DIR/no-unsized-arguments.rs:48:5
+   |
+LL |     become unsized_argument(*b);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: unsized argument of type `[u8]`
+
+error: aborting due to 3 previous errors
+
+For more information about this error, try `rustc --explain E0277`.

tests/ui/explicit-tail-calls/no-unsized-arguments.rs

@@ -0,0 +1,50 @@
+//@ add-minicore
+//@ ignore-backends: gcc
+//@ min-llvm-version: 22
+//
+//@ revisions: x86 x86_64 aarch64
+//
+//@ [x86] compile-flags: --target=i686-unknown-linux-gnu
+//@ [x86] needs-llvm-components: x86
+//@ [x86_64] compile-flags: --target=x86_64-unknown-linux-gnu
+//@ [x86_64] needs-llvm-components: x86
+//@ [aarch64] compile-flags: --target=aarch64-unknown-linux-gnu
+//@ [aarch64] needs-llvm-components: aarch64
+#![feature(explicit_tail_calls, rust_tail_cc, unsized_fn_params, no_core)]
+#![allow(incomplete_features, internal_features)]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+extern "C" {
+    fn extract(_: [u8]) -> u8;
+}
+
+fn vanilla(b: [u8]) -> u8 {
+    fn unsized_argument(x: [u8]) -> u8 {
+        unsafe { extract(x) }
+    }
+
+    // Non-tail call.
+    let _ = unsized_argument(b);
+
+    become unsized_argument(b);
+    //~^ ERROR unsized arguments cannot be used in a tail call
+}
+
+extern "tail" fn tailcc(b: &[u8]) -> u8 {
+    // `extern "tail"` is special because we also can't unsized parameters in standard definitions
+    // and calls.
+    extern "tail" fn unsized_argument(x: [u8]) -> u8 {
+        //~^ ERROR the size for values of type `[u8]` cannot be known at compilation time
+        unsafe { extract(x) }
+    }
+
+    // Vanilla call.
+    let _ = unsized_argument(*b);
+
+    become unsized_argument(*b);
+    //~^ ERROR unsized arguments cannot be used in a tail call
+}

tests/ui/explicit-tail-calls/no-unsized-arguments.x86.stderr

@@ -0,0 +1,31 @@
+error[E0277]: the size for values of type `[u8]` cannot be known at compilation time
+  --> $DIR/no-unsized-arguments.rs:40:42
+   |
+LL |     extern "tail" fn unsized_argument(x: [u8]) -> u8 {
+   |                                          ^^^^ doesn't have a size known at compile-time
+   |
+   = help: the trait `Sized` is not implemented for `[u8]`
+help: function arguments must have a statically known size, borrowed slices always have a known size
+   |
+LL |     extern "tail" fn unsized_argument(x: &[u8]) -> u8 {
+   |                                          +
+
+error: unsized arguments cannot be used in a tail call
+  --> $DIR/no-unsized-arguments.rs:33:5
+   |
+LL |     become unsized_argument(b);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: unsized argument of type `[u8]`
+
+error: unsized arguments cannot be used in a tail call
+  --> $DIR/no-unsized-arguments.rs:48:5
+   |
+LL |     become unsized_argument(*b);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: unsized argument of type `[u8]`
+
+error: aborting due to 3 previous errors
+
+For more information about this error, try `rustc --explain E0277`.

tests/ui/explicit-tail-calls/no-unsized-arguments.x86_64.stderr

@@ -0,0 +1,31 @@
+error[E0277]: the size for values of type `[u8]` cannot be known at compilation time
+  --> $DIR/no-unsized-arguments.rs:40:42
+   |
+LL |     extern "tail" fn unsized_argument(x: [u8]) -> u8 {
+   |                                          ^^^^ doesn't have a size known at compile-time
+   |
+   = help: the trait `Sized` is not implemented for `[u8]`
+help: function arguments must have a statically known size, borrowed slices always have a known size
+   |
+LL |     extern "tail" fn unsized_argument(x: &[u8]) -> u8 {
+   |                                          +
+
+error: unsized arguments cannot be used in a tail call
+  --> $DIR/no-unsized-arguments.rs:33:5
+   |
+LL |     become unsized_argument(b);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: unsized argument of type `[u8]`
+
+error: unsized arguments cannot be used in a tail call
+  --> $DIR/no-unsized-arguments.rs:48:5
+   |
+LL |     become unsized_argument(*b);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: unsized argument of type `[u8]`
+
+error: aborting due to 3 previous errors
+
+For more information about this error, try `rustc --explain E0277`.

tests/ui/explicit-tail-calls/signature-mismatch.rs

@@ -1,5 +1,5 @@
 #![expect(incomplete_features)]
-#![feature(explicit_tail_calls)]
+#![feature(explicit_tail_calls, rust_tail_cc)]
 #![feature(c_variadic)]
 
 fn _f0((): ()) {
@@ -8,26 +8,30 @@ fn _f0((): ()) {
 
 fn _g0() {}
 
-
 fn _f1() {
     become _g1(()); //~ error: mismatched signatures
 }
 
 fn _g1((): ()) {}
 
-
 extern "C" fn _f2() {
     become _g2(); //~ error: mismatched function ABIs
 }
 
 fn _g2() {}
 
-
 fn _f3() {
     become _g3(); //~ error: mismatched function ABIs
 }
 
 extern "C" fn _g3() {}
 
+extern "tail" fn _tailcc() {}
+
+fn _f4() {
+    // tailcc does not need the signatures to match,
+    // but only tailcc can tail call tailcc.
+    become _tailcc(); //~ error: mismatched function ABIs
+}
 
 fn main() {}