Auto merge of #158395 - JonathanBrouwer:rollup-d7BTVnf, r=<try>

Rollup of 17 pull requests


try-job: dist-various-1
try-job: test-various
try-job: x86_64-gnu-aux
try-job: x86_64-gnu-llvm-21-3
try-job: x86_64-msvc-1
try-job: aarch64-apple
try-job: x86_64-mingw-1
try-job: i686-msvc-2

Author: bors

compiler/rustc_abi/src/extern_abi.rs

@@ -343,10 +343,16 @@ impl ExternAbi {
                 // This ABI does not support calls at all (except via assembly).
                 false
             }
+            Self::RustCall => {
+                // Argument untupling requires additional support for tail calls to be possible,
+                // see <https://github.com/rust-lang/rust/pull/158248>. There is no real uses of
+                // tail calling `extern "rust-call"` functions
+                false
+            }
+
             Self::C { .. }
             | Self::System { .. }
             | Self::Rust
-            | Self::RustCall
             | Self::RustCold
             | Self::RustInvalid
             | Self::Unadjusted

compiler/rustc_ast/src/ast.rs

@@ -3059,21 +3059,23 @@ impl FnDecl {
     }
 
     /// The marker index for "no splatted arguments".
+    /// Higher values are also not supported, for performance reasons.
+    ///
     /// Must have the same value as `FnSigKind::NO_SPLATTED_ARG_INDEX` and `FnDeclFlags::NO_SPLATTED_ARG_INDEX`.
-    pub const NO_SPLATTED_ARG_INDEX: u16 = u16::MAX;
+    pub const NO_SPLATTED_ARG_INDEX: u8 = u8::MAX;
 
     /// Returns a splatted argument index, if any are present.
-    pub fn splatted(&self) -> Option<u16> {
+    pub fn splatted(&self) -> Option<u8> {
         self.inputs.iter().enumerate().find_map(|(index, arg)| {
-            if index == Self::NO_SPLATTED_ARG_INDEX as usize {
+            if index >= usize::from(Self::NO_SPLATTED_ARG_INDEX) {
                 // AST validation has already checked the splatted argument index is valid, so just
                 // ignore invalid indexes here.
                 None
             } else {
                 arg.attrs
                     .iter()
                     .any(|attr| attr.has_name(sym::splat))
-                    .then_some(u16::try_from(index).unwrap())
+                    .then_some(u8::try_from(index).unwrap())
             }
         })
     }

compiler/rustc_ast_lowering/src/delegation.rs

@@ -93,7 +93,7 @@ struct ParamInfo {
     pub c_variadic: bool,
 
     /// The index of the splatted parameter, if any.
-    pub splatted: Option<u16>,
+    pub splatted: Option<u8>,
 }
 
 const PARENT_ID: hir::ItemLocalId = hir::ItemLocalId::ZERO;
@@ -364,11 +364,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
     fn param_info(&self, def_id: DefId) -> ParamInfo {
         let sig = self.tcx.fn_sig(def_id).skip_binder().skip_binder();
 
-        // FIXME(splat): use `sig.splatted()` once FnSig has it
         ParamInfo {
             param_count: sig.inputs().len() + usize::from(sig.c_variadic()),
             c_variadic: sig.c_variadic(),
-            splatted: None,
+            splatted: sig.splatted(),
         }
     }
 

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -412,11 +412,11 @@ impl<'a> AstValidator<'a> {
             })
             .unzip();
 
-        // A splatted argument at the "no splatted" marker index is not supported (this is an
-        // unlikely edge case).
+        // A splatted argument greater than or equal to the "no splatted" marker index is not
+        // supported.
         if let (Some(&splatted_arg_index), Some(&splatted_span)) =
             (splatted_arg_indexes.last(), splatted_spans.last())
-            && splatted_arg_index == FnDecl::NO_SPLATTED_ARG_INDEX
+            && splatted_arg_index >= u16::from(FnDecl::NO_SPLATTED_ARG_INDEX)
         {
             self.dcx().emit_err(diagnostics::InvalidSplattedArg {
                 splatted_arg_index,

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -271,6 +271,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
         .map(|local| {
             let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty);
 
+            // FIXME(splat): un-tuple splatted arguments in codegen, for performance
             // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482
             if Some(local) == fx.mir.spread_arg {
                 // This argument (e.g. the last argument in the "rust-call" ABI)

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1459,6 +1459,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         );
     }
 
+    fn vscale(&mut self, _: Self::Type) -> Self::Value {
+        unimplemented!("`rustc_codegen_gcc` doesn't support scalable vectors yet")
+    }
+
     fn select(
         &mut self,
         cond: RValue<'gcc>,

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1226,6 +1226,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
+    fn vscale(&mut self, ty: &'ll Type) -> &'ll Value {
+        unsafe { llvm::LLVMRustBuildVScale(self.llbuilder, ty) }
+    }
+
     fn select(
         &mut self,
         cond: &'ll Value,

compiler/rustc_codegen_llvm/src/builder/autodiff.rs

@@ -3,6 +3,9 @@ use std::ptr;
 use rustc_ast::expand::autodiff_attrs::{DiffActivity, DiffMode};
 use rustc_ast::expand::typetree::FncTree;
 use rustc_codegen_ssa::common::TypeKind;
+use rustc_codegen_ssa::mir::IntrinsicResult;
+use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
+use rustc_codegen_ssa::mir::place::PlaceValue;
 use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods};
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_hir::attrs::RustcAutodiff;
@@ -11,7 +14,7 @@ use rustc_middle::{bug, ty};
 use rustc_target::callconv::PassMode;
 use tracing::debug;
 
-use crate::builder::{Builder, PlaceRef, UNNAMED};
+use crate::builder::{Builder, UNNAMED};
 use crate::context::SimpleCx;
 use crate::declare::declare_simple_fn;
 use crate::llvm::{self, TRUE, Type, Value};
@@ -296,9 +299,10 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>(
     ret_ty: &'ll Type,
     fn_args: &[&'ll Value],
     attrs: &RustcAutodiff,
-    dest: PlaceRef<'tcx, &'ll Value>,
+    dest_layout: ty::layout::TyAndLayout<'tcx>,
+    dest_place: Option<PlaceValue<&'ll Value>>,
     fnc_tree: FncTree,
-) {
+) -> IntrinsicResult<'tcx, &'ll Value> {
     // We have to pick the name depending on whether we want forward or reverse mode autodiff.
     let mut ad_name: String = match attrs.mode {
         DiffMode::Forward => "__enzyme_fwddiff",
@@ -381,11 +385,18 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>(
     let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None);
 
     let fn_ret_ty = builder.cx.val_ty(call);
-    if fn_ret_ty != builder.cx.type_void() && fn_ret_ty != builder.cx.type_struct(&[], false) {
+    if fn_ret_ty == builder.cx.type_void() || fn_ret_ty == builder.cx.type_struct(&[], false) {
         // If we return void or an empty struct, then our caller (due to how we generated it)
         // does not expect a return value. As such, we have no pointer (or place) into which
         // we could store our value, and would store into an undef, which would cause UB.
         // As such, we just ignore the return value in those cases.
-        builder.store_to_place(call, dest.val);
+        IntrinsicResult::Operand(OperandValue::ZeroSized)
+    } else if let Some(dest_place) = dest_place {
+        builder.store_to_place(call, dest_place);
+        IntrinsicResult::WroteIntoPlace
+    } else {
+        IntrinsicResult::Operand(
+            OperandRef::from_immediate_or_packed_pair(builder, call, dest_layout).val,
+        )
     }
 }

compiler/rustc_codegen_llvm/src/context.rs

@@ -978,10 +978,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     }
 
     fn intrinsic_call_expects_place_always(&self, name: Symbol) -> bool {
-        matches!(
-            name,
-            sym::autodiff | sym::volatile_load | sym::unaligned_volatile_load | sym::black_box
-        )
+        matches!(name, sym::volatile_load | sym::unaligned_volatile_load | sym::black_box)
     }
 }
 

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -43,6 +43,7 @@ use crate::errors::{
     AutoDiffWithoutEnable, AutoDiffWithoutLto, IntrinsicSignatureMismatch, IntrinsicWrongArch,
     OffloadWithoutEnable, OffloadWithoutFatLTO, UnknownIntrinsic,
 };
+use crate::intrinsic::ty::typetree::fnc_typetrees;
 use crate::llvm::{self, Type, Value};
 use crate::type_of::LayoutLlvmExt;
 use crate::va_arg::emit_va_arg;
@@ -223,12 +224,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 )
             }
             sym::autodiff => {
-                let result = PlaceRef {
-                    val: result_place.unwrap(),
-                    layout: result_layout,
-                };
-                codegen_autodiff(self, tcx, instance, args, result);
-                return IntrinsicResult::WroteIntoPlace;
+                return codegen_autodiff(self, tcx, instance, args, result_layout, result_place);
             }
             sym::offload => {
                 if tcx.sess.opts.unstable_opts.offload.is_empty() {
@@ -1728,8 +1724,9 @@ fn codegen_autodiff<'ll, 'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: ty::Instance<'tcx>,
     args: &[OperandRef<'tcx, &'ll Value>],
-    result: PlaceRef<'tcx, &'ll Value>,
-) {
+    result_layout: ty::layout::TyAndLayout<'tcx>,
+    result_place: Option<PlaceValue<&'ll Value>>,
+) -> IntrinsicResult<'tcx, &'ll Value> {
     if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) {
         let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable);
     }
@@ -1769,9 +1766,9 @@ fn codegen_autodiff<'ll, 'tcx>(
             diff_id,
             diff_args
         ),
-        Err(_) => {
+        Err(err) => {
             // An error has already been emitted
-            return;
+            return IntrinsicResult::Err(err);
         }
     };
 
@@ -1791,7 +1788,7 @@ fn codegen_autodiff<'ll, 'tcx>(
         &mut diff_attrs.input_activity,
     );
 
-    let fnc_tree = rustc_middle::ty::fnc_typetrees(tcx, source_fn_ptr_ty);
+    let fnc_tree = fnc_typetrees(tcx, source_fn_ptr_ty);
 
     // Build body
     generate_enzyme_call(
@@ -1802,9 +1799,10 @@ fn codegen_autodiff<'ll, 'tcx>(
         llret_ty,
         &val_arr,
         &diff_attrs,
-        result,
+        result_layout,
+        result_place,
         fnc_tree,
-    );
+    )
 }
 
 // Generates the LLVM code to offload a Rust function to a target device (e.g., GPU).