Support u128/i128 c-variadic arguments

on some platforms

Author: folkertdev

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -3,8 +3,7 @@ use std::ffi::c_uint;
 use std::{assert_matches, iter, ptr};
 
 use rustc_abi::{
-    Align, BackendRepr, Float, HasDataLayout, Integer, NumScalableVectors, Primitive, Size,
-    WrappingRange,
+    Align, BackendRepr, Float, HasDataLayout, NumScalableVectors, Primitive, Size, WrappingRange,
 };
 use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
@@ -298,11 +297,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     Primitive::Pointer(_) => {
                         // Pointers are always OK.
                     }
-                    Primitive::Int(Integer::I128, _) => {
-                        // FIXME: maybe we should support these? At least on 32-bit powerpc
-                        // the logic in LLVM does not handle i128 correctly though.
-                        bug!("the va_arg intrinsic does not support `i128`/`u128`")
-                    }
                     Primitive::Int(..) => {
                         let int_width = self.cx().size_of(result.layout.ty).bits();
                         let target_c_int_width = self.cx().sess().target.options.c_int_width;

compiler/rustc_codegen_llvm/src/va_arg.rs

@@ -1,4 +1,4 @@
-use rustc_abi::{Align, BackendRepr, Endian, HasDataLayout, Primitive, Size};
+use rustc_abi::{Align, BackendRepr, Endian, Float, HasDataLayout, Integer, Primitive, Size};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::OperandRef;
@@ -8,7 +8,7 @@ use rustc_codegen_ssa::traits::{
 use rustc_middle::bug;
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
-use rustc_target::spec::{Arch, Env, LlvmAbi, RustcAbi};
+use rustc_target::spec::{Arch, Env, HasTargetSpec, LlvmAbi, RustcAbi};
 
 use crate::builder::Builder;
 use crate::llvm::{Type, Value};
@@ -30,11 +30,9 @@ fn round_pointer_up_to_alignment<'ll>(
     ptr_ty: &'ll Type,
 ) -> &'ll Value {
     let ptr = bx.inbounds_ptradd(addr, bx.const_i32(align.bytes() as i32 - 1));
-    bx.call_intrinsic(
-        "llvm.ptrmask",
-        &[ptr_ty, bx.type_i32()],
-        &[ptr, bx.const_int(bx.isize_ty, -(align.bytes() as isize) as i64)],
-    )
+    let pointer_width = bx.target_spec().pointer_width;
+    let mask = align.bytes().wrapping_neg() & (u64::MAX >> (64 - pointer_width));
+    bx.call_intrinsic("llvm.ptrmask", &[ptr_ty, bx.type_isize()], &[ptr, bx.const_usize(mask)])
 }
 
 fn emit_direct_ptr_va_arg<'ll, 'tcx>(
@@ -75,6 +73,35 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     }
 }
 
+/// Some backends apply special alignment rules to c-variadic arguments.
+fn get_param_type_alignment<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    layout: TyAndLayout<'tcx>,
+) -> Align {
+    let BackendRepr::Scalar(scalar) = layout.backend_repr else {
+        bug!("unexpected backend repr {:?}", layout.backend_repr);
+    };
+
+    match bx.cx.tcx.sess.target.arch {
+        Arch::PowerPC64 => match scalar.primitive() {
+            Primitive::Int(integer, _) => match integer {
+                Integer::I8 | Integer::I16 => unreachable!(),
+                Integer::I32 | Integer::I64 => { /* fall through */ }
+                Integer::I128 => return Align::EIGHT,
+            },
+            Primitive::Float(float) => match float {
+                Float::F16 | Float::F32 => unreachable!(),
+                Float::F64 => { /* fall through */ }
+                Float::F128 => return Align::from_bytes(16).unwrap(),
+            },
+            Primitive::Pointer(_) => { /* fall through */ }
+        },
+        _ => { /* fall through */ }
+    }
+
+    layout.align.abi
+}
+
 enum PassMode {
     Direct,
     Indirect,
@@ -115,23 +142,23 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
         (
             bx.cx.layout_of(Ty::new_imm_ptr(bx.cx.tcx, target_ty)).llvm_type(bx.cx),
             bx.cx.data_layout().pointer_size(),
-            bx.cx.data_layout().pointer_align(),
+            bx.cx.data_layout().pointer_align().abi,
         )
     } else {
-        (layout.llvm_type(bx.cx), layout.size, layout.align)
+        (layout.llvm_type(bx.cx), layout.size, get_param_type_alignment(bx, layout))
     };
     let (addr, addr_align) = emit_direct_ptr_va_arg(
         bx,
         list,
         size,
-        align.abi,
+        align,
         slot_size,
         allow_higher_align,
         force_right_adjust,
     );
     if indirect {
         let tmp_ret = bx.load(llty, addr, addr_align);
-        bx.load(layout.llvm_type(bx.cx), tmp_ret, align.abi)
+        bx.load(layout.llvm_type(bx.cx), tmp_ret, align)
     } else {
         bx.load(llty, addr, addr_align)
     }
@@ -765,9 +792,16 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
     // byte boundary if alignment needed by type exceeds 8 byte boundary.
     // It isn't stated explicitly in the standard, but in practice we use
     // alignment greater than 16 where necessary.
-    if layout.layout.align.bytes() > 8 {
-        unreachable!("all instances of VaArgSafe have an alignment <= 8");
-    }
+    let overflow_arg_area_v = if layout.layout.align.bytes() > 8 {
+        round_pointer_up_to_alignment(
+            bx,
+            overflow_arg_area_v,
+            layout.layout.align.abi,
+            bx.type_ptr(),
+        )
+    } else {
+        overflow_arg_area_v
+    };
 
     // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
     let mem_addr = overflow_arg_area_v;
@@ -1075,6 +1109,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             PassMode::Direct,
             SlotSize::Bytes8,
             AllowHigherAlign::Yes,
+            // ForceRightAdjust only takes effect on big-endian architectures.
             ForceRightAdjust::Yes,
         ),
         Arch::RiscV32 if target.llvm_abiname == LlvmAbi::Ilp32e => {

library/core/src/ffi/va_list.rs

@@ -269,11 +269,13 @@ mod sealed {
     impl Sealed for i16 {}
     impl Sealed for i32 {}
     impl Sealed for i64 {}
+    impl Sealed for i128 {}
     impl Sealed for isize {}
 
     impl Sealed for u16 {}
     impl Sealed for u32 {}
     impl Sealed for u64 {}
+    impl Sealed for u128 {}
     impl Sealed for usize {}
 
     impl Sealed for f32 {}
@@ -287,15 +289,27 @@ mod sealed {
 ///
 /// # Safety
 ///
-/// The standard library implements this trait for primitive types that are
-/// expected to have a variable argument application-binary interface (ABI) on all
-/// platforms.
+/// The standard library implements this trait for primitive types that have a variable
+/// argument application-binary interface (ABI) on the current platform. The trait is always
+/// implemented for:
+///
+/// - [`c_int`], [`c_long`] and [`c_longlong`]
+/// - [`c_uint`], [`c_ulong`] and [`c_ulonglong`]
+/// - [`c_double`]
 ///
 /// When C passes variable arguments, integers smaller than [`c_int`] and floats smaller
 /// than [`c_double`] are implicitly promoted to [`c_int`] and [`c_double`] respectively.
 /// Implementing this trait for types that are subject to this promotion rule is invalid.
 ///
+/// This trait is only implemented for 128-bit integers when the platform defines the `__int128`
+/// type.
+///
 /// [`c_int`]: core::ffi::c_int
+/// [`c_long`]: core::ffi::c_long
+/// [`c_longlong`]: core::ffi::c_longlong
+/// [`c_uint`]: core::ffi::c_uint
+/// [`c_ulong`]: core::ffi::c_ulong
+/// [`c_ulonglong`]: core::ffi::c_ulonglong
 /// [`c_double`]: core::ffi::c_double
 // We may unseal this trait in the future, but currently our `va_arg` implementations don't support
 // types with an alignment larger than 8, or with a non-scalar layout. Inline assembly can be used
@@ -340,6 +354,34 @@ unsafe impl VaArgSafe for u32 {}
 unsafe impl VaArgSafe for u64 {}
 unsafe impl VaArgSafe for usize {}
 
+// NOTE: this is conservative, we can add more targets later if we're sure 128-bit integers are FFI
+// safe there. This is a subset of targets that define __int128.
+crate::cfg_select! {
+    target_pointer_width = "32" => {
+        // GCC says <https://gcc.gnu.org/onlinedocs/gcc-15.2.0/gcc/_005f_005fint128.html>
+        //
+        // > There is no support in GCC for expressing an integer constant of type __int128 for targets
+        // > with long long integer less than 128 bits wide.
+    }
+    target_env = "msvc" => {
+        // Per https://learn.microsoft.com/en-us/cpp/cpp/data-type-ranges?view=msvc-170, __int128 is
+        // not recognized by msvc.
+    }
+    any(
+        target_arch = "x86_64",
+        target_arch = "aarch64",
+        target_arch = "riscv64",
+        target_arch = "loongarch64",
+        target_arch = "s390x",
+        target_arch = "powerpc64"
+    ) => {
+        unsafe impl VaArgSafe for i128 {}
+        unsafe impl VaArgSafe for u128 {}
+    }
+    _ => {
+    }
+}
+
 unsafe impl VaArgSafe for f64 {}
 
 unsafe impl<T> VaArgSafe for *mut T {}
@@ -352,9 +394,11 @@ const _: () = {
     va_arg_safe_check::<crate::ffi::c_int>();
     va_arg_safe_check::<crate::ffi::c_uint>();
     va_arg_safe_check::<crate::ffi::c_long>();
+
     va_arg_safe_check::<crate::ffi::c_ulong>();
     va_arg_safe_check::<crate::ffi::c_longlong>();
     va_arg_safe_check::<crate::ffi::c_ulonglong>();
+
     va_arg_safe_check::<crate::ffi::c_double>();
 };
 

tests/run-make/c-link-to-rust-va-list-fn/checkrust.rs

@@ -57,6 +57,34 @@ pub unsafe extern "C" fn check_list_copy_0(mut ap: VaList) -> usize {
     if compare_c_str(ap.arg::<*const c_char>(), c"Correct") { 0 } else { 0xff }
 }
 
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn check_list_i128(mut ap: VaList) -> usize {
+    cfg_select! {
+        target_pointer_width = "32" => {}
+        target_env = "msvc" => {}
+        any(
+            target_arch = "x86_64",
+            target_arch = "aarch64",
+            target_arch = "riscv64",
+            target_arch = "loongarch64",
+            target_arch = "s390x",
+            target_arch = "powerpc64"
+        ) => {
+            continue_if!(ap.arg::<i128>() == -42);
+            // use a 32-bit value here to test the alignment logic.
+            continue_if!(ap.arg::<c_int>() == 0xAAAA_AAAAu32.cast_signed());
+            continue_if!(ap.arg::<u128>() == u128::MAX);
+            return 0;
+        }
+        _ => {}
+    }
+
+    // We're running this test on a platform where rustc does not implement
+    // VaArgSafe for i128. The implementation should be added to std if this
+    // comes up.
+    0xFF
+}
+
 #[unsafe(no_mangle)]
 pub unsafe extern "C" fn check_varargs_0(_: c_int, mut ap: ...) -> usize {
     continue_if!(ap.arg::<c_int>() == 42);

tests/run-make/c-link-to-rust-va-list-fn/test.c

@@ -8,6 +8,7 @@ extern size_t check_list_0(va_list ap);
 extern size_t check_list_1(va_list ap);
 extern size_t check_list_2(va_list ap);
 extern size_t check_list_copy_0(va_list ap);
+extern size_t check_list_i128(va_list ap);
 extern size_t check_varargs_0(int fixed, ...);
 extern size_t check_varargs_1(int fixed, ...);
 extern size_t check_varargs_2(int fixed, ...);
@@ -38,6 +39,10 @@ int main(int argc, char* argv[]) {
 
     assert(test_rust(check_list_copy_0, 6.28, 16, 'A', "Skip Me!", "Correct") == 0);
 
+#if defined(__SIZEOF_INT128__)
+    assert(test_rust(check_list_i128, (__int128)-42, 0xAAAAAAAA, (unsigned __int128)-1) == 0);
+#endif
+
     assert(check_varargs_0(0, 42, "Hello, World!") == 0);
 
     assert(check_varargs_1(0, 3.14, 12l, 'A', 0x1LL) == 0);

tests/ui/c-variadic/roundtrip.rs

@@ -12,7 +12,8 @@ use std::ffi::*;
 const unsafe extern "C" fn variadic<T: VaArgSafe>(mut ap: ...) -> (T, T) {
     let x = ap.arg::<T>();
     // Intersperse a small type to test alignment logic. A `u32` (i.e. `c_uint`) is the smallest
-    // type that implements `VaArgSafe`: smaller types would automatically be promoted.
+    // type that implements `VaArgSafe` (except on some 16-bit targets): smaller types would
+    // automatically be promoted.
     assert!(ap.arg::<u32>() == 0xAAAA_AAAA);
     let y = ap.arg::<T>();
 
@@ -74,5 +75,24 @@ fn main() {
         static mut B: u32 = 2u32;
         roundtrip_ptr!(*const u32, &raw const A, &raw const B);
         roundtrip_ptr!(*mut u32, &raw mut A, &raw mut B);
+
+        // The 128-bit integers only implement VaArgSafe on some targets, a subset of those that
+        // define `__int128`. We test some of those targets here.
+        cfg_select! {
+            target_pointer_width = "32" => {}
+            target_env = "msvc" => {}
+            any(
+                target_arch = "x86_64",
+                target_arch = "aarch64",
+                target_arch = "riscv64",
+                target_arch = "loongarch64",
+                target_arch = "s390x",
+                target_arch = "powerpc64"
+            ) => {
+                roundtrip!(i128, -1, -2);
+                roundtrip!(u128, 1, 2);
+            }
+            _ => {}
+        }
     }
 }