Support u128/i128 c-variadic arguments

On platforms where `clang` defines `__int128`.

Author: folkertdev

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -3,8 +3,7 @@ use std::ffi::c_uint;
 use std::{assert_matches, iter, ptr};
 
 use rustc_abi::{
-    Align, BackendRepr, Float, HasDataLayout, Integer, NumScalableVectors, Primitive, Size,
-    WrappingRange,
+    Align, BackendRepr, Float, HasDataLayout, NumScalableVectors, Primitive, Size, WrappingRange,
 };
 use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
@@ -298,11 +297,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     Primitive::Pointer(_) => {
                         // Pointers are always OK.
                     }
-                    Primitive::Int(Integer::I128, _) => {
-                        // FIXME: maybe we should support these? At least on 32-bit powerpc
-                        // the logic in LLVM does not handle i128 correctly though.
-                        bug!("the va_arg intrinsic does not support `i128`/`u128`")
-                    }
                     Primitive::Int(..) => {
                         let int_width = self.cx().size_of(result.layout.ty).bits();
                         let target_c_int_width = self.cx().sess().target.options.c_int_width;

compiler/rustc_codegen_llvm/src/va_arg.rs

@@ -1,4 +1,4 @@
-use rustc_abi::{Align, BackendRepr, Endian, HasDataLayout, Primitive, Size};
+use rustc_abi::{Align, BackendRepr, Endian, Float, HasDataLayout, Integer, Primitive, Size};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::OperandRef;
@@ -77,6 +77,35 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     }
 }
 
+/// Some backends apply special alignment rules to c-variadic arguments.
+fn get_param_type_alignment<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    layout: TyAndLayout<'tcx>,
+) -> Align {
+    let BackendRepr::Scalar(scalar) = layout.backend_repr else {
+        bug!("unexpected backend repr {:?}", layout.backend_repr);
+    };
+
+    match bx.cx.tcx.sess.target.arch {
+        Arch::PowerPC64 => match scalar.primitive() {
+            Primitive::Int(integer, _) => match integer {
+                Integer::I8 | Integer::I16 => unreachable!(),
+                Integer::I32 | Integer::I64 => { /* fall through */ }
+                Integer::I128 => return Align::EIGHT,
+            },
+            Primitive::Float(float) => match float {
+                Float::F16 | Float::F32 => unreachable!(),
+                Float::F64 => { /* fall through */ }
+                Float::F128 => return Align::from_bytes(16).unwrap(),
+            },
+            Primitive::Pointer(_) => { /* fall through */ }
+        },
+        _ => { /* fall through */ }
+    }
+
+    layout.align.abi
+}
+
 enum PassMode {
     Direct,
     Indirect,
@@ -136,23 +165,23 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
         (
             bx.cx.layout_of(Ty::new_imm_ptr(bx.cx.tcx, target_ty)).llvm_type(bx.cx),
             bx.cx.data_layout().pointer_size(),
-            bx.cx.data_layout().pointer_align(),
+            bx.cx.data_layout().pointer_align().abi,
         )
     } else {
-        (layout.llvm_type(bx.cx), layout.size, layout.align)
+        (layout.llvm_type(bx.cx), layout.size, get_param_type_alignment(bx, layout))
     };
     let (addr, addr_align) = emit_direct_ptr_va_arg(
         bx,
         list,
         size,
-        align.abi,
+        align,
         slot_size,
         allow_higher_align,
         force_right_adjust,
     );
     if indirect {
         let tmp_ret = bx.load(llty, addr, addr_align);
-        bx.load(layout.llvm_type(bx.cx), tmp_ret, align.abi)
+        bx.load(layout.llvm_type(bx.cx), tmp_ret, align)
     } else {
         bx.load(llty, addr, addr_align)
     }
@@ -585,8 +614,10 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
     // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
     // l->fp_offset > 176 - num_fp * 16 go to step 7.
 
+    // We support x86_64-unknown-linux-gnux32 which uses 4-byte pointers.
     let unsigned_int_offset = 4;
-    let ptr_offset = 8;
+    let ptr_offset = bx.tcx().data_layout.pointer_size().bytes();
+
     let gp_offset_ptr = va_list_addr;
     let fp_offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(unsigned_int_offset));
 
@@ -660,7 +691,7 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
                     let reg_hi_addr = bx.inbounds_ptradd(reg_lo_addr, bx.const_i32(16));
 
                     let align = layout.layout.align().abi;
-                    let tmp = bx.alloca(layout.layout.size(), align);
+                    let tmp = bx.alloca_with_ty(layout);
 
                     let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
                     let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
@@ -683,7 +714,7 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
                         Primitive::Int(_, _) | Primitive::Pointer(_) => (gp_addr, fp_addr),
                     };
 
-                    let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
+                    let tmp = bx.alloca_with_ty(layout);
 
                     let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
                     let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
@@ -751,7 +782,7 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>(
     src_align: Align,
 ) -> &'ll Value {
     if layout.layout.align.abi > src_align {
-        let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
+        let tmp = bx.alloca_with_ty(layout);
         bx.memcpy(
             tmp,
             layout.layout.align.abi,
@@ -782,9 +813,11 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
     // byte boundary if alignment needed by type exceeds 8 byte boundary.
     // It isn't stated explicitly in the standard, but in practice we use
     // alignment greater than 16 where necessary.
-    if layout.layout.align.bytes() > 8 {
-        unreachable!("all instances of VaArgSafe have an alignment <= 8");
-    }
+    let overflow_arg_area_v = if layout.layout.align.bytes() > 8 {
+        round_pointer_up_to_alignment(bx, overflow_arg_area_v, layout.layout.align.abi)
+    } else {
+        overflow_arg_area_v
+    };
 
     // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
     let mem_addr = overflow_arg_area_v;
@@ -1091,6 +1124,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             PassMode::Direct,
             SlotSize::Bytes8,
             AllowHigherAlign::Yes,
+            // ForceRightAdjust only takes effect on big-endian architectures.
             ForceRightAdjust::Yes,
         ),
         Arch::RiscV32 if target.llvm_abiname == LlvmAbi::Ilp32e => {

library/core/src/ffi/va_list.rs

@@ -280,6 +280,15 @@ mod sealed {
     impl Sealed for f32 {}
     impl Sealed for f64 {}
 
+    // The unstable annotation here is not needed, but makes sure that the feature gate is defined
+    // on all targets, even though the actual instances of VaArgSafe for i128/u128 are not.
+    #[unstable_feature_bound(c_variadic_int128)]
+    #[unstable(feature = "c_variadic_int128", issue = "155752")]
+    impl Sealed for i128 {}
+    #[unstable_feature_bound(c_variadic_int128)]
+    #[unstable(feature = "c_variadic_int128", issue = "155752")]
+    impl Sealed for u128 {}
+
     impl<T> Sealed for *mut T {}
     impl<T> Sealed for *const T {}
 }
@@ -304,6 +313,9 @@ mod sealed {
 /// and [`c_float`] is promoted to [`c_double`]. Implementing this trait for types that are
 /// subject to this promotion rule is invalid.
 ///
+/// This trait is only implemented for 128-bit integers when the platform defines the `__int128`
+/// type.
+///
 /// [`c_int`]: core::ffi::c_int
 /// [`c_long`]: core::ffi::c_long
 /// [`c_longlong`]: core::ffi::c_longlong
@@ -357,6 +369,51 @@ unsafe impl VaArgSafe for u32 {}
 unsafe impl VaArgSafe for u64 {}
 unsafe impl VaArgSafe for usize {}
 
+// Implement `VaArgSafe` for 128-bit integers on targets where clang provides `__int128`.
+//
+// GCC does not implement `__int128` for any 16-bit/32-bit target:
+//
+// https://gcc.gnu.org/onlinedocs/gcc-15.2.0/gcc/_005f_005fint128.html
+//
+// > There is no support in GCC for expressing an integer constant of type __int128 for targets
+// > with long long integer less than 128 bits wide.
+//
+// Per https://learn.microsoft.com/en-us/cpp/cpp/data-type-ranges?view=msvc-170, MSVC does not
+// define `__int128`.
+//
+// Clang is slightly more permissive: it defines `__int128` on wasm32 (a 32-bit target) and also
+// does provide `__int128` on 64-bit `*-pc-windows-msvc`, and we follow suit.
+cfg_select! {
+    any(
+        target_arch = "aarch64",
+        target_arch = "amdgpu",
+        target_arch = "arm64ec",
+        target_arch = "bpf",
+        target_arch = "loongarch64",
+        target_arch = "mips64",
+        target_arch = "mips64r6",
+        target_arch = "nvptx64",
+        target_arch = "powerpc64",
+        target_arch = "riscv64",
+        target_arch = "s390x",
+        target_arch = "sparc64",
+        target_arch = "wasm32",
+        target_arch = "wasm64",
+        target_arch = "x86_64",
+    ) => {
+        #[cfg(not(any(target_arch = "wasm32", target_abi = "x32", target_pointer_width = "64")))]
+        compile_error!("unexpected target architecture for 128-bit c-variadic");
+
+        #[unstable_feature_bound(c_variadic_int128)]
+        #[unstable(feature = "c_variadic_int128", issue = "155752")]
+        unsafe impl VaArgSafe for i128 {}
+        #[unstable_feature_bound(c_variadic_int128)]
+        #[unstable(feature = "c_variadic_int128", issue = "155752")]
+        unsafe impl VaArgSafe for u128 {}
+    }
+    _ => {}
+}
+
 unsafe impl VaArgSafe for f64 {}
 
 unsafe impl<T> VaArgSafe for *mut T {}

tests/assembly-llvm/c-variadic/sparc.rs

@@ -20,6 +20,8 @@ pub unsafe trait VaArgSafe {}
 
 unsafe impl VaArgSafe for i32 {}
 unsafe impl VaArgSafe for i64 {}
+#[cfg(target_pointer_width = "64")]
+unsafe impl VaArgSafe for i128 {}
 unsafe impl VaArgSafe for f64 {}
 unsafe impl<T> VaArgSafe for *const T {}
 
@@ -104,6 +106,22 @@ unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
     va_arg(ap)
 }
 
+#[unsafe(no_mangle)]
+#[cfg(target_pointer_width = "64")]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // SPARC64-LABEL: read_i128
+    // SPARC64: ldx [%o0], %o1
+    // SPARC64-NEXT: add %o1, 15, %o1
+    // SPARC64-NEXT: and %o1, -16, %o1
+    // SPARC64-NEXT: add %o1, 16, %o2
+    // SPARC64-NEXT: stx %o2, [%o0]
+    // SPARC64-NEXT: ldx [%o1], %o0
+    // SPARC64-NEXT: or %o1, 8, %o1
+    // SPARC64-NEXT: retl
+    // SPARC64-NEXT: ldx [%o1], %o1
+    va_arg(ap)
+}
+
 #[unsafe(no_mangle)]
 unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
     // SPARC: read_ptr = read_i32

tests/run-make/c-link-to-rust-va-list-fn/checkrust.rs

@@ -1,5 +1,5 @@
 #![crate_type = "staticlib"]
-#![feature(c_variadic)]
+#![feature(c_variadic, c_variadic_int128)]
 
 use core::ffi::{CStr, VaList, c_char, c_double, c_int, c_long, c_longlong};
 
@@ -57,6 +57,49 @@ pub unsafe extern "C" fn check_list_copy_0(mut ap: VaList) -> usize {
     if compare_c_str(ap.next_arg::<*const c_char>(), c"Correct") { 0 } else { 0xff }
 }
 
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn check_list_i128(mut ap: VaList) -> usize {
+    cfg_select! {
+        any(
+            target_arch = "aarch64",
+            target_arch = "amdgpu",
+            target_arch = "arm64ec",
+            target_arch = "bpf",
+            target_arch = "loongarch64",
+            target_arch = "mips64",
+            target_arch = "mips64r6",
+            target_arch = "nvptx64",
+            target_arch = "powerpc64",
+            target_arch = "riscv64",
+            target_arch = "s390x",
+            target_arch = "sparc64",
+            target_arch = "wasm32",
+            target_arch = "wasm64",
+            target_arch = "x86_64",
+        ) => {
+            #[cfg(not(any(
+                target_arch = "wasm32",
+                target_abi = "x32",
+                target_pointer_width = "64",
+            )))]
+            compile_error!("unexpected target architecture for 128-bit c-variadic");
+
+            continue_if!(ap.next_arg::<i128>() == -42);
+            // use a 32-bit value here to test the alignment logic.
+            continue_if!(ap.next_arg::<c_int>() == 0xAAAA_AAAAu32.cast_signed());
+            continue_if!(ap.next_arg::<u128>() == u128::MAX);
+
+            return 0;
+        }
+        _ => {
+            // This function was called a platform where rustc does not implement
+            // VaArgSafe for i128 but clang does define __int128. Rustc should add
+            // the implementation if this comes up.
+            0xFF
+        }
+    }
+}
+
 #[unsafe(no_mangle)]
 pub unsafe extern "C" fn check_varargs_0(_: c_int, mut ap: ...) -> usize {
     continue_if!(ap.next_arg::<c_int>() == 42);

tests/run-make/c-link-to-rust-va-list-fn/test.c

@@ -8,6 +8,7 @@ extern size_t check_list_0(va_list ap);
 extern size_t check_list_1(va_list ap);
 extern size_t check_list_2(va_list ap);
 extern size_t check_list_copy_0(va_list ap);
+extern size_t check_list_i128(va_list ap);
 extern size_t check_varargs_0(int fixed, ...);
 extern size_t check_varargs_1(int fixed, ...);
 extern size_t check_varargs_2(int fixed, ...);
@@ -38,6 +39,10 @@ int main(int argc, char* argv[]) {
 
     assert(test_rust(check_list_copy_0, 6.28, 16, 'A', "Skip Me!", "Correct") == 0);
 
+#if defined(__SIZEOF_INT128__)
+    assert(test_rust(check_list_i128, (__int128)-42, 0xAAAAAAAA, (unsigned __int128)-1) == 0);
+#endif
+
     assert(check_varargs_0(0, 42, "Hello, World!") == 0);
 
     assert(check_varargs_1(0, 3.14, 12l, 'A', 0x1LL) == 0);