Support u128/i128 c-variadic arguments

Author: folkertdev

compiler/rustc_codegen_llvm/src/va_arg.rs

@@ -1,4 +1,4 @@
-use rustc_abi::{Align, BackendRepr, Endian, HasDataLayout, Primitive, Size};
+use rustc_abi::{Align, BackendRepr, Endian, Float, HasDataLayout, Integer, Primitive, Size};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::OperandRef;
@@ -8,7 +8,7 @@ use rustc_codegen_ssa::traits::{
 use rustc_middle::bug;
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
-use rustc_target::spec::{Arch, Env, LlvmAbi, RustcAbi};
+use rustc_target::spec::{Arch, Env, HasTargetSpec, LlvmAbi, RustcAbi};
 
 use crate::builder::Builder;
 use crate::llvm::{Type, Value};
@@ -30,11 +30,9 @@ fn round_pointer_up_to_alignment<'ll>(
     ptr_ty: &'ll Type,
 ) -> &'ll Value {
     let ptr = bx.inbounds_ptradd(addr, bx.const_i32(align.bytes() as i32 - 1));
-    bx.call_intrinsic(
-        "llvm.ptrmask",
-        &[ptr_ty, bx.type_i32()],
-        &[ptr, bx.const_int(bx.isize_ty, -(align.bytes() as isize) as i64)],
-    )
+    let pointer_width = bx.target_spec().pointer_width;
+    let mask = align.bytes().wrapping_neg() & (u64::MAX >> (64 - pointer_width));
+    bx.call_intrinsic("llvm.ptrmask", &[ptr_ty, bx.type_isize()], &[ptr, bx.const_usize(mask)])
 }
 
 fn emit_direct_ptr_va_arg<'ll, 'tcx>(
@@ -75,6 +73,35 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     }
 }
 
+/// Some backends apply special alignment rules to c-variadic arguments.
+fn get_param_type_alignment<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    layout: TyAndLayout<'tcx>,
+) -> Align {
+    let BackendRepr::Scalar(scalar) = layout.backend_repr else {
+        bug!("unexpected backend repr {:?}", layout.backend_repr);
+    };
+
+    match bx.cx.tcx.sess.target.arch {
+        Arch::PowerPC64 => match scalar.primitive() {
+            Primitive::Int(integer, _) => match integer {
+                Integer::I8 | Integer::I16 => unreachable!(),
+                Integer::I32 | Integer::I64 => { /* fall through */ }
+                Integer::I128 => return Align::EIGHT,
+            },
+            Primitive::Float(float) => match float {
+                Float::F16 | Float::F32 => unreachable!(),
+                Float::F64 => { /* fall through */ }
+                Float::F128 => return Align::from_bytes(16).unwrap(),
+            },
+            Primitive::Pointer(_) => { /* fall through */ }
+        },
+        _ => { /* fall through */ }
+    }
+
+    layout.align.abi
+}
+
 enum PassMode {
     Direct,
     Indirect,
@@ -115,23 +142,23 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
         (
             bx.cx.layout_of(Ty::new_imm_ptr(bx.cx.tcx, target_ty)).llvm_type(bx.cx),
             bx.cx.data_layout().pointer_size(),
-            bx.cx.data_layout().pointer_align(),
+            bx.cx.data_layout().pointer_align().abi,
         )
     } else {
-        (layout.llvm_type(bx.cx), layout.size, layout.align)
+        (layout.llvm_type(bx.cx), layout.size, get_param_type_alignment(bx, layout))
     };
     let (addr, addr_align) = emit_direct_ptr_va_arg(
         bx,
         list,
         size,
-        align.abi,
+        align,
         slot_size,
         allow_higher_align,
         force_right_adjust,
     );
     if indirect {
         let tmp_ret = bx.load(llty, addr, addr_align);
-        bx.load(layout.llvm_type(bx.cx), tmp_ret, align.abi)
+        bx.load(layout.llvm_type(bx.cx), tmp_ret, align)
     } else {
         bx.load(llty, addr, addr_align)
     }
@@ -765,9 +792,16 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
     // byte boundary if alignment needed by type exceeds 8 byte boundary.
     // It isn't stated explicitly in the standard, but in practice we use
     // alignment greater than 16 where necessary.
-    if layout.layout.align.bytes() > 8 {
-        unreachable!("all instances of VaArgSafe have an alignment <= 8");
-    }
+    let overflow_arg_area_v = if layout.layout.align.bytes() > 8 {
+        round_pointer_up_to_alignment(
+            bx,
+            overflow_arg_area_v,
+            layout.layout.align.abi,
+            bx.type_ptr(),
+        )
+    } else {
+        overflow_arg_area_v
+    };
 
     // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
     let mem_addr = overflow_arg_area_v;
@@ -1075,6 +1109,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             PassMode::Direct,
             SlotSize::Bytes8,
             AllowHigherAlign::Yes,
+            // ForceRightAdjust only takes effect on big-endian architectures.
             ForceRightAdjust::Yes,
         ),
         Arch::RiscV32 if target.llvm_abiname == LlvmAbi::Ilp32e => {

library/core/src/ffi/va_list.rs

@@ -268,10 +268,12 @@ mod sealed {
 
     impl Sealed for i32 {}
     impl Sealed for i64 {}
+    impl Sealed for i128 {}
     impl Sealed for isize {}
 
     impl Sealed for u32 {}
     impl Sealed for u64 {}
+    impl Sealed for u128 {}
     impl Sealed for usize {}
 
     impl Sealed for f64 {}
@@ -302,11 +304,13 @@ pub unsafe trait VaArgSafe: sealed::Sealed {}
 // i8 and i16 are implicitly promoted to c_int in C, and cannot implement `VaArgSafe`.
 unsafe impl VaArgSafe for i32 {}
 unsafe impl VaArgSafe for i64 {}
+unsafe impl VaArgSafe for i128 {}
 unsafe impl VaArgSafe for isize {}
 
 // u8 and u16 are implicitly promoted to c_int in C, and cannot implement `VaArgSafe`.
 unsafe impl VaArgSafe for u32 {}
 unsafe impl VaArgSafe for u64 {}
+unsafe impl VaArgSafe for u128 {}
 unsafe impl VaArgSafe for usize {}
 
 // f32 is implicitly promoted to c_double in C, and cannot implement `VaArgSafe`.

tests/run-make/c-link-to-rust-va-list-fn/checkrust.rs

@@ -57,6 +57,15 @@ pub unsafe extern "C" fn check_list_copy_0(mut ap: VaList) -> usize {
     if compare_c_str(ap.arg::<*const c_char>(), c"Correct") { 0 } else { 0xff }
 }
 
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn check_list_i128(mut ap: VaList) -> usize {
+    continue_if!(ap.arg::<i128>() == -42);
+    // use a 32-bit value here to test the alignment logic.
+    continue_if!(ap.arg::<c_int>() == 0xAAAA_AAAAu32.cast_signed());
+    continue_if!(ap.arg::<u128>() == u128::MAX);
+    0
+}
+
 #[unsafe(no_mangle)]
 pub unsafe extern "C" fn check_varargs_0(_: c_int, mut ap: ...) -> usize {
     continue_if!(ap.arg::<c_int>() == 42);
@@ -156,7 +165,7 @@ extern "C" fn run_test_variadic() -> usize {
 
 #[unsafe(no_mangle)]
 extern "C" fn run_test_va_list_by_value() -> usize {
-    unsafe extern "C" fn helper(mut ap: ...) -> usize {
+    unsafe extern "C" fn helper(ap: ...) -> usize {
         unsafe { test_va_list_by_value(ap) }
     }
 

tests/run-make/c-link-to-rust-va-list-fn/test.c

@@ -8,6 +8,7 @@ extern size_t check_list_0(va_list ap);
 extern size_t check_list_1(va_list ap);
 extern size_t check_list_2(va_list ap);
 extern size_t check_list_copy_0(va_list ap);
+extern size_t check_list_i128(va_list ap);
 extern size_t check_varargs_0(int fixed, ...);
 extern size_t check_varargs_1(int fixed, ...);
 extern size_t check_varargs_2(int fixed, ...);
@@ -38,6 +39,10 @@ int main(int argc, char* argv[]) {
 
     assert(test_rust(check_list_copy_0, 6.28, 16, 'A', "Skip Me!", "Correct") == 0);
 
+#if defined(__SIZEOF_INT128__)
+    assert(test_rust(check_list_i128, (__int128)-42, 0xAAAAAAAA, (unsigned __int128)-1) == 0);
+#endif
+
     assert(check_varargs_0(0, 42, "Hello, World!") == 0);
 
     assert(check_varargs_1(0, 3.14, 12l, 'A', 0x1LL) == 0);