c-variadic: more precise compatibility check in const-eval

folkertdev · folkertdev · commit 6aedccd296de · 2026-04-26T18:48:18.000+02:00
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -8,6 +8,7 @@ use std::assert_matches;
 
 use rustc_abi::{FieldIdx, HasDataLayout, Size, VariantIdx};
 use rustc_apfloat::ieee::{Double, Half, Quad, Single};
+use rustc_ast::{IntTy, UintTy};
 use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
 use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
 use rustc_middle::ty::layout::TyAndLayout;
@@ -21,9 +22,9 @@ use super::util::ensure_monomorphic_enough;
 use super::{
     AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer,
     PointerArithmetic, Projectable, Provenance, Scalar, err_ub_format, err_unsup_format, interp_ok,
-    throw_inval, throw_ub, throw_ub_format, throw_unsup_format,
+    throw_inval, throw_ub, throw_ub_format,
 };
-use crate::interpret::Writeable;
+use crate::interpret::{MPlaceTy, Writeable};
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 enum MulAddType {
@@ -56,6 +57,14 @@ pub(crate) enum MinMax {
     MaximumNumberNsz,
 }
 
+enum VarArgCompatible {
+    Identical,
+    Incompatible,
+    CastSignedToUnsigned(IntTy),
+    CastUnsignedToSigned(UintTy),
+    ValidPtrCast,
+}
+
 /// Directly returns an `Allocation` containing an absolute path representation of the given type.
 pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
     let path = crate::util::type_name(tcx, ty);
@@ -739,18 +748,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     throw_ub!(VaArgOutOfBounds);
                 };
 
-                // NOTE: In C some type conversions are allowed (e.g. casting between signed and
-                // unsigned integers). For now we require c-variadic arguments to be read with the
-                // exact type they were passed as.
-                if arg_mplace.layout.ty != dest.layout.ty {
-                    throw_unsup_format!(
-                        "va_arg type mismatch: requested `{}`, but next argument is `{}`",
-                        dest.layout.ty,
-                        arg_mplace.layout.ty
-                    );
-                }
-                // Copy the argument.
-                self.copy_op(&arg_mplace, dest)?;
+                // Error when the caller's argument is not c-variadic compatible with the type
+                // requested by the callee.
+                self.validate_c_variadic_compatible(&arg_mplace, dest.layout)?;
+
+                // Copy the argument, allowing a transmute and relying on the compatibility check
+                // rejecting conversions between types of different size.
+                self.copy_op_allow_transmute(&arg_mplace, dest)?;
 
                 // Update the VaList pointer.
                 let new_key = self.va_list_ptr(varargs);
@@ -766,6 +770,149 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         interp_ok(true)
     }
 
+    /// Validate whether the value and type passed by the caller are compatible with the type
+    /// requested by the callee. Based on section 7.16.1.1 of the C23 specification.
+    ///
+    /// The caller requesting a value of a type is valid when that type is compatible with the type
+    /// provided by the caller (see `validate_c_variadic_compatible_ty`) and, if both types are
+    /// integers of different signedness, the passed value must be representable in both types.
+    fn validate_c_variadic_compatible(
+        &mut self,
+        arg_mplace: &MPlaceTy<'tcx, M::Provenance>,
+        callee_type: TyAndLayout<'tcx>,
+    ) -> InterpResult<'tcx> {
+        // Identical types are clearly compatible.
+        if arg_mplace.layout.ty == callee_type.ty {
+            return interp_ok(());
+        }
+
+        // Types of different sizes can never be compatible.
+        if arg_mplace.layout.size != callee_type.size {
+            throw_ub_format!(
+                "va_arg type mismatch: requested `{}` is incompatible with next argument of type `{}`",
+                callee_type.ty,
+                arg_mplace.layout.ty,
+            )
+        }
+
+        fn validate_cast<'tcx, T, U>(x: T, target_ty: Ty<'tcx>) -> InterpResult<'tcx>
+        where
+            T: std::fmt::Display + Copy,
+            U: std::fmt::Display + TryFrom<T>,
+        {
+            if U::try_from(x).is_ok() {
+                return interp_ok(());
+            }
+
+            throw_ub_format!(
+                "va_arg value mismatch: value `{x}` cannot be represented by type {target_ty}",
+            )
+        }
+
+        match self.validate_c_variadic_compatible_ty(arg_mplace.layout.ty, callee_type.ty) {
+            VarArgCompatible::Identical => interp_ok(()),
+            VarArgCompatible::ValidPtrCast => interp_ok(()),
+            VarArgCompatible::Incompatible => throw_ub_format!(
+                "va_arg type mismatch: requested `{}` is incompatible with next argument of type `{}`",
+                callee_type.ty,
+                arg_mplace.layout.ty,
+            ),
+            VarArgCompatible::CastSignedToUnsigned(int_ty) => {
+                // Check that the value can be represented in the target type.
+                let callee_ty = callee_type.ty;
+                let scalar = self.read_scalar(arg_mplace)?;
+                match int_ty {
+                    IntTy::Isize => match self.data_layout().pointer_size().bits() {
+                        16 => validate_cast::<i16, u16>(scalar.to_i16()?, callee_ty)?,
+                        32 => validate_cast::<i32, u32>(scalar.to_i32()?, callee_ty)?,
+                        64 => validate_cast::<i64, u64>(scalar.to_i64()?, callee_ty)?,
+                        _ => unreachable!("unexpected pointer size"),
+                    },
+                    IntTy::I8 => validate_cast::<i8, u8>(scalar.to_i8()?, callee_ty)?,
+                    IntTy::I16 => validate_cast::<i16, u16>(scalar.to_i16()?, callee_ty)?,
+                    IntTy::I32 => validate_cast::<i32, u32>(scalar.to_i32()?, callee_ty)?,
+                    IntTy::I64 => validate_cast::<i64, u64>(scalar.to_i64()?, callee_ty)?,
+                    IntTy::I128 => validate_cast::<i128, u128>(scalar.to_i128()?, callee_ty)?,
+                };
+
+                interp_ok(())
+            }
+            VarArgCompatible::CastUnsignedToSigned(uint_ty) => {
+                // Check that the value can be represented in the target type.
+                let callee_ty = callee_type.ty;
+                let scalar = self.read_scalar(arg_mplace)?;
+                match uint_ty {
+                    UintTy::Usize => match self.data_layout().pointer_size().bits() {
+                        16 => validate_cast::<u16, i16>(scalar.to_u16()?, callee_ty)?,
+                        32 => validate_cast::<u32, i32>(scalar.to_u32()?, callee_ty)?,
+                        64 => validate_cast::<u64, i64>(scalar.to_u64()?, callee_ty)?,
+                        _ => unreachable!("unexpected pointer size"),
+                    },
+                    UintTy::U8 => validate_cast::<u8, i8>(scalar.to_u8()?, callee_ty)?,
+                    UintTy::U16 => validate_cast::<u16, i16>(scalar.to_u16()?, callee_ty)?,
+                    UintTy::U32 => validate_cast::<u32, i32>(scalar.to_u32()?, callee_ty)?,
+                    UintTy::U64 => validate_cast::<u64, i64>(scalar.to_u64()?, callee_ty)?,
+                    UintTy::U128 => validate_cast::<u128, i128>(scalar.to_u128()?, callee_ty)?,
+                };
+
+                interp_ok(())
+            }
+        }
+    }
+
+    /// Check whether the caller and callee type are compatible for c-variadic calls. Further
+    /// validation of the argument value may be needed to detect all UB.
+    ///
+    /// Types `T` and `U` are compatible when:
+    ///
+    /// - `T` and `U` are the same type.
+    /// - `T` is a signed integer and `U` the corresponding unsigned integer,
+    /// - `T` is an unsigned integer and `U` the corresponding signed integer,
+    /// - `T` and `U` are both pointers, and their target types are compatible.
+    /// - `T` is a pointer to `c_void` and `U` is a pointer to `i8` or `u8`.
+    /// - `T` is a pointer to `i8` or `u8` and `U` is a pointer to `c_void`.
+    fn validate_c_variadic_compatible_ty(
+        &mut self,
+        caller_type: Ty<'tcx>,
+        callee_type: Ty<'tcx>,
+    ) -> VarArgCompatible {
+        if caller_type == callee_type {
+            return VarArgCompatible::Identical;
+        }
+
+        // The signedness of c_char is irrelevant here.
+        let is_c_char = |ty: Ty<'_>| matches!(ty.kind(), ty::Uint(UintTy::U8) | ty::Int(IntTy::I8));
+
+        match (caller_type.kind(), callee_type.kind()) {
+            (ty::RawPtr(caller_target_ty, _), ty::RawPtr(callee_target_ty, _)) => {
+                // Accept the cast if one type is pointer to qualified or unqualified void,
+                // and the other is a pointer to a qualified or unqualified character type.
+                if caller_target_ty.is_c_void(self.tcx.tcx) && is_c_char(*callee_target_ty) {
+                    return VarArgCompatible::ValidPtrCast;
+                }
+                if callee_target_ty.is_c_void(self.tcx.tcx) && is_c_char(*caller_target_ty) {
+                    return VarArgCompatible::ValidPtrCast;
+                }
+
+                // Accept the cast if both types are pointers to qualified or unqualified versions
+                // of compatible types.
+                match self.validate_c_variadic_compatible_ty(*caller_target_ty, *callee_target_ty) {
+                    VarArgCompatible::Incompatible => VarArgCompatible::Incompatible,
+                    _ => VarArgCompatible::ValidPtrCast,
+                }
+            }
+            (ty::Int(int_ty), ty::Uint(uint_ty)) => {
+                assert_eq!(int_ty.bit_width(), uint_ty.bit_width());
+                VarArgCompatible::CastSignedToUnsigned(*int_ty)
+            }
+            (ty::Uint(uint_ty), ty::Int(int_ty)) => {
+                assert_eq!(uint_ty.bit_width(), int_ty.bit_width());
+                VarArgCompatible::CastUnsignedToSigned(*uint_ty)
+            }
+            _ => VarArgCompatible::Incompatible,
+        }
+    }
+
     pub(super) fn eval_nondiverging_intrinsic(
         &mut self,
         intrinsic: &NonDivergingIntrinsic<'tcx>,
diff --git a/library/core/src/ffi/va_list.rs b/library/core/src/ffi/va_list.rs
@@ -391,10 +391,24 @@ impl<'f> VaList<'f> {
     /// # Safety
     ///
     /// This function is only sound to call when there is another argument to read, and that
-    /// argument is a properly initialized value of the type `T`.
+    /// argument is a properly initialized value compatible with the type `T`.
     ///
     /// Calling this function with an incompatible type, an invalid value, or when there
     /// are no more variable arguments, is unsound.
+    ///
+    /// Types `T` and `U` are compatible when:
+    ///
+    /// - `T` and `U` are the same type.
+    /// - `T` is a signed integer and `U` the corresponding unsigned integer,
+    /// - `T` is an unsigned integer and `U` the corresponding signed integer,
+    /// - `T` and `U` are both pointers, and their target types are compatible.
+    /// - `T` is a pointer to [`c_void`] and `U` is a pointer to [`i8]` or [`u8`].
+    /// - `T` is a pointer to [`i8]` or [`u8`] and `U` is a pointer to [`c_void`].
+    ///
+    /// When `T` and `U` are both integer types, the value that the caller passes must be
+    /// representable by `U`.
+    ///
+    /// [`c_void`]: core::ffi::c_void
     #[inline] // Avoid codegen when not used to help backends that don't support VaList.
     #[rustc_const_unstable(feature = "const_c_variadic", issue = "151787")]
     pub const unsafe fn next_arg<T: VaArgSafe>(&mut self) -> T {
diff --git a/tests/ui/consts/const-eval/c-variadic-fail.rs b/tests/ui/consts/const-eval/c-variadic-fail.rs
@@ -6,7 +6,7 @@
 #![feature(const_destruct)]
 #![feature(const_clone)]
 
-use std::ffi::VaList;
+use std::ffi::{VaList, c_char, c_void};
 use std::mem::MaybeUninit;
 
 const unsafe extern "C" fn read_n<const N: usize>(mut ap: ...) {
@@ -37,7 +37,7 @@ const unsafe extern "C" fn read_as<T: core::ffi::VaArgSafe>(mut ap: ...) -> T {
     ap.next_arg::<T>()
 }
 
-unsafe fn read_cast() {
+unsafe fn read_cast_numeric() {
     const { read_as::<i32>(1i32) };
     const { read_as::<u32>(1u32) };
 
@@ -47,20 +47,48 @@ unsafe fn read_cast() {
     const { read_as::<i64>(1i64) };
     const { read_as::<u64>(1u64) };
 
+    // A cast between signed and unsigned is OK so long as both types can represent the value.
     const { read_as::<u32>(1i32) };
-    //~^ ERROR va_arg type mismatch: requested `u32`, but next argument is `i32`
-
     const { read_as::<i32>(1u32) };
-    //~^ ERROR va_arg type mismatch: requested `i32`, but next argument is `u32`
+
+    const { read_as::<u32>(-1i32) };
+    //~^ ERROR va_arg value mismatch: value `-1` cannot be represented by type u32
+    const { read_as::<i32>(u32::MAX) };
+    //~^ ERROR va_arg value mismatch: value `4294967295` cannot be represented by type i32
 
     const { read_as::<i32>(1u64) };
-    //~^ ERROR va_arg type mismatch: requested `i32`, but next argument is `u64`
+    //~^ ERROR va_arg type mismatch: requested `i32` is incompatible with next argument of type `u64`
 
     const { read_as::<f64>(1i32) };
-    //~^ ERROR va_arg type mismatch: requested `f64`, but next argument is `i32`
+    //~^ ERROR va_arg type mismatch: requested `f64` is incompatible with next argument of type `i32`
+}
 
-    const { read_as::<*const u8>(1i32) };
-    //~^ ERROR va_arg type mismatch: requested `*const u8`, but next argument is `i32`
+unsafe fn read_cast_pointer() {
+    // A pointer mutability cast is OK.
+    const { read_as::<*const i32>(std::ptr::dangling_mut::<i32>()) };
+    const { read_as::<*mut i32>(std::ptr::dangling::<i32>()) };
+
+    // A pointer cast is OK between compatible types.
+    const { read_as::<*const i32>(std::ptr::dangling::<u32>()) };
+    const { read_as::<*const i32>(std::ptr::dangling_mut::<u32>()) };
+    const { read_as::<*mut i32>(std::ptr::dangling::<u32>()) };
+    const { read_as::<*mut i32>(std::ptr::dangling_mut::<u32>()) };
+
+    // Casting between pointers to i8/u8 and c_void is OK.
+    const { read_as::<*const c_char>(std::ptr::dangling::<c_void>()) };
+    const { read_as::<*const c_void>(std::ptr::dangling::<c_char>()) };
+    const { read_as::<*const i8>(std::ptr::dangling::<c_void>()) };
+    const { read_as::<*const c_void>(std::ptr::dangling::<i8>()) };
+    const { read_as::<*const u8>(std::ptr::dangling::<c_void>()) };
+    const { read_as::<*const c_void>(std::ptr::dangling::<u8>()) };
+
+    const { read_as::<*const u16>(std::ptr::dangling::<c_void>()) };
+    //~^ ERROR va_arg type mismatch: requested `*const u16` is incompatible with next argument of type `*const c_void`
+    const { read_as::<*const c_void>(std::ptr::dangling::<u16>()) };
+    //~^ ERROR va_arg type mismatch: requested `*const c_void` is incompatible with next argument of type `*const u16`
+
+    const { read_as::<*const u8>(1usize) };
+    //~^ ERROR requested `*const u8` is incompatible with next argument of type `usize`
 }
 
 fn use_after_free() {
@@ -138,7 +166,8 @@ fn drop_of_invalid() {
 fn main() {
     unsafe {
         read_too_many();
-        read_cast();
+        read_cast_numeric();
+        read_cast_pointer();
         manual_copy_read();
         manual_copy_drop();
         manual_copy_forget();
diff --git a/tests/ui/consts/const-eval/c-variadic-fail.stderr b/tests/ui/consts/const-eval/c-variadic-fail.stderr
