interpret: properly check for inhabitedness of nested references

Author: RalfJung

compiler/rustc_const_eval/src/interpret/validity.rs

@@ -35,6 +35,7 @@ use super::{
     format_interp_error,
 };
 use crate::enter_trace_span;
+use crate::interpret::ensure_monomorphic_enough;
 
 // for the validation errors
 #[rustfmt::skip]
@@ -734,11 +735,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
             )
         }
         // Do not allow references to uninhabited types.
-        if place.layout.is_uninhabited() {
+        if !place.layout.ty.is_opsem_inhabited(*self.ecx.tcx, self.ecx.typing_env) {
             let ty = place.layout.ty;
             throw_validation_failure!(
                 self.path,
-                format!("encountered a {ptr_kind} pointing to uninhabited type {ty}")
+                format!("encountered a {ptr_kind} pointing to uninhabited type `{ty}`")
             )
         }
 
@@ -1568,8 +1569,9 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
         }
 
         // Assert that we checked everything there is to check about this type.
+        // `is_opsem_inhabited` implies that the layout is inhabited (checked by layout invariants).
         assert!(
-            !val.layout.is_uninhabited(),
+            val.layout.ty.is_opsem_inhabited(*self.ecx.tcx, self.ecx.typing_env),
             "a value of type `{}` passed validation but that type is uninhabited",
             val.layout.ty
         );
@@ -1627,6 +1629,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     ) -> InterpResult<'tcx> {
         trace!("validate_operand_internal: {:?}, {:?}", *val, val.layout.ty);
 
+        // We can't check validity if there are any generics left.
+        ensure_monomorphic_enough(*self.tcx, val.layout.ty)?;
+
         // Run the visitor.
         self.run_for_validation_mut(|ecx| {
             let reset_padding = reset_provenance_and_padding && {

compiler/rustc_middle/src/queries.rs

@@ -2145,6 +2145,11 @@ rustc_queries! {
         desc { "computing the uninhabited predicate of `{}`", key }
     }
 
+    /// Do not call this query directly: invoke `Ty::is_opsem_inhabited` instead.
+    query is_opsem_inhabited_raw(env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>) -> bool {
+        desc { "computing whether `{}` is inhabited on the opsem level", env.value }
+    }
+
     query crate_dep_kind(_: CrateNum) -> CrateDepKind {
         eval_always
         desc { "fetching what a dependency looks like" }

compiler/rustc_middle/src/ty/inhabitedness/mod.rs

@@ -43,6 +43,9 @@
 //! This code should only compile in modules where the uninhabitedness of `Foo`
 //! is visible.
 
+use std::assert_matches;
+
+use rustc_data_structures::fx::FxHashSet;
 use rustc_type_ir::TyKind::*;
 use tracing::instrument;
 
@@ -54,7 +57,12 @@ pub mod inhabited_predicate;
 pub use inhabited_predicate::InhabitedPredicate;
 
 pub(crate) fn provide(providers: &mut Providers) {
-    *providers = Providers { inhabited_predicate_adt, inhabited_predicate_type, ..*providers };
+    *providers = Providers {
+        inhabited_predicate_adt,
+        inhabited_predicate_type,
+        is_opsem_inhabited_raw,
+        ..*providers
+    };
 }
 
 /// Returns an `InhabitedPredicate` that is generic over type parameters and
@@ -186,14 +194,27 @@ impl<'tcx> Ty<'tcx> {
         self.inhabited_predicate(tcx).apply(tcx, typing_env, module)
     }
 
-    /// Returns true if the type is uninhabited without regard to visibility
+    /// Returns true if the type is uninhabited without regard to visibility.
+    ///
+    /// This is still conservative; for instance, a `#[non_exhaustive]` enum *in another crate*
+    /// is always considered inhabited.
     pub fn is_privately_uninhabited(
         self,
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
     ) -> bool {
         !self.inhabited_predicate(tcx).apply_ignore_module(tcx, typing_env)
     }
+
+    /// Returns whether `self` is considered inhabited on the opsem level, i.e., its validity
+    /// invariant might be satisfiable. `self` is expected to be monomorphic and normalized.
+    pub fn is_opsem_inhabited(self, tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>) -> bool {
+        // Handle simple cases directly, use the query with its cache for the rest.
+        is_opsem_inhabited_recursor(self, tcx, &mut (), /* stop_at_ref */ false, &|ty, _, _| {
+            // ADT handler: stop recursing, invoke the query.
+            tcx.is_opsem_inhabited_raw(typing_env.as_query_input(ty))
+        })
+    }
 }
 
 /// N.B. this query should only be called through `Ty::inhabited_predicate`
@@ -216,3 +237,159 @@ fn inhabited_predicate_type<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> InhabitedP
         _ => bug!("unexpected TyKind, use `Ty::inhabited_predicate`"),
     }
 }
+
+/// Recurse over a type to determine whether it is inhabited on the opsem level.
+/// Key constraints are:
+/// - if a type's validity invariant is satisfiable, it must be opsem-inhabited.
+/// - if a type's layout is marked uninhabited, it must be opsem-uninhabited.
+///
+/// Beyond that, the value returned by this function is not a stable guarantee.
+///
+/// When we encounter an ADT, we call `adt_handler`, giving it as its last argument a closure that
+/// it can invoke to continue the recursion. This lets us share the logic for "simple" cases
+/// (i.e., everything except for ADTs) between `Ty::is_opsem_inhabited` and the query.
+///
+/// `seen` is used to detect infinite recursion: the set contains all ADTs that we encountered
+/// on our path to the current type.
+/// If `stop_at_ref` is true, we stop recursing at the next reference we encounter.
+fn is_opsem_inhabited_recursor<'tcx, SEEN>(
+    ty: Ty<'tcx>,
+    tcx: TyCtxt<'tcx>,
+    seen: &mut SEEN,
+    stop_at_ref: bool,
+    adt_handler: &impl Fn(
+        Ty<'tcx>,
+        &mut SEEN,
+        &dyn Fn(Ty<'tcx>, &mut SEEN, /* stop_at_ref */ bool) -> bool,
+    ) -> bool,
+) -> bool {
+    match *ty.kind() {
+        // Trivially (un)inhabited types
+        ty::Int(_)
+        | ty::Uint(_)
+        | ty::Float(_)
+        | ty::Bool
+        | ty::Char
+        | ty::Str
+        | ty::Foreign(..)
+        | ty::RawPtr(..)
+        | ty::FnPtr(..)
+        | ty::FnDef(..) => true,
+        ty::Dynamic(..) => true, // We can't reason about traits, assume they are inhabited
+        ty::Slice(..) => true,   // Slices can always be empty
+        ty::Never => false,
+
+        // Types where we recurse
+        ty::Ref(_, pointee, _) => {
+            if stop_at_ref {
+                // Bailing out here is safe as the layout code always considers references
+                // inhabited, so the implication ("layout uninhabited => opsem uninhabited")
+                // is upheld.
+                return true;
+            }
+            is_opsem_inhabited_recursor(pointee, tcx, seen, stop_at_ref, adt_handler)
+        }
+        ty::Tuple(tys) => tys
+            .iter()
+            .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler)),
+        ty::Array(elem, len) => {
+            len.try_to_target_usize(tcx).unwrap() == 0
+                || is_opsem_inhabited_recursor(elem, tcx, seen, stop_at_ref, adt_handler)
+        }
+        ty::Pat(inner, _pat) => {
+            is_opsem_inhabited_recursor(inner, tcx, seen, stop_at_ref, adt_handler)
+        }
+        ty::Closure(_def, args) => {
+            let args = args.as_closure();
+            args.upvar_tys()
+                .iter()
+                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler))
+        }
+        ty::Coroutine(_def, args) => {
+            let args = args.as_coroutine();
+            args.upvar_tys()
+                .iter()
+                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler))
+        }
+        ty::CoroutineClosure(_def, args) => {
+            let args = args.as_coroutine_closure();
+            args.upvar_tys()
+                .iter()
+                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler))
+        }
+        ty::UnsafeBinder(base) => {
+            let base = tcx.instantiate_bound_regions_with_erased((*base).into());
+            is_opsem_inhabited_recursor(base, tcx, seen, stop_at_ref, adt_handler)
+        }
+        ty::Adt(..) => {
+            // ADTs need a special handler to avoid infinite recursion. That handler is meant to
+            // call back into the recursor. Ideally it'd just call `is_opsem_inhabited_recursor` but
+            // then it would have to pass itself as the adt_handler argument which is not possible
+            // in Rust... so we provide the handler with a callback that it can use to continue the
+            // recursion with the same `adt_handler`.
+            adt_handler(ty, seen, &|ty, seen, stop_at_ref| {
+                is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler)
+            })
+        }
+
+        ty::Error(_)
+        | ty::Infer(..)
+        | ty::Placeholder(..)
+        | ty::Bound(..)
+        | ty::Param(..)
+        | ty::Alias(..)
+        | ty::CoroutineWitness(..) => {
+            bug!("non-normalized type in `is_opsem_uninhabited`: `{ty}`")
+        }
+    }
+}
+
+fn is_opsem_inhabited_raw<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>,
+) -> bool {
+    let (ty, typing_env) = (env.value, env.typing_env);
+    assert_matches!(
+        ty.kind(),
+        ty::Adt(..),
+        "the query should only be invoked by `Ty::is_opsem_inhabited`"
+    );
+
+    is_opsem_inhabited_recursor(
+        ty,
+        tcx,
+        &mut FxHashSet::<DefId>::default(),
+        /* stop_at_ref */ false,
+        &|ty, seen, rec| {
+            let ty::Adt(adt_def, adt_args) = *ty.kind() else {
+                unreachable! {}
+            };
+            if adt_def.is_union() {
+                // Unions are always inhabited.
+                return true;
+            }
+
+            let new_adt = seen.insert(adt_def.did());
+            // If we have seen this ADT before, stop at the next reference to avoid infinite
+            // recursion. We can't stop here since we have to ensure that "layout inhabited"
+            // implies "opsem inhabited".
+            let stop_at_ref = !new_adt;
+
+            // We are inhabited if in some variant all fields are inhabited.
+            let inhabited = adt_def.variants().iter().any(|variant| {
+                variant.fields.iter().all(|field| {
+                    let ty = field.ty(tcx, adt_args);
+                    let ty = tcx.normalize_erasing_regions(typing_env, ty);
+                    rec(ty, seen, stop_at_ref)
+                })
+            });
+
+            // Remove the type again so that we allow it to appear on other branches.
+            if new_adt {
+                seen.remove(&adt_def.did());
+            }
+
+            inhabited
+        },
+    )
+}

compiler/rustc_ty_utils/src/layout/invariant.rs

@@ -1,6 +1,7 @@
 use std::assert_matches;
 
 use rustc_abi::{BackendRepr, FieldsShape, Scalar, Size, TagEncoding, Variants};
+use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutCx, TyAndLayout};
 use rustc_middle::{bug, ty};
 
@@ -34,6 +35,15 @@ pub(super) fn layout_sanity_check<'tcx>(cx: &LayoutCx<'tcx>, layout: &TyAndLayou
             layout.ty
         );
     }
+    // ABI uninhabitedness should imply opsem uninhabitedness. However, we can only check that if
+    // the type is really monomorphic (while we can compute a layout for some generic types).
+    if layout.is_uninhabited() && !layout.ty.has_param() {
+        assert!(
+            !layout.ty.is_opsem_inhabited(tcx, cx.typing_env),
+            "{:?} is ABI-uninhabited but not opsem-uninhabited?",
+            layout.ty
+        );
+    }
 
     /// Yields non-ZST fields of the type
     fn non_zst_fields<'tcx, 'a>(

src/tools/miri/tests/fail/validity/ref_to_uninhabited1.rs

@@ -3,7 +3,7 @@ use std::mem::{forget, transmute};
 
 fn main() {
     unsafe {
-        let x: Box<!> = transmute(&mut 42); //~ERROR: encountered a box pointing to uninhabited type !
+        let x: Box<!> = transmute(&mut 42); //~ERROR: encountered a box pointing to uninhabited type `!`
         forget(x);
     }
 }

src/tools/miri/tests/fail/validity/ref_to_uninhabited1.stderr

@@ -1,4 +1,4 @@
-error: Undefined Behavior: constructing invalid value of type std::boxed::Box<!>: encountered a box pointing to uninhabited type !
+error: Undefined Behavior: constructing invalid value of type std::boxed::Box<!>: encountered a box pointing to uninhabited type `!`
   --> tests/fail/validity/ref_to_uninhabited1.rs:LL:CC
    |
 LL |         let x: Box<!> = transmute(&mut 42);

src/tools/miri/tests/fail/validity/ref_to_uninhabited2.rs

@@ -4,6 +4,6 @@ enum Void {}
 
 fn main() {
     unsafe {
-        let _x: &(i32, Void) = transmute(&42); //~ERROR: encountered a reference pointing to uninhabited type (i32, Void)
+        let _x: &&(i32, Void) = transmute(&&42); //~ERROR: encountered a reference pointing to uninhabited type `&(i32, Void)`
     }
 }

src/tools/miri/tests/fail/validity/ref_to_uninhabited2.stderr

@@ -1,8 +1,8 @@
-error: Undefined Behavior: constructing invalid value of type &(i32, Void): encountered a reference pointing to uninhabited type (i32, Void)
+error: Undefined Behavior: constructing invalid value of type &&(i32, Void): encountered a reference pointing to uninhabited type `&(i32, Void)`
   --> tests/fail/validity/ref_to_uninhabited2.rs:LL:CC
    |
-LL |         let _x: &(i32, Void) = transmute(&42);
-   |                                ^^^^^^^^^^^^^^ Undefined Behavior occurred here
+LL |         let _x: &&(i32, Void) = transmute(&&42);
+   |                                 ^^^^^^^^^^^^^^^ Undefined Behavior occurred here
    |
    = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information

tests/ui/consts/const-eval/raw-bytes.32bit.stderr

@@ -218,7 +218,7 @@ LL | const DATA_FN_PTR: fn() = unsafe { mem::transmute(&13) };
                ╾ALLOC_ID╼                                     │ ╾──╼
            }
 
-error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type Bar
+error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type `Bar`
   --> $DIR/raw-bytes.rs:110:1
    |
 LL | const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
@@ -458,7 +458,7 @@ LL | const RAW_TRAIT_OBJ_VTABLE_INVALID: *const dyn Trait = unsafe { mem::transm
                ╾ALLOC_ID╼ ╾ALLOC_ID╼                         │ ╾──╼╾──╼
            }
 
-error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type [!; 1]
+error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type `[!; 1]`
   --> $DIR/raw-bytes.rs:188:1
    |
 LL | const _: &[!; 1] = unsafe { &*(1_usize as *const [!; 1]) };

tests/ui/consts/const-eval/raw-bytes.64bit.stderr

@@ -218,7 +218,7 @@ LL | const DATA_FN_PTR: fn() = unsafe { mem::transmute(&13) };
                ╾ALLOC_ID╼                         │ ╾──────╼
            }
 
-error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type Bar
+error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type `Bar`
   --> $DIR/raw-bytes.rs:110:1
    |
 LL | const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
@@ -458,7 +458,7 @@ LL | const RAW_TRAIT_OBJ_VTABLE_INVALID: *const dyn Trait = unsafe { mem::transm
                ╾ALLOC_ID╼ ╾ALLOC_ID╼ │ ╾──────╼╾──────╼
            }
 
-error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type [!; 1]
+error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type `[!; 1]`
   --> $DIR/raw-bytes.rs:188:1
    |
 LL | const _: &[!; 1] = unsafe { &*(1_usize as *const [!; 1]) };