interpret: properly check for inhabitedness of nested references

Author: RalfJung

compiler/rustc_const_eval/src/interpret/validity.rs

@@ -35,6 +35,7 @@ use super::{
     format_interp_error,
 };
 use crate::enter_trace_span;
+use crate::interpret::ensure_monomorphic_enough;
 
 // for the validation errors
 #[rustfmt::skip]
@@ -734,11 +735,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
             )
         }
         // Do not allow references to uninhabited types.
-        if place.layout.is_uninhabited() {
+        if !place.layout.ty.is_opsem_inhabited(*self.ecx.tcx, self.ecx.typing_env) {
             let ty = place.layout.ty;
             throw_validation_failure!(
                 self.path,
-                format!("encountered a {ptr_kind} pointing to uninhabited type {ty}")
+                format!("encountered a {ptr_kind} pointing to uninhabited type `{ty}`")
             )
         }
 
@@ -1573,6 +1574,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
             );
         }
         if cfg!(debug_assertions) {
+            assert!(
+                val.layout.ty.is_opsem_inhabited(*self.ecx.tcx, self.ecx.typing_env),
+                "a value of type `{}` somehow passed validity but is not actually inhabited",
+                val.layout.ty,
+            );
             // Check that we don't miss any new changes to layout computation in our checks above.
             match val.layout.backend_repr {
                 BackendRepr::Scalar(scalar_layout) => {
@@ -1626,6 +1632,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     ) -> InterpResult<'tcx> {
         trace!("validate_operand_internal: {:?}, {:?}", *val, val.layout.ty);
 
+        // We can't check validity if there are any generics left.
+        ensure_monomorphic_enough(*self.tcx, val.layout.ty)?;
+
         // Run the visitor.
         self.run_for_validation_mut(|ecx| {
             let reset_padding = reset_provenance_and_padding && {

compiler/rustc_middle/src/queries.rs

@@ -2171,6 +2171,11 @@ rustc_queries! {
         desc { "computing the uninhabited predicate of `{}`", key }
     }
 
+    /// Do not call this query directly: invoke `Ty::is_opsem_inhabited` instead.
+    query is_opsem_inhabited_raw(env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>) -> bool {
+        desc { "computing whether `{}` is inhabited on the opsem level", env.value }
+    }
+
     query crate_dep_kind(_: CrateNum) -> CrateDepKind {
         eval_always
         desc { "fetching what a dependency looks like" }

compiler/rustc_middle/src/ty/inhabitedness/mod.rs

@@ -43,6 +43,7 @@
 //! This code should only compile in modules where the uninhabitedness of `Foo`
 //! is visible.
 
+use rustc_data_structures::fx::FxHashSet;
 use rustc_type_ir::TyKind::*;
 use tracing::instrument;
 
@@ -54,7 +55,12 @@ pub mod inhabited_predicate;
 pub use inhabited_predicate::InhabitedPredicate;
 
 pub(crate) fn provide(providers: &mut Providers) {
-    *providers = Providers { inhabited_predicate_adt, inhabited_predicate_type, ..*providers };
+    *providers = Providers {
+        inhabited_predicate_adt,
+        inhabited_predicate_type,
+        is_opsem_inhabited_raw,
+        ..*providers
+    };
 }
 
 /// Returns an `InhabitedPredicate` that is generic over type parameters and
@@ -186,14 +192,26 @@ impl<'tcx> Ty<'tcx> {
         self.inhabited_predicate(tcx).apply(tcx, typing_env, module)
     }
 
-    /// Returns true if the type is uninhabited without regard to visibility
+    /// Returns true if the type is uninhabited without regard to visibility.
+    ///
+    /// This is still conservative; for instance, a `#[non_exhaustive]` enum *in another crate*
+    /// is always considered inhabited.
     pub fn is_privately_uninhabited(
         self,
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
     ) -> bool {
         !self.inhabited_predicate(tcx).apply_ignore_module(tcx, typing_env)
     }
+
+    /// Returns whether `self` is considered inhabited on the opsem level, i.e., its validity
+    /// invariant might be satisfiable. `self` is expected to be monomorphic and normalized.
+    pub fn is_opsem_inhabited(self, tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>) -> bool {
+        // Handle simple cases directly, use the query with its cache for the rest.
+        is_opsem_inhabited_recursor(self, tcx, &mut FxHashSet::default(), &|ty, _, _| {
+            tcx.is_opsem_inhabited_raw(typing_env.as_query_input(ty))
+        })
+    }
 }
 
 /// N.B. this query should only be called through `Ty::inhabited_predicate`
@@ -216,3 +234,125 @@ fn inhabited_predicate_type<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> InhabitedP
         _ => bug!("unexpected TyKind, use `Ty::inhabited_predicate`"),
     }
 }
+
+/// `seen` is used to bound the recursion: the set contains all ADTs that we encountered
+/// on our path to the current type.
+///
+/// When we encounter an ADT, we call `adt_handler`, giving it as its last argument a closure that
+/// it can invoke to continue the recursion.
+fn is_opsem_inhabited_recursor<'tcx>(
+    ty: Ty<'tcx>,
+    tcx: TyCtxt<'tcx>,
+    seen: &mut FxHashSet<Ty<'tcx>>,
+    adt_handler: &impl Fn(
+        Ty<'tcx>,
+        &mut FxHashSet<Ty<'tcx>>,
+        &dyn Fn(Ty<'tcx>, &mut FxHashSet<Ty<'tcx>>) -> bool,
+    ) -> bool,
+) -> bool {
+    match *ty.kind() {
+        // Trivially (un)inhabited types
+        ty::Int(_)
+        | ty::Uint(_)
+        | ty::Float(_)
+        | ty::Bool
+        | ty::Char
+        | ty::Str
+        | ty::Foreign(..)
+        | ty::RawPtr(..)
+        | ty::FnPtr(..)
+        | ty::FnDef(..) => true,
+        ty::Dynamic(..) => true, // We can't reason about traits, assume they are inhabited
+        ty::Slice(..) => true,   // Slices can always be empty
+        ty::Never => false,
+
+        // Types where we recurse
+        ty::Ref(_, pointee, _) => is_opsem_inhabited_recursor(pointee, tcx, seen, adt_handler),
+        ty::Tuple(tys) => {
+            tys.iter().all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, adt_handler))
+        }
+        ty::Array(elem, len) => {
+            len.try_to_target_usize(tcx).unwrap() == 0
+                || is_opsem_inhabited_recursor(elem, tcx, seen, adt_handler)
+        }
+        ty::Pat(inner, _pat) => is_opsem_inhabited_recursor(inner, tcx, seen, adt_handler),
+        ty::Closure(_def, args) => {
+            let args = args.as_closure();
+            args.upvar_tys()
+                .iter()
+                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, adt_handler))
+        }
+        ty::Coroutine(_def, args) => {
+            let args = args.as_coroutine();
+            args.upvar_tys()
+                .iter()
+                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, adt_handler))
+        }
+        ty::CoroutineClosure(_def, args) => {
+            let args = args.as_coroutine_closure();
+            args.upvar_tys()
+                .iter()
+                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, adt_handler))
+        }
+        ty::UnsafeBinder(..) => {
+            true // FIXME(unsafe_binder) should these really be trivially inhabited?
+        }
+        ty::Adt(..) => {
+            // ADTs need a special handler to avoid infinite recursion.
+            adt_handler(ty, seen, &|ty, seen| {
+                is_opsem_inhabited_recursor(ty, tcx, seen, adt_handler)
+            })
+        }
+
+        ty::Error(_)
+        | ty::Infer(..)
+        | ty::Placeholder(..)
+        | ty::Bound(..)
+        | ty::Param(..)
+        | ty::Alias(..)
+        | ty::CoroutineWitness(..) => {
+            bug!("non-normalized type in `is_opsem_uninhabited_raw::rec`: `{ty}`")
+        }
+    }
+}
+
+fn is_opsem_inhabited_raw<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>,
+) -> bool {
+    let (ty, typing_env) = (env.value, env.typing_env);
+    is_opsem_inhabited_recursor(ty, tcx, &mut FxHashSet::default(), &|ty, seen, rec| {
+        let ty::Adt(adt_def, adt_args) = *ty.kind() else {
+            unreachable! {}
+        };
+        if adt_def.is_union() {
+            // Unions are always inhabited.
+            return true;
+        }
+
+        if !seen.insert(ty) {
+            // We recursed to the same type. Coinductively assume that the type is inhabited.
+            return true;
+        }
+        if !tcx.recursion_limit().value_within_limit(seen.len()) {
+            // We seem to be stuck in an infinite recursion.
+            // Bailing out here is unfortunate as it means that the recursion limit affects the
+            // operational semantics... but what else could we do?
+            return true;
+        }
+
+        // We are inhabited if in some variant all fields are inhabited.
+        let inhabited = adt_def.variants().iter().any(|variant| {
+            variant.fields.iter().all(|field| {
+                let ty = field.ty(tcx, adt_args);
+                let ty = tcx.normalize_erasing_regions(typing_env, ty);
+                rec(ty, seen)
+            })
+        });
+
+        // Remove the type again so that we allow it to appear on other branches.
+        seen.remove(&ty);
+
+        inhabited
+    })
+}

compiler/rustc_ty_utils/src/layout/invariant.rs

@@ -1,6 +1,7 @@
 use std::assert_matches;
 
 use rustc_abi::{BackendRepr, FieldsShape, Scalar, Size, TagEncoding, Variants};
+use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutCx, TyAndLayout};
 use rustc_middle::{bug, ty};
 
@@ -34,6 +35,15 @@ pub(super) fn layout_sanity_check<'tcx>(cx: &LayoutCx<'tcx>, layout: &TyAndLayou
             layout.ty
         );
     }
+    // ABI uninhabitedness should imply opsem uninhabitedness. However, we can only check that if
+    // the type is really monomorphic (while we can compute a layout for some generic types).
+    if layout.is_uninhabited() && !layout.ty.has_param() {
+        assert!(
+            !layout.ty.is_opsem_inhabited(tcx, cx.typing_env),
+            "{:?} is ABI-uninhabited but not opsem-uninhabited?",
+            layout.ty
+        );
+    }
 
     /// Yields non-ZST fields of the type
     fn non_zst_fields<'tcx, 'a>(

src/tools/miri/tests/fail/validity/ref_to_uninhabited1.rs

@@ -3,7 +3,7 @@ use std::mem::{forget, transmute};
 
 fn main() {
     unsafe {
-        let x: Box<!> = transmute(&mut 42); //~ERROR: encountered a box pointing to uninhabited type !
+        let x: Box<!> = transmute(&mut 42); //~ERROR: encountered a box pointing to uninhabited type `!`
         forget(x);
     }
 }

src/tools/miri/tests/fail/validity/ref_to_uninhabited1.stderr

@@ -1,4 +1,4 @@
-error: Undefined Behavior: constructing invalid value of type std::boxed::Box<!>: encountered a box pointing to uninhabited type !
+error: Undefined Behavior: constructing invalid value of type std::boxed::Box<!>: encountered a box pointing to uninhabited type `!`
   --> tests/fail/validity/ref_to_uninhabited1.rs:LL:CC
    |
 LL |         let x: Box<!> = transmute(&mut 42);

src/tools/miri/tests/fail/validity/ref_to_uninhabited2.rs

@@ -4,6 +4,6 @@ enum Void {}
 
 fn main() {
     unsafe {
-        let _x: &(i32, Void) = transmute(&42); //~ERROR: encountered a reference pointing to uninhabited type (i32, Void)
+        let _x: &&(i32, Void) = transmute(&&42); //~ERROR: encountered a reference pointing to uninhabited type `&(i32, Void)`
     }
 }

src/tools/miri/tests/fail/validity/ref_to_uninhabited2.stderr

@@ -1,8 +1,8 @@
-error: Undefined Behavior: constructing invalid value of type &(i32, Void): encountered a reference pointing to uninhabited type (i32, Void)
+error: Undefined Behavior: constructing invalid value of type &&(i32, Void): encountered a reference pointing to uninhabited type `&(i32, Void)`
   --> tests/fail/validity/ref_to_uninhabited2.rs:LL:CC
    |
-LL |         let _x: &(i32, Void) = transmute(&42);
-   |                                ^^^^^^^^^^^^^^ Undefined Behavior occurred here
+LL |         let _x: &&(i32, Void) = transmute(&&42);
+   |                                 ^^^^^^^^^^^^^^^ Undefined Behavior occurred here
    |
    = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information

tests/ui/consts/const-eval/raw-bytes.32bit.stderr

@@ -218,7 +218,7 @@ LL | const DATA_FN_PTR: fn() = unsafe { mem::transmute(&13) };
                ╾ALLOC_ID╼                                     │ ╾──╼
            }
 
-error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type Bar
+error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type `Bar`
   --> $DIR/raw-bytes.rs:110:1
    |
 LL | const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
@@ -458,7 +458,7 @@ LL | const RAW_TRAIT_OBJ_VTABLE_INVALID: *const dyn Trait = unsafe { mem::transm
                ╾ALLOC_ID╼ ╾ALLOC_ID╼                         │ ╾──╼╾──╼
            }
 
-error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type [!; 1]
+error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type `[!; 1]`
   --> $DIR/raw-bytes.rs:188:1
    |
 LL | const _: &[!; 1] = unsafe { &*(1_usize as *const [!; 1]) };

tests/ui/consts/const-eval/raw-bytes.64bit.stderr

@@ -218,7 +218,7 @@ LL | const DATA_FN_PTR: fn() = unsafe { mem::transmute(&13) };
                ╾ALLOC_ID╼                         │ ╾──────╼
            }
 
-error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type Bar
+error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type `Bar`
   --> $DIR/raw-bytes.rs:110:1
    |
 LL | const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
@@ -458,7 +458,7 @@ LL | const RAW_TRAIT_OBJ_VTABLE_INVALID: *const dyn Trait = unsafe { mem::transm
                ╾ALLOC_ID╼ ╾ALLOC_ID╼ │ ╾──────╼╾──────╼
            }
 
-error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type [!; 1]
+error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type `[!; 1]`
   --> $DIR/raw-bytes.rs:188:1
    |
 LL | const _: &[!; 1] = unsafe { &*(1_usize as *const [!; 1]) };