interpret: properly check for inhabitedness of nested references

Author: RalfJung

compiler/rustc_const_eval/src/interpret/validity.rs

@@ -734,11 +734,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
             )
         }
         // Do not allow references to uninhabited types.
-        if place.layout.is_uninhabited() {
+        if !place.layout.ty.is_opsem_inhabited(*self.ecx.tcx, self.ecx.typing_env) {
             let ty = place.layout.ty;
             throw_validation_failure!(
                 self.path,
-                format!("encountered a {ptr_kind} pointing to uninhabited type {ty}")
+                format!("encountered a {ptr_kind} pointing to uninhabited type `{ty}`")
             )
         }
 
@@ -1573,6 +1573,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
             );
         }
         if cfg!(debug_assertions) {
+            assert!(
+                val.layout.ty.is_opsem_inhabited(*self.ecx.tcx, self.ecx.typing_env),
+                "a value of type `{}` somehow passed validity but is not actually inhabited",
+                val.layout.ty,
+            );
             // Check that we don't miss any new changes to layout computation in our checks above.
             match val.layout.backend_repr {
                 BackendRepr::Scalar(scalar_layout) => {

compiler/rustc_middle/src/queries.rs

@@ -2171,6 +2171,11 @@ rustc_queries! {
         desc { "computing the uninhabited predicate of `{}`", key }
     }
 
+    /// Do not call this query directly: invoke `Ty::is_opsem_inhabited` instead.
+    query is_opsem_inhabited_raw(env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>) -> bool {
+        desc { "computing whether `{}` is inhabited on the opsem level", env.value }
+    }
+
     query crate_dep_kind(_: CrateNum) -> CrateDepKind {
         eval_always
         desc { "fetching what a dependency looks like" }

compiler/rustc_middle/src/ty/inhabitedness/mod.rs

@@ -54,7 +54,12 @@ pub mod inhabited_predicate;
 pub use inhabited_predicate::InhabitedPredicate;
 
 pub(crate) fn provide(providers: &mut Providers) {
-    *providers = Providers { inhabited_predicate_adt, inhabited_predicate_type, ..*providers };
+    *providers = Providers {
+        inhabited_predicate_adt,
+        inhabited_predicate_type,
+        is_opsem_inhabited_raw,
+        ..*providers
+    };
 }
 
 /// Returns an `InhabitedPredicate` that is generic over type parameters and
@@ -186,14 +191,26 @@ impl<'tcx> Ty<'tcx> {
         self.inhabited_predicate(tcx).apply(tcx, typing_env, module)
     }
 
-    /// Returns true if the type is uninhabited without regard to visibility
+    /// Returns true if the type is uninhabited without regard to visibility.
+    ///
+    /// This is still conservative; for instance, a `#[non_exhaustive]` enum *in another crate*
+    /// is always considered inhabited.
     pub fn is_privately_uninhabited(
         self,
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
     ) -> bool {
         !self.inhabited_predicate(tcx).apply_ignore_module(tcx, typing_env)
     }
+
+    /// Returns whether `self` is considered inhabited on the opsem level, i.e., its validity
+    /// invariant might be satisfiable. `self` is expected to be monomorphic and normalized.
+    pub fn is_opsem_inhabited(self, tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>) -> bool {
+        // Handle simple cases directly, use the query with its cache for the rest.
+        is_opsem_inhabited_recursor(self, tcx, None, &|ty, _, _| {
+            tcx.is_opsem_inhabited_raw(typing_env.as_query_input(ty))
+        })
+    }
 }
 
 /// N.B. this query should only be called through `Ty::inhabited_predicate`
@@ -216,3 +233,107 @@ fn inhabited_predicate_type<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> InhabitedP
         _ => bug!("unexpected TyKind, use `Ty::inhabited_predicate`"),
     }
 }
+
+/// `root` is used to bound the recursion: the first time we encounter an ADT, we
+/// store its `DefId` as the "root"; if we ever hit the same ADT again we stop.
+/// All recursive types must go through an ADT so this catches all possible recursion.
+///
+/// When we git an ADT, we call `adt_handler`, giving it as its last argument a closure that it
+/// can invoke to continue the recursion.
+fn is_opsem_inhabited_recursor<'tcx>(
+    ty: Ty<'tcx>,
+    tcx: TyCtxt<'tcx>,
+    root: Option<DefId>,
+    adt_handler: &impl Fn(Ty<'tcx>, Option<DefId>, &dyn Fn(Ty<'tcx>, Option<DefId>) -> bool) -> bool,
+) -> bool {
+    match *ty.kind() {
+        // Trivially (un)inhabited types
+        ty::Int(_)
+        | ty::Uint(_)
+        | ty::Float(_)
+        | ty::Bool
+        | ty::Char
+        | ty::Str
+        | ty::Foreign(..)
+        | ty::RawPtr(..)
+        | ty::FnPtr(..)
+        | ty::FnDef(..) => true,
+        ty::Dynamic(..) => true, // We can't reason about traits, assume they are inhabited
+        ty::Slice(..) => true,   // Slices can always be empty
+        ty::Never => false,
+
+        // Types where we recurse
+        ty::Ref(_, pointee, _) => is_opsem_inhabited_recursor(pointee, tcx, root, adt_handler),
+        ty::Tuple(tys) => {
+            tys.iter().all(|ty| is_opsem_inhabited_recursor(ty, tcx, root, adt_handler))
+        }
+        ty::Array(elem, len) => {
+            len.try_to_target_usize(tcx).unwrap() == 0
+                || is_opsem_inhabited_recursor(elem, tcx, root, adt_handler)
+        }
+        ty::Pat(inner, _pat) => is_opsem_inhabited_recursor(inner, tcx, root, adt_handler),
+        ty::Closure(_closure_def, closure_args) => {
+            let closure_args = closure_args.as_closure();
+            closure_args
+                .upvar_tys()
+                .iter()
+                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, root, adt_handler))
+        }
+        ty::Coroutine(..) => {
+            true // FIXME should these really be trivially inhabited?
+        }
+        ty::CoroutineClosure(..) => {
+            true // FIXME should these really be trivially inhabited?
+        }
+        ty::UnsafeBinder(..) => {
+            true // FIXME should these really be trivially inhabited?
+        }
+        ty::Adt(..) => {
+            // ADTs need a special handler to avoid infinite recursion.
+            adt_handler(ty, root, &|ty, root| {
+                is_opsem_inhabited_recursor(ty, tcx, root, adt_handler)
+            })
+        }
+
+        ty::Error(_)
+        | ty::Infer(..)
+        | ty::Placeholder(..)
+        | ty::Bound(..)
+        | ty::Param(..)
+        | ty::Alias(..)
+        | ty::CoroutineWitness(..) => {
+            bug!("non-normalized type in `is_opsem_uninhabited_raw::rec`: `{ty}`")
+        }
+    }
+}
+
+fn is_opsem_inhabited_raw<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>,
+) -> bool {
+    let (ty, typing_env) = (env.value, env.typing_env);
+    is_opsem_inhabited_recursor(ty, tcx, None, &|ty, root, rec| {
+        let ty::Adt(adt_def, adt_args) = *ty.kind() else {
+            unreachable! {}
+        };
+        if adt_def.is_union() {
+            // Unions are always inhabited.
+            return true;
+        }
+        if Some(adt_def.did()) == root {
+            // We recursed to (possibly another instance of) the same type.
+            // Coinductively assume that the type is inhabited.
+            return true;
+        }
+        let root = root.unwrap_or(adt_def.did());
+
+        // We are inhabited if in some variant all fields are inhabited.
+        adt_def.variants().iter().any(|variant| {
+            variant.fields.iter().all(|field| {
+                let ty = field.ty(tcx, adt_args);
+                let ty = tcx.normalize_erasing_regions(typing_env, ty);
+                rec(ty, Some(root))
+            })
+        })
+    })
+}

compiler/rustc_ty_utils/src/layout/invariant.rs

@@ -34,6 +34,14 @@ pub(super) fn layout_sanity_check<'tcx>(cx: &LayoutCx<'tcx>, layout: &TyAndLayou
             layout.ty
         );
     }
+    // ABI uninhabitedness should omply opsem uninhabitedness.
+    if layout.is_uninhabited() {
+        assert!(
+            layout.ty.is_opsem_inhabited(tcx, cx.typing_env),
+            "{:?} is ABI-uninhabited but not opsem-uninhabited?",
+            layout.ty
+        );
+    }
 
     /// Yields non-ZST fields of the type
     fn non_zst_fields<'tcx, 'a>(

src/tools/miri/tests/fail/validity/ref_to_uninhabited1.rs

@@ -3,7 +3,7 @@ use std::mem::{forget, transmute};
 
 fn main() {
     unsafe {
-        let x: Box<!> = transmute(&mut 42); //~ERROR: encountered a box pointing to uninhabited type !
+        let x: Box<!> = transmute(&mut 42); //~ERROR: encountered a box pointing to uninhabited type `!`
         forget(x);
     }
 }

src/tools/miri/tests/fail/validity/ref_to_uninhabited1.stderr

@@ -1,4 +1,4 @@
-error: Undefined Behavior: constructing invalid value of type std::boxed::Box<!>: encountered a box pointing to uninhabited type !
+error: Undefined Behavior: constructing invalid value of type std::boxed::Box<!>: encountered a box pointing to uninhabited type `!`
   --> tests/fail/validity/ref_to_uninhabited1.rs:LL:CC
    |
 LL |         let x: Box<!> = transmute(&mut 42);

src/tools/miri/tests/fail/validity/ref_to_uninhabited2.rs

@@ -4,6 +4,6 @@ enum Void {}
 
 fn main() {
     unsafe {
-        let _x: &(i32, Void) = transmute(&42); //~ERROR: encountered a reference pointing to uninhabited type (i32, Void)
+        let _x: &&(i32, Void) = transmute(&&42); //~ERROR: encountered a reference pointing to uninhabited type `&(i32, Void)`
     }
 }

src/tools/miri/tests/fail/validity/ref_to_uninhabited2.stderr

@@ -1,8 +1,8 @@
-error: Undefined Behavior: constructing invalid value of type &(i32, Void): encountered a reference pointing to uninhabited type (i32, Void)
+error: Undefined Behavior: constructing invalid value of type &&(i32, Void): encountered a reference pointing to uninhabited type `&(i32, Void)`
   --> tests/fail/validity/ref_to_uninhabited2.rs:LL:CC
    |
-LL |         let _x: &(i32, Void) = transmute(&42);
-   |                                ^^^^^^^^^^^^^^ Undefined Behavior occurred here
+LL |         let _x: &&(i32, Void) = transmute(&&42);
+   |                                 ^^^^^^^^^^^^^^^ Undefined Behavior occurred here
    |
    = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information

tests/ui/consts/const-eval/raw-bytes.64bit.stderr

@@ -218,7 +218,7 @@ LL | const DATA_FN_PTR: fn() = unsafe { mem::transmute(&13) };
                ╾ALLOC_ID╼                         │ ╾──────╼
            }
 
-error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type Bar
+error[E0080]: constructing invalid value of type &Bar: encountered a reference pointing to uninhabited type `Bar`
   --> $DIR/raw-bytes.rs:110:1
    |
 LL | const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
@@ -458,7 +458,7 @@ LL | const RAW_TRAIT_OBJ_VTABLE_INVALID: *const dyn Trait = unsafe { mem::transm
                ╾ALLOC_ID╼ ╾ALLOC_ID╼ │ ╾──────╼╾──────╼
            }
 
-error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type [!; 1]
+error[E0080]: constructing invalid value of type &[!; 1]: encountered a reference pointing to uninhabited type `[!; 1]`
   --> $DIR/raw-bytes.rs:188:1
    |
 LL | const _: &[!; 1] = unsafe { &*(1_usize as *const [!; 1]) };

tests/ui/consts/const-eval/ub-uninhabit.rs

@@ -18,19 +18,25 @@ union MaybeUninit<T: Copy> {
 }
 
 const BAD_BAD_BAD: Bar = unsafe { MaybeUninit { uninit: () }.init };
-//~^ ERROR constructing invalid value
+//~^ ERROR value of uninhabited type `Bar`
 
 const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
-//~^ ERROR constructing invalid value
+//~^ ERROR reference pointing to uninhabited type `Bar`
 
 const BAD_BAD_ARRAY: [Bar; 1] = unsafe { MaybeUninit { uninit: () }.init };
-//~^ ERROR constructing invalid value
+//~^ ERROR value of uninhabited type `Bar`
 
 const READ_NEVER: () = unsafe {
     let mem = [0u32; 8];
     let ptr = mem.as_ptr().cast::<!>();
     let _val = intrinsics::read_via_copy(ptr);
-    //~^ ERROR constructing invalid value
+    //~^ ERROR value of the never type
 };
 
+const BAD_NESTED_REF: &&! = unsafe { mem::transmute(&&0) };
+//~^ ERROR reference pointing to uninhabited type `&!`
+
+const BAD_NESTED_UNSIZED_REF: &&(!, [i32]) = unsafe { mem::transmute(&(&[0] as &[i32])) };
+//~^ ERROR reference pointing to uninhabited type `&(!, [i32])`
+
 fn main() {}