Rollup merge of #156235 - rajgandhi1:compiler_stack_overflow_fix, r=adwinwhite

fix: Guard SizeSkeleton::compute against stack overflow

Fixes #156137

Fix: extract the recursion into a private `compute_inner` that carries a depth counter. When depth exceeds the crate's recursion limit, return `LayoutError::Unknown` and let the existing transmute size-check produce a normal error instead of crashing.

A regression test is included in `tests/ui/transmute/`.

Author: jhpratt

compiler/rustc_hir_typeck/src/intrinsicck.rs

@@ -73,14 +73,14 @@ fn check_transmute<'tcx>(
     to: Ty<'tcx>,
     hir_id: HirId,
 ) {
-    let span = || tcx.hir_span(hir_id);
+    let span = tcx.hir_span(hir_id);
     let normalize = |ty| {
         if let Ok(ty) = tcx.try_normalize_erasing_regions(typing_env, Unnormalized::new_wip(ty)) {
             ty
         } else {
             Ty::new_error_with_message(
                 tcx,
-                span(),
+                span,
                 format!("tried to normalize non-wf type {ty:#?} in check_transmute"),
             )
         }
@@ -95,8 +95,8 @@ fn check_transmute<'tcx>(
         return;
     }
 
-    let sk_from = SizeSkeleton::compute(from, tcx, typing_env);
-    let sk_to = SizeSkeleton::compute(to, tcx, typing_env);
+    let sk_from = SizeSkeleton::compute(from, tcx, typing_env, span);
+    let sk_to = SizeSkeleton::compute(to, tcx, typing_env, span);
     trace!(?sk_from, ?sk_to);
 
     // Check for same size using the skeletons.
@@ -114,7 +114,7 @@ fn check_transmute<'tcx>(
             && let SizeSkeleton::Known(size_to, _) = sk_to
             && size_to == Pointer(tcx.data_layout.instruction_address_space).size(&tcx)
         {
-            struct_span_code_err!(tcx.sess.dcx(), span(), E0591, "can't transmute zero-sized type")
+            struct_span_code_err!(tcx.sess.dcx(), span, E0591, "can't transmute zero-sized type")
                 .with_note(format!("source type: {from}"))
                 .with_note(format!("target type: {to}"))
                 .with_help("cast with `as` to a pointer instead")
@@ -125,7 +125,7 @@ fn check_transmute<'tcx>(
 
     let mut err = struct_span_code_err!(
         tcx.sess.dcx(),
-        span(),
+        span,
         E0512,
         "cannot transmute between types of different sizes, or dependently-sized types"
     );

compiler/rustc_lint/src/types.rs

@@ -8,7 +8,7 @@ use rustc_middle::bug;
 use rustc_middle::ty::layout::{LayoutOf, SizeSkeleton};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt, Unnormalized};
 use rustc_session::{declare_lint, declare_lint_pass, impl_lint_pass};
-use rustc_span::{Span, Symbol, sym};
+use rustc_span::{DUMMY_SP, Span, Symbol, sym};
 use tracing::debug;
 
 mod improper_ctypes; // these files do the implementation for ImproperCTypesDefinitions,ImproperCTypesDeclarations
@@ -877,7 +877,8 @@ pub(crate) fn repr_nullable_ptr<'tcx>(
             // At this point, the field's type is known to be nonnull and the parent enum is Option-like.
             // If the computed size for the field and the enum are different, the nonnull optimization isn't
             // being applied (and we've got a problem somewhere).
-            let compute_size_skeleton = |t| SizeSkeleton::compute(t, tcx, typing_env).ok();
+            let compute_size_skeleton =
+                |t| SizeSkeleton::compute(t, tcx, typing_env, DUMMY_SP).ok();
             if !compute_size_skeleton(ty)?.same_size(compute_size_skeleton(field_ty)?) {
                 bug!("improper_ctypes: Option nonnull optimization not applied?");
             }

compiler/rustc_middle/src/error.rs

@@ -62,6 +62,18 @@ pub(crate) struct RecursionLimitReached<'tcx> {
     pub suggested_limit: rustc_hir::limit::Limit,
 }
 
+#[derive(Diagnostic)]
+#[diag("reached the recursion limit while computing the size of `{$ty}`")]
+#[help(
+    "consider increasing the recursion limit by adding a `#![recursion_limit = \"{$suggested_limit}\"]`"
+)]
+pub(crate) struct RecursionLimitReachedSizeSkeleton<'tcx> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'tcx>,
+    pub suggested_limit: rustc_hir::limit::Limit,
+}
+
 #[derive(Diagnostic)]
 #[diag("constant evaluation of enum discriminant resulted in non-integer")]
 pub(crate) struct ConstEvalNonIntError {

compiler/rustc_middle/src/ty/layout.rs

@@ -332,9 +332,38 @@ impl<'tcx> SizeSkeleton<'tcx> {
         ty: Ty<'tcx>,
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
+        span: Span,
+    ) -> Result<SizeSkeleton<'tcx>, &'tcx LayoutError<'tcx>> {
+        Self::compute_inner(ty, tcx, typing_env, span, 0)
+    }
+
+    fn compute_inner(
+        ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        typing_env: ty::TypingEnv<'tcx>,
+        span: Span,
+        depth: usize,
     ) -> Result<SizeSkeleton<'tcx>, &'tcx LayoutError<'tcx>> {
         debug_assert!(!ty.has_non_region_infer());
 
+        // Bail out if we've recursed too deeply (issue #156137); a cyclic type
+        // alias can otherwise blow the stack here. Using `>=` rather than `>`
+        // means we fire exactly at the limit, which lets us report the
+        // cycle-root type (`Thing<T>`) instead of an innocent field type.
+        let recursion_limit = tcx.recursion_limit();
+        if depth >= recursion_limit.0 {
+            let suggested_limit = match recursion_limit {
+                hir::limit::Limit(0) => hir::limit::Limit(2),
+                limit => limit * 2,
+            };
+            let reported = tcx.dcx().emit_err(crate::error::RecursionLimitReachedSizeSkeleton {
+                span,
+                ty,
+                suggested_limit,
+            });
+            return Err(tcx.arena.alloc(LayoutError::ReferencesError(reported)));
+        }
+
         // First try computing a static layout.
         let err = match tcx.layout_of(typing_env.as_query_input(ty)) {
             Ok(layout) => {
@@ -407,7 +436,7 @@ impl<'tcx> SizeSkeleton<'tcx> {
                     return Ok(SizeSkeleton::Known(Size::from_bytes(0), None));
                 }
 
-                match SizeSkeleton::compute(inner, tcx, typing_env)? {
+                match SizeSkeleton::compute_inner(inner, tcx, typing_env, span, depth + 1)? {
                     // This may succeed because the multiplication of two types may overflow
                     // but a single size of a nested array will not.
                     SizeSkeleton::Known(s, a) => {
@@ -434,10 +463,15 @@ impl<'tcx> SizeSkeleton<'tcx> {
                 // Get a zero-sized variant or a pointer newtype.
                 let zero_or_ptr_variant = |i| {
                     let i = VariantIdx::from_usize(i);
-                    let fields =
-                        def.variant(i).fields.iter().map(|field| {
-                            SizeSkeleton::compute(field.ty(tcx, args), tcx, typing_env)
-                        });
+                    let fields = def.variant(i).fields.iter().map(|field| {
+                        SizeSkeleton::compute_inner(
+                            field.ty(tcx, args),
+                            tcx,
+                            typing_env,
+                            span,
+                            depth + 1,
+                        )
+                    });
                     let mut ptr = None;
                     for field in fields {
                         let field = field?;
@@ -487,13 +521,13 @@ impl<'tcx> SizeSkeleton<'tcx> {
                 if ty == normalized {
                     Err(err)
                 } else {
-                    SizeSkeleton::compute(normalized, tcx, typing_env)
+                    SizeSkeleton::compute_inner(normalized, tcx, typing_env, span, depth + 1)
                 }
             }
 
             ty::Pat(base, pat) => {
                 // Pattern types are always the same size as their base.
-                let base = SizeSkeleton::compute(base, tcx, typing_env);
+                let base = SizeSkeleton::compute_inner(base, tcx, typing_env, span, depth + 1);
                 match *pat {
                     ty::PatternKind::Range { .. } | ty::PatternKind::Or(_) => base,
                     // But in the case of `!null` patterns we need to note that in the

tests/ui/transmute/recursive-size-skeleton.rs

@@ -0,0 +1,25 @@
+// Regression test for issue #156137.
+// Computing the `SizeSkeleton` of a type whose layout depends on itself
+// through a normalizing type alias used to recurse without bound and
+// blow the stack. We now bail out via the recursion limit and emit a
+// regular error instead of ICE-ing.
+
+use std::mem::transmute;
+
+trait Trait {
+    type Assoc;
+}
+impl<T> Trait for T {
+    type Assoc = T;
+}
+type Alias<T> = <T as Trait>::Assoc;
+
+pub struct Thing<T: ?Sized>(*const T, Alias<Thing<T>>);
+
+pub fn weird<T: ?Sized>(value: Thing<T>) {
+    let _: i32 = unsafe { transmute(value) };
+    //~^ ERROR reached the recursion limit while computing the size of `Thing<T>`
+    //~| ERROR cannot transmute between types of different sizes, or dependently-sized types
+}
+
+fn main() {}

tests/ui/transmute/recursive-size-skeleton.stderr

@@ -0,0 +1,20 @@
+error: reached the recursion limit while computing the size of `Thing<T>`
+  --> $DIR/recursive-size-skeleton.rs:20:27
+   |
+LL |     let _: i32 = unsafe { transmute(value) };
+   |                           ^^^^^^^^^
+   |
+   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]`
+
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/recursive-size-skeleton.rs:20:27
+   |
+LL |     let _: i32 = unsafe { transmute(value) };
+   |                           ^^^^^^^^^
+   |
+   = note: source type: `Thing<T>` (the type has an unknown layout)
+   = note: target type: `i32` (32 bits)
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0512`.