Fix malformed transmute handling during MIR build

It used to just error on wrong transmutes, without any body tainting.
That resulted in ICE during subsequent MIR optimizations.
Fixed by tainting the body right before the MIR optimizations

Author: Human9000-bit

compiler/rustc_hir_typeck/src/intrinsicck.rs

@@ -8,6 +8,7 @@ use rustc_index::Idx;
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{LayoutError, SizeSkeleton};
 use rustc_middle::ty::{self, Ty, TyCtxt, Unnormalized};
+use rustc_span::ErrorGuaranteed;
 use rustc_span::def_id::LocalDefId;
 use tracing::trace;
 
@@ -72,7 +73,7 @@ fn check_transmute<'tcx>(
     from: Ty<'tcx>,
     to: Ty<'tcx>,
     hir_id: HirId,
-) {
+) -> Result<(), ErrorGuaranteed> {
     let span = tcx.hir_span(hir_id);
     let normalize = |ty| {
         if let Ok(ty) = tcx.try_normalize_erasing_regions(typing_env, Unnormalized::new_wip(ty)) {
@@ -92,7 +93,7 @@ fn check_transmute<'tcx>(
 
     // Transmutes that are only changing lifetimes are always ok.
     if from == to {
-        return;
+        return Ok(());
     }
 
     let sk_from = SizeSkeleton::compute(from, tcx, typing_env, span);
@@ -104,7 +105,7 @@ fn check_transmute<'tcx>(
         && let Ok(sk_to) = sk_to
     {
         if sk_from.same_size(sk_to) {
-            return;
+            return Ok(());
         }
 
         // Special-case transmuting from `typeof(function)` and
@@ -119,7 +120,7 @@ fn check_transmute<'tcx>(
                 .with_note(format!("target type: {to}"))
                 .with_help("cast with `as` to a pointer instead")
                 .emit();
-            return;
+            return Ok(());
         }
     }
 
@@ -131,21 +132,25 @@ fn check_transmute<'tcx>(
     );
     if from == to {
         err.note(format!("`{from}` does not have a fixed size"));
-        err.emit();
+        Err(err.emit())
     } else {
         err.note(format!("source type: `{}` ({})", from, skeleton_string(from, sk_from)));
         err.note(format!("target type: `{}` ({})", to, skeleton_string(to, sk_to)));
-        err.emit();
+        Err(err.emit())
     }
 }
 
-pub(crate) fn check_transmutes(tcx: TyCtxt<'_>, owner: LocalDefId) {
+pub(crate) fn check_transmutes(tcx: TyCtxt<'_>, owner: LocalDefId) -> Result<(), ErrorGuaranteed> {
     assert!(!tcx.is_typeck_child(owner.to_def_id()));
     let typeck_results = tcx.typeck(owner);
-    let None = typeck_results.tainted_by_errors else { return };
+    if let Some(e) = typeck_results.tainted_by_errors {
+        return Err(e);
+    };
 
     let typing_env = ty::TypingEnv::post_analysis(tcx, owner);
+    let mut result = Ok(());
     for &(from, to, hir_id) in &typeck_results.transmutes_to_check {
-        check_transmute(tcx, typing_env, from, to, hir_id);
+        result = result.and(check_transmute(tcx, typing_env, from, to, hir_id));
     }
+    result
 }

compiler/rustc_middle/src/queries.rs

@@ -1118,7 +1118,7 @@ rustc_queries! {
     }
 
     /// Unsafety-check this `LocalDefId`.
-    query check_transmutes(key: LocalDefId) {
+    query check_transmutes(key: LocalDefId) -> Result<(), ErrorGuaranteed> {
         desc { "check transmute calls inside `{}`", tcx.def_path_str(key) }
     }
 

compiler/rustc_mir_transform/src/lib.rs

@@ -541,11 +541,15 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
         body.tainted_by_errors = Some(error_reported);
     }
 
+    let root = tcx.typeck_root_def_id_local(def);
+    if let Err(e) = tcx.check_transmutes(root) {
+        body.tainted_by_errors = Some(e);
+    }
+
     // Also taint the body if it's within a top-level item that is not well formed.
     //
     // We do this check here and not during `mir_promoted` because that may result
     // in borrowck cycles if WF requires looking into an opaque hidden type.
-    let root = tcx.typeck_root_def_id_local(def);
     match tcx.def_kind(root) {
         DefKind::Fn
         | DefKind::AssocFn

tests/rustdoc-ui/issues/issue-79494.rs

@@ -1,4 +1,4 @@
 //@ only-64bit
 
 pub const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
-//~^ ERROR transmuting from 8-byte type to 16-byte type
+//~^ ERROR: cannot transmute between types of different sizes, or dependently-sized types [E0512]

tests/rustdoc-ui/issues/issue-79494.stderr

@@ -1,9 +1,12 @@
-error[E0080]: transmuting from 8-byte type to 16-byte type: `usize` -> `&[u8]`
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
   --> $DIR/issue-79494.rs:3:33
    |
 LL | pub const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
-   |                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `ZST` failed here
+   |                                 ^^^^^^^^^^^^^^^^^^^
+   |
+   = note: source type: `usize` (64 bits)
+   = note: target type: `&[u8]` (128 bits)
 
 error: aborting due to 1 previous error
 
-For more information about this error, try `rustc --explain E0080`.
+For more information about this error, try `rustc --explain E0512`.

tests/ui/consts/transmute-size-mismatch-before-typeck.rs

@@ -14,5 +14,4 @@ fn main() {
 }
 
 const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
-//~^ ERROR transmuting from
-//~| ERROR cannot transmute between types of different sizes
+//~^ ERROR cannot transmute between types of different sizes

tests/ui/consts/transmute-size-mismatch-before-typeck.stderr

@@ -1,9 +1,3 @@
-error[E0080]: transmuting from word size type to 2 * word size type: `usize` -> `&[u8]`
-  --> $DIR/transmute-size-mismatch-before-typeck.rs:16:29
-   |
-LL | const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
-   |                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^ evaluation of `ZST` failed here
-
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
   --> $DIR/transmute-size-mismatch-before-typeck.rs:16:29
    |
@@ -13,7 +7,6 @@ LL | const ZST: &[u8] = unsafe { std::mem::transmute(1usize) };
    = note: source type: `usize` (word size)
    = note: target type: `&[u8]` (2 * word size)
 
-error: aborting due to 2 previous errors
+error: aborting due to 1 previous error
 
-Some errors have detailed explanations: E0080, E0512.
-For more information about an error, try `rustc --explain E0080`.
+For more information about this error, try `rustc --explain E0512`.

tests/ui/layout/base-layout-is-sized-ice-123078.rs

@@ -8,8 +8,7 @@ struct S {
 }
 
 const C: S = unsafe { std::mem::transmute(()) };
-//~^ ERROR the type `S` has an unknown layout
-//~| ERROR cannot transmute between types of different sizes, or dependently-sized types
+//~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
 
 const _: [(); {
     C;

tests/ui/layout/base-layout-is-sized-ice-123078.stderr

@@ -16,12 +16,6 @@ help: the `Box` type always has a statically known size and allocates its conten
 LL |     a: Box<[u8]>,
    |        ++++    +
 
-error[E0080]: the type `S` has an unknown layout
-  --> $DIR/base-layout-is-sized-ice-123078.rs:10:1
-   |
-LL | const C: S = unsafe { std::mem::transmute(()) };
-   | ^^^^^^^^^^ evaluation of `C` failed here
-
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
   --> $DIR/base-layout-is-sized-ice-123078.rs:10:23
    |
@@ -31,7 +25,7 @@ LL | const C: S = unsafe { std::mem::transmute(()) };
    = note: source type: `()` (0 bits)
    = note: target type: `S` (the type `S` has an unknown layout)
 
-error: aborting due to 3 previous errors
+error: aborting due to 2 previous errors
 
-Some errors have detailed explanations: E0080, E0277, E0512.
-For more information about an error, try `rustc --explain E0080`.
+Some errors have detailed explanations: E0277, E0512.
+For more information about an error, try `rustc --explain E0277`.

tests/ui/transmute/transmute-in-async-fn.rs

@@ -0,0 +1,12 @@
+//@compile-flags: -Zmir-enable-passes=+DataflowConstProp --crate-type lib
+//@ edition:2021
+pub async fn a() -> u32 {
+    unsafe { std::mem::transmute(1u64) }
+    //~^error: cannot transmute between types of different sizes, or dependently-sized types
+}
+
+pub async fn b() -> u32 {
+    let closure = || unsafe { std::mem::transmute(1u64) };
+    //~^ ERROR: cannot transmute between types of different sizes, or dependently-sized types [E0512]
+    closure()
+}