Auto merge of #154144 - Human9000-bit:const-prop-transmute-async-fix, r=<try>

Fix dataflow const prop behavior when propagating malsized transmutes

Author: bors

compiler/rustc_hir_typeck/src/intrinsicck.rs

@@ -8,6 +8,7 @@ use rustc_index::Idx;
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{LayoutError, SizeSkeleton};
 use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::ErrorGuaranteed;
 use rustc_span::def_id::LocalDefId;
 use tracing::trace;
 
@@ -75,7 +76,7 @@ fn check_transmute<'tcx>(
     from: Ty<'tcx>,
     to: Ty<'tcx>,
     hir_id: HirId,
-) {
+) -> Result<(), ErrorGuaranteed> {
     let span = || tcx.hir_span(hir_id);
     let normalize = |ty| {
         if let Ok(ty) = tcx.try_normalize_erasing_regions(typing_env, ty) {
@@ -95,7 +96,7 @@ fn check_transmute<'tcx>(
 
     // Transmutes that are only changing lifetimes are always ok.
     if from == to {
-        return;
+        return Ok(());
     }
 
     let sk_from = SizeSkeleton::compute(from, tcx, typing_env);
@@ -107,7 +108,7 @@ fn check_transmute<'tcx>(
         && let Ok(sk_to) = sk_to
     {
         if sk_from.same_size(sk_to) {
-            return;
+            return Ok(());
         }
 
         // Special-case transmuting from `typeof(function)` and
@@ -122,7 +123,7 @@ fn check_transmute<'tcx>(
                 .with_note(format!("target type: {to}"))
                 .with_help("cast with `as` to a pointer instead")
                 .emit();
-            return;
+            return Ok(());
         }
     }
 
@@ -134,21 +135,25 @@ fn check_transmute<'tcx>(
     );
     if from == to {
         err.note(format!("`{from}` does not have a fixed size"));
-        err.emit();
+        Err(err.emit())
     } else {
         err.note(format!("source type: `{}` ({})", from, skeleton_string(from, sk_from)));
         err.note(format!("target type: `{}` ({})", to, skeleton_string(to, sk_to)));
-        err.emit();
+        Err(err.emit())
     }
 }
 
-pub(crate) fn check_transmutes(tcx: TyCtxt<'_>, owner: LocalDefId) {
+pub(crate) fn check_transmutes(tcx: TyCtxt<'_>, owner: LocalDefId) -> Result<(), ErrorGuaranteed> {
     assert!(!tcx.is_typeck_child(owner.to_def_id()));
     let typeck_results = tcx.typeck(owner);
-    let None = typeck_results.tainted_by_errors else { return };
+    if let Some(e) = typeck_results.tainted_by_errors {
+        return Err(e);
+    };
 
     let typing_env = ty::TypingEnv::post_analysis(tcx, owner);
+    let mut result = Ok(());
     for &(from, to, hir_id) in &typeck_results.transmutes_to_check {
-        check_transmute(tcx, typing_env, from, to, hir_id);
+        result = check_transmute(tcx, typing_env, from, to, hir_id);
     }
+    result
 }

compiler/rustc_middle/src/queries.rs

@@ -1134,7 +1134,7 @@ rustc_queries! {
     }
 
     /// Unsafety-check this `LocalDefId`.
-    query check_transmutes(key: LocalDefId) {
+    query check_transmutes(key: LocalDefId) -> Result<(), ErrorGuaranteed> {
         desc { "check transmute calls inside `{}`", tcx.def_path_str(key) }
     }
 

compiler/rustc_mir_transform/src/dataflow_const_prop.rs

@@ -861,6 +861,11 @@ fn try_write_constant<'tcx>(
     if layout.backend_repr.is_scalar()
         && let Some(value) = propagatable_scalar(place, state, map)
     {
+        // Size is always present for scalars
+        if value.size() != layout.backend_repr.scalar_size(ecx).unwrap() {
+            throw_machine_stop_str!("mismatched sizes");
+        }
+
         return ecx.write_immediate(Immediate::Scalar(value), dest);
     }
 

compiler/rustc_mir_transform/src/lib.rs

@@ -827,6 +827,11 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> {
         return body;
     }
 
+    // If check_transmutes found faulty transmutes, then return early to avoid ICEs during optimizations
+    if !tcx.is_typeck_child(did.to_def_id()) && tcx.check_transmutes(did).is_err() {
+        return body;
+    }
+
     run_optimization_passes(tcx, &mut body);
 
     body

tests/ui/transmute/transmute-in-async-fn.rs

@@ -0,0 +1,6 @@
+//@compile-flags: -Zmir-enable-passes=+DataflowConstProp --crate-type lib
+//@ edition:2021
+pub async fn a() -> u32 {
+    unsafe { std::mem::transmute(1u64) }
+    //~^error: cannot transmute between types of different sizes, or dependently-sized types
+}

tests/ui/transmute/transmute-in-async-fn.stderr

@@ -0,0 +1,12 @@
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/transmute-in-async-fn.rs:4:14
+   |
+LL |     unsafe { std::mem::transmute(1u64) }
+   |              ^^^^^^^^^^^^^^^^^^^
+   |
+   = note: source type: `u64` (64 bits)
+   = note: target type: `u32` (32 bits)
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0512`.