transmute: fix check for whether newtypes have equal size

Author: RalfJung

compiler/rustc_middle/src/ty/layout.rs

@@ -426,13 +426,35 @@ impl<'tcx> SizeSkeleton<'tcx> {
             }
 
             ty::Adt(def, args) => {
-                // Only newtypes and enums w/ nullable pointer optimization.
+                // Only newtypes and enums w/ nullable pointer optimization (NPO).
                 if def.is_union() || def.variants().is_empty() || def.variants().len() > 2 {
                     return Err(err);
                 }
+                // Only default repr types.
+                {
+                    // We can ignore the seed and some particular flags that can never affect the
+                    // layout of newtypes / NPO types, but we have to check everything else.
+                    // If you are adding a new field to `ReprOptions`, make sure to extend the check
+                    // below so that we bail out if it is not at its default value!
+                    let ReprOptions { int, align, pack, flags, scalable, field_shuffle_seed: _ } =
+                        def.repr();
+                    let ignored_flags = ReprFlags::IS_TRANSPARENT
+                        | ReprFlags::IS_LINEAR
+                        | ReprFlags::RANDOMIZE_LAYOUT;
+                    if int.is_some()
+                        || align.is_some()
+                        || pack.is_some()
+                        || flags.difference(ignored_flags) != ReprFlags::default()
+                        || scalable.is_some()
+                    {
+                        return Err(err);
+                    }
+                }
 
                 // Get a zero-sized variant or a pointer newtype.
-                let zero_or_ptr_variant = |i| {
+                // Returns `Ok(None)` for 1-ZST types, `Ok(Some)` if (ignoring all 1-ZST fields)
+                // there's just a single pointer, and `Err` otherwise.
+                let zero_or_ptr_variant = |i| -> Result<Option<SizeSkeleton<'tcx>>, _> {
                     let i = VariantIdx::from_usize(i);
                     let fields =
                         def.variant(i).fields.iter().map(|field| {

tests/ui/transmute/raw-ptr-non-null.rs

@@ -48,4 +48,23 @@ pub unsafe fn shouldnt_work2<T: ?Sized>(from: *mut T) -> PtrAndEmptyArray<T> {
     //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
 }
 
+#[repr(align(16))]
+struct OverAlignWrap<T>(T);
+
+fn shouldnt_work3<T: ?Sized>(x: OverAlignWrap<*const T>) -> *const T {
+    unsafe { transmute(x) }
+    //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
+}
+
+#[repr(C)]
+enum NotNullPointerOptimized<T> {
+    Some(T),
+    None
+}
+
+fn shouldnt_work4<T: ?Sized>(x: &T) -> NotNullPointerOptimized<&T> {
+    unsafe { transmute(x) }
+    //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
+}
+
 fn main() {}

tests/ui/transmute/transmute-different-sizes.rs

@@ -43,6 +43,24 @@ LL |     std::mem::transmute(from)
    = note: source type: `*mut T` (pointer to `T`)
    = note: target type: `PtrAndEmptyArray<T>` (size can vary because of <T as Pointee>::Metadata)
 
-error: aborting due to 5 previous errors
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/transmute-different-sizes.rs:55:14
+   |
+LL |     unsafe { transmute(x) }
+   |              ^^^^^^^^^
+   |
+   = note: source type: `OverAlignWrap<*const T>` (size can vary because of <T as Pointee>::Metadata)
+   = note: target type: `*const T` (pointer to `T`)
+
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/transmute-different-sizes.rs:66:14
+   |
+LL |     unsafe { transmute(x) }
+   |              ^^^^^^^^^
+   |
+   = note: source type: `&T` (pointer to `T`)
+   = note: target type: `NotNullPointerOptimized<&T>` (size can vary because of <T as Pointee>::Metadata)
+
+error: aborting due to 7 previous errors
 
 For more information about this error, try `rustc --explain E0512`.

tests/ui/transmute/transmute-different-sizes.stderr

@@ -5,6 +5,7 @@
 #![allow(dead_code)]
 
 use std::mem::transmute;
+use std::ptr::NonNull;
 
 fn a<T, U: ?Sized>(x: &[T]) -> &U {
     unsafe { transmute(x) } //~ ERROR cannot transmute between types of different sizes
@@ -15,11 +16,11 @@ fn b<T: ?Sized, U: ?Sized>(x: &T) -> &U {
 }
 
 fn c<T, U>(x: &T) -> &U {
-    unsafe { transmute(x) }
+    unsafe { transmute(x) } // Ok!
 }
 
 fn d<T, U>(x: &[T]) -> &[U] {
-    unsafe { transmute(x) }
+    unsafe { transmute(x) } // Ok!
 }
 
 fn e<T: ?Sized, U>(x: &T) -> &U {
@@ -30,4 +31,23 @@ fn f<T, U: ?Sized>(x: &T) -> &U {
     unsafe { transmute(x) } //~ ERROR cannot transmute between types of different sizes
 }
 
+// We can transmute even between pointers of unknown size as long as the metadata of
+// input and output type are the same. This even accounts for null pointer optimizations.
+fn g1<T: ?Sized>(x: &T) -> Option<&T> {
+    unsafe { transmute(x) } // Ok!
+}
+
+fn g2<T: ?Sized>(x: Option<NonNull<T>>) -> NonNull<T> {
+    unsafe { transmute(x) } // Ok!
+}
+
+fn g3<T: ?Sized>(x: *const T) -> Option<NonNull<T>> {
+    unsafe { transmute(x) } // Ok!
+}
+
+// Make sure we can see through all the layers of `Box`.
+fn h<T: ?Sized>(x: Box<T>) -> &'static T {
+    unsafe { transmute(x) } // Ok!
+}
+
 fn main() { }

tests/ui/transmute/transmute-fat-pointers.rs

@@ -1,5 +1,5 @@
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
-  --> $DIR/transmute-fat-pointers.rs:10:14
+  --> $DIR/transmute-fat-pointers.rs:11:14
    |
 LL |     unsafe { transmute(x) }
    |              ^^^^^^^^^
@@ -8,7 +8,7 @@ LL |     unsafe { transmute(x) }
    = note: target type: `&U` (pointer to `U`)
 
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
-  --> $DIR/transmute-fat-pointers.rs:14:14
+  --> $DIR/transmute-fat-pointers.rs:15:14
    |
 LL |     unsafe { transmute(x) }
    |              ^^^^^^^^^
@@ -17,7 +17,7 @@ LL |     unsafe { transmute(x) }
    = note: target type: `&U` (pointer to `U`)
 
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
-  --> $DIR/transmute-fat-pointers.rs:26:14
+  --> $DIR/transmute-fat-pointers.rs:27:14
    |
 LL |     unsafe { transmute(x) }
    |              ^^^^^^^^^
@@ -26,7 +26,7 @@ LL |     unsafe { transmute(x) }
    = note: target type: `&U` (N bits)
 
 error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
-  --> $DIR/transmute-fat-pointers.rs:30:14
+  --> $DIR/transmute-fat-pointers.rs:31:14
    |
 LL |     unsafe { transmute(x) }
    |              ^^^^^^^^^