repr(transparent): don't consider most length-0 arrays trivial

With this PR, an array type is considered trivial
for the purpose of `repr(transparent)` only if its
element type is—we emit the `repr_transparent_non_zst_fields` FCW
otherwise.

This has two benefits:

## Forbid non-portable definitions

Some types have alignment 1 only on certain platforms.
Prior to this PR, the following snippet would compile on AVR,
and *only* on AVR:

```rust
#[repr(transparent)]
struct Foo(i32, [u16; 0]);
```

After this PR, the above now fails to compile on any target.

## FFI and CFI compatibility

We want to add support for Control Flow Integrity to Rust at some point.
There are some good reasons to want CFI to consider `*const [u8; 0]`
and `*const [u8; 1]` compatible with one another. But that means we must
consider `*const [u8; 0]` and `*const ()` to be CFI-incompatible.
Declaring `[u8; 0]` non-trivial for `repr(transparent)` makes that
easier to achieve. See discussion on Zulip:
<https://rust-lang.zulipchat.com/#narrow/channel/136281-t-opsem/topic/ABI-compatibility.20rules.20of.20ZST.20types/near/591412488>

Author: Jules-Bertholet

compiler/rustc_hir_analysis/src/check/check.rs

@@ -1709,6 +1709,10 @@ pub(super) fn check_transparent<'tcx>(tcx: TyCtxt<'tcx>, adt: ty::AdtDef<'tcx>)
                     "`repr(C)` types",
                     "is a `#[repr(C)]` type, so it is not guaranteed to be zero-sized on all targets.",
                 ),
+                UnsuitedReason::Array => (
+                    "array types with non-trivial element types",
+                    "is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.",
+                ),
             };
             Diag::new(
                 dcx,
@@ -1785,6 +1789,7 @@ pub(super) fn check_transparent<'tcx>(tcx: TyCtxt<'tcx>, adt: ty::AdtDef<'tcx>)
         NonExhaustive,
         PrivateField,
         ReprC,
+        Array,
     }
 
     fn check_unsuited<'tcx>(
@@ -1797,7 +1802,15 @@ pub(super) fn check_transparent<'tcx>(tcx: TyCtxt<'tcx>, adt: ty::AdtDef<'tcx>)
             tcx.try_normalize_erasing_regions(typing_env, Unnormalized::new_wip(ty)).unwrap_or(ty);
         match ty.kind() {
             ty::Tuple(list) => list.iter().try_for_each(|t| check_unsuited(tcx, typing_env, t)),
-            ty::Array(ty, _) => check_unsuited(tcx, typing_env, *ty),
+            ty::Array(elem_ty, _) => {
+                let elem_layout = tcx.layout_of(typing_env.as_query_input(*elem_ty));
+                let elem_trivial = elem_layout.is_ok_and(|layout| layout.is_1zst());
+                if elem_trivial {
+                    check_unsuited(tcx, typing_env, *elem_ty)
+                } else {
+                    return ControlFlow::Break(UnsuitedInfo { ty, reason: UnsuitedReason::Array });
+                }
+            }
             ty::Adt(def, args) => {
                 if !def.did().is_local() && !find_attr!(tcx, def.did(), RustcPubTransparent(_)) {
                     let non_exhaustive = def.is_variant_list_non_exhaustive()

tests/ui/abi/compatibility.rs

@@ -145,11 +145,13 @@ macro_rules! test_abi_compatible {
             use super::*;
             // Declaring a `type` doesn't even check well-formedness, so we also declare a function.
             fn check_wf(_x: $t1, _y: $t2) {}
-            // Test argument and return value, `Rust` and `C` ABIs.
+            // Test argument and return value in various ABIs.
             #[rustc_abi(assert_eq)]
             type TestRust = (fn($t1) -> $t1, fn($t2) -> $t2);
             #[rustc_abi(assert_eq)]
             type TestC = (extern "C" fn($t1) -> $t1, extern "C" fn($t2) -> $t2);
+            #[rustc_abi(assert_eq)]
+            type TestSystem = (extern "system" fn($t1) -> $t1, extern "system" fn($t2) -> $t2);
         }
     };
 }
@@ -200,7 +202,8 @@ test_abi_compatible!(isize_int, isize, i64);
 
 // Compatibility of 1-ZST.
 test_abi_compatible!(zst_unit, Zst, ());
-test_abi_compatible!(zst_array, Zst, [u8; 0]);
+test_abi_compatible!(zst_array, Zst, [(); 0]);
+test_abi_compatible!(zst_array_2, Zst, [(); 42]);
 test_abi_compatible!(nonzero_int, NonZero<i32>, i32);
 
 // `#[repr(C)]` enums should not change ABI based on individual variant inhabitedness.
@@ -220,7 +223,7 @@ struct TransparentWrapper1<T: ?Sized>(T);
 #[repr(transparent)]
 struct TransparentWrapper2<T: ?Sized>((), Zst, T);
 #[repr(transparent)]
-struct TransparentWrapper3<T>(T, [u8; 0], PhantomData<u64>);
+struct TransparentWrapper3<T>(T, [Zst; 42], PhantomData<u64>);
 #[repr(transparent)]
 union TransparentWrapperUnion<T> {
     nothing: (),
@@ -299,14 +302,14 @@ macro_rules! test_nonnull {
             test_abi_compatible!(result_ok_unit, Result<(), $t>, $t);
             test_abi_compatible!(result_err_zst, Result<$t, Zst>, $t);
             test_abi_compatible!(result_ok_zst, Result<Zst, $t>, $t);
-            test_abi_compatible!(result_err_arr, Result<$t, [i8; 0]>, $t);
-            test_abi_compatible!(result_ok_arr, Result<[i8; 0], $t>, $t);
+            test_abi_compatible!(result_err_arr, Result<$t, [(); 42]>, $t);
+            test_abi_compatible!(result_ok_arr, Result<[(); 42], $t>, $t);
             test_abi_compatible!(result_err_void, Result<$t, Void>, $t);
             test_abi_compatible!(result_ok_void, Result<Void, $t>, $t);
             test_abi_compatible!(either_err_zst, Either<$t, Zst>, $t);
             test_abi_compatible!(either_ok_zst, Either<Zst, $t>, $t);
             test_abi_compatible!(either2_err_zst, Either2<$t, Zst>, $t);
-            test_abi_compatible!(either2_err_arr, Either2<$t, [i8; 0]>, $t);
+            test_abi_compatible!(either2_err_arr, Either2<$t, [(); 42]>, $t);
         }
     }
 }

tests/ui/layout/homogeneous-aggr-transparent.rs

@@ -14,7 +14,7 @@ struct Wrapper1<T>(T);
 #[repr(transparent)]
 struct Wrapper2<T>((), Zst, T);
 #[repr(transparent)]
-struct Wrapper3<T>(T, [u8; 0], PhantomData<u64>);
+struct Wrapper3<T>(T, [Zst; 42], PhantomData<u64>);
 #[repr(transparent)]
 union WrapperUnion<T: Copy> {
     nothing: (),

tests/ui/repr/repr-transparent-array.rs

@@ -0,0 +1,41 @@
+#![deny(repr_transparent_non_zst_fields)]
+
+type SusArray = [u8; 0];
+
+type GoodArray = [[(); 0]; 42];
+
+pub type Sized = i32;
+
+#[repr(transparent)]
+pub struct T1(SusArray);
+#[repr(transparent)]
+pub struct T2(GoodArray, SusArray);
+#[repr(transparent)]
+pub struct T3(SusArray, GoodArray);
+
+#[repr(transparent)]
+pub struct T5(Sized, SusArray);
+//~^ ERROR zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+//~| WARN this was previously accepted by the compiler
+
+#[repr(transparent)]
+pub struct T6(SusArray, Sized);
+//~^ ERROR zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+//~| WARN this was previously accepted by the compiler
+
+#[repr(transparent)]
+pub struct T7(T1, SusArray); // still wrong, even when the array is hidden inside another type
+//~^ ERROR zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+//~| WARN this was previously accepted by the compiler
+
+#[repr(transparent)]
+pub struct T8(SusArray, SusArray); // still wrong
+//~^ ERROR zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+//~| WARN this was previously accepted by the compiler
+
+#[repr(transparent)]
+pub struct T9([[u8; 0]; 0], SusArray); // still wrong
+//~^ ERROR zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+//~| WARN this was previously accepted by the compiler
+
+fn main() {}

tests/ui/repr/repr-transparent-array.stderr

@@ -0,0 +1,137 @@
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:17:22
+   |
+LL | pub struct T5(Sized, SusArray);
+   |                      ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+note: the lint level is defined here
+  --> $DIR/repr-transparent-array.rs:1:9
+   |
+LL | #![deny(repr_transparent_non_zst_fields)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:22:15
+   |
+LL | pub struct T6(SusArray, Sized);
+   |               ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:27:19
+   |
+LL | pub struct T7(T1, SusArray); // still wrong, even when the array is hidden inside another type
+   |                   ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:32:25
+   |
+LL | pub struct T8(SusArray, SusArray); // still wrong
+   |                         ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:37:29
+   |
+LL | pub struct T9([[u8; 0]; 0], SusArray); // still wrong
+   |                             ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+
+error: aborting due to 5 previous errors
+
+Future incompatibility report: Future breakage diagnostic:
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:17:22
+   |
+LL | pub struct T5(Sized, SusArray);
+   |                      ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+note: the lint level is defined here
+  --> $DIR/repr-transparent-array.rs:1:9
+   |
+LL | #![deny(repr_transparent_non_zst_fields)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Future breakage diagnostic:
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:22:15
+   |
+LL | pub struct T6(SusArray, Sized);
+   |               ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+note: the lint level is defined here
+  --> $DIR/repr-transparent-array.rs:1:9
+   |
+LL | #![deny(repr_transparent_non_zst_fields)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Future breakage diagnostic:
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:27:19
+   |
+LL | pub struct T7(T1, SusArray); // still wrong, even when the array is hidden inside another type
+   |                   ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+note: the lint level is defined here
+  --> $DIR/repr-transparent-array.rs:1:9
+   |
+LL | #![deny(repr_transparent_non_zst_fields)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Future breakage diagnostic:
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:32:25
+   |
+LL | pub struct T8(SusArray, SusArray); // still wrong
+   |                         ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+note: the lint level is defined here
+  --> $DIR/repr-transparent-array.rs:1:9
+   |
+LL | #![deny(repr_transparent_non_zst_fields)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Future breakage diagnostic:
+error: zero-sized fields in `repr(transparent)` cannot contain array types with non-trivial element types
+  --> $DIR/repr-transparent-array.rs:37:29
+   |
+LL | pub struct T9([[u8; 0]; 0], SusArray); // still wrong
+   |                             ^^^^^^^^
+   |
+   = note: this field contains `[u8; 0]`, which is an array with a non-trivial element type, so it is not guaranteed to have a trivial ABI in all situations.
+   = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+   = note: for more information, see issue #78586 <https://github.com/rust-lang/rust/issues/78586>
+note: the lint level is defined here
+  --> $DIR/repr-transparent-array.rs:1:9
+   |
+LL | #![deny(repr_transparent_non_zst_fields)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+