fix: Adapts new enum layout to work in all editions.

The new implementation was only working for future edition, but since
currently enum layout is not guaranteed, it adds the changes
to all editions.

Signed-off-by: Miguel Marques <miguel.m.marques@tecnico.ulisboa.pt>
Co-authored-by: Vicente Gusmão <vicente.gusmao@tecnico.ulisboa.pt>

Author: Pinguimmar

compiler/rustc_abi/src/layout.rs

@@ -680,11 +680,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
 
                                 return true;
                             }
-                            // Repacking is currently only on future edition. For editions where it is disabled,
-                            // fall back to the original niche-filling behaviour.
-                            if !repr.can_repack_variant_around_niche() {
-                                return false;
-                            }
+
                             // The ordinary niche-filling path can only move a non-largest variant as a
                             // whole before or after the chosen niche. If that fails, try placing the
                             // variant's fields individually while treating the niche bytes as reserved.
@@ -764,27 +760,20 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
 
                     Some(layout)
                 };
-            if repr.can_repack_variant_around_niche() {
-                let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?;
-                // Chooses the first max-sized niche-providing variant whose layout succeeds.
-                for (largest_variant_index, _) in
-                    variant_layouts.iter_enumerated().filter(|&(_, layout)| {
-                        layout.size == max_variant_size && layout.largest_niche.is_some()
-                    })
-                {
-                    if let Some(layout) = try_niche_variant(largest_variant_index) {
-                        return Some(layout);
-                    }
-                }
 
-                None
-            } else {
-                let largest_variant_index = variant_layouts
-                    .iter_enumerated()
-                    .max_by_key(|(_i, layout)| layout.size.bytes())
-                    .map(|(i, _layout)| i)?;
-                try_niche_variant(largest_variant_index)
+            let max_variant_size = variant_layouts.iter().map(|layout| layout.size).max()?;
+            // Chooses the first max-sized niche-providing variant whose layout succeeds.
+            for (largest_variant_index, _) in
+                variant_layouts.iter_enumerated().filter(|&(_, layout)| {
+                    layout.size == max_variant_size && layout.largest_niche.is_some()
+                })
+            {
+                if let Some(layout) = try_niche_variant(largest_variant_index) {
+                    return Some(layout);
+                }
             }
+
+            None
         };
 
         let niche_filling_layout = calculate_niche_filling_layout();

compiler/rustc_abi/src/lib.rs

@@ -94,9 +94,6 @@ bitflags! {
         /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details.
         const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5;
         const IS_SCALABLE        = 1 << 6;
-        /// If true, enum niche-filling may repack variant fields around the reserved niche.
-        const CAN_REPACK_VARIANT_AROUND_NICHE = 1 << 7;
-        // Any of these flags being set prevent field reordering optimisation.
         const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
                                  | ReprFlags::IS_SCALABLE.bits()
@@ -228,12 +225,6 @@ impl ReprOptions {
         !self.inhibit_struct_field_reordering() && self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT)
     }
 
-    /// Returns `true` if enum niche-filling can try to pack variant fields around
-    /// the reserved niche when whole-variant placement fails.
-    pub fn can_repack_variant_around_niche(&self) -> bool {
-        self.flags.contains(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE)
-    }
-
     /// Returns `true` if this `#[repr()]` should inhibit union ABI optimisations.
     pub fn inhibits_union_abi_opt(&self) -> bool {
         self.c()

compiler/rustc_middle/src/ty/mod.rs

@@ -1603,10 +1603,6 @@ impl<'tcx> TyCtxt<'tcx> {
             flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
         }
 
-        if self.def_span(did).edition().at_least_edition_future() {
-            flags.insert(ReprFlags::CAN_REPACK_VARIANT_AROUND_NICHE);
-        }
-
         // box is special, on the one hand the compiler assumes an ordered layout, with the pointer
         // always at offset zero. On the other hand we want scalar abi optimizations.
         let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox);

tests/ui/structs-enums/enum-niche-fill-around-tag.rs

@@ -1,13 +1,8 @@
-//@ revisions: old future
 //@ run-pass
-//@ [old] edition:2024
-//@ [future] edition:future
-//@ [future] compile-flags: -Z unstable-options
 
 #![allow(dead_code)]
-#![cfg_attr(future, feature(offset_of_enum))]
+#![feature(offset_of_enum)]
 
-#[cfg(future)]
 use std::mem::{align_of, offset_of};
 use std::mem::size_of;
 use std::num::NonZero;
@@ -77,64 +72,55 @@ enum ChooseFittingNiche {
 fn main() {
     assert_eq!(size_of::<Inner>(), 8);
 
-    #[cfg(old)]
-    {
-        assert_eq!(size_of::<Outer>(), 12);
-        assert_eq!(size_of::<RepackPadding>(), 12);
-    }
-
-    #[cfg(future)]
-    {
-        assert_eq!(size_of::<Outer>(), 8);
-        assert_eq!(size_of::<RepackPadding>(), 8);
-
-        assert_eq!(size_of::<RepackedBase>(), 8);
-        assert_eq!(align_of::<RepackedBase>(), 4);
-        assert_eq!(offset_of!(RepackedBase, int32), 0);
-        assert_eq!(offset_of!(RepackedBase, boolean), 4);
-
-        assert_eq!(size_of::<RepackedAroundNiche>(), 8);
-        assert_eq!(align_of::<RepackedAroundNiche>(), 4);
-        assert_eq!(size_of::<Option<RepackedAroundNiche>>(), 8);
-        assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0);
-        assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0);
-        assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5);
-        assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6);
-
-        assert_eq!(size_of::<NestedRepackedAroundNiche>(), 8);
-        assert_eq!(align_of::<NestedRepackedAroundNiche>(), 4);
-        assert_eq!(size_of::<Option<NestedRepackedAroundNiche>>(), 8);
-        assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0);
-        assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0);
-        assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5);
-        assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6);
-
-        assert_eq!(size_of::<RepackedPair>(), 4);
-        assert_eq!(align_of::<RepackedPair>(), 2);
-        assert_eq!(offset_of!(RepackedPair, field2), 0);
-        assert_eq!(offset_of!(RepackedPair, field3), 2);
-
-        assert_eq!(size_of::<RepackedNestedAggregate>(), 8);
-        assert_eq!(align_of::<RepackedNestedAggregate>(), 4);
-        assert_eq!(size_of::<Option<RepackedNestedAggregate>>(), 8);
-        assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0);
-        assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0);
-        assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6);
-
-        assert_eq!(size_of::<FittingNichePayload>(), 8);
-        assert_eq!(align_of::<FittingNichePayload>(), 4);
-        assert_eq!(offset_of!(FittingNichePayload, word), 0);
-        assert_eq!(offset_of!(FittingNichePayload, flag), 4);
-
-        assert_eq!(size_of::<ChooseFittingNiche>(), 8);
-        assert_eq!(align_of::<ChooseFittingNiche>(), 4);
-        assert_eq!(size_of::<Option<ChooseFittingNiche>>(), 8);
-        assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0);
-        assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0);
-        assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5);
-        assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6);
-        assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0);
-        assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5);
-        assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6);
-    }
+    assert_eq!(size_of::<Outer>(), 8);
+    assert_eq!(size_of::<RepackPadding>(), 8);
+
+    assert_eq!(size_of::<RepackedBase>(), 8);
+    assert_eq!(align_of::<RepackedBase>(), 4);
+    assert_eq!(offset_of!(RepackedBase, int32), 0);
+    assert_eq!(offset_of!(RepackedBase, boolean), 4);
+
+    assert_eq!(size_of::<RepackedAroundNiche>(), 8);
+    assert_eq!(align_of::<RepackedAroundNiche>(), 4);
+    assert_eq!(size_of::<Option<RepackedAroundNiche>>(), 8);
+    assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0);
+    assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0);
+    assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5);
+    assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6);
+
+    assert_eq!(size_of::<NestedRepackedAroundNiche>(), 8);
+    assert_eq!(align_of::<NestedRepackedAroundNiche>(), 4);
+    assert_eq!(size_of::<Option<NestedRepackedAroundNiche>>(), 8);
+    assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0);
+    assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0);
+    assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5);
+    assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6);
+
+    assert_eq!(size_of::<RepackedPair>(), 4);
+    assert_eq!(align_of::<RepackedPair>(), 2);
+    assert_eq!(offset_of!(RepackedPair, field2), 0);
+    assert_eq!(offset_of!(RepackedPair, field3), 2);
+
+    assert_eq!(size_of::<RepackedNestedAggregate>(), 8);
+    assert_eq!(align_of::<RepackedNestedAggregate>(), 4);
+    assert_eq!(size_of::<Option<RepackedNestedAggregate>>(), 8);
+    assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0);
+    assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0);
+    assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6);
+
+    assert_eq!(size_of::<FittingNichePayload>(), 8);
+    assert_eq!(align_of::<FittingNichePayload>(), 4);
+    assert_eq!(offset_of!(FittingNichePayload, word), 0);
+    assert_eq!(offset_of!(FittingNichePayload, flag), 4);
+
+    assert_eq!(size_of::<ChooseFittingNiche>(), 8);
+    assert_eq!(align_of::<ChooseFittingNiche>(), 4);
+    assert_eq!(size_of::<Option<ChooseFittingNiche>>(), 8);
+    assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6);
 }