11//! Experimental support for emitting retags as function calls in generated code.
2+ //!
3+ //! We attempt to retag every argument and return value of a function, and every rvalue
4+ //! of an assignment. The first step to retagging is to generate a [`RetagPlan`], which
5+ //! describes which pointers within the place or operand can be retagged.
26
7+ #![ allow( unused) ]
8+ use rustc_abi:: { BackendRepr , FieldIdx , FieldsShape , VariantIdx , Variants } ;
9+ use rustc_ast:: Mutability ;
10+ use rustc_data_structures:: fx:: FxIndexMap ;
311use rustc_middle:: mir:: { Rvalue , WithRetag } ;
12+ use rustc_middle:: ty;
13+ use rustc_middle:: ty:: layout:: TyAndLayout ;
414
15+ use crate :: RetagInfo ;
516use crate :: mir:: FunctionCx ;
617use crate :: mir:: operand:: OperandRef ;
718use crate :: mir:: place:: PlaceRef ;
@@ -12,6 +23,160 @@ pub(crate) fn rvalue_needs_retag(rvalue: &Rvalue<'_>) -> bool {
1223 !matches ! ( rvalue, Rvalue :: Ref ( ..) ) && !matches ! ( rvalue, Rvalue :: Use ( .., WithRetag :: No ) )
1324}
1425
26+ /// A description of the pointers within a type that need to be retagged.
27+ #[ derive( Debug ) ]
28+ enum RetagPlan < V > {
29+ /// Indicates that a pointer should be retagged.
30+ EmitRetag ( RetagInfo < V > ) ,
31+ /// Indicates that one or more fields or variants of this type
32+ /// contain pointers that need to be retagged.
33+ Recurse {
34+ field_plans : FxIndexMap < FieldIdx , RetagPlan < V > > ,
35+ variant_plans : FxIndexMap < VariantIdx , RetagPlan < V > > ,
36+ } ,
37+ }
38+
39+ impl < V > RetagPlan < V > {
40+ /// A helper function to move a [`RetagPlan`] into a particular field.
41+ fn for_field ( self , ix : FieldIdx ) -> Self {
42+ let mut field_plans = FxIndexMap :: default ( ) ;
43+ field_plans. insert ( ix, self ) ;
44+ RetagPlan :: Recurse { field_plans, variant_plans : FxIndexMap :: default ( ) }
45+ }
46+ }
47+
48+ impl < ' a , ' tcx , V > RetagPlan < V > {
49+ /// Attempts to create a [`RetagPlan`] for a place or operand with the given layout.
50+ fn build < Bx : BuilderMethods < ' a , ' tcx > > (
51+ bx : & mut Bx ,
52+ layout : TyAndLayout < ' tcx > ,
53+ is_fn_entry : bool ,
54+ ) -> Option < RetagPlan < Bx :: Value > > {
55+ // If the value being retagged is smaller than a pointer, then it can't contain any
56+ // pointers we need to retag, so we can stop recursion early. This optimization is
57+ // crucial for ZSTs, because they can contain way more fields than we can ever visit.
58+ if layout. is_sized ( ) && layout. size < bx. tcx ( ) . data_layout . pointer_size ( ) {
59+ return None ;
60+ }
61+ // SIMD vectors may only contain raw pointers, integers, and floating point values,
62+ // which do not need to be retagged.
63+ if matches ! ( layout. backend_repr, BackendRepr :: SimdVector { .. } ) {
64+ return None ;
65+ }
66+
67+ // Check the type of this value to see what to do with it (retag, or recurse).
68+ match layout. ty . kind ( ) {
69+ & ty:: Ref ( _, pointee, mt) => {
70+ let pointee_layout = bx. layout_of ( pointee) ;
71+ Self :: emit_retag ( bx, pointee_layout, Some ( mt) , is_fn_entry)
72+ }
73+ & ty:: RawPtr ( _, _) => None ,
74+ // `Box` needs special handling, since the innermost pointer is what gets retagged, but
75+ // the outermost `Box` is what determines the permission that gets created.
76+ ty:: Adt ( adt, _) if adt. is_box ( ) => Self :: visit_box ( bx, layout, is_fn_entry) ,
77+ // Skip traversing for everything inside of `MaybeDangling`
78+ ty:: Adt ( adt, _) if adt. is_maybe_dangling ( ) => None ,
79+ _ => Self :: walk_value ( bx, layout, is_fn_entry) ,
80+ }
81+ }
82+
83+ /// Recurses through the fields and variants of a value in memory order to create a [`RetagPlan`].
84+ fn walk_value < Bx : BuilderMethods < ' a , ' tcx > > (
85+ bx : & mut Bx ,
86+ layout : TyAndLayout < ' tcx > ,
87+ is_fn_entry : bool ,
88+ ) -> Option < RetagPlan < Bx :: Value > > {
89+ let mut field_plans = FxIndexMap :: default ( ) ;
90+ let mut variant_plans = FxIndexMap :: default ( ) ;
91+
92+ match & layout. fields {
93+ FieldsShape :: Union ( _) | FieldsShape :: Primitive => { }
94+ _ => {
95+ for ix in layout. fields . index_by_increasing_offset ( ) {
96+ let field_layout = layout. field ( bx, ix) ;
97+ if let Some ( plan) = Self :: build ( bx, field_layout, is_fn_entry) {
98+ field_plans. insert ( FieldIdx :: from_usize ( ix) , plan) ;
99+ }
100+ }
101+ }
102+ }
103+
104+ match & layout. variants {
105+ Variants :: Single { .. } | Variants :: Empty => { }
106+ Variants :: Multiple { variants, .. } => {
107+ for ix in variants. indices ( ) {
108+ let variant_layout = layout. for_variant ( bx, ix) ;
109+ if let Some ( plan) = Self :: build ( bx, variant_layout, is_fn_entry) {
110+ variant_plans. insert ( ix, plan) ;
111+ }
112+ }
113+ }
114+ }
115+
116+ ( !field_plans. is_empty ( ) || !variant_plans. is_empty ( ) )
117+ . then ( || RetagPlan :: Recurse { field_plans, variant_plans } )
118+ }
119+
120+ /// Emits a retag for a `Box`.
121+ fn visit_box < Bx : BuilderMethods < ' a , ' tcx > > (
122+ bx : & mut Bx ,
123+ layout : TyAndLayout < ' tcx > ,
124+ is_fn_entry : bool ,
125+ ) -> Option < RetagPlan < Bx :: Value > > {
126+ assert ! ( layout. ty. is_box( ) ) ;
127+ assert_eq ! ( layout. fields. count( ) , 2 , "`Box` must have exactly 2 fields" ) ;
128+ let mut field_plans = FxIndexMap :: default ( ) ;
129+
130+ // Only retag the inner pointer of a `Box` if it came from the global allocator.
131+ if layout. ty . is_box_global ( bx. tcx ( ) ) {
132+ let boxed_ty = layout. ty . expect_boxed_ty ( ) ;
133+ let boxed_layout = bx. layout_of ( boxed_ty) ;
134+ if let Some ( mut plan) = Self :: emit_retag ( bx, boxed_layout, None , is_fn_entry) {
135+ // `Unique<T>`
136+ let unique = layout. field ( bx, 0 ) ;
137+ assert_eq ! ( unique. fields. count( ) , 2 ) ;
138+ plan = plan. for_field ( FieldIdx :: ZERO ) ;
139+
140+ // `NonNull<T>`
141+ let nonnull = unique. field ( bx, 0 ) ;
142+ assert_eq ! ( nonnull. fields. count( ) , 1 ) ;
143+ plan = plan. for_field ( FieldIdx :: ZERO ) ;
144+
145+ // `*mut T is !null`
146+ let pattern = nonnull. field ( bx, 0 ) ;
147+ let ty:: Pat ( base, _) = pattern. ty . kind ( ) else {
148+ unreachable ! ( "`NonNull` should contain a pattern type" )
149+ } ;
150+ assert_eq ! ( base. builtin_deref( true ) , Some ( boxed_ty) ) ;
151+
152+ field_plans. insert ( FieldIdx :: ZERO , plan) ;
153+ }
154+ }
155+
156+ // We always try to retag the second field (the allocator)
157+ let field_layout = layout. field ( bx, 1 ) ;
158+ if let Some ( plan) = Self :: build ( bx, field_layout, is_fn_entry) {
159+ field_plans. insert ( FieldIdx :: ONE , plan) ;
160+ }
161+
162+ ( !field_plans. is_empty ( ) )
163+ . then ( || RetagPlan :: Recurse { field_plans, variant_plans : FxIndexMap :: default ( ) } )
164+ }
165+
166+ /// Determines if a pointer needs to be retagged, when it points to
167+ /// a type with the given layout. Returns `None` for mutable pointers
168+ /// to types that are entirely covered by `UnsafePinned`, for which retags
169+ /// are a no-op.
170+ fn emit_retag < Bx : BuilderMethods < ' a , ' tcx > > (
171+ _bx : & mut Bx ,
172+ _pointee_layout : TyAndLayout < ' tcx > ,
173+ _ptr_kind : Option < Mutability > ,
174+ _is_fn_entry : bool ,
175+ ) -> Option < RetagPlan < Bx :: Value > > {
176+ None
177+ }
178+ }
179+
15180impl < ' a , ' tcx , Bx : BuilderMethods < ' a , ' tcx > > FunctionCx < ' a , ' tcx , Bx > {
16181 /// Retags the pointers within an [`OperandRef`].
17182pub ( crate ) fn codegen_retag_operand (
0 commit comments