Skip to content

Commit 2ee725b

Browse files
committed
ty_utils: lower tuples to ScalableVector repr
Instead of just using regular struct lowering for these types, which results in an incorrect ABI (e.g. returning indirectly), use `BackendRepr::ScalableVector` which will lower to the correct type and be passed in registers. This also enables some simplifications for generating alloca of scalable vectors and greater re-use of `scalable_vector_parts`. A LLVM codegen test demonstrating the changed IR this generates is included in the next commit alongside some intrinsics that make these tuples usable.
1 parent 6d6d14d commit 2ee725b

12 files changed

Lines changed: 191 additions & 78 deletions

File tree

compiler/rustc_abi/src/layout.rs

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ use tracing::{debug, trace};
1010

1111
use crate::{
1212
AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
13-
LayoutData, Niche, NonZeroUsize, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding,
14-
TargetDataLayout, Variants, WrappingRange,
13+
LayoutData, Niche, NonZeroUsize, NumScalableVectors, Primitive, ReprOptions, Scalar, Size,
14+
StructKind, TagEncoding, TargetDataLayout, Variants, WrappingRange,
1515
};
1616

1717
mod coroutine;
@@ -204,13 +204,19 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
204204
&self,
205205
element: F,
206206
count: u64,
207+
number_of_vectors: NumScalableVectors,
207208
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
208209
where
209210
FieldIdx: Idx,
210211
VariantIdx: Idx,
211212
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
212213
{
213-
vector_type_layout(SimdVectorKind::Scalable, self.cx.data_layout(), element, count)
214+
vector_type_layout(
215+
SimdVectorKind::Scalable(number_of_vectors),
216+
self.cx.data_layout(),
217+
element,
218+
count,
219+
)
214220
}
215221

216222
pub fn simd_type<FieldIdx, VariantIdx, F>(
@@ -1526,7 +1532,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
15261532

15271533
enum SimdVectorKind {
15281534
/// `#[rustc_scalable_vector]`
1529-
Scalable,
1535+
Scalable(NumScalableVectors),
15301536
/// `#[repr(simd, packed)]`
15311537
PackedFixed,
15321538
/// `#[repr(simd)]`
@@ -1559,9 +1565,10 @@ where
15591565
let size =
15601566
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
15611567
let (repr, align) = match kind {
1562-
SimdVectorKind::Scalable => {
1563-
(BackendRepr::ScalableVector { element, count }, dl.llvmlike_vector_align(size))
1564-
}
1568+
SimdVectorKind::Scalable(number_of_vectors) => (
1569+
BackendRepr::ScalableVector { element, count, number_of_vectors },
1570+
dl.llvmlike_vector_align(size),
1571+
),
15651572
// Non-power-of-two vectors have padding up to the next power-of-two.
15661573
// If we're a packed repr, remove the padding while keeping the alignment as close
15671574
// to a vector as possible.

compiler/rustc_abi/src/lib.rs

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1717,6 +1717,54 @@ impl AddressSpace {
17171717
pub const ZERO: Self = AddressSpace(0);
17181718
}
17191719

1720+
/// How many scalable vectors are in a `BackendRepr::ScalableVector`?
1721+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
1722+
#[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
1723+
pub enum NumScalableVectors {
1724+
/// A single scalable vector
1725+
///
1726+
/// ```rust,ignore (example)
1727+
/// #[rustc_scalable_vector(4)]
1728+
/// struct svuint32_t(i32);
1729+
/// ```
1730+
One,
1731+
/// Tuple of two scalable vectors
1732+
///
1733+
/// ```rust,ignore (example)
1734+
/// #[rustc_scalable_vector]
1735+
/// struct svuint32x2_t(svuint32_t, svuint32_t);
1736+
/// ```
1737+
Two,
1738+
/// Tuple of three scalable vectors
1739+
///
1740+
/// ```rust,ignore (example)
1741+
/// #[rustc_scalable_vector]
1742+
/// struct svuint32x3_t(svuint32_t, svuint32_t, svuint32_t);
1743+
/// ```
1744+
Three,
1745+
/// Tuple of four scalable vectors
1746+
///
1747+
/// ```rust,ignore (example)
1748+
/// #[rustc_scalable_vector]
1749+
/// struct svuint32x4_t(svuint32_t, svuint32_t, svuint32_t, svuint32_t);
1750+
/// ```
1751+
Four,
1752+
}
1753+
1754+
impl NumScalableVectors {
1755+
// Returns `NumScalableVectors` for values of two, three or four, which are a valid number of
1756+
// fields for a tuple of scalable vectors to have. `1` is a valid value of `NumScalableVectors`
1757+
// but not for a tuple which would have a field count.
1758+
pub fn from_field_count(count: usize) -> Option<Self> {
1759+
match count {
1760+
2 => Some(NumScalableVectors::Two),
1761+
3 => Some(NumScalableVectors::Three),
1762+
4 => Some(NumScalableVectors::Four),
1763+
_ => None,
1764+
}
1765+
}
1766+
}
1767+
17201768
/// The way we represent values to the backend
17211769
///
17221770
/// Previously this was conflated with the "ABI" a type is given, as in the platform-specific ABI.
@@ -1735,6 +1783,7 @@ pub enum BackendRepr {
17351783
ScalableVector {
17361784
element: Scalar,
17371785
count: u64,
1786+
number_of_vectors: NumScalableVectors,
17381787
},
17391788
SimdVector {
17401789
element: Scalar,
@@ -1841,8 +1890,12 @@ impl BackendRepr {
18411890
BackendRepr::SimdVector { element: element.to_union(), count }
18421891
}
18431892
BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
1844-
BackendRepr::ScalableVector { element, count } => {
1845-
BackendRepr::ScalableVector { element: element.to_union(), count }
1893+
BackendRepr::ScalableVector { element, count, number_of_vectors } => {
1894+
BackendRepr::ScalableVector {
1895+
element: element.to_union(),
1896+
count,
1897+
number_of_vectors,
1898+
}
18461899
}
18471900
}
18481901
}
@@ -2181,18 +2234,10 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
21812234
}
21822235

21832236
/// Returns `true` if the size of the type is only known at runtime.
2184-
pub fn is_runtime_sized(&self) -> bool {
2237+
pub fn is_scalable_vector(&self) -> bool {
21852238
matches!(self.backend_repr, BackendRepr::ScalableVector { .. })
21862239
}
21872240

2188-
/// Returns the elements count of a scalable vector.
2189-
pub fn scalable_vector_element_count(&self) -> Option<u64> {
2190-
match self.backend_repr {
2191-
BackendRepr::ScalableVector { count, .. } => Some(count),
2192-
_ => None,
2193-
}
2194-
}
2195-
21962241
/// Returns `true` if the type is a ZST and not unsized.
21972242
///
21982243
/// Note that this does *not* imply that the type is irrelevant for layout! It can still have

compiler/rustc_codegen_gcc/src/builder.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use rustc_data_structures::fx::FxHashSet;
2424
use rustc_middle::bug;
2525
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
2626
use rustc_middle::ty::layout::{
27-
FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers,
27+
FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers, TyAndLayout,
2828
};
2929
use rustc_middle::ty::{self, AtomicOrdering, Instance, Ty, TyCtxt};
3030
use rustc_span::Span;
@@ -942,8 +942,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
942942
.get_address(self.location)
943943
}
944944

945-
fn scalable_alloca(&mut self, _elt: u64, _align: Align, _element_ty: Ty<'_>) -> RValue<'gcc> {
946-
todo!()
945+
fn alloca_with_ty(&mut self, ty: TyAndLayout<'tcx>) -> RValue<'gcc> {
946+
self.alloca(ty.layout.size, ty.layout.align.abi)
947947
}
948948

949949
fn load(&mut self, pointee_ty: Type<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {

compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ pub(crate) mod autodiff;
77
pub(crate) mod gpu_offload;
88

99
use libc::{c_char, c_uint};
10-
use rustc_abi as abi;
11-
use rustc_abi::{Align, Size, WrappingRange};
10+
use rustc_abi::{self as abi, Align, Size, WrappingRange};
1211
use rustc_codegen_ssa::MemFlags;
1312
use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
1413
use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
@@ -615,21 +614,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
615614
}
616615
}
617616

618-
fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value {
617+
fn alloca_with_ty(&mut self, layout: TyAndLayout<'tcx>) -> Self::Value {
619618
let mut bx = Builder::with_cx(self.cx);
620619
bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
621-
let llvm_ty = match element_ty.kind() {
622-
ty::Bool => bx.type_i1(),
623-
ty::Int(int_ty) => self.cx.type_int_from_ty(*int_ty),
624-
ty::Uint(uint_ty) => self.cx.type_uint_from_ty(*uint_ty),
625-
ty::Float(float_ty) => self.cx.type_float_from_ty(*float_ty),
626-
_ => unreachable!("scalable vectors can only contain a bool, int, uint or float"),
627-
};
620+
let scalable_vector_ty = layout.llvm_type(self.cx);
628621

629622
unsafe {
630-
let ty = llvm::LLVMScalableVectorType(llvm_ty, elt.try_into().unwrap());
631-
let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
632-
llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
623+
let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, scalable_vector_ty, UNNAMED);
624+
llvm::LLVMSetAlignment(alloca, layout.align.abi.bytes() as c_uint);
633625
alloca
634626
}
635627
}

compiler/rustc_codegen_llvm/src/type_of.rs

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use std::fmt::Write;
22

33
use rustc_abi::Primitive::{Float, Int, Pointer};
4-
use rustc_abi::{Align, BackendRepr, FieldsShape, Scalar, Size, Variants};
4+
use rustc_abi::{Align, BackendRepr, FieldsShape, NumScalableVectors, Scalar, Size, Variants};
55
use rustc_codegen_ssa::traits::*;
66
use rustc_middle::bug;
77
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
@@ -24,14 +24,24 @@ fn uncached_llvm_type<'a, 'tcx>(
2424
let element = layout.scalar_llvm_type_at(cx, element);
2525
return cx.type_vector(element, count);
2626
}
27-
BackendRepr::ScalableVector { ref element, count } => {
27+
BackendRepr::ScalableVector { ref element, count, number_of_vectors } => {
2828
let element = if element.is_bool() {
2929
cx.type_i1()
3030
} else {
3131
layout.scalar_llvm_type_at(cx, *element)
3232
};
3333

34-
return cx.type_scalable_vector(element, count);
34+
let vector_type = cx.type_scalable_vector(element, count);
35+
return match number_of_vectors {
36+
NumScalableVectors::One => vector_type,
37+
NumScalableVectors::Two => cx.type_struct(&[vector_type, vector_type], false),
38+
NumScalableVectors::Three => {
39+
cx.type_struct(&[vector_type, vector_type, vector_type], false)
40+
}
41+
NumScalableVectors::Four => {
42+
cx.type_struct(&[vector_type, vector_type, vector_type, vector_type], false)
43+
}
44+
};
3545
}
3646
BackendRepr::Memory { .. } | BackendRepr::ScalarPair(..) => {}
3747
}

compiler/rustc_codegen_ssa/src/mir/debuginfo.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -438,8 +438,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
438438
if operand.layout.ty.is_scalable_vector()
439439
&& bx.sess().target.arch == rustc_target::spec::Arch::AArch64
440440
{
441-
let (count, element_ty) =
442-
operand.layout.ty.scalable_vector_element_count_and_type(bx.tcx());
441+
let (count, element_ty, _) =
442+
operand.layout.ty.scalable_vector_parts(bx.tcx()).unwrap();
443443
// i.e. `<vscale x N x i1>` when `N != 16`
444444
if element_ty.is_bool() && count != 16 {
445445
return;

compiler/rustc_codegen_ssa/src/mir/place.rs

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use std::ops::Deref as _;
2+
13
use rustc_abi::{
24
Align, BackendRepr, FieldIdx, FieldsShape, Size, TagEncoding, VariantIdx, Variants,
35
};
@@ -109,8 +111,8 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
109111
bx: &mut Bx,
110112
layout: TyAndLayout<'tcx>,
111113
) -> Self {
112-
if layout.is_runtime_sized() {
113-
Self::alloca_runtime_sized(bx, layout)
114+
if layout.deref().is_scalable_vector() {
115+
Self::alloca_scalable(bx, layout)
114116
} else {
115117
Self::alloca_size(bx, layout.size, layout)
116118
}
@@ -151,16 +153,11 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
151153
}
152154
}
153155

154-
fn alloca_runtime_sized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
156+
fn alloca_scalable<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
155157
bx: &mut Bx,
156158
layout: TyAndLayout<'tcx>,
157159
) -> Self {
158-
let (element_count, ty) = layout.ty.scalable_vector_element_count_and_type(bx.tcx());
159-
PlaceValue::new_sized(
160-
bx.scalable_alloca(element_count as u64, layout.align.abi, ty),
161-
layout.align.abi,
162-
)
163-
.with_type(layout)
160+
PlaceValue::new_sized(bx.alloca_with_ty(layout), layout.align.abi).with_type(layout)
164161
}
165162
}
166163

compiler/rustc_codegen_ssa/src/traits/builder.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ pub trait BuilderMethods<'a, 'tcx>:
235235
fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value;
236236

237237
fn alloca(&mut self, size: Size, align: Align) -> Self::Value;
238-
fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value;
238+
fn alloca_with_ty(&mut self, layout: TyAndLayout<'tcx>) -> Self::Value;
239239

240240
fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value;
241241
fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value;

compiler/rustc_middle/src/ty/sty.rs

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use std::borrow::Cow;
66
use std::ops::{ControlFlow, Range};
77

88
use hir::def::{CtorKind, DefKind};
9-
use rustc_abi::{FIRST_VARIANT, FieldIdx, ScalableElt, VariantIdx};
9+
use rustc_abi::{FIRST_VARIANT, FieldIdx, NumScalableVectors, ScalableElt, VariantIdx};
1010
use rustc_data_structures::debug_assert_matches;
1111
use rustc_errors::{ErrorGuaranteed, MultiSpan};
1212
use rustc_hir as hir;
@@ -1261,17 +1261,27 @@ impl<'tcx> Ty<'tcx> {
12611261
}
12621262
}
12631263

1264-
pub fn scalable_vector_element_count_and_type(self, tcx: TyCtxt<'tcx>) -> (u16, Ty<'tcx>) {
1264+
pub fn scalable_vector_parts(
1265+
self,
1266+
tcx: TyCtxt<'tcx>,
1267+
) -> Option<(u16, Ty<'tcx>, NumScalableVectors)> {
12651268
let Adt(def, args) = self.kind() else {
1266-
bug!("`scalable_vector_size_and_type` called on invalid type")
1269+
return None;
12671270
};
1268-
let Some(ScalableElt::ElementCount(element_count)) = def.repr().scalable else {
1269-
bug!("`scalable_vector_size_and_type` called on non-scalable vector type");
1271+
let (num_vectors, vec_def) = match def.repr().scalable? {
1272+
ScalableElt::ElementCount(_) => (NumScalableVectors::One, *def),
1273+
ScalableElt::Container => (
1274+
NumScalableVectors::from_field_count(def.non_enum_variant().fields.len())?,
1275+
def.non_enum_variant().fields[FieldIdx::ZERO].ty(tcx, args).ty_adt_def()?,
1276+
),
12701277
};
1271-
let variant = def.non_enum_variant();
1278+
let Some(ScalableElt::ElementCount(element_count)) = vec_def.repr().scalable else {
1279+
return None;
1280+
};
1281+
let variant = vec_def.non_enum_variant();
12721282
assert_eq!(variant.fields.len(), 1);
12731283
let field_ty = variant.fields[FieldIdx::ZERO].ty(tcx, args);
1274-
(element_count, field_ty)
1284+
Some((element_count, field_ty, num_vectors))
12751285
}
12761286

12771287
pub fn simd_size_and_type(self, tcx: TyCtxt<'tcx>) -> (u64, Ty<'tcx>) {

compiler/rustc_public/src/abi.rs

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,39 @@ pub enum TagEncoding {
232232
},
233233
}
234234

235+
/// How many scalable vectors are in a `ValueAbi::ScalableVector`?
236+
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)]
237+
pub enum NumScalableVectors {
238+
/// A single scalable vector
239+
///
240+
/// ```rust,ignore (example)
241+
/// #[rustc_scalable_vector(4)]
242+
/// struct svuint32_t(i32);
243+
/// ```
244+
One,
245+
/// Tuple of two scalable vectors
246+
///
247+
/// ```rust,ignore (example)
248+
/// #[rustc_scalable_vector]
249+
/// struct svuint32x2_t(svuint32_t, svuint32_t);
250+
/// ```
251+
Two,
252+
/// Tuple of three scalable vectors
253+
///
254+
/// ```rust,ignore (example)
255+
/// #[rustc_scalable_vector]
256+
/// struct svuint32x3_t(svuint32_t, svuint32_t, svuint32_t);
257+
/// ```
258+
Three,
259+
/// Tuple of four scalable vectors
260+
///
261+
/// ```rust,ignore (example)
262+
/// #[rustc_scalable_vector]
263+
/// struct svuint32x4_t(svuint32_t, svuint32_t, svuint32_t, svuint32_t);
264+
/// ```
265+
Four,
266+
}
267+
235268
/// Describes how values of the type are passed by target ABIs,
236269
/// in terms of categories of C types there are ABI rules for.
237270
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)]
@@ -245,6 +278,7 @@ pub enum ValueAbi {
245278
ScalableVector {
246279
element: Scalar,
247280
count: u64,
281+
number_of_vectors: NumScalableVectors,
248282
},
249283
Aggregate {
250284
/// If true, the size is exact, otherwise it's only a lower bound.

0 commit comments

Comments
 (0)