Skip to content

Commit 4fbcb03

Browse files
davidtwcodavidtwco
committed
cg_llvm: sve_tuple_{create,get,set} intrinsics
Clang changed to representing tuples of scalable vectors as structs rather than as wide vectors (that is, scalable vector types where the `N` part of the `<vscale x N x ty>` type was multiplied by the number of vectors). rustc mirrored this in the initial implementation of scalable vectors. Earlier versions of our patches used the wide vector representation and our intrinsic patches used the legacy `llvm.aarch64.sve.tuple.{create,get,set}{2,3,4}` intrinsics for creating these tuples/getting/setting the vectors, which were only supported due to LLVM's `AutoUpgrade` pass converting these intrinsics into `llvm.vector.insert`. `AutoUpgrade` only supports these legacy intrinsics with the wide vector representation. With the current struct representation, Clang has special handling in codegen for generating `insertvalue`/`extractvalue` instructions for these operations, which must be replicated by rustc's codegen for our intrinsics to use. This patch implements new intrinsics in `core::intrinsics::scalable` (mirroring the structure of `core::intrinsics::simd`) which rustc lowers to the appropriate `insertvalue`/`extractvalue` instructions.
1 parent a2f7f3c commit 4fbcb03

8 files changed

Lines changed: 298 additions & 4 deletions

File tree

‎compiler/rustc_codegen_llvm/src/intrinsic.rs‎

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ use std::ffi::c_uint;
33
use std::{assert_matches, ptr};
44

55
use rustc_abi::{
6-
Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange,
6+
Align, BackendRepr, ExternAbi, Float, HasDataLayout, NumScalableVectors, Primitive, Size,
7+
WrappingRange,
78
};
89
use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
910
use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
@@ -605,6 +606,115 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
605606
self.pointercast(val, self.type_ptr())
606607
}
607608

609+
sym::sve_tuple_create2 => {
610+
assert_matches!(
611+
self.layout_of(fn_args.type_at(0)).backend_repr,
612+
BackendRepr::SimdScalableVector {
613+
number_of_vectors: NumScalableVectors(1),
614+
..
615+
}
616+
);
617+
let tuple_ty = self.layout_of(fn_args.type_at(1));
618+
assert_matches!(
619+
tuple_ty.backend_repr,
620+
BackendRepr::SimdScalableVector {
621+
number_of_vectors: NumScalableVectors(2),
622+
..
623+
}
624+
);
625+
let ret = self.const_poison(self.backend_type(tuple_ty));
626+
let ret = self.insert_value(ret, args[0].immediate(), 0);
627+
self.insert_value(ret, args[1].immediate(), 1)
628+
}
629+
630+
sym::sve_tuple_create3 => {
631+
assert_matches!(
632+
self.layout_of(fn_args.type_at(0)).backend_repr,
633+
BackendRepr::SimdScalableVector {
634+
number_of_vectors: NumScalableVectors(1),
635+
..
636+
}
637+
);
638+
let tuple_ty = self.layout_of(fn_args.type_at(1));
639+
assert_matches!(
640+
tuple_ty.backend_repr,
641+
BackendRepr::SimdScalableVector {
642+
number_of_vectors: NumScalableVectors(3),
643+
..
644+
}
645+
);
646+
let ret = self.const_poison(self.backend_type(tuple_ty));
647+
let ret = self.insert_value(ret, args[0].immediate(), 0);
648+
let ret = self.insert_value(ret, args[1].immediate(), 1);
649+
self.insert_value(ret, args[2].immediate(), 2)
650+
}
651+
652+
sym::sve_tuple_create4 => {
653+
assert_matches!(
654+
self.layout_of(fn_args.type_at(0)).backend_repr,
655+
BackendRepr::SimdScalableVector {
656+
number_of_vectors: NumScalableVectors(1),
657+
..
658+
}
659+
);
660+
let tuple_ty = self.layout_of(fn_args.type_at(1));
661+
assert_matches!(
662+
tuple_ty.backend_repr,
663+
BackendRepr::SimdScalableVector {
664+
number_of_vectors: NumScalableVectors(4),
665+
..
666+
}
667+
);
668+
let ret = self.const_poison(self.backend_type(tuple_ty));
669+
let ret = self.insert_value(ret, args[0].immediate(), 0);
670+
let ret = self.insert_value(ret, args[1].immediate(), 1);
671+
let ret = self.insert_value(ret, args[2].immediate(), 2);
672+
self.insert_value(ret, args[3].immediate(), 3)
673+
}
674+
675+
sym::sve_tuple_get => {
676+
assert_matches!(
677+
self.layout_of(fn_args.type_at(0)).backend_repr,
678+
BackendRepr::SimdScalableVector {
679+
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
680+
..
681+
}
682+
);
683+
assert_matches!(
684+
self.layout_of(fn_args.type_at(1)).backend_repr,
685+
BackendRepr::SimdScalableVector {
686+
number_of_vectors: NumScalableVectors(1),
687+
..
688+
}
689+
);
690+
self.extract_value(
691+
args[0].immediate(),
692+
fn_args.const_at(2).to_leaf().to_i32() as u64,
693+
)
694+
}
695+
696+
sym::sve_tuple_set => {
697+
assert_matches!(
698+
self.layout_of(fn_args.type_at(0)).backend_repr,
699+
BackendRepr::SimdScalableVector {
700+
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
701+
..
702+
}
703+
);
704+
assert_matches!(
705+
self.layout_of(fn_args.type_at(1)).backend_repr,
706+
BackendRepr::SimdScalableVector {
707+
number_of_vectors: NumScalableVectors(1),
708+
..
709+
}
710+
);
711+
self.insert_value(
712+
args[0].immediate(),
713+
args[1].immediate(),
714+
fn_args.const_at(2).to_leaf().to_i32() as u64,
715+
)
716+
}
717+
608718
_ if name.as_str().starts_with("simd_") => {
609719
// Unpack non-power-of-2 #[repr(packed, simd)] arguments.
610720
// This gives them the expected layout of a regular #[repr(simd)] vector.

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -783,6 +783,12 @@ pub(crate) fn check_intrinsic_type(
783783
sym::simd_shuffle => (3, 0, vec![param(0), param(0), param(1)], param(2)),
784784
sym::simd_shuffle_const_generic => (2, 1, vec![param(0), param(0)], param(1)),
785785

786+
sym::sve_tuple_create2 => (2, 0, vec![param(0), param(0)], param(1)),
787+
sym::sve_tuple_create3 => (2, 0, vec![param(0), param(0), param(0)], param(1)),
788+
sym::sve_tuple_create4 => (2, 0, vec![param(0), param(0), param(0), param(0)], param(1)),
789+
sym::sve_tuple_get => (2, 1, vec![param(0)], param(1)),
790+
sym::sve_tuple_set => (2, 1, vec![param(0), param(1)], param(0)),
791+
786792
sym::atomic_cxchg | sym::atomic_cxchgweak => (
787793
1,
788794
2,

‎compiler/rustc_span/src/symbol.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1979,6 +1979,11 @@ symbols! {
19791979
suggestion,
19801980
super_let,
19811981
supertrait_item_shadowing,
1982+
sve_tuple_create2,
1983+
sve_tuple_create3,
1984+
sve_tuple_create4,
1985+
sve_tuple_get,
1986+
sve_tuple_set,
19821987
sym,
19831988
sync,
19841989
synthetic,

library/core/src/intrinsics/simd.rs renamed to library/core/src/intrinsics/simd/mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
//!
33
//! In this module, a "vector" is any `repr(simd)` type.
44
5+
pub mod scalable;
6+
57
use crate::marker::ConstParamTy;
68

79
/// Inserts an element into a vector, returning the updated vector.

‎library/core/src/intrinsics/simd/scalable.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
//! Scalable vector compiler intrinsics.
2+
//!
3+
//! In this module, a "vector" is any `#[rustc_scalable_vector]`-annotated type.
4+
5+
/// Create a tuple of two vectors.
6+
///
7+
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
8+
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
9+
/// type `SVec`.
10+
///
11+
/// Corresponds to Clang's `__builtin_sve_svcreate2*` builtins.
12+
#[cfg(target_arch = "aarch64")]
13+
#[rustc_nounwind]
14+
#[rustc_intrinsic]
15+
pub unsafe fn sve_tuple_create2<SVec, SVecTup>(x0: SVec, x1: SVec) -> SVecTup;
16+
17+
/// Create a tuple of three vectors.
18+
///
19+
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
20+
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
21+
/// type `SVec`.
22+
///
23+
/// Corresponds to Clang's `__builtin_sve_svcreate3*` builtins.
24+
#[cfg(target_arch = "aarch64")]
25+
#[rustc_intrinsic]
26+
#[rustc_nounwind]
27+
pub unsafe fn sve_tuple_create3<SVec, SVecTup>(x0: SVec, x1: SVec, x2: SVec) -> SVecTup;
28+
29+
/// Create a tuple of four vectors.
30+
///
31+
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
32+
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
33+
/// type `SVec`.
34+
///
35+
/// Corresponds to Clang's `__builtin_sve_svcreate4*` builtins.
36+
#[cfg(target_arch = "aarch64")]
37+
#[rustc_intrinsic]
38+
#[rustc_nounwind]
39+
pub unsafe fn sve_tuple_create4<SVec, SVecTup>(x0: SVec, x1: SVec, x2: SVec, x3: SVec) -> SVecTup;
40+
41+
/// Get one vector from a tuple of vectors.
42+
///
43+
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
44+
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
45+
/// type `SVec`.
46+
///
47+
/// Corresponds to Clang's `__builtin_sve_svget*` builtins.
48+
///
49+
/// # Safety
50+
///
51+
/// `IDX` must be in-bounds of the tuple.
52+
#[cfg(target_arch = "aarch64")]
53+
#[rustc_intrinsic]
54+
#[rustc_nounwind]
55+
pub unsafe fn sve_tuple_get<SVecTup, SVec, const IDX: i32>(tuple: SVecTup) -> SVec;
56+
57+
/// Change one vector in a tuple of vectors.
58+
///
59+
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
60+
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
61+
/// type `SVec`.
62+
///
63+
/// Corresponds to Clang's `__builtin_sve_svset*` builtins.
64+
///
65+
/// # Safety
66+
///
67+
/// `IDX` must be in-bounds of the tuple.
68+
#[cfg(target_arch = "aarch64")]
69+
#[rustc_intrinsic]
70+
#[rustc_nounwind]
71+
pub unsafe fn sve_tuple_set<SVecTup, SVec, const IDX: i32>(tuple: SVecTup, x: SVec) -> SVecTup;

‎tests/codegen-llvm/scalable-vectors/tuple-intrinsics.rs‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
//@ build-pass
2+
//@ only-aarch64
3+
#![crate_type = "lib"]
4+
#![allow(incomplete_features, internal_features)]
5+
#![feature(abi_unadjusted, core_intrinsics, link_llvm_intrinsics, rustc_attrs)]
6+
7+
// Tests that tuples of scalable vectors are passed as immediates and that the intrinsics for
8+
// creating/getting/setting tuples of scalable vectors generate the correct assembly
9+
10+
#[derive(Copy, Clone)]
11+
#[rustc_scalable_vector(4)]
12+
#[allow(non_camel_case_types)]
13+
pub struct svfloat32_t(f32);
14+
15+
#[derive(Copy, Clone)]
16+
#[rustc_scalable_vector]
17+
#[allow(non_camel_case_types)]
18+
pub struct svfloat32x2_t(svfloat32_t, svfloat32_t);
19+
20+
#[derive(Copy, Clone)]
21+
#[rustc_scalable_vector]
22+
#[allow(non_camel_case_types)]
23+
pub struct svfloat32x3_t(svfloat32_t, svfloat32_t, svfloat32_t);
24+
25+
#[derive(Copy, Clone)]
26+
#[rustc_scalable_vector]
27+
#[allow(non_camel_case_types)]
28+
pub struct svfloat32x4_t(svfloat32_t, svfloat32_t, svfloat32_t, svfloat32_t);
29+
30+
#[inline(never)]
31+
#[target_feature(enable = "sve")]
32+
pub fn svdup_n_f32(op: f32) -> svfloat32_t {
33+
extern "C" {
34+
#[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4f32")]
35+
fn _svdup_n_f32(op: f32) -> svfloat32_t;
36+
}
37+
unsafe { _svdup_n_f32(op) }
38+
}
39+
40+
// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float> } @svcreate2_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1)
41+
#[no_mangle]
42+
#[target_feature(enable = "sve")]
43+
pub fn svcreate2_f32(x0: svfloat32_t, x1: svfloat32_t) -> svfloat32x2_t {
44+
// CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
45+
// CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
46+
unsafe { std::intrinsics::simd::scalable::sve_tuple_create2(x0, x1) }
47+
}
48+
49+
// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @svcreate3_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1, <vscale x 4 x float> %x2)
50+
#[no_mangle]
51+
#[target_feature(enable = "sve")]
52+
pub fn svcreate3_f32(x0: svfloat32_t, x1: svfloat32_t, x2: svfloat32_t) -> svfloat32x3_t {
53+
// CHECK-LABEL: @_RNvCsk3YxfLN8zWY_6tuples13svcreate3_f32
54+
// CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
55+
// CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
56+
// CHECK-NEXT: %3 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %2, <vscale x 4 x float> %x2, 2
57+
unsafe { std::intrinsics::simd::scalable::sve_tuple_create3(x0, x1, x2) }
58+
}
59+
60+
// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @svcreate4_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1, <vscale x 4 x float> %x2, <vscale x 4 x float> %x3)
61+
#[no_mangle]
62+
#[target_feature(enable = "sve")]
63+
pub fn svcreate4_f32(
64+
x0: svfloat32_t,
65+
x1: svfloat32_t,
66+
x2: svfloat32_t,
67+
x3: svfloat32_t,
68+
) -> svfloat32x4_t {
69+
// CHECK-LABEL: @_RNvCsk3YxfLN8zWY_6tuples13svcreate4_f32
70+
// CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
71+
// CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
72+
// CHECK-NEXT: %3 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %2, <vscale x 4 x float> %x2, 2
73+
// CHECK-NEXT: %4 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %3, <vscale x 4 x float> %x3, 3
74+
unsafe { std::intrinsics::simd::scalable::sve_tuple_create4(x0, x1, x2, x3) }
75+
}
76+
77+
// CHECK: define <vscale x 4 x float> @svget2_f32({ <vscale x 4 x float>, <vscale x 4 x float> } %tup)
78+
#[no_mangle]
79+
#[target_feature(enable = "sve")]
80+
pub fn svget2_f32<const IDX: i32>(tup: svfloat32x2_t) -> svfloat32_t {
81+
// CHECK: %1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %tup, 0
82+
unsafe { std::intrinsics::simd::scalable::sve_tuple_get::<_, _, { IDX }>(tup) }
83+
}
84+
85+
// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float> } @svset2_f32({ <vscale x 4 x float>, <vscale x 4 x float> } %tup, <vscale x 4 x float> %x)
86+
#[no_mangle]
87+
#[target_feature(enable = "sve")]
88+
pub fn svset2_f32<const IDX: i32>(tup: svfloat32x2_t, x: svfloat32_t) -> svfloat32x2_t {
89+
// CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } %tup, <vscale x 4 x float> %x, 0
90+
unsafe { std::intrinsics::simd::scalable::sve_tuple_set::<_, _, { IDX }>(tup, x) }
91+
}
92+
93+
// This function exists only so there are calls to the generic functions
94+
#[target_feature(enable = "sve")]
95+
pub fn test() {
96+
let x = svdup_n_f32(2f32);
97+
let tup = svcreate2_f32(x, x);
98+
let x = svget2_f32::<0>(tup);
99+
let tup = svset2_f32::<0>(tup, x);
100+
}

‎tests/ui/simd/masked-load-store-check-fail.stderr‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ LL | | Simd::<u8, 4>([9; 4]),
2121
LL | | );
2222
| |_________^
2323
note: function defined here
24-
--> $SRC_DIR/core/src/intrinsics/simd.rs:LL:COL
24+
--> $SRC_DIR/core/src/intrinsics/simd/mod.rs:LL:COL
2525

2626
error[E0308]: mismatched types
2727
--> $DIR/masked-load-store-check-fail.rs:25:13
@@ -46,7 +46,7 @@ LL | | default,
4646
LL | | );
4747
| |_________^
4848
note: function defined here
49-
--> $SRC_DIR/core/src/intrinsics/simd.rs:LL:COL
49+
--> $SRC_DIR/core/src/intrinsics/simd/mod.rs:LL:COL
5050

5151
error: aborting due to 2 previous errors
5252

‎triagebot.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1077,7 +1077,7 @@ cc = ["@Amanieu", "@folkertdev", "@sayantn"]
10771077
message = "Some changes occurred in `std_detect`"
10781078
cc = ["@Amanieu", "@folkertdev", "@sayantn"]
10791079

1080-
[mentions."library/core/src/intrinsics/simd.rs"]
1080+
[mentions."library/core/src/intrinsics/simd/mod.rs"]
10811081
message = """
10821082
Some changes occurred to the platform-builtins intrinsics. Make sure the
10831083
LLVM backend as well as portable-simd gets adapted for the changes.

0 commit comments

Comments
 (0)