cg_llvm: sve_tuple_{create,get,set} intrinsics

Clang changed from representing tuples of scalable vectors as
structs rather than as wide vectors (that is, scalable vector types
where the `N` part of the `<vscale x N x ty>` type was multiplied by
the number of vectors). rustc mirrored this in the initial implementation
of scalable vectors.

Earlier versions of our patches used the wide vector representation and
our intrinsic patches used the legacy
`llvm.aarch64.sve.tuple.{create,get,set}{2,3,4}` intrinsics for creating
these tuples/getting/setting the vectors, which were only supported
due to LLVM's `AutoUpgrade` pass converting these intrinsics into
`llvm.vector.insert`. `AutoUpgrade` only supports these legacy intrinsics
with the wide vector representation.

With the current struct representation, Clang has special handling in
codegen for generating `insertvalue`/`extractvalue` instructions for
these operations, which must be replicated by rustc's codegen for our
intrinsics to use. This patch implements new intrinsics in
`core::intrinsics::scalable` (mirroring the structure of
`core::intrinsics::simd`) which rustc lowers to the appropriate
`insertvalue`/`extractvalue` instructions.

Author: davidtwco

compiler/rustc_codegen_gcc/src/builder.rs

@@ -24,7 +24,8 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::bug;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::ty::layout::{
-    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers, TyAndLayout,
+    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError,
+    LayoutOfHelpers, TyAndLayout,
 };
 use rustc_middle::ty::{self, AtomicOrdering, Instance, Ty, TyCtxt};
 use rustc_span::Span;

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -3,7 +3,8 @@ use std::ffi::c_uint;
 use std::ptr;
 
 use rustc_abi::{
-    Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange,
+    Align, BackendRepr, ExternAbi, Float, HasDataLayout, NumScalableVectors, Primitive, Size,
+    WrappingRange,
 };
 use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
@@ -581,6 +582,98 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 self.pointercast(val, self.type_ptr())
             }
 
+            sym::sve_tuple_create2 => {
+                assert_matches!(
+                    self.layout_of(fn_args.type_at(0)).backend_repr,
+                    BackendRepr::ScalableVector { number_of_vectors: NumScalableVectors::One, .. }
+                );
+                let tuple_ty = self.layout_of(fn_args.type_at(1));
+                assert_matches!(
+                    tuple_ty.backend_repr,
+                    BackendRepr::ScalableVector { number_of_vectors: NumScalableVectors::Two, .. }
+                );
+                let ret = self.const_poison(self.backend_type(tuple_ty));
+                let ret = self.insert_value(ret, args[0].immediate(), 0);
+                self.insert_value(ret, args[1].immediate(), 1)
+            }
+
+            sym::sve_tuple_create3 => {
+                assert_matches!(
+                    self.layout_of(fn_args.type_at(0)).backend_repr,
+                    BackendRepr::ScalableVector { number_of_vectors: NumScalableVectors::One, .. }
+                );
+                let tuple_ty = self.layout_of(fn_args.type_at(1));
+                assert_matches!(
+                    tuple_ty.backend_repr,
+                    BackendRepr::ScalableVector {
+                        number_of_vectors: NumScalableVectors::Three,
+                        ..
+                    }
+                );
+                let ret = self.const_poison(self.backend_type(tuple_ty));
+                let ret = self.insert_value(ret, args[0].immediate(), 0);
+                let ret = self.insert_value(ret, args[1].immediate(), 1);
+                self.insert_value(ret, args[2].immediate(), 2)
+            }
+
+            sym::sve_tuple_create4 => {
+                assert_matches!(
+                    self.layout_of(fn_args.type_at(0)).backend_repr,
+                    BackendRepr::ScalableVector { number_of_vectors: NumScalableVectors::One, .. }
+                );
+                let tuple_ty = self.layout_of(fn_args.type_at(1));
+                assert_matches!(
+                    tuple_ty.backend_repr,
+                    BackendRepr::ScalableVector { number_of_vectors: NumScalableVectors::Four, .. }
+                );
+                let ret = self.const_poison(self.backend_type(tuple_ty));
+                let ret = self.insert_value(ret, args[0].immediate(), 0);
+                let ret = self.insert_value(ret, args[1].immediate(), 1);
+                let ret = self.insert_value(ret, args[2].immediate(), 2);
+                self.insert_value(ret, args[3].immediate(), 3)
+            }
+
+            sym::sve_tuple_get => {
+                assert_matches!(
+                    self.layout_of(fn_args.type_at(0)).backend_repr,
+                    BackendRepr::ScalableVector {
+                        number_of_vectors: NumScalableVectors::Two
+                            | NumScalableVectors::Three
+                            | NumScalableVectors::Four,
+                        ..
+                    }
+                );
+                assert_matches!(
+                    self.layout_of(fn_args.type_at(1)).backend_repr,
+                    BackendRepr::ScalableVector { number_of_vectors: NumScalableVectors::One, .. }
+                );
+                self.extract_value(
+                    args[0].immediate(),
+                    fn_args.const_at(2).to_leaf().to_i32() as u64,
+                )
+            }
+
+            sym::sve_tuple_set => {
+                assert_matches!(
+                    self.layout_of(fn_args.type_at(0)).backend_repr,
+                    BackendRepr::ScalableVector {
+                        number_of_vectors: NumScalableVectors::Two
+                            | NumScalableVectors::Three
+                            | NumScalableVectors::Four,
+                        ..
+                    }
+                );
+                assert_matches!(
+                    self.layout_of(fn_args.type_at(1)).backend_repr,
+                    BackendRepr::ScalableVector { number_of_vectors: NumScalableVectors::One, .. }
+                );
+                self.insert_value(
+                    args[0].immediate(),
+                    args[1].immediate(),
+                    fn_args.const_at(2).to_leaf().to_i32() as u64,
+                )
+            }
+
             _ if name.as_str().starts_with("simd_") => {
                 // Unpack non-power-of-2 #[repr(packed, simd)] arguments.
                 // This gives them the expected layout of a regular #[repr(simd)] vector.

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -785,6 +785,12 @@ pub(crate) fn check_intrinsic_type(
         sym::simd_shuffle => (3, 0, vec![param(0), param(0), param(1)], param(2)),
         sym::simd_shuffle_const_generic => (2, 1, vec![param(0), param(0)], param(1)),
 
+        sym::sve_tuple_create2 => (2, 0, vec![param(0), param(0)], param(1)),
+        sym::sve_tuple_create3 => (2, 0, vec![param(0), param(0), param(0)], param(1)),
+        sym::sve_tuple_create4 => (2, 0, vec![param(0), param(0), param(0), param(0)], param(1)),
+        sym::sve_tuple_get => (2, 1, vec![param(0)], param(1)),
+        sym::sve_tuple_set => (2, 1, vec![param(0), param(1)], param(0)),
+
         sym::atomic_cxchg | sym::atomic_cxchgweak => (
             1,
             2,

compiler/rustc_span/src/symbol.rs

@@ -1984,6 +1984,11 @@ symbols! {
         suggestion,
         super_let,
         supertrait_item_shadowing,
+        sve_tuple_create2,
+        sve_tuple_create3,
+        sve_tuple_create4,
+        sve_tuple_get,
+        sve_tuple_set,
         sym,
         sync,
         synthetic,

library/core/src/intrinsics/simd.rs library/core/src/intrinsics/simd/mod.rslibrary/core/src/intrinsics/simd.rs renamed to library/core/src/intrinsics/simd/mod.rs

@@ -2,6 +2,8 @@
 //!
 //! In this module, a "vector" is any `repr(simd)` type.
 
+pub mod scalable;
+
 use crate::marker::ConstParamTy;
 
 /// Inserts an element into a vector, returning the updated vector.

library/core/src/intrinsics/simd/scalable.rs

@@ -0,0 +1,74 @@
+//! Scalable vector compiler intrinsics.
+//!
+//! In this module, a "vector" is any `#[rustc_scalable_vector]`-annotated type.
+
+/// Create a tuple of two vectors.
+///
+/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
+/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
+/// type `SVec`.
+///
+/// Corresponds to Clang's `__builtin_sve_svcreate2*` builtins.
+#[rustc_nounwind]
+#[rustc_intrinsic]
+pub unsafe fn sve_tuple_create2<SVec, SVecTup>(x0: SVec, x1: SVec) -> SVecTup;
+
+/// Create a tuple of three vectors.
+///
+/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
+/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
+/// type `SVec`.
+///
+/// Corresponds to Clang's `__builtin_sve_svcreate3*` builtins.
+#[cfg(target_arch = "aarch64")]
+#[rustc_intrinsic]
+#[rustc_nounwind]
+#[target_feature(enable = "sve")]
+pub unsafe fn sve_tuple_create3<SVec, SVecTup>(x0: SVec, x1: SVec, x2: SVec) -> SVecTup;
+
+/// Create a tuple of four vectors.
+///
+/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
+/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
+/// type `SVec`.
+///
+/// Corresponds to Clang's `__builtin_sve_svcreate4*` builtins.
+#[cfg(target_arch = "aarch64")]
+#[rustc_intrinsic]
+#[rustc_nounwind]
+#[target_feature(enable = "sve")]
+pub unsafe fn sve_tuple_create4<SVec, SVecTup>(x0: SVec, x1: SVec, x2: SVec, x3: SVec) -> SVecTup;
+
+/// Get one vector from a tuple of vectors.
+///
+/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
+/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
+/// type `SVec`.
+///
+/// Corresponds to Clang's `__builtin_sve_svget*` builtins.
+///
+/// # Safety
+///
+/// `IDX` must be in-bounds of the tuple.
+#[cfg(target_arch = "aarch64")]
+#[rustc_intrinsic]
+#[rustc_nounwind]
+#[target_feature(enable = "sve")]
+pub unsafe fn sve_tuple_get<SVecTup, SVec, const IDX: i32>(tuple: SVecTup) -> SVec;
+
+/// Change one vector in a tuple of vectors.
+///
+/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
+/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
+/// type `SVec`.
+///
+/// Corresponds to Clang's `__builtin_sve_svset*` builtins.
+///
+/// # Safety
+///
+/// `IDX` must be in-bounds of the tuple.
+#[cfg(target_arch = "aarch64")]
+#[rustc_intrinsic]
+#[rustc_nounwind]
+#[target_feature(enable = "sve")]
+pub unsafe fn sve_tuple_set<SVecTup, SVec, const IDX: i32>(tuple: SVecTup, x: SVec) -> SVecTup;

tests/codegen-llvm/scalable-vectors/tuple-intrinsics.rs

@@ -0,0 +1,100 @@
+//@ build-pass
+//@ only-aarch64
+#![crate_type = "lib"]
+#![allow(incomplete_features, internal_features)]
+#![feature(abi_unadjusted, core_intrinsics, link_llvm_intrinsics, rustc_attrs)]
+
+// Tests that tuples of scalable vectors are passed as immediates and that the intrinsics for
+// creating/getting/setting tuples of scalable vectors generate the correct assembly
+
+#[derive(Copy, Clone)]
+#[rustc_scalable_vector(4)]
+#[allow(non_camel_case_types)]
+pub struct svfloat32_t(f32);
+
+#[derive(Copy, Clone)]
+#[rustc_scalable_vector]
+#[allow(non_camel_case_types)]
+pub struct svfloat32x2_t(svfloat32_t, svfloat32_t);
+
+#[derive(Copy, Clone)]
+#[rustc_scalable_vector]
+#[allow(non_camel_case_types)]
+pub struct svfloat32x3_t(svfloat32_t, svfloat32_t, svfloat32_t);
+
+#[derive(Copy, Clone)]
+#[rustc_scalable_vector]
+#[allow(non_camel_case_types)]
+pub struct svfloat32x4_t(svfloat32_t, svfloat32_t, svfloat32_t, svfloat32_t);
+
+#[inline(never)]
+#[target_feature(enable = "sve")]
+pub fn svdup_n_f32(op: f32) -> svfloat32_t {
+    extern "C" {
+        #[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4f32")]
+        fn _svdup_n_f32(op: f32) -> svfloat32_t;
+    }
+    unsafe { _svdup_n_f32(op) }
+}
+
+// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float> } @svcreate2_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1)
+#[no_mangle]
+#[target_feature(enable = "sve")]
+pub fn svcreate2_f32(x0: svfloat32_t, x1: svfloat32_t) -> svfloat32x2_t {
+    // CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
+    // CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
+    unsafe { std::intrinsics::simd::scalable::sve_tuple_create2(x0, x1) }
+}
+
+// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @svcreate3_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1, <vscale x 4 x float> %x2)
+#[no_mangle]
+#[target_feature(enable = "sve")]
+pub fn svcreate3_f32(x0: svfloat32_t, x1: svfloat32_t, x2: svfloat32_t) -> svfloat32x3_t {
+    // CHECK-LABEL: @_RNvCsk3YxfLN8zWY_6tuples13svcreate3_f32
+    // CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
+    // CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
+    // CHECK-NEXT: %3 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %2, <vscale x 4 x float> %x2, 2
+    unsafe { std::intrinsics::simd::scalable::sve_tuple_create3(x0, x1, x2) }
+}
+
+// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @svcreate4_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1, <vscale x 4 x float> %x2, <vscale x 4 x float> %x3)
+#[no_mangle]
+#[target_feature(enable = "sve")]
+pub fn svcreate4_f32(
+    x0: svfloat32_t,
+    x1: svfloat32_t,
+    x2: svfloat32_t,
+    x3: svfloat32_t,
+) -> svfloat32x4_t {
+    // CHECK-LABEL: @_RNvCsk3YxfLN8zWY_6tuples13svcreate4_f32
+    // CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
+    // CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
+    // CHECK-NEXT: %3 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %2, <vscale x 4 x float> %x2, 2
+    // CHECK-NEXT: %4 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %3, <vscale x 4 x float> %x3, 3
+    unsafe { std::intrinsics::simd::scalable::sve_tuple_create4(x0, x1, x2, x3) }
+}
+
+// CHECK: define <vscale x 4 x float> @svget2_f32({ <vscale x 4 x float>, <vscale x 4 x float> } %tup)
+#[no_mangle]
+#[target_feature(enable = "sve")]
+pub fn svget2_f32<const IDX: i32>(tup: svfloat32x2_t) -> svfloat32_t {
+    // CHECK: %1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %tup, 0
+    unsafe { std::intrinsics::simd::scalable::sve_tuple_get::<_, _, { IDX }>(tup) }
+}
+
+// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float> } @svset2_f32({ <vscale x 4 x float>, <vscale x 4 x float> } %tup, <vscale x 4 x float> %x)
+#[no_mangle]
+#[target_feature(enable = "sve")]
+pub fn svset2_f32<const IDX: i32>(tup: svfloat32x2_t, x: svfloat32_t) -> svfloat32x2_t {
+    // CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } %tup, <vscale x 4 x float> %x, 0
+    unsafe { std::intrinsics::simd::scalable::sve_tuple_set::<_, _, { IDX }>(tup, x) }
+}
+
+// This function exists only so there are calls to the generic functions
+#[target_feature(enable = "sve")]
+pub fn test() {
+    let x = svdup_n_f32(2f32);
+    let tup = svcreate2_f32(x, x);
+    let x = svget2_f32::<0>(tup);
+    let tup = svset2_f32::<0>(tup, x);
+}