[SLP] Vectorize struct-returning intrinsics

Allow SLP to combine across lanes calls that return a literal struct
(llvm.sincos, llvm.*.with.overflow, llvm.frexp, ...) into a single
call returning a struct of vectors, by widening {T, T, ...} to
{<VF x T>, ...} via VectorTypeUtils and emitting extractvalue +
extractelement for external uses.

Original Pull Request: llvm#195521

Original Pull Request2: llvm#196756

Recommit after revert llvm#198265 (comment)

Added check for valid vectorizable type, small corner cases fixes

Reviewers: 

Pull Request: llvm#199433

Author: alexey-bataev

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S --passes=slp-vectorizer -mtriple=aarch64-pc-windows-gnu -mcpu=cortex-x3 -slp-revec < %s | FileCheck %s
+
+define <vscale x 16 x i1> @test() {
+; CHECK-LABEL: define <vscale x 16 x i1> @test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[VECTOR_PH:.*]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ zeroinitializer, %[[VECTOR_PH]] ]
+; CHECK-NEXT:    ret <vscale x 16 x i1> [[ACTIVE_LANE_MASK]]
+;
+vector.ph:
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %vector.ph ]
+  %active.lane.mask = phi <vscale x 16 x i1> [ zeroinitializer, %vector.ph ]
+  ret <vscale x 16 x i1> %active.lane.mask
+}

llvm/test/Transforms/SLPVectorizer/AArch64/scalable-type-revec.ll

@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S --passes=slp-vectorizer -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LAND_RHS_I_I_I1756:.*]]
+; CHECK:       [[LAND_RHS_I_I_I1756]]:
+; CHECK-NEXT:    [[VALUE_SROA_8_026_I_I_I1760:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP4:%.*]], %[[LAND_RHS_I_I_I1756]] ]
+; CHECK-NEXT:    [[VALUE_SROA_12_025_I_I_I1761:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP3:%.*]], %[[LAND_RHS_I_I_I1756]] ]
+; CHECK-NEXT:    [[SHR_1_I_I_I_I1784:%.*]] = lshr i32 [[VALUE_SROA_8_026_I_I_I1760]], 0
+; CHECK-NEXT:    [[ADD3_1_I_I_I_I1799:%.*]] = or i32 0, [[SHR_1_I_I_I_I1784]]
+; CHECK-NEXT:    [[SHR_2_I_I_I_I1786:%.*]] = lshr i32 [[VALUE_SROA_12_025_I_I_I1761]], 0
+; CHECK-NEXT:    [[ADD3_2_I_I_I_I1801:%.*]] = or i32 [[ADD3_1_I_I_I_I1799]], [[SHR_2_I_I_I_I1786]]
+; CHECK-NEXT:    [[SHR_5_I_I_I_I1792:%.*]] = lshr i32 0, 0
+; CHECK-NEXT:    [[ADD3_5_I_I_I_I1807:%.*]] = or i32 [[ADD3_2_I_I_I_I1801]], [[SHR_5_I_I_I_I1792]]
+; CHECK-NEXT:    [[CMP4_I_I_I_I_I1815:%.*]] = icmp eq i32 [[ADD3_5_I_I_I_I1807]], 0
+; CHECK-NEXT:    [[TMP0:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 0, i32 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i1 } [[TMP0]], 1
+; CHECK-NEXT:    [[OR_I40_2_I_I_I_I1917:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 0, i32 [[OR_I40_2_I_I_I_I1917]])
+; CHECK-NEXT:    [[TMP3]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4]] = extractvalue { i32, i1 } [[TMP0]], 0
+; CHECK-NEXT:    br label %[[LAND_RHS_I_I_I1756]]
+;
+entry:
+  br label %land.rhs.i.i.i1756
+
+land.rhs.i.i.i1756:
+  %value.sroa.8.026.i.i.i1760 = phi i32 [ 0, %entry ], [ %4, %land.rhs.i.i.i1756 ]
+  %value.sroa.12.025.i.i.i1761 = phi i32 [ 0, %entry ], [ %3, %land.rhs.i.i.i1756 ]
+  %shr.1.i.i.i.i1784 = lshr i32 %value.sroa.8.026.i.i.i1760, 0
+  %add3.1.i.i.i.i1799 = or i32 0, %shr.1.i.i.i.i1784
+  %shr.2.i.i.i.i1786 = lshr i32 %value.sroa.12.025.i.i.i1761, 0
+  %add3.2.i.i.i.i1801 = or i32 %add3.1.i.i.i.i1799, %shr.2.i.i.i.i1786
+  %shr.5.i.i.i.i1792 = lshr i32 0, 0
+  %add3.5.i.i.i.i1807 = or i32 %add3.2.i.i.i.i1801, %shr.5.i.i.i.i1792
+  %cmp4.i.i.i.i.i1815 = icmp eq i32 %add3.5.i.i.i.i1807, 0
+  %0 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 0, i32 0)
+  %1 = extractvalue { i32, i1 } %0, 1
+  %or.i40.2.i.i.i.i1917 = zext i1 %1 to i32
+  %2 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 0, i32 %or.i40.2.i.i.i.i1917)
+  %3 = extractvalue { i32, i1 } %2, 0
+  %4 = extractvalue { i32, i1 } %0, 0
+  br label %land.rhs.i.i.i1756
+}
+
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)

llvm/test/Transforms/SLPVectorizer/AMDGPU/transform-node-gather-struct.ll

@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S --passes=slp-vectorizer -mtriple=riscv64-unknown-unknown-elf -mattr=+v < %s | FileCheck %s
+
+define target("riscv.vector.tuple", <vscale x 32 x i8>, 2) @test() {
+; CHECK-LABEL: define target("riscv.vector.tuple", <vscale x 32 x i8>, 2) @test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[CALL:%.*]] = tail call <vscale x 4 x i64> @llvm.riscv.vand.nxv4i64.i64.i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i64 0, i64 0)
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call target("riscv.vector.tuple", <vscale x 32 x i8>, 2) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv32i8_2t.nxv4i64(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) poison, <vscale x 4 x i64> [[CALL]], i32 0)
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call target("riscv.vector.tuple", <vscale x 32 x i8>, 2) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv32i8_2t.nxv4i64(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) [[CALL1]], <vscale x 4 x i64> zeroinitializer, i32 0)
+; CHECK-NEXT:    ret target("riscv.vector.tuple", <vscale x 32 x i8>, 2) [[CALL2]]
+;
+bb:
+  %call = tail call <vscale x 4 x i64> @llvm.riscv.vand.nxv4i64.i64.i64(<vscale x 4 x i64> zeroinitializer, <vscale x 4 x i64> zeroinitializer, i64 0, i64 0)
+  %call1 = tail call target("riscv.vector.tuple", <vscale x 32 x i8>, 2) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv32i8_2t.nxv4i64(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) poison, <vscale x 4 x i64> %call, i32 0)
+  %call2 = tail call target("riscv.vector.tuple", <vscale x 32 x i8>, 2) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv32i8_2t.nxv4i64(target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %call1, <vscale x 4 x i64> zeroinitializer, i32 0)
+  ret target("riscv.vector.tuple", <vscale x 32 x i8>, 2) %call2
+}
+
+declare <vscale x 4 x i64> @llvm.riscv.vand.nxv4i64.i64.i64(<vscale x 4 x i64>, <vscale x 4 x i64>, i64, i64)
+
+declare target("riscv.vector.tuple", <vscale x 32 x i8>, 2) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv32i8_2t.nxv4i64(target("riscv.vector.tuple", <vscale x 32 x i8>, 2), <vscale x 4 x i64>, i32 immarg)