Skip to content

Commit 2c90c5e

Browse files
zonglinpengZonglin Peng
zonglinpeng
and
Zonglin Peng
authored
AI: [G3] dequantize_per_tensor asym SIMD: integer zero-point subtract (#20703) (#20703)
Summary: Optimizes the per-tensor `dequantize_per_tensor_out` SIMD fast path on Fusion-G3 for int8 / uint8 / int16 / uint16 -> float32. What is optimized: the zero-point subtract is kept in the int32 domain (`PDX_SUB_MX32`) so it issues on the integer unit and overlaps the float pipe, and the widened int32 codes are fed straight into the mixed-type `PDX_MUL_MXF32(int32, scale)` (the idiom the vendor NNLib dequantize kernel uses). Note: `PDX_MUL_MXF32` takes an `xb_vecMxf32`, so the compiler still emits an int->float convert -- writing the multiply mixed-type keeps the source concise but does not eliminate the convert (thanks mvartani-meta for the correction). The measured win comes from keeping the subtract in the integer domain instead of the float domain, which frees the float pipe and lets the compiler schedule the convert as the multiply's operand rather than as a separate explicit cast. Bit-identical to the previous code for 8/16-bit inputs: `q - zp` cannot overflow int32 (inputs are <=16-bit and zp is in the same quant range) and converts exactly to float, matching `float(q) - float(zp)`. What is improved from 00b95cf2b4: that revision moved the subtract into the integer domain but kept an explicit `(xb_vecMxf32)` convert, so the asymmetric FP pipe still did an explicit convert + multiply and the symmetric path was left untouched. This diff feeds the int32 codes directly into the mixed-type multiply on both paths. That clears the residual large-tensor asymmetric regression 00b95cf2b4 still had -- e.g. `objv1_1x400x400x1_u8` (160000 elems) goes 0.75x -> 1.00x vs stock and `dpev26_4x60x4x4_u16` goes 0.89x -> 1.17x -- while keeping every symmetric case at parity or better, and the object file is ~144 B smaller. Reviewed By: mvartani-meta, DrJessop Differential Revision: D110529287 --------- Co-authored-by: Zonglin Peng <zonglinpeng@fb.com>
1 parent 2524691 commit 2c90c5e

1 file changed

Lines changed: 96 additions & 96 deletions

File tree

‎backends/cadence/fusion_g3/operators/op_dequantize.cpp‎

Lines changed: 96 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -611,34 +611,40 @@ Tensor& dequantize_per_tensor_out(
611611
const int8_t* __restrict__ input_data = input.const_data_ptr<int8_t>();
612612
#if defined(__XTENSA__)
613613
// Direct inline PDX SIMD dequantization for per-tensor int8->float32.
614-
// 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave
615-
// independent loads, converts, and FP MACs across unrolled iterations,
616-
// hiding load-to-use latency and reducing loop overhead.
614+
// The zero-point subtract is kept in the int32 domain (PDX_SUB_MX32) so
615+
// it issues on the integer unit and overlaps the float pipe; the widened
616+
// int32 codes feed straight into the mixed-type PDX_MUL_MXF32. That
617+
// intrinsic takes an xb_vecMxf32, so the compiler still emits an
618+
// int->float convert -- writing it mixed-type only keeps the source
619+
// concise, it does not remove the convert. q - zp cannot overflow int32
620+
// (inputs are <=16-bit and zp is in the same quant range) and converts
621+
// exactly to float, so the result is bit-identical to
622+
// float(q) - float(zp). 4x-unrolled to hide load-to-use latency.
617623
if (dequant_simd_aligned(input_data, out_data)) {
618624
const int numel = inp_shape[0];
619625
auto vIn = reinterpret_cast<const xb_vecMx8*>(input_data);
620626
auto vOut = reinterpret_cast<xb_vecMxf32*>(out_data);
621627
const xb_vecMxf32 v_scale{
622628
scale_data, scale_data, scale_data, scale_data};
623629
int i = 0;
630+
// 4x unrolled main loop: 16 elements per iteration
631+
const int e16 = (numel >> 4) << 4;
624632
if (zero_point_data != 0) {
633+
const xb_vecMx32 v_zp{
634+
zero_point_data,
635+
zero_point_data,
636+
zero_point_data,
637+
zero_point_data};
625638
const float zp_f = static_cast<float>(zero_point_data);
626-
const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f};
627-
// 4x unrolled main loop: 16 elements per iteration
628-
const int e16 = (numel >> 4) << 4;
629639
for (; i < e16; i += 16) {
630640
xb_vecMx32 vP0 = PDX_LV32_MX8_I(vIn, 0);
631641
xb_vecMx32 vP1 = PDX_LV32_MX8_I(vIn + 1, 0);
632642
xb_vecMx32 vP2 = PDX_LV32_MX8_I(vIn + 2, 0);
633643
xb_vecMx32 vP3 = PDX_LV32_MX8_I(vIn + 3, 0);
634-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
635-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
636-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
637-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
638-
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale);
639-
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale);
640-
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale);
641-
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale);
644+
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale);
645+
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale);
646+
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale);
647+
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale);
642648
vIn += 4;
643649
vOut += 4;
644650
}
@@ -648,21 +654,15 @@ Tensor& dequantize_per_tensor_out(
648654
(static_cast<float>(input_data[i]) - zp_f) * scale_data;
649655
}
650656
} else {
651-
// 4x unrolled main loop: 16 elements per iteration
652-
const int e16 = (numel >> 4) << 4;
653657
for (; i < e16; i += 16) {
654658
xb_vecMx32 vP0 = PDX_LV32_MX8_I(vIn, 0);
655659
xb_vecMx32 vP1 = PDX_LV32_MX8_I(vIn + 1, 0);
656660
xb_vecMx32 vP2 = PDX_LV32_MX8_I(vIn + 2, 0);
657661
xb_vecMx32 vP3 = PDX_LV32_MX8_I(vIn + 3, 0);
658-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
659-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
660-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
661-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
662-
vOut[0] = PDX_MUL_MXF32(vF0, v_scale);
663-
vOut[1] = PDX_MUL_MXF32(vF1, v_scale);
664-
vOut[2] = PDX_MUL_MXF32(vF2, v_scale);
665-
vOut[3] = PDX_MUL_MXF32(vF3, v_scale);
662+
vOut[0] = PDX_MUL_MXF32(vP0, v_scale);
663+
vOut[1] = PDX_MUL_MXF32(vP1, v_scale);
664+
vOut[2] = PDX_MUL_MXF32(vP2, v_scale);
665+
vOut[3] = PDX_MUL_MXF32(vP3, v_scale);
666666
vIn += 4;
667667
vOut += 4;
668668
}
@@ -703,34 +703,40 @@ Tensor& dequantize_per_tensor_out(
703703
const uint8_t* __restrict__ input_data = input.const_data_ptr<uint8_t>();
704704
#if defined(__XTENSA__)
705705
// Direct inline PDX SIMD dequantization for per-tensor uint8->float32.
706-
// 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave
707-
// independent loads, converts, and FP MACs across unrolled iterations,
708-
// hiding load-to-use latency and reducing loop overhead.
706+
// The zero-point subtract is kept in the int32 domain (PDX_SUB_MX32) so
707+
// it issues on the integer unit and overlaps the float pipe; the widened
708+
// int32 codes feed straight into the mixed-type PDX_MUL_MXF32. That
709+
// intrinsic takes an xb_vecMxf32, so the compiler still emits an
710+
// int->float convert -- writing it mixed-type only keeps the source
711+
// concise, it does not remove the convert. q - zp cannot overflow int32
712+
// (inputs are <=16-bit and zp is in the same quant range) and converts
713+
// exactly to float, so the result is bit-identical to
714+
// float(q) - float(zp). 4x-unrolled to hide load-to-use latency.
709715
if (dequant_simd_aligned(input_data, out_data)) {
710716
const int numel = inp_shape[0];
711717
auto vIn = reinterpret_cast<const xb_vecMxu8*>(input_data);
712718
auto vOut = reinterpret_cast<xb_vecMxf32*>(out_data);
713719
const xb_vecMxf32 v_scale{
714720
scale_data, scale_data, scale_data, scale_data};
715721
int i = 0;
722+
// 4x unrolled main loop: 16 elements per iteration
723+
const int e16 = (numel >> 4) << 4;
716724
if (zero_point_data != 0) {
725+
const xb_vecMxu32 v_zp{
726+
static_cast<uint32_t>(zero_point_data),
727+
static_cast<uint32_t>(zero_point_data),
728+
static_cast<uint32_t>(zero_point_data),
729+
static_cast<uint32_t>(zero_point_data)};
717730
const float zp_f = static_cast<float>(zero_point_data);
718-
const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f};
719-
// 4x unrolled main loop: 16 elements per iteration
720-
const int e16 = (numel >> 4) << 4;
721731
for (; i < e16; i += 16) {
722732
xb_vecMxu32 vP0 = PDX_LVU32_MX8_I(vIn, 0);
723733
xb_vecMxu32 vP1 = PDX_LVU32_MX8_I(vIn + 1, 0);
724734
xb_vecMxu32 vP2 = PDX_LVU32_MX8_I(vIn + 2, 0);
725735
xb_vecMxu32 vP3 = PDX_LVU32_MX8_I(vIn + 3, 0);
726-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
727-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
728-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
729-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
730-
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale);
731-
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale);
732-
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale);
733-
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale);
736+
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale);
737+
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale);
738+
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale);
739+
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale);
734740
vIn += 4;
735741
vOut += 4;
736742
}
@@ -740,21 +746,15 @@ Tensor& dequantize_per_tensor_out(
740746
(static_cast<float>(input_data[i]) - zp_f) * scale_data;
741747
}
742748
} else {
743-
// 4x unrolled main loop: 16 elements per iteration
744-
const int e16 = (numel >> 4) << 4;
745749
for (; i < e16; i += 16) {
746750
xb_vecMxu32 vP0 = PDX_LVU32_MX8_I(vIn, 0);
747751
xb_vecMxu32 vP1 = PDX_LVU32_MX8_I(vIn + 1, 0);
748752
xb_vecMxu32 vP2 = PDX_LVU32_MX8_I(vIn + 2, 0);
749753
xb_vecMxu32 vP3 = PDX_LVU32_MX8_I(vIn + 3, 0);
750-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
751-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
752-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
753-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
754-
vOut[0] = PDX_MUL_MXF32(vF0, v_scale);
755-
vOut[1] = PDX_MUL_MXF32(vF1, v_scale);
756-
vOut[2] = PDX_MUL_MXF32(vF2, v_scale);
757-
vOut[3] = PDX_MUL_MXF32(vF3, v_scale);
754+
vOut[0] = PDX_MUL_MXF32(vP0, v_scale);
755+
vOut[1] = PDX_MUL_MXF32(vP1, v_scale);
756+
vOut[2] = PDX_MUL_MXF32(vP2, v_scale);
757+
vOut[3] = PDX_MUL_MXF32(vP3, v_scale);
758758
vIn += 4;
759759
vOut += 4;
760760
}
@@ -795,34 +795,40 @@ Tensor& dequantize_per_tensor_out(
795795
const int16_t* __restrict__ input_data = input.const_data_ptr<int16_t>();
796796
#if defined(__XTENSA__)
797797
// Direct inline PDX SIMD dequantization for per-tensor int16->float32.
798-
// 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave
799-
// independent loads, converts, and FP MACs across unrolled iterations,
800-
// hiding load-to-use latency and reducing loop overhead.
798+
// The zero-point subtract is kept in the int32 domain (PDX_SUB_MX32) so
799+
// it issues on the integer unit and overlaps the float pipe; the widened
800+
// int32 codes feed straight into the mixed-type PDX_MUL_MXF32. That
801+
// intrinsic takes an xb_vecMxf32, so the compiler still emits an
802+
// int->float convert -- writing it mixed-type only keeps the source
803+
// concise, it does not remove the convert. q - zp cannot overflow int32
804+
// (inputs are <=16-bit and zp is in the same quant range) and converts
805+
// exactly to float, so the result is bit-identical to
806+
// float(q) - float(zp). 4x-unrolled to hide load-to-use latency.
801807
if (dequant_simd_aligned(input_data, out_data)) {
802808
const int numel = inp_shape[0];
803809
auto vIn = reinterpret_cast<const xb_vecMx16*>(input_data);
804810
auto vOut = reinterpret_cast<xb_vecMxf32*>(out_data);
805811
const xb_vecMxf32 v_scale{
806812
scale_data, scale_data, scale_data, scale_data};
807813
int i = 0;
814+
// 4x unrolled main loop: 16 elements per iteration
815+
const int e16 = (numel >> 4) << 4;
808816
if (zero_point_data != 0) {
817+
const xb_vecMx32 v_zp{
818+
zero_point_data,
819+
zero_point_data,
820+
zero_point_data,
821+
zero_point_data};
809822
const float zp_f = static_cast<float>(zero_point_data);
810-
const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f};
811-
// 4x unrolled main loop: 16 elements per iteration
812-
const int e16 = (numel >> 4) << 4;
813823
for (; i < e16; i += 16) {
814824
xb_vecMx32 vP0 = PDX_LV32_MX16_I(vIn, 0);
815825
xb_vecMx32 vP1 = PDX_LV32_MX16_I(vIn + 1, 0);
816826
xb_vecMx32 vP2 = PDX_LV32_MX16_I(vIn + 2, 0);
817827
xb_vecMx32 vP3 = PDX_LV32_MX16_I(vIn + 3, 0);
818-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
819-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
820-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
821-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
822-
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale);
823-
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale);
824-
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale);
825-
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale);
828+
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale);
829+
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale);
830+
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale);
831+
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale);
826832
vIn += 4;
827833
vOut += 4;
828834
}
@@ -832,21 +838,15 @@ Tensor& dequantize_per_tensor_out(
832838
(static_cast<float>(input_data[i]) - zp_f) * scale_data;
833839
}
834840
} else {
835-
// 4x unrolled main loop: 16 elements per iteration
836-
const int e16 = (numel >> 4) << 4;
837841
for (; i < e16; i += 16) {
838842
xb_vecMx32 vP0 = PDX_LV32_MX16_I(vIn, 0);
839843
xb_vecMx32 vP1 = PDX_LV32_MX16_I(vIn + 1, 0);
840844
xb_vecMx32 vP2 = PDX_LV32_MX16_I(vIn + 2, 0);
841845
xb_vecMx32 vP3 = PDX_LV32_MX16_I(vIn + 3, 0);
842-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
843-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
844-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
845-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
846-
vOut[0] = PDX_MUL_MXF32(vF0, v_scale);
847-
vOut[1] = PDX_MUL_MXF32(vF1, v_scale);
848-
vOut[2] = PDX_MUL_MXF32(vF2, v_scale);
849-
vOut[3] = PDX_MUL_MXF32(vF3, v_scale);
846+
vOut[0] = PDX_MUL_MXF32(vP0, v_scale);
847+
vOut[1] = PDX_MUL_MXF32(vP1, v_scale);
848+
vOut[2] = PDX_MUL_MXF32(vP2, v_scale);
849+
vOut[3] = PDX_MUL_MXF32(vP3, v_scale);
850850
vIn += 4;
851851
vOut += 4;
852852
}
@@ -888,34 +888,40 @@ Tensor& dequantize_per_tensor_out(
888888
input.const_data_ptr<uint16_t>();
889889
#if defined(__XTENSA__)
890890
// Direct inline PDX SIMD dequantization for per-tensor uint16->float32.
891-
// 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave
892-
// independent loads, converts, and FP MACs across unrolled iterations,
893-
// hiding load-to-use latency and reducing loop overhead.
891+
// The zero-point subtract is kept in the int32 domain (PDX_SUB_MX32) so
892+
// it issues on the integer unit and overlaps the float pipe; the widened
893+
// int32 codes feed straight into the mixed-type PDX_MUL_MXF32. That
894+
// intrinsic takes an xb_vecMxf32, so the compiler still emits an
895+
// int->float convert -- writing it mixed-type only keeps the source
896+
// concise, it does not remove the convert. q - zp cannot overflow int32
897+
// (inputs are <=16-bit and zp is in the same quant range) and converts
898+
// exactly to float, so the result is bit-identical to
899+
// float(q) - float(zp). 4x-unrolled to hide load-to-use latency.
894900
if (dequant_simd_aligned(input_data, out_data)) {
895901
const int numel = inp_shape[0];
896902
auto vIn = reinterpret_cast<const xb_vecMxu16*>(input_data);
897903
auto vOut = reinterpret_cast<xb_vecMxf32*>(out_data);
898904
const xb_vecMxf32 v_scale{
899905
scale_data, scale_data, scale_data, scale_data};
900906
int i = 0;
907+
// 4x unrolled main loop: 16 elements per iteration
908+
const int e16 = (numel >> 4) << 4;
901909
if (zero_point_data != 0) {
910+
const xb_vecMxu32 v_zp{
911+
static_cast<uint32_t>(zero_point_data),
912+
static_cast<uint32_t>(zero_point_data),
913+
static_cast<uint32_t>(zero_point_data),
914+
static_cast<uint32_t>(zero_point_data)};
902915
const float zp_f = static_cast<float>(zero_point_data);
903-
const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f};
904-
// 4x unrolled main loop: 16 elements per iteration
905-
const int e16 = (numel >> 4) << 4;
906916
for (; i < e16; i += 16) {
907917
xb_vecMxu32 vP0 = PDX_LVU32_MX16_I(vIn, 0);
908918
xb_vecMxu32 vP1 = PDX_LVU32_MX16_I(vIn + 1, 0);
909919
xb_vecMxu32 vP2 = PDX_LVU32_MX16_I(vIn + 2, 0);
910920
xb_vecMxu32 vP3 = PDX_LVU32_MX16_I(vIn + 3, 0);
911-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
912-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
913-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
914-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
915-
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale);
916-
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale);
917-
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale);
918-
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale);
921+
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale);
922+
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale);
923+
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale);
924+
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale);
919925
vIn += 4;
920926
vOut += 4;
921927
}
@@ -925,21 +931,15 @@ Tensor& dequantize_per_tensor_out(
925931
(static_cast<float>(input_data[i]) - zp_f) * scale_data;
926932
}
927933
} else {
928-
// 4x unrolled main loop: 16 elements per iteration
929-
const int e16 = (numel >> 4) << 4;
930934
for (; i < e16; i += 16) {
931935
xb_vecMxu32 vP0 = PDX_LVU32_MX16_I(vIn, 0);
932936
xb_vecMxu32 vP1 = PDX_LVU32_MX16_I(vIn + 1, 0);
933937
xb_vecMxu32 vP2 = PDX_LVU32_MX16_I(vIn + 2, 0);
934938
xb_vecMxu32 vP3 = PDX_LVU32_MX16_I(vIn + 3, 0);
935-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
936-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
937-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
938-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
939-
vOut[0] = PDX_MUL_MXF32(vF0, v_scale);
940-
vOut[1] = PDX_MUL_MXF32(vF1, v_scale);
941-
vOut[2] = PDX_MUL_MXF32(vF2, v_scale);
942-
vOut[3] = PDX_MUL_MXF32(vF3, v_scale);
939+
vOut[0] = PDX_MUL_MXF32(vP0, v_scale);
940+
vOut[1] = PDX_MUL_MXF32(vP1, v_scale);
941+
vOut[2] = PDX_MUL_MXF32(vP2, v_scale);
942+
vOut[3] = PDX_MUL_MXF32(vP3, v_scale);
943943
vIn += 4;
944944
vOut += 4;
945945
}

0 commit comments

Comments
 (0)