Skip to content

Commit 2af81d7

Browse files
committed
ASIMD optimizations for lerp functions
1 parent ea025fe commit 2af81d7

7 files changed

Lines changed: 335 additions & 28 deletions

File tree

include/private/dsp/arch/aarch64/asimd/pmath.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include <private/dsp/arch/aarch64/asimd/pmath/fmop_kx.h>
3434
#include <private/dsp/arch/aarch64/asimd/pmath/fmop_vv.h>
3535
#include <private/dsp/arch/aarch64/asimd/pmath/lanczos.h>
36+
#include <private/dsp/arch/aarch64/asimd/pmath/lerp.h>
3637
#include <private/dsp/arch/aarch64/asimd/pmath/log.h>
3738
#include <private/dsp/arch/aarch64/asimd/pmath/lramp.h>
3839
#include <private/dsp/arch/aarch64/asimd/pmath/minmax.h>
Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
/*
2+
* Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
3+
* (C) 2026 Vladimir Sadovnikov <sadko4u@gmail.com>
4+
*
5+
* This file is part of lsp-dsp-lib
6+
* Created on: 3 июл. 2026 г.
7+
*
8+
* lsp-dsp-lib is free software: you can redistribute it and/or modify
9+
* it under the terms of the GNU Lesser General Public License as published by
10+
* the Free Software Foundation, either version 3 of the License, or
11+
* any later version.
12+
*
13+
* lsp-dsp-lib is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* GNU Lesser General Public License for more details.
17+
*
18+
* You should have received a copy of the GNU Lesser General Public License
19+
* along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
20+
*/
21+
22+
#ifndef INCLUDE_PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LERP_H_
23+
#define INCLUDE_PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LERP_H_
24+
25+
#ifndef PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL
26+
#error "This header should not be included directly"
27+
#endif /* PRIVATE_DSP_ARCH_AARCH64_ASIMD_IMPL */
28+
29+
#include <private/dsp/arch/aarch64/asimd/pmath/sin.h>
30+
31+
namespace lsp
32+
{
33+
namespace asimd
34+
{
35+
void lerp_vvv(float *dst, const float *a, const float *b, const float *k, size_t count)
36+
{
37+
ARCH_AARCH64_ASM
38+
(
39+
// x32 blocks
40+
__ASM_EMIT("subs %[count], %[count], #32")
41+
__ASM_EMIT("b.lo 2f")
42+
__ASM_EMIT("1:")
43+
__ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = a
44+
__ASM_EMIT("ldp q2, q3, [%[a], #0x20]")
45+
__ASM_EMIT("ldp q4, q5, [%[a], #0x40]")
46+
__ASM_EMIT("ldp q6, q7, [%[a], #0x60]")
47+
__ASM_EMIT("ldp q16, q17, [%[b], #0x00]") // v16 = b
48+
__ASM_EMIT("ldp q18, q19, [%[b], #0x20]")
49+
__ASM_EMIT("ldp q20, q21, [%[b], #0x40]")
50+
__ASM_EMIT("ldp q22, q23, [%[b], #0x60]")
51+
__ASM_EMIT("ldp q24, q25, [%[k], #0x00]") // v24 = k
52+
__ASM_EMIT("ldp q26, q27, [%[k], #0x20]")
53+
__ASM_EMIT("ldp q28, q29, [%[k], #0x40]")
54+
__ASM_EMIT("ldp q30, q31, [%[k], #0x60]")
55+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
56+
__ASM_EMIT("fsub v17.4s, v17.4s, v1.4s")
57+
__ASM_EMIT("fsub v18.4s, v18.4s, v2.4s")
58+
__ASM_EMIT("fsub v19.4s, v19.4s, v3.4s")
59+
__ASM_EMIT("fsub v20.4s, v20.4s, v4.4s")
60+
__ASM_EMIT("fsub v21.4s, v21.4s, v5.4s")
61+
__ASM_EMIT("fsub v22.4s, v22.4s, v6.4s")
62+
__ASM_EMIT("fsub v23.4s, v23.4s, v7.4s")
63+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
64+
__ASM_EMIT("fmla v1.4s, v17.4s, v25.4s")
65+
__ASM_EMIT("fmla v2.4s, v18.4s, v26.4s")
66+
__ASM_EMIT("fmla v3.4s, v19.4s, v27.4s")
67+
__ASM_EMIT("fmla v4.4s, v20.4s, v28.4s")
68+
__ASM_EMIT("fmla v5.4s, v21.4s, v29.4s")
69+
__ASM_EMIT("fmla v6.4s, v22.4s, v30.4s")
70+
__ASM_EMIT("fmla v7.4s, v23.4s, v31.4s")
71+
__ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
72+
__ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
73+
__ASM_EMIT("stp q4, q5, [%[dst], #0x40]")
74+
__ASM_EMIT("stp q6, q7, [%[dst], #0x60]")
75+
__ASM_EMIT("subs %[count], %[count], #32")
76+
__ASM_EMIT("add %[a], %[a], #0x80")
77+
__ASM_EMIT("add %[b], %[b], #0x80")
78+
__ASM_EMIT("add %[k], %[k], #0x80")
79+
__ASM_EMIT("add %[dst], %[dst], #0x80")
80+
__ASM_EMIT("b.hs 1b")
81+
// x16 block
82+
__ASM_EMIT("2:")
83+
__ASM_EMIT("adds %[count], %[count], #16")
84+
__ASM_EMIT("b.lt 4f")
85+
__ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = a
86+
__ASM_EMIT("ldp q2, q3, [%[a], #0x20]")
87+
__ASM_EMIT("ldp q16, q17, [%[b], #0x00]") // v16 = b
88+
__ASM_EMIT("ldp q18, q19, [%[b], #0x20]")
89+
__ASM_EMIT("ldp q24, q25, [%[k], #0x00]") // v24 = k
90+
__ASM_EMIT("ldp q26, q27, [%[k], #0x20]")
91+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
92+
__ASM_EMIT("fsub v17.4s, v17.4s, v1.4s")
93+
__ASM_EMIT("fsub v18.4s, v18.4s, v2.4s")
94+
__ASM_EMIT("fsub v19.4s, v19.4s, v3.4s")
95+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
96+
__ASM_EMIT("fmla v1.4s, v17.4s, v25.4s")
97+
__ASM_EMIT("fmla v2.4s, v18.4s, v26.4s")
98+
__ASM_EMIT("fmla v3.4s, v19.4s, v27.4s")
99+
__ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
100+
__ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
101+
__ASM_EMIT("sub %[count], %[count], #16")
102+
__ASM_EMIT("add %[a], %[a], #0x40")
103+
__ASM_EMIT("add %[b], %[b], #0x40")
104+
__ASM_EMIT("add %[k], %[k], #0x40")
105+
__ASM_EMIT("add %[dst], %[dst], #0x40")
106+
// x8 block
107+
__ASM_EMIT("4:")
108+
__ASM_EMIT("adds %[count], %[count], #8")
109+
__ASM_EMIT("b.lt 6f")
110+
__ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = a
111+
__ASM_EMIT("ldp q16, q17, [%[b], #0x00]") // v16 = b
112+
__ASM_EMIT("ldp q24, q25, [%[k], #0x00]") // v24 = k
113+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
114+
__ASM_EMIT("fsub v17.4s, v17.4s, v1.4s")
115+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
116+
__ASM_EMIT("fmla v1.4s, v17.4s, v25.4s")
117+
__ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
118+
__ASM_EMIT("sub %[count], %[count], #8")
119+
__ASM_EMIT("add %[a], %[a], #0x20")
120+
__ASM_EMIT("add %[b], %[b], #0x20")
121+
__ASM_EMIT("add %[k], %[k], #0x20")
122+
__ASM_EMIT("add %[dst], %[dst], #0x20")
123+
// x4 blocks
124+
__ASM_EMIT("6:")
125+
__ASM_EMIT("adds %[count], %[count], #4")
126+
__ASM_EMIT("b.lt 8f")
127+
__ASM_EMIT("ldr q0, [%[a], #0x00]") // v0 = a
128+
__ASM_EMIT("ldr q16, [%[b], #0x00]") // v16 = b
129+
__ASM_EMIT("ldr q24, [%[k], #0x00]") // v24 = k
130+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
131+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
132+
__ASM_EMIT("str q0, [%[dst], #0x00]")
133+
__ASM_EMIT("sub %[count], %[count], #4")
134+
__ASM_EMIT("add %[a], %[a], #0x10")
135+
__ASM_EMIT("add %[b], %[b], #0x10")
136+
__ASM_EMIT("add %[k], %[k], #0x10")
137+
__ASM_EMIT("add %[dst], %[dst], #0x10")
138+
// x1 blocks
139+
__ASM_EMIT("8:")
140+
__ASM_EMIT("adds %[count], %[count], #3")
141+
__ASM_EMIT("b.lt 10f")
142+
__ASM_EMIT("9:")
143+
__ASM_EMIT("ld1r {v0.4s}, [%[a]]") // v0 = a
144+
__ASM_EMIT("ld1r {v16.4s}, [%[b]]") // v16 = b
145+
__ASM_EMIT("ld1r {v24.4s}, [%[k]]") // v24 = k
146+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
147+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
148+
__ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
149+
__ASM_EMIT("subs %[count], %[count], #1")
150+
__ASM_EMIT("add %[a], %[a], #0x04")
151+
__ASM_EMIT("add %[b], %[b], #0x04")
152+
__ASM_EMIT("add %[k], %[k], #0x04")
153+
__ASM_EMIT("add %[dst], %[dst], #0x04")
154+
__ASM_EMIT("b.ge 9b")
155+
// end
156+
__ASM_EMIT("10:")
157+
158+
: [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b), [k] "+r" (k),
159+
[count] "+r" (count)
160+
:
161+
: "cc", "memory",
162+
"v0", "v1", "v2", "v3",
163+
"v4", "v5", "v6", "v7",
164+
"v16", "v17", "v18", "v19",
165+
"v20", "v21", "v22", "v23",
166+
"v24", "v25", "v26", "v27",
167+
"v28", "v29", "v30", "v31"
168+
);
169+
}
170+
171+
void lerp_vvk(float *dst, const float *a, const float *b, float k, size_t count)
172+
{
173+
ARCH_AARCH64_ASM
174+
(
175+
// x32 blocks
176+
__ASM_EMIT("subs %[count], %[count], #32")
177+
__ASM_EMIT("ld1r {v24.4s}, [%[k]]")
178+
__ASM_EMIT("b.lo 2f")
179+
__ASM_EMIT("1:")
180+
__ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = a
181+
__ASM_EMIT("ldp q2, q3, [%[a], #0x20]")
182+
__ASM_EMIT("ldp q4, q5, [%[a], #0x40]")
183+
__ASM_EMIT("ldp q6, q7, [%[a], #0x60]")
184+
__ASM_EMIT("ldp q16, q17, [%[b], #0x00]") // v16 = b
185+
__ASM_EMIT("ldp q18, q19, [%[b], #0x20]")
186+
__ASM_EMIT("ldp q20, q21, [%[b], #0x40]")
187+
__ASM_EMIT("ldp q22, q23, [%[b], #0x60]")
188+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
189+
__ASM_EMIT("fsub v17.4s, v17.4s, v1.4s")
190+
__ASM_EMIT("fsub v18.4s, v18.4s, v2.4s")
191+
__ASM_EMIT("fsub v19.4s, v19.4s, v3.4s")
192+
__ASM_EMIT("fsub v20.4s, v20.4s, v4.4s")
193+
__ASM_EMIT("fsub v21.4s, v21.4s, v5.4s")
194+
__ASM_EMIT("fsub v22.4s, v22.4s, v6.4s")
195+
__ASM_EMIT("fsub v23.4s, v23.4s, v7.4s")
196+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
197+
__ASM_EMIT("fmla v1.4s, v17.4s, v24.4s")
198+
__ASM_EMIT("fmla v2.4s, v18.4s, v24.4s")
199+
__ASM_EMIT("fmla v3.4s, v19.4s, v24.4s")
200+
__ASM_EMIT("fmla v4.4s, v20.4s, v24.4s")
201+
__ASM_EMIT("fmla v5.4s, v21.4s, v24.4s")
202+
__ASM_EMIT("fmla v6.4s, v22.4s, v24.4s")
203+
__ASM_EMIT("fmla v7.4s, v23.4s, v24.4s")
204+
__ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
205+
__ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
206+
__ASM_EMIT("stp q4, q5, [%[dst], #0x40]")
207+
__ASM_EMIT("stp q6, q7, [%[dst], #0x60]")
208+
__ASM_EMIT("subs %[count], %[count], #32")
209+
__ASM_EMIT("add %[a], %[a], #0x80")
210+
__ASM_EMIT("add %[b], %[b], #0x80")
211+
__ASM_EMIT("add %[dst], %[dst], #0x80")
212+
__ASM_EMIT("b.hs 1b")
213+
// x16 block
214+
__ASM_EMIT("2:")
215+
__ASM_EMIT("adds %[count], %[count], #16")
216+
__ASM_EMIT("b.lt 4f")
217+
__ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = a
218+
__ASM_EMIT("ldp q2, q3, [%[a], #0x20]")
219+
__ASM_EMIT("ldp q16, q17, [%[b], #0x00]") // v16 = b
220+
__ASM_EMIT("ldp q18, q19, [%[b], #0x20]")
221+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
222+
__ASM_EMIT("fsub v17.4s, v17.4s, v1.4s")
223+
__ASM_EMIT("fsub v18.4s, v18.4s, v2.4s")
224+
__ASM_EMIT("fsub v19.4s, v19.4s, v3.4s")
225+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
226+
__ASM_EMIT("fmla v1.4s, v17.4s, v24.4s")
227+
__ASM_EMIT("fmla v2.4s, v18.4s, v24.4s")
228+
__ASM_EMIT("fmla v3.4s, v19.4s, v24.4s")
229+
__ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
230+
__ASM_EMIT("stp q2, q3, [%[dst], #0x20]")
231+
__ASM_EMIT("sub %[count], %[count], #16")
232+
__ASM_EMIT("add %[a], %[a], #0x40")
233+
__ASM_EMIT("add %[b], %[b], #0x40")
234+
__ASM_EMIT("add %[dst], %[dst], #0x40")
235+
// x8 block
236+
__ASM_EMIT("4:")
237+
__ASM_EMIT("adds %[count], %[count], #8")
238+
__ASM_EMIT("b.lt 6f")
239+
__ASM_EMIT("ldp q0, q1, [%[a], #0x00]") // v0 = a
240+
__ASM_EMIT("ldp q16, q17, [%[b], #0x00]") // v16 = b
241+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
242+
__ASM_EMIT("fsub v17.4s, v17.4s, v1.4s")
243+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
244+
__ASM_EMIT("fmla v1.4s, v17.4s, v24.4s")
245+
__ASM_EMIT("stp q0, q1, [%[dst], #0x00]")
246+
__ASM_EMIT("sub %[count], %[count], #8")
247+
__ASM_EMIT("add %[a], %[a], #0x20")
248+
__ASM_EMIT("add %[b], %[b], #0x20")
249+
__ASM_EMIT("add %[dst], %[dst], #0x20")
250+
// x4 blocks
251+
__ASM_EMIT("6:")
252+
__ASM_EMIT("adds %[count], %[count], #4")
253+
__ASM_EMIT("b.lt 8f")
254+
__ASM_EMIT("ldr q0, [%[a], #0x00]") // v0 = a
255+
__ASM_EMIT("ldr q16, [%[b], #0x00]") // v16 = b
256+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
257+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
258+
__ASM_EMIT("str q0, [%[dst], #0x00]")
259+
__ASM_EMIT("sub %[count], %[count], #4")
260+
__ASM_EMIT("add %[a], %[a], #0x10")
261+
__ASM_EMIT("add %[b], %[b], #0x10")
262+
__ASM_EMIT("add %[dst], %[dst], #0x10")
263+
// x1 blocks
264+
__ASM_EMIT("8:")
265+
__ASM_EMIT("adds %[count], %[count], #3")
266+
__ASM_EMIT("b.lt 10f")
267+
__ASM_EMIT("9:")
268+
__ASM_EMIT("ld1r {v0.4s}, [%[a]]") // v0 = a
269+
__ASM_EMIT("ld1r {v16.4s}, [%[b]]") // v16 = b
270+
__ASM_EMIT("fsub v16.4s, v16.4s, v0.4s") // v16 = b-a
271+
__ASM_EMIT("fmla v0.4s, v16.4s, v24.4s") // v0 = a + (b-a)*k
272+
__ASM_EMIT("st1 {v0.s}[0], [%[dst]]")
273+
__ASM_EMIT("subs %[count], %[count], #1")
274+
__ASM_EMIT("add %[a], %[a], #0x04")
275+
__ASM_EMIT("add %[b], %[b], #0x04")
276+
__ASM_EMIT("add %[dst], %[dst], #0x04")
277+
__ASM_EMIT("b.ge 9b")
278+
// end
279+
__ASM_EMIT("10:")
280+
281+
: [dst] "+r" (dst), [a] "+r" (a), [b] "+r" (b),
282+
[count] "+r" (count)
283+
: [k] "r" (&k)
284+
: "cc", "memory",
285+
"v0", "v1", "v2", "v3",
286+
"v4", "v5", "v6", "v7",
287+
"v16", "v17", "v18", "v19",
288+
"v20", "v21", "v22", "v23",
289+
"v24"
290+
);
291+
}
292+
} /* namespace asimd */
293+
} /* namespace lsp */
294+
295+
296+
297+
298+
#endif /* INCLUDE_PRIVATE_DSP_ARCH_AARCH64_ASIMD_PMATH_LERP_H_ */

src/main/aarch64/asimd.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,14 @@
516516
EXPORT1(pmix_v2);
517517
EXPORT1(pmix_k1);
518518
EXPORT1(pmix_k2);
519+
520+
EXPORT1(lerp_vvv);
521+
EXPORT1(lerp_vvk);
522+
// EXPORT1(lerp_vkv);
523+
// EXPORT1(lerp_vkk);
524+
// EXPORT1(lerp_kvv);
525+
// EXPORT1(lerp_kvk);
526+
// EXPORT1(lerp_kkv);
519527
}
520528
} /* namespace asimd */
521529
} /* namespace lsp */

src/test/ptest/pmath/lerp_vvk.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,12 @@ namespace lsp
6060
}
6161
)
6262

63-
// IF_ARCH_AARCH64(
64-
// namespace asimd
65-
// {
66-
// void lerp_vvk(float *dst, const float *a, const float *b, float k, size_t count);
67-
// }
68-
// )
63+
IF_ARCH_AARCH64(
64+
namespace asimd
65+
{
66+
void lerp_vvk(float *dst, const float *a, const float *b, float k, size_t count);
67+
}
68+
)
6969

7070
typedef void (* lerp_vvk_t)(float *dst, const float *a, const float *b, float k, size_t count);
7171
}
@@ -113,7 +113,7 @@ PTEST_BEGIN("dsp.pmath", lerp_vvk, 5, 1000)
113113
IF_ARCH_X86(CALL(avx::lerp_vvk_fma3));
114114
IF_ARCH_X86(CALL(avx512::lerp_vvk));
115115
IF_ARCH_ARM(CALL(neon_d32::lerp_vvk));
116-
// IF_ARCH_AARCH64(CALL(asimd::lerp_vvk));
116+
IF_ARCH_AARCH64(CALL(asimd::lerp_vvk));
117117
PTEST_SEPARATOR;
118118
}
119119

src/test/ptest/pmath/lerp_vvv.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,12 @@ namespace lsp
6060
}
6161
)
6262

63-
// IF_ARCH_AARCH64(
64-
// namespace asimd
65-
// {
66-
// void lerp_vvv(float *dst, const float *a, const float *b, const float *k, size_t count);
67-
// }
68-
// )
63+
IF_ARCH_AARCH64(
64+
namespace asimd
65+
{
66+
void lerp_vvv(float *dst, const float *a, const float *b, const float *k, size_t count);
67+
}
68+
)
6969

7070
typedef void (* lerp_vvv_t)(float *dst, const float *a, const float *b, const float *k, size_t count);
7171
}
@@ -114,7 +114,7 @@ PTEST_BEGIN("dsp.pmath", lerp_vvv, 5, 1000)
114114
IF_ARCH_X86(CALL(avx::lerp_vvv_fma3));
115115
IF_ARCH_X86(CALL(avx512::lerp_vvv));
116116
IF_ARCH_ARM(CALL(neon_d32::lerp_vvv));
117-
// IF_ARCH_AARCH64(CALL(asimd::lerp_vvv));
117+
IF_ARCH_AARCH64(CALL(asimd::lerp_vvv));
118118
PTEST_SEPARATOR;
119119
}
120120

0 commit comments

Comments
 (0)