ntt_aarch64_asm.S

/* Copyright (c) 2022 Arm Limited
 * Copyright (c) 2022 Hanno Becker
 * Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
 * Copyright (c) The mlkem-native project authors
 * Copyright (c) The mldsa-native project authors
 * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
 */

/* References
 * ==========
 *
 * - [NeonNTT]
 *   Neon NTT: Faster Dilithium, Kyber, and Saber on Cortex-A72 and Apple M1
 *   Becker, Hwang, Kannwischer, Yang, Yang
 *   https://eprint.iacr.org/2021/986
 *
 * - [SLOTHY_Paper]
 *   Fast and Clean: Auditable high-performance assembly via constraint solving
 *   Abdulrahman, Becker, Kannwischer, Klein
 *   https://eprint.iacr.org/2022/1303
 */

/* AArch64 ML-DSA forward NTT following @[NeonNTT] and @[SLOTHY_Paper] */

#include "../../../common.h"
#if defined(MLD_ARITH_BACKEND_AARCH64) &&  !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)
/* simpasm: header-end */

.macro mulmodq dst, src, const, idx0, idx1
        // Signed barrett multiplication @[NeonNTT, Section 3.1.2] using
        // round-to-nearest-even-integer approximation. Following loc.cit.,
        // this is functionally the same as a signed Montgomery multiplication
        // with a suitable constant of absolute value < q.
        sqrdmulh t2.4s,      \src\().4s, \const\().s[\idx1\()]
        mul      \dst\().4s, \src\().4s, \const\().s[\idx0\()]
        mls      \dst\().4s, t2.4s,      consts.s[0]
        // Bounds: |dst| < BARMUL_BOUND

        // (See the end of dev/aarch64_clean/src/intt.S for the exact value of
        // BARMUL_BOUND)
.endm

.macro mulmod dst, src, const, const_twisted
        // Same as mulmodq, but with different multiplier across each lanes.
        sqrdmulh t2.4s,   \src\().4s, \const_twisted\().4s
        mul      \dst\().4s, \src\().4s, \const\().4s
        mls      \dst\().4s, t2.4s,   consts.s[0]
        // Bounds: |dst| < BARMUL_BOUND
.endm

.macro ct_butterfly a, b, root, idx0, idx1
        // Compute a' = a + barmul(b, zeta), b' = a - barmul(b, zeta), commonly
        // known as the Cooley-Tukey butterfly.
        mulmodq tmp, \b, \root, \idx0, \idx1
        sub \b\().4s, \a\().4s, tmp.4s
        add \a\().4s, \a\().4s, tmp.4s
        // Bounds: |a'|, |b'| < |a| + BARMUL_BOUND <= |a| + q

        // In conclusion, the magnitudes of all coefficients grow by at most q
        // after each layer.
.endm

.macro ct_butterfly_v a, b, root, root_twisted
        // Same as ct_butterfly, but with different zetas across each lanes.
        mulmod tmp, \b, \root, \root_twisted
        sub \b\().4s, \a\().4s, tmp.4s
        add \a\().4s, \a\().4s, tmp.4s
        // Bounds: |a'|, |b'| < |a| + BARMUL_BOUND <= |a| + q
.endm

.macro load_roots_123
        ldr q_root0, [r012345_ptr], #(4*16)
        ldr q_root1, [r012345_ptr, #(-4*16 + 1*16)]
        ldr q_root2, [r012345_ptr, #(-4*16 + 2*16)]
        ldr q_root3, [r012345_ptr, #(-4*16 + 3*16)]
.endm

.macro load_next_roots_456
       ldr q_root0, [r012345_ptr], #(4*16)
       ldr q_root1, [r012345_ptr, #(-4*16 + 1*16)]
       ldr q_root2, [r012345_ptr, #(-4*16 + 2*16)]
       ldr q_root3, [r012345_ptr, #(-4*16 + 3*16)]
.endm

.macro load_next_roots_78_1
        ldr q_root0,    [r67_ptr], #(12*16)
        ldr q_root0_tw, [r67_ptr, #(-12*16 + 1*16)]
        ldr q_root1,    [r67_ptr, #(-12*16 + 2*16)]
        ldr q_root1_tw, [r67_ptr, #(-12*16 + 3*16)]
        ldr q_root2,    [r67_ptr, #(-12*16 + 4*16)]
        ldr q_root2_tw, [r67_ptr, #(-12*16 + 5*16)]
.endm

.macro load_next_roots_78_2
        ldr q_root0,    [r67_ptr, #(-12*16 + 6*16)]
        ldr q_root0_tw, [r67_ptr, #(-12*16 + 7*16)]
        ldr q_root1,    [r67_ptr, #(-12*16 + 8*16)]
        ldr q_root1_tw, [r67_ptr, #(-12*16 + 9*16)]
        ldr q_root2,    [r67_ptr, #(-12*16 + 10*16)]
        ldr q_root2_tw, [r67_ptr, #(-12*16 + 11*16)]
.endm

.macro transpose4 data0, data1, data2, data3
        trn1 t0.4s, \data0\().4s, \data1\().4s
        trn2 t1.4s, \data0\().4s, \data1\().4s
        trn1 t2.4s, \data2\().4s, \data3\().4s
        trn2 t3.4s, \data2\().4s, \data3\().4s

        trn2 \data2\().2d, t0.2d, t2.2d
        trn2 \data3\().2d, t1.2d, t3.2d
        trn1 \data0\().2d, t0.2d, t2.2d
        trn1 \data1\().2d, t1.2d, t3.2d
.endm

.macro save_vregs
        sub sp, sp, #(16*4)
        stp  d8,  d9, [sp, #16*0]
        stp d10, d11, [sp, #16*1]
        stp d12, d13, [sp, #16*2]
        stp d14, d15, [sp, #16*3]
.endm

.macro restore_vregs
        ldp  d8,  d9, [sp, #16*0]
        ldp d10, d11, [sp, #16*1]
        ldp d12, d13, [sp, #16*2]
        ldp d14, d15, [sp, #16*3]
        add sp, sp, #(16*4)
.endm

.macro push_stack
        save_vregs
.endm

.macro pop_stack
        restore_vregs
.endm

        // Arguments
        in          .req x0  // Input/output buffer
        r012345_ptr .req x1  // twiddles for layer 0,1,2,3,4,5
        r67_ptr     .req x2  // twiddles for layer 6,7

        inp     .req x3
        count   .req x4
        wtmp    .req w5

        data0  .req v8
        data1  .req v9
        data2  .req v10
        data3  .req v11
        data4  .req v12
        data5  .req v13
        data6  .req v14
        data7  .req v15

        q_data0  .req q8
        q_data1  .req q9
        q_data2  .req q10
        q_data3  .req q11
        q_data4  .req q12
        q_data5  .req q13
        q_data6  .req q14
        q_data7  .req q15

        root0    .req v0
        root1    .req v1
        root2    .req v2
        root3    .req v3
        root0_tw .req v4
        root1_tw .req v5
        root2_tw .req v6

        q_root0    .req q0
        q_root1    .req q1
        q_root2    .req q2
        q_root3    .req q3
        q_root0_tw .req q4
        q_root1_tw .req q5
        q_root2_tw .req q6

        consts    .req v7

        tmp .req v24
        t0  .req v25
        t1  .req v26
        t2  .req v27
        t3  .req v28

        .text
        .global MLD_ASM_NAMESPACE(ntt_aarch64_asm)
        .balign 4
MLD_ASM_FN_SYMBOL(ntt_aarch64_asm)
        push_stack

        // Load MLDSA_Q = 8380417
        movz wtmp, #0xe001
        movk wtmp, #0x7f, lsl #16
        dup consts.4s, wtmp

        mov inp, in
        mov count, #8

        load_roots_123

        .p2align 2

                                                // Instructions:    77
                                                // Expected cycles: 91
                                                // Expected IPC:    0.85

                                                // Cycle bound:     91.0
                                                // IPC bound:       0.85

                                                // Wall time:     2.15s
                                                // User time:     2.15s

                                                // ------------------------------------ cycle (expected) ------------------------------------>
                                                // 0                        25                       50                       75
                                                // |------------------------|------------------------|------------------------|---------------
        ldr q23, [x0, #912]                     // *..........................................................................................
        ldr q13, [x0, #896]                     // *..........................................................................................
        ldr q22, [x0, #128]                     // .*.........................................................................................
        ldr q26, [x0, #400]                     // .*.........................................................................................
        ldr q8, [x0, #640]                      // ..*........................................................................................
        ldr q6, [x0, #528]                      // ...*.......................................................................................
        mul v10.4S, v13.4S, v0.S[0]             // ....*......................................................................................
        sqrdmulh v13.4S, v13.4S, v0.S[1]        // ......*....................................................................................
        mul v12.4S, v8.4S, v0.S[0]              // ........*..................................................................................
        sqrdmulh v27.4S, v8.4S, v0.S[1]         // ..........*................................................................................
        mul v4.4S, v6.4S, v0.S[0]               // ............*..............................................................................
        mls v10.4S, v13.4S, v7.S[0]             // ..............*............................................................................
        ldr q13, [x0, #384]                     // ...............*...........................................................................
        sqrdmulh v14.4S, v23.4S, v0.S[1]        // ................*..........................................................................
        mls v12.4S, v27.4S, v7.S[0]             // ..................*........................................................................
        add v31.4S, v13.4S, v10.4S              // ...................*.......................................................................
        sub v13.4S, v13.4S, v10.4S              // ....................*......................................................................
        mul v10.4S, v23.4S, v0.S[0]             // ....................*......................................................................
        sqrdmulh v8.4S, v13.4S, v1.S[1]         // ......................*....................................................................
        sub v18.4S, v22.4S, v12.4S              // .......................*...................................................................
        mls v10.4S, v14.4S, v7.S[0]             // ........................*..................................................................
        mul v13.4S, v13.4S, v1.S[0]             // ..........................*................................................................
        mls v13.4S, v8.4S, v7.S[0]              // ............................*..............................................................
        sub v29.4S, v26.4S, v10.4S              // .............................*.............................................................
        add v25.4S, v26.4S, v10.4S              // ..............................*............................................................
        mul v10.4S, v31.4S, v0.S[2]             // ..............................*............................................................
        mul v14.4S, v25.4S, v0.S[2]             // ................................*..........................................................
        add v17.4S, v18.4S, v13.4S              // .................................*.........................................................
        sub v15.4S, v18.4S, v13.4S              // ..................................*........................................................
        sqrdmulh v13.4S, v31.4S, v0.S[3]        // ..................................*........................................................
        sqrdmulh v20.4S, v15.4S, v3.S[1]        // ....................................*......................................................
        sqrdmulh v5.4S, v17.4S, v2.S[3]         // ......................................*....................................................
        mls v10.4S, v13.4S, v7.S[0]             // ........................................*..................................................
        ldr q13, [x0, #768]                     // .........................................*.................................................
        mul v18.4S, v17.4S, v2.S[2]             // ..........................................*................................................
        add v31.4S, v22.4S, v12.4S              // ...........................................*...............................................
        mul v23.4S, v15.4S, v3.S[0]             // ............................................*..............................................
        ldr q17, [x0, #144]                     // .............................................*.............................................
        add v19.4S, v31.4S, v10.4S              // .............................................*.............................................
        sub v16.4S, v31.4S, v10.4S              // ..............................................*............................................
        mul v10.4S, v13.4S, v0.S[0]             // ..............................................*............................................
        sqrdmulh v13.4S, v13.4S, v0.S[1]        // ................................................*..........................................
        sqrdmulh v27.4S, v16.4S, v2.S[1]        // ..................................................*........................................
        mul v11.4S, v16.4S, v2.S[0]             // ....................................................*......................................
        mls v10.4S, v13.4S, v7.S[0]             // ......................................................*....................................
        ldr q13, [x0, #656]                     // ......................................................*....................................
        ldr q22, [x0, #256]                     // .......................................................*...................................
        mls v11.4S, v27.4S, v7.S[0]             // ........................................................*..................................
        sqrdmulh v15.4S, v13.4S, v0.S[1]        // ..........................................................*................................
        sub v12.4S, v22.4S, v10.4S              // ...........................................................*...............................
        add v30.4S, v22.4S, v10.4S              // ............................................................*..............................
        mul v10.4S, v13.4S, v0.S[0]             // ............................................................*..............................
        ldr q28, [x0, #0]                       // .............................................................*.............................
        sqrdmulh v13.4S, v25.4S, v0.S[3]        // ..............................................................*............................
        sqrdmulh v27.4S, v30.4S, v0.S[3]        // ................................................................*..........................
        mls v10.4S, v15.4S, v7.S[0]             // ..................................................................*........................
        mls v14.4S, v13.4S, v7.S[0]             // ....................................................................*......................
        ldr q13, [x0, #512]                     // ....................................................................*......................
        sqrdmulh v25.4S, v12.4S, v1.S[1]        // ......................................................................*....................
        add v24.4S, v17.4S, v10.4S              // .......................................................................*...................
        sub v21.4S, v17.4S, v10.4S              // ........................................................................*..................
        sqrdmulh v8.4S, v13.4S, v0.S[1]         // ........................................................................*..................
        sub v9.4S, v24.4S, v14.4S               // .........................................................................*.................
        mul v26.4S, v12.4S, v1.S[0]             // ..........................................................................*................
        mul v13.4S, v13.4S, v0.S[0]             // ............................................................................*..............
        mls v13.4S, v8.4S, v7.S[0]              // ..............................................................................*............
        mul v8.4S, v30.4S, v0.S[2]              // ................................................................................*..........
        mls v8.4S, v27.4S, v7.S[0]              // ..................................................................................*........
        add v16.4S, v28.4S, v13.4S              // ...................................................................................*.......
        sub v10.4S, v28.4S, v13.4S              // ....................................................................................*......
        mls v26.4S, v25.4S, v7.S[0]             // ....................................................................................*......
        sqrdmulh v12.4S, v19.4S, v1.S[3]        // ......................................................................................*....
        sub v25.4S, v16.4S, v8.4S               // .......................................................................................*...
        mls v23.4S, v20.4S, v7.S[0]             // ........................................................................................*..
        sub v22.4S, v25.4S, v11.4S              // .........................................................................................*.
        sqrdmulh v20.4S, v9.4S, v2.S[1]         // ..........................................................................................*
        sub v15.4S, v10.4S, v26.4S              // ..........................................................................................*

                                                 // ------------------------------------ cycle (expected) ------------------------------------>
                                                 // 0                        25                       50                       75
                                                 // |------------------------|------------------------|------------------------|---------------
        // ldr q30, [x0, #640]                    // ..*........................................................................................
        // ldr q13, [x0, #896]                    // *..........................................................................................
        // sqrdmulh v27.4S, v30.4S, v0.S[1]       // ..........*................................................................................
        // ldr q9, [x0, #384]                     // ...............*...........................................................................
        // sqrdmulh v19.4S, v13.4S, v0.S[1]       // ......*....................................................................................
        // mul v8.4S, v13.4S, v0.S[0]             // ....*......................................................................................
        // mls v8.4S, v19.4S, v7.S[0]             // ..............*............................................................................
        // add v22.4S, v9.4S, v8.4S               // ...................*.......................................................................
        // ldr q6, [x0, #512]                     // ....................................................................*......................
        // sub v29.4S, v9.4S, v8.4S               // ....................*......................................................................
        // mul v17.4S, v30.4S, v0.S[0]            // ........*..................................................................................
        // sqrdmulh v13.4S, v22.4S, v0.S[3]       // ..................................*........................................................
        // mul v14.4S, v22.4S, v0.S[2]            // ..............................*............................................................
        // mls v14.4S, v13.4S, v7.S[0]            // ........................................*..................................................
        // ldr q22, [x0, #128]                    // .*.........................................................................................
        // mul v4.4S, v6.4S, v0.S[0]              // ............................................................................*..............
        // mls v17.4S, v27.4S, v7.S[0]            // ..................*........................................................................
        // add v24.4S, v22.4S, v17.4S             // ...........................................*...............................................
        // sub v21.4S, v22.4S, v17.4S             // .......................*...................................................................
        // sub v9.4S, v24.4S, v14.4S              // ..............................................*............................................
        // sqrdmulh v20.4S, v9.4S, v2.S[1]        // ..................................................*........................................
        // ldr q30, [x0, #656]                    // ......................................................*....................................
        // mul v23.4S, v29.4S, v1.S[0]            // ..........................*................................................................
        // mul v11.4S, v9.4S, v2.S[0]             // ....................................................*......................................
        // ldr q28, [x0, #0]                      // .............................................................*.............................
        // ldr q13, [x0, #912]                    // *..........................................................................................
        // sqrdmulh v27.4S, v30.4S, v0.S[1]       // ..........................................................*................................
        // ldr q9, [x0, #400]                     // .*.........................................................................................
        // sqrdmulh v10.4S, v6.4S, v0.S[1]        // ........................................................................*..................
        // sqrdmulh v19.4S, v13.4S, v0.S[1]       // ................*..........................................................................
        // mul v8.4S, v13.4S, v0.S[0]             // ....................*......................................................................
        // sqrdmulh v13.4S, v29.4S, v1.S[1]       // ......................*....................................................................
        // mls v8.4S, v19.4S, v7.S[0]             // ........................*..................................................................
        // mls v11.4S, v20.4S, v7.S[0]            // ........................................................*..................................
        // mls v23.4S, v13.4S, v7.S[0]            // ............................*..............................................................
        // add v22.4S, v9.4S, v8.4S               // ..............................*............................................................
        // ldr q6, [x0, #528]                     // ...*.......................................................................................
        // sub v29.4S, v9.4S, v8.4S               // .............................*.............................................................
        // mul v17.4S, v30.4S, v0.S[0]            // ............................................................*..............................
        // ldr q9, [x0, #768]                     // .........................................*.................................................
        // sqrdmulh v13.4S, v22.4S, v0.S[3]       // ..............................................................*............................
        // add v18.4S, v21.4S, v23.4S             // .................................*.........................................................
        // mls v4.4S, v10.4S, v7.S[0]             // ..............................................................................*............
        // sub v31.4S, v21.4S, v23.4S             // ..................................*........................................................
        // sqrdmulh v16.4S, v31.4S, v3.S[1]       // ....................................*......................................................
        // add v19.4S, v24.4S, v14.4S             // .............................................*.............................................
        // mul v14.4S, v22.4S, v0.S[2]            // ................................*..........................................................
        // sub v10.4S, v28.4S, v4.4S              // ....................................................................................*......
        // mls v14.4S, v13.4S, v7.S[0]            // ....................................................................*......................
        // ldr q13, [x0, #256]                    // .......................................................*...................................
        // sqrdmulh v22.4S, v9.4S, v0.S[1]        // ................................................*..........................................
        // mul v8.4S, v9.4S, v0.S[0]              // ..............................................*............................................
        // mul v23.4S, v31.4S, v3.S[0]            // ............................................*..............................................
        // mls v8.4S, v22.4S, v7.S[0]             // ......................................................*....................................
        // mls v23.4S, v16.4S, v7.S[0]            // ........................................................................................*..
        // add v16.4S, v28.4S, v4.4S              // ...................................................................................*.......
        // ldr q22, [x0, #144]                    // .............................................*.............................................
        // mul v4.4S, v6.4S, v0.S[0]              // ............*..............................................................................
        // mls v17.4S, v27.4S, v7.S[0]            // ..................................................................*........................
        // add v21.4S, v13.4S, v8.4S              // ............................................................*..............................
        // sub v27.4S, v13.4S, v8.4S              // ...........................................................*...............................
        // sqrdmulh v31.4S, v21.4S, v0.S[3]       // ................................................................*..........................
        // mul v8.4S, v21.4S, v0.S[2]             // ................................................................................*..........
        // add v24.4S, v22.4S, v17.4S             // .......................................................................*...................
        // sub v21.4S, v22.4S, v17.4S             // ........................................................................*..................
        // sqrdmulh v5.4S, v18.4S, v2.S[3]        // ......................................*....................................................
        // mls v8.4S, v31.4S, v7.S[0]             // ..................................................................................*........
        // sub v9.4S, v24.4S, v14.4S              // .........................................................................*.................
        // sqrdmulh v20.4S, v27.4S, v1.S[1]       // ......................................................................*....................
        // mul v26.4S, v27.4S, v1.S[0]            // ..........................................................................*................
        // sub v25.4S, v16.4S, v8.4S              // .......................................................................................*...
        // mul v18.4S, v18.4S, v2.S[2]            // ..........................................*................................................
        // sub v22.4S, v25.4S, v11.4S             // .........................................................................................*.
        // mls v26.4S, v20.4S, v7.S[0]            // ....................................................................................*......
        // sqrdmulh v20.4S, v9.4S, v2.S[1]        // ..........................................................................................*
        // sqrdmulh v12.4S, v19.4S, v1.S[3]       // ......................................................................................*....
        // sub v15.4S, v10.4S, v26.4S             // ..........................................................................................*

        sub count, count, #2
ntt_layer123_start:
                                                // Instructions:    76
                                                // Expected cycles: 72
                                                // Expected IPC:    1.06

                                                // Cycle bound:     72.0
                                                // IPC bound:       1.06

                                                // Wall time:     7.52s
                                                // User time:     7.52s

                                                // -------------------------- cycle (expected) --------------------------->
                                                // 0                        25                       50
                                                // |------------------------|------------------------|---------------------
        add v31.4S, v10.4S, v26.4S              // l.......................................................................
        mul v17.4S, v19.4S, v1.S[2]             // l.......................................................................
        add v26.4S, v15.4S, v23.4S              // .l......................................................................
        ldr q30, [x0, #672]                     // .e......................................................................
        sub v13.4S, v15.4S, v23.4S              // ..l.....................................................................
        mul v23.4S, v29.4S, v1.S[0]             // ..*.....................................................................
        add v25.4S, v25.4S, v11.4S              // ...l....................................................................
        str q22, [x0, #384]                     // ...l....................................................................
        mul v11.4S, v9.4S, v2.S[0]              // ....*...................................................................
        str q13, [x0, #896]                     // ....l...................................................................
        ldr q28, [x0, #16]                      // .....*..................................................................
        add v10.4S, v16.4S, v8.4S               // .....l..................................................................
        mls v17.4S, v12.4S, v7.S[0]             // ......l.................................................................
        ldr q13, [x0, #928]                     // ......e.................................................................
        str q26, [x0, #768]                     // .......l................................................................
        sqrdmulh v27.4S, v30.4S, v0.S[1]        // ........e...............................................................
        mls v18.4S, v5.4S, v7.S[0]              // ..........l.............................................................
        ldr q9, [x0, #416]                      // ..........e.............................................................
        sub v16.4S, v10.4S, v17.4S              // ...........l............................................................
        add v15.4S, v10.4S, v17.4S              // ............l...........................................................
        sqrdmulh v10.4S, v6.4S, v0.S[1]         // ............*...........................................................
        str q16, [x0, #128]                     // .............l..........................................................
        str q15, [x0], #(16)                    // ..............l.........................................................
        sqrdmulh v19.4S, v13.4S, v0.S[1]        // ..............e.........................................................
        sub v15.4S, v31.4S, v18.4S              // ...............l........................................................
        mul v8.4S, v13.4S, v0.S[0]              // ................e.......................................................
        add v26.4S, v31.4S, v18.4S              // ................l.......................................................
        str q15, [x0, #624]                     // .................l......................................................
        sqrdmulh v13.4S, v29.4S, v1.S[1]        // ..................*.....................................................
        str q26, [x0, #496]                     // ..................l.....................................................
        mls v8.4S, v19.4S, v7.S[0]              // ....................e...................................................
        mls v11.4S, v20.4S, v7.S[0]             // ......................*.................................................
        mls v23.4S, v13.4S, v7.S[0]             // ........................*...............................................
        add v22.4S, v9.4S, v8.4S                // .........................e..............................................
        ldr q6, [x0, #528]                      // .........................e..............................................
        sub v29.4S, v9.4S, v8.4S                // ..........................e.............................................
        mul v17.4S, v30.4S, v0.S[0]             // ..........................e.............................................
        ldr q9, [x0, #768]                      // ...........................*............................................
        sqrdmulh v13.4S, v22.4S, v0.S[3]        // ............................e...........................................
        add v18.4S, v21.4S, v23.4S              // .............................*..........................................
        mls v4.4S, v10.4S, v7.S[0]              // ..............................*.........................................
        sub v31.4S, v21.4S, v23.4S              // ..............................*.........................................
        sqrdmulh v16.4S, v31.4S, v3.S[1]        // ................................*.......................................
        add v19.4S, v24.4S, v14.4S              // ................................*.......................................
        mul v14.4S, v22.4S, v0.S[2]             // ..................................e.....................................
        sub v10.4S, v28.4S, v4.4S               // ...................................*....................................
        mls v14.4S, v13.4S, v7.S[0]             // ....................................e...................................
        ldr q13, [x0, #256]                     // ....................................*...................................
        sqrdmulh v22.4S, v9.4S, v0.S[1]         // ......................................*.................................
        mul v8.4S, v9.4S, v0.S[0]               // ........................................*...............................
        mul v23.4S, v31.4S, v3.S[0]             // ..........................................*.............................
        mls v8.4S, v22.4S, v7.S[0]              // ............................................*...........................
        mls v23.4S, v16.4S, v7.S[0]             // ..............................................*.........................
        add v16.4S, v28.4S, v4.4S               // ..............................................*.........................
        ldr q22, [x0, #144]                     // ...............................................e........................
        mul v4.4S, v6.4S, v0.S[0]               // ................................................e.......................
        mls v17.4S, v27.4S, v7.S[0]             // ..................................................e.....................
        add v21.4S, v13.4S, v8.4S               // ..................................................*.....................
        sub v27.4S, v13.4S, v8.4S               // ....................................................*...................
        sqrdmulh v31.4S, v21.4S, v0.S[3]        // ....................................................*...................
        str q25, [x0, #240]                     // ......................................................l.................
        mul v8.4S, v21.4S, v0.S[2]              // ......................................................*.................
        add v24.4S, v22.4S, v17.4S              // .......................................................e................
        sub v21.4S, v22.4S, v17.4S              // ........................................................e...............
        sqrdmulh v5.4S, v18.4S, v2.S[3]         // ........................................................*...............
        mls v8.4S, v31.4S, v7.S[0]              // ..........................................................*.............
        sub v9.4S, v24.4S, v14.4S               // ...........................................................e............
        sqrdmulh v20.4S, v27.4S, v1.S[1]        // ............................................................*...........
        mul v26.4S, v27.4S, v1.S[0]             // ..............................................................*.........
        sub v25.4S, v16.4S, v8.4S               // ...............................................................*........
        mul v18.4S, v18.4S, v2.S[2]             // ................................................................*.......
        sub v22.4S, v25.4S, v11.4S              // .................................................................*......
        mls v26.4S, v20.4S, v7.S[0]             // ..................................................................*.....
        sqrdmulh v20.4S, v9.4S, v2.S[1]         // ....................................................................e...
        sqrdmulh v12.4S, v19.4S, v1.S[3]        // ......................................................................*.
        sub v15.4S, v10.4S, v26.4S              // .......................................................................*

                                                      // ----------------------------------------------------------------------------------------- cycle (expected) ------------------------------------------------------------------------------------------>
                                                      // 0                        25                       50                       75                       100                      125                      150                      175
                                                      // |------------------------|------------------------|------------------------|------------------------|------------------------|------------------------|------------------------|----------------------
        // ldr q8, [x0, #0]                            // ....~..................................................................'....*..................................................................'....~.................................................
        // ldr q9, [x0, #(1*(1024/8))]                 // ..............................................e........................'..............................................~........................'..............................................~.......
        // ldr q10, [x0, #(2*(1024/8))]                // ...................................~...................................'...................................*...................................'...................................~..................
        // ldr q11, [x0, #(3*(1024/8))]                // .........e.............................................................'.........~.............................................................'.........~............................................
        // ldr q12, [x0, #(4*(1024/8))]                // ........................e..............................................'........................~..............................................'........................~.............................
        // ldr q13, [x0, #(5*(1024/8))]                // e......................................................................'~......................................................................'~.....................................................
        // ldr q14, [x0, #(6*(1024/8))]                // ..........................~............................................'..........................*............................................'..........................~...........................
        // ldr q15, [x0, #(7*(1024/8))]                // .....e.................................................................'.....~.................................................................'.....~................................................
        // sqrdmulh v27.4s,      v12.4s, v0.s[1]       // ...........~...........................................................'...........*...........................................................'...........~..........................................
        // mul      v24.4s, v12.4s, v0.s[0]            // ...............................................e.......................'...............................................~.......................'...............................................~......
        // mls      v24.4s, v27.4s,      v7.s[0]       // .............................~.........................................'.............................*.........................................'.............................~........................
        // sub v12.4s, v8.4s, v24.4s                   // ..................................~....................................'..................................*....................................'..................................~...................
        // add v8.4s, v8.4s, v24.4s                    // .............................................~.........................'.............................................*.........................'.............................................~........
        // sqrdmulh v27.4s,      v13.4s, v0.s[1]       // .......e...............................................................'.......~...............................................................'.......~..............................................
        // mul      v24.4s, v13.4s, v0.s[0]            // .........................e.............................................'.........................~.............................................'.........................~............................
        // mls      v24.4s, v27.4s,      v7.s[0]       // .................................................e.....................'.................................................~.....................'.................................................~....
        // sub v13.4s, v9.4s, v24.4s                   // .......................................................e...............'.......................................................~...............'......................................................
        // add v9.4s, v9.4s, v24.4s                    // ......................................................e................'......................................................~................'......................................................
        // sqrdmulh v27.4s,      v14.4s, v0.s[1]       // .....................................~.................................'.....................................*.................................'.....................................~................
        // mul      v24.4s, v14.4s, v0.s[0]            // .......................................~...............................'.......................................*...............................'.......................................~..............
        // mls      v24.4s, v27.4s,      v7.s[0]       // ...........................................~...........................'...........................................*...........................'...........................................~..........
        // sub v14.4s, v10.4s, v24.4s                  // ...................................................~...................'...................................................*...................'...................................................~..
        // add v10.4s, v10.4s, v24.4s                  // .................................................~.....................'.................................................*.....................'.................................................~....
        // sqrdmulh v27.4s,      v15.4s, v0.s[1]       // .............e.........................................................'.............~.........................................................'.............~........................................
        // mul      v24.4s, v15.4s, v0.s[0]            // ...............e.......................................................'...............~.......................................................'...............~......................................
        // mls      v24.4s, v27.4s,      v7.s[0]       // ...................e...................................................'...................~...................................................'...................~..................................
        // sub v15.4s, v11.4s, v24.4s                  // .........................e.............................................'.........................~.............................................'.........................~............................
        // add v11.4s, v11.4s, v24.4s                  // ........................e..............................................'........................~..............................................'........................~.............................
        // sqrdmulh v27.4s,      v10.4s, v0.s[3]       // ...................................................~...................'...................................................*...................'...................................................~..
        // mul      v24.4s, v10.4s, v0.s[2]            // .....................................................~.................'.....................................................*.................'......................................................
        // mls      v24.4s, v27.4s,      v7.s[0]       // .........................................................~.............'.........................................................*.............'......................................................
        // sub v10.4s, v8.4s, v24.4s                   // ..............................................................~........'..............................................................*........'......................................................
        // add v8.4s, v8.4s, v24.4s                    // ....~..................................................................'....~..................................................................'....l.................................................
        // sqrdmulh v27.4s,      v11.4s, v0.s[3]       // ...........................e...........................................'...........................~...........................................'...........................~..........................
        // mul      v24.4s, v11.4s, v0.s[2]            // .................................e.....................................'.................................~.....................................'.................................~....................
        // mls      v24.4s, v27.4s,      v7.s[0]       // ...................................e...................................'...................................~...................................'...................................~..................
        // sub v11.4s, v9.4s, v24.4s                   // ..........................................................e............'..........................................................~............'......................................................
        // add v9.4s, v9.4s, v24.4s                    // ...............................~.......................................'...............................*.......................................'...............................~......................
        // sqrdmulh v27.4s,      v14.4s, v1.s[1]       // ...........................................................~...........'...........................................................*...........'......................................................
        // mul      v24.4s, v14.4s, v1.s[0]            // .............................................................~.........'.............................................................*.........'......................................................
        // mls      v24.4s, v27.4s,      v7.s[0]       // .................................................................~.....'.................................................................*.....'......................................................
        // sub v14.4s, v12.4s, v24.4s                  // ......................................................................~'......................................................................*'......................................................
        // add v12.4s, v12.4s, v24.4s                  // .......................................................................~.......................................................................l......................................................
        // sqrdmulh v27.4s,      v15.4s, v1.s[1]       // .................~.....................................................'.................*.....................................................'.................~....................................
        // mul      v24.4s, v15.4s, v1.s[0]            // .~.....................................................................'.*.....................................................................'.~....................................................
        // mls      v24.4s, v27.4s,      v7.s[0]       // .......................~...............................................'.......................*...............................................'.......................~..............................
        // sub v15.4s, v13.4s, v24.4s                  // .............................~.........................................'.............................*.........................................'.............................~........................
        // add v13.4s, v13.4s, v24.4s                  // ............................~..........................................'............................*..........................................'............................~.........................
        // sqrdmulh v27.4s,      v9.4s, v1.s[3]        // .....................................................................~.'.....................................................................*.'......................................................
        // mul      v24.4s, v9.4s, v1.s[2]             // .......................................................................~.......................................................................l......................................................
        // mls      v24.4s, v27.4s,      v7.s[0]       // .....~.................................................................'.....~.................................................................'.....l................................................
        // sub v9.4s, v8.4s, v24.4s                    // ..........~............................................................'..........~............................................................'..........l...........................................
        // add v8.4s, v8.4s, v24.4s                    // ...........~...........................................................'...........~...........................................................'...........l..........................................
        // sqrdmulh v27.4s,      v11.4s, v2.s[1]       // ...................................................................e...'...................................................................~...'......................................................
        // mul      v24.4s, v11.4s, v2.s[0]            // ...~...................................................................'...*...................................................................'...~..................................................
        // mls      v24.4s, v27.4s,      v7.s[0]       // .....................~.................................................'.....................*.................................................'.....................~................................
        // sub v11.4s, v10.4s, v24.4s                  // ................................................................~......'................................................................*......'......................................................
        // add v10.4s, v10.4s, v24.4s                  // ..~....................................................................'..~....................................................................'..l...................................................
        // sqrdmulh v27.4s,      v13.4s, v2.s[3]       // .......................................................~...............'.......................................................*...............'......................................................
        // mul      v24.4s, v13.4s, v2.s[2]            // ...............................................................~.......'...............................................................*.......'......................................................
        // mls      v24.4s, v27.4s,      v7.s[0]       // .........~.............................................................'.........~.............................................................'.........l............................................
        // sub v13.4s, v12.4s, v24.4s                  // ..............~........................................................'..............~........................................................'..............l.......................................
        // add v12.4s, v12.4s, v24.4s                  // ...............~.......................................................'...............~.......................................................'...............l......................................
        // sqrdmulh v27.4s,      v15.4s, v3.s[1]       // ...............................~.......................................'...............................*.......................................'...............................~......................
        // mul      v24.4s, v15.4s, v3.s[0]            // .........................................~.............................'.........................................*.............................'.........................................~............
        // mls      v24.4s, v27.4s,      v7.s[0]       // .............................................~.........................'.............................................*.........................'.............................................~........
        // sub v15.4s, v14.4s, v24.4s                  // .~.....................................................................'.~.....................................................................'.l....................................................
        // add v14.4s, v14.4s, v24.4s                  // ~......................................................................'~......................................................................'l.....................................................
        // str q8, [x0], #(16)                         // .............~.........................................................'.............~.........................................................'.............l........................................
        // str q9, [x0, #(-16 + 1*(1024/8))]           // ............~..........................................................'............~..........................................................'............l.........................................
        // str q10, [x0, #(-16 + 2*(1024/8))]          // .....................................................~.................'.....................................................~.................'.....................................................l
        // str q11, [x0, #(-16 + 3*(1024/8))]          // ..~....................................................................'..~....................................................................'..l...................................................
        // str q12, [x0, #(-16 + 4*(1024/8))]          // .................~.....................................................'.................~.....................................................'.................l....................................
        // str q13, [x0, #(-16 + 5*(1024/8))]          // ................~......................................................'................~......................................................'................l.....................................
        // str q14, [x0, #(-16 + 6*(1024/8))]          // ......~................................................................'......~................................................................'......l...............................................
        // str q15, [x0, #(-16 + 7*(1024/8))]          // ...~...................................................................'...~...................................................................'...l..................................................

        subs count, count, 1
        cbnz count, ntt_layer123_start
                                                // Instructions:    75
                                                // Expected cycles: 62
                                                // Expected IPC:    1.21

                                                // Cycle bound:     62.0
                                                // IPC bound:       1.21

                                                // Wall time:     2.32s
                                                // User time:     2.32s

                                                // --------------------- cycle (expected) ---------------------->
                                                // 0                        25                       50
                                                // |------------------------|------------------------|-----------
        add v13.4S, v10.4S, v26.4S              // *.............................................................
        mls v18.4S, v5.4S, v7.S[0]              // *.............................................................
        str q22, [x0, #384]                     // .*............................................................
        add v27.4S, v16.4S, v8.4S               // .*............................................................
        mul v22.4S, v19.4S, v1.S[2]             // ..*...........................................................
        add v26.4S, v24.4S, v14.4S              // ..*...........................................................
        ldr q31, [x0, #272]                     // ...*..........................................................
        sub v14.4S, v15.4S, v23.4S              // ...*..........................................................
        add v17.4S, v15.4S, v23.4S              // ....*.........................................................
        mls v22.4S, v12.4S, v7.S[0]             // ....*.........................................................
        add v28.4S, v13.4S, v18.4S              // .....*........................................................
        str q14, [x0, #896]                     // .....*........................................................
        sqrdmulh v24.4S, v6.4S, v0.S[1]         // ......*.......................................................
        add v5.4S, v25.4S, v11.4S               // ......*.......................................................
        sub v19.4S, v13.4S, v18.4S              // .......*......................................................
        str q17, [x0, #768]                     // .......*......................................................
        str q5, [x0, #256]                      // ........*.....................................................
        mul v16.4S, v9.4S, v2.S[0]              // ........*.....................................................
        ldr q18, [x0, #784]                     // .........*....................................................
        str q19, [x0, #640]                     // .........*....................................................
        mls v16.4S, v20.4S, v7.S[0]             // ..........*...................................................
        str q28, [x0, #512]                     // ..........*...................................................
        add v13.4S, v27.4S, v22.4S              // ...........*..................................................
        ldr q15, [x0, #16]                      // ...........*..................................................
        sub v10.4S, v27.4S, v22.4S              // ............*.................................................
        mls v4.4S, v24.4S, v7.S[0]              // ............*.................................................
        str q13, [x0], #(16)                    // .............*................................................
        str q10, [x0, #112]                     // ..............*...............................................
        sqrdmulh v12.4S, v29.4S, v1.S[1]        // ..............*...............................................
        mul v23.4S, v29.4S, v1.S[0]             // ................*.............................................
        mul v8.4S, v26.4S, v1.S[2]              // ..................*...........................................
        add v20.4S, v15.4S, v4.4S               // ...................*..........................................
        sub v6.4S, v15.4S, v4.4S                // ....................*.........................................
        mls v23.4S, v12.4S, v7.S[0]             // ....................*.........................................
        sqrdmulh v22.4S, v18.4S, v0.S[1]        // ......................*.......................................
        mul v5.4S, v18.4S, v0.S[0]              // ........................*.....................................
        sub v28.4S, v21.4S, v23.4S              // ..........................*...................................
        sqrdmulh v10.4S, v26.4S, v1.S[3]        // ..........................*...................................
        mls v5.4S, v22.4S, v7.S[0]              // ............................*.................................
        sqrdmulh v30.4S, v28.4S, v3.S[1]        // ..............................*...............................
        add v4.4S, v21.4S, v23.4S               // ................................*.............................
        mls v8.4S, v10.4S, v7.S[0]              // ................................*.............................
        add v12.4S, v31.4S, v5.4S               // .................................*............................
        sub v9.4S, v31.4S, v5.4S                // ..................................*...........................
        sqrdmulh v25.4S, v4.4S, v2.S[3]         // ..................................*...........................
        sqrdmulh v15.4S, v9.4S, v1.S[1]         // ....................................*.........................
        sqrdmulh v31.4S, v12.4S, v0.S[3]        // ......................................*.......................
        mul v18.4S, v12.4S, v0.S[2]             // ........................................*.....................
        mul v11.4S, v9.4S, v1.S[0]              // ..........................................*...................
        mls v18.4S, v31.4S, v7.S[0]             // ............................................*.................
        mul v29.4S, v4.4S, v2.S[2]              // ..............................................*...............
        mls v29.4S, v25.4S, v7.S[0]             // ................................................*.............
        add v23.4S, v20.4S, v18.4S              // .................................................*............
        mls v11.4S, v15.4S, v7.S[0]             // ..................................................*...........
        sub v31.4S, v20.4S, v18.4S              // ..................................................*...........
        add v17.4S, v23.4S, v8.4S               // ...................................................*..........
        add v5.4S, v31.4S, v16.4S               // ....................................................*.........
        mul v24.4S, v28.4S, v3.S[0]             // ....................................................*.........
        str q17, [x0], #(16)                    // .....................................................*........
        sub v19.4S, v31.4S, v16.4S              // ......................................................*.......
        mls v24.4S, v30.4S, v7.S[0]             // ......................................................*.......
        str q5, [x0, #240]                      // .......................................................*......
        add v31.4S, v6.4S, v11.4S               // .......................................................*......
        sub v26.4S, v23.4S, v8.4S               // ........................................................*.....
        str q19, [x0, #368]                     // ........................................................*.....
        add v4.4S, v31.4S, v29.4S               // .........................................................*....
        sub v13.4S, v6.4S, v11.4S               // .........................................................*....
        str q26, [x0, #112]                     // ..........................................................*...
        sub v11.4S, v31.4S, v29.4S              // ..........................................................*...
        sub v22.4S, v13.4S, v24.4S              // ...........................................................*..
        add v23.4S, v13.4S, v24.4S              // ...........................................................*..
        str q4, [x0, #496]                      // ............................................................*.
        str q11, [x0, #624]                     // ............................................................*.
        str q23, [x0, #752]                     // .............................................................*
        str q22, [x0, #880]                     // .............................................................*

                                                 // --------------------- cycle (expected) ---------------------->
                                                 // 0                        25                       50
                                                 // |------------------------|------------------------|-----------
        // add v31.4S, v10.4S, v26.4S             // *.............................................................
        // mul v17.4S, v19.4S, v1.S[2]            // ..*...........................................................
        // add v26.4S, v15.4S, v23.4S             // ....*.........................................................
        // sub v13.4S, v15.4S, v23.4S             // ...*..........................................................
        // mul v23.4S, v29.4S, v1.S[0]            // ................*.............................................
        // add v25.4S, v25.4S, v11.4S             // ......*.......................................................
        // str q22, [x0, #384]                    // .*............................................................
        // mul v11.4S, v9.4S, v2.S[0]             // ........*.....................................................
        // str q13, [x0, #896]                    // .....*........................................................
        // ldr q28, [x0, #16]                     // ...........*..................................................
        // add v10.4S, v16.4S, v8.4S              // .*............................................................
        // mls v17.4S, v12.4S, v7.S[0]            // ....*.........................................................
        // str q26, [x0, #768]                    // .......*......................................................
        // mls v18.4S, v5.4S, v7.S[0]             // *.............................................................
        // sub v16.4S, v10.4S, v17.4S             // ............*.................................................
        // add v15.4S, v10.4S, v17.4S             // ...........*..................................................
        // sqrdmulh v10.4S, v6.4S, v0.S[1]        // ......*.......................................................
        // str q16, [x0, #128]                    // ..............*...............................................
        // str q15, [x0], #(16)                   // .............*................................................
        // sub v15.4S, v31.4S, v18.4S             // .......*......................................................
        // add v26.4S, v31.4S, v18.4S             // .....*........................................................
        // str q15, [x0, #624]                    // .........*....................................................
        // sqrdmulh v13.4S, v29.4S, v1.S[1]       // ..............*...............................................
        // str q26, [x0, #496]                    // ..........*...................................................
        // mls v11.4S, v20.4S, v7.S[0]            // ..........*...................................................
        // mls v23.4S, v13.4S, v7.S[0]            // ....................*.........................................
        // ldr q9, [x0, #768]                     // .........*....................................................
        // add v18.4S, v21.4S, v23.4S             // ................................*.............................
        // mls v4.4S, v10.4S, v7.S[0]             // ............*.................................................
        // sub v31.4S, v21.4S, v23.4S             // ..........................*...................................
        // sqrdmulh v16.4S, v31.4S, v3.S[1]       // ..............................*...............................
        // add v19.4S, v24.4S, v14.4S             // ..*...........................................................
        // sub v10.4S, v28.4S, v4.4S              // ....................*.........................................
        // ldr q13, [x0, #256]                    // ...*..........................................................
        // sqrdmulh v22.4S, v9.4S, v0.S[1]        // ......................*.......................................
        // mul v8.4S, v9.4S, v0.S[0]              // ........................*.....................................
        // mul v23.4S, v31.4S, v3.S[0]            // ....................................................*.........
        // mls v8.4S, v22.4S, v7.S[0]             // ............................*.................................
        // mls v23.4S, v16.4S, v7.S[0]            // ......................................................*.......
        // add v16.4S, v28.4S, v4.4S              // ...................*..........................................
        // add v21.4S, v13.4S, v8.4S              // .................................*............................
        // sub v27.4S, v13.4S, v8.4S              // ..................................*...........................
        // sqrdmulh v31.4S, v21.4S, v0.S[3]       // ......................................*.......................
        // str q25, [x0, #240]                    // ........*.....................................................
        // mul v8.4S, v21.4S, v0.S[2]             // ........................................*.....................
        // sqrdmulh v5.4S, v18.4S, v2.S[3]        // ..................................*...........................
        // mls v8.4S, v31.4S, v7.S[0]             // ............................................*.................
        // sqrdmulh v20.4S, v27.4S, v1.S[1]       // ....................................*.........................
        // mul v26.4S, v27.4S, v1.S[0]            // ..........................................*...................
        // sub v25.4S, v16.4S, v8.4S              // ..................................................*...........
        // mul v18.4S, v18.4S, v2.S[2]            // ..............................................*...............
        // sub v22.4S, v25.4S, v11.4S             // ......................................................*.......
        // mls v26.4S, v20.4S, v7.S[0]            // ..................................................*...........
        // sqrdmulh v12.4S, v19.4S, v1.S[3]       // ..........................*...................................
        // sub v15.4S, v10.4S, v26.4S             // .........................................................*....
        // add v31.4S, v10.4S, v26.4S             // .......................................................*......
        // mul v17.4S, v19.4S, v1.S[2]            // ..................*...........................................
        // add v26.4S, v15.4S, v23.4S             // ...........................................................*..
        // sub v13.4S, v15.4S, v23.4S             // ...........................................................*..
        // add v25.4S, v25.4S, v11.4S             // ....................................................*.........
        // str q22, [x0, #384]                    // ........................................................*.....
        // str q13, [x0, #896]                    // .............................................................*
        // add v10.4S, v16.4S, v8.4S              // .................................................*............
        // mls v17.4S, v12.4S, v7.S[0]            // ................................*.............................
        // str q26, [x0, #768]                    // .............................................................*
        // mls v18.4S, v5.4S, v7.S[0]             // ................................................*.............
        // sub v16.4S, v10.4S, v17.4S             // ........................................................*.....
        // add v15.4S, v10.4S, v17.4S             // ...................................................*..........
        // str q16, [x0, #128]                    // ..........................................................*...
        // str q15, [x0], #(16)                   // .....................................................*........
        // sub v15.4S, v31.4S, v18.4S             // ..........................................................*...
        // add v26.4S, v31.4S, v18.4S             // .........................................................*....
        // str q15, [x0, #624]                    // ............................................................*.
        // str q26, [x0, #496]                    // ............................................................*.
        // str q25, [x0, #240]                    // .......................................................*......


        mov in, inp
        mov count, #8

        .p2align 2
        ldr q9, [x0, #64]                        // *......................................................................................................................................
        ldr q23, [x1], #(4*16)                   // *......................................................................................................................................
        ldr q21, [x2, #96]                       // .*.....................................................................................................................................
        ldr q1, [x0, #32]                        // .*.....................................................................................................................................
        ldr q14, [x1, #-48]                      // ..*....................................................................................................................................
        ldr q13, [x0, #0]                        // ...*...................................................................................................................................
        ldr q11, [x2, #80]                       // ...*...................................................................................................................................
        sqrdmulh v16.4S, v9.4S, v23.S[1]         // ....*..................................................................................................................................
        ldr q17, [x0, #80]                       // ....*..................................................................................................................................
        mul v15.4S, v9.4S, v23.S[0]              // ......*................................................................................................................................
        ldr q30, [x0, #112]                      // ......*................................................................................................................................
        ldr q27, [x0, #96]                       // .......*...............................................................................................................................
        ldr q8, [x2, #48]                        // ........*..............................................................................................................................
        sqrdmulh v12.4S, v17.4S, v23.S[1]        // ........*..............................................................................................................................
        ldr q6, [x0, #48]                        // .........*.............................................................................................................................
        mls v15.4S, v16.4S, v7.S[0]              // ..........*............................................................................................................................
        sqrdmulh v18.4S, v27.4S, v23.S[1]        // ............*..........................................................................................................................
        sqrdmulh v19.4S, v30.4S, v23.S[1]        // ..............*........................................................................................................................
        add v5.4S, v13.4S, v15.4S                // ...............*.......................................................................................................................
        mul v25.4S, v27.4S, v23.S[0]             // ................*......................................................................................................................
        sub v26.4S, v13.4S, v15.4S               // ................*......................................................................................................................
        mls v25.4S, v18.4S, v7.S[0]              // ..................*....................................................................................................................
        mul v10.4S, v17.4S, v23.S[0]             // ....................*..................................................................................................................
        mls v10.4S, v12.4S, v7.S[0]              // ......................*................................................................................................................
        mul v4.4S, v30.4S, v23.S[0]              // ........................*..............................................................................................................
        sub v22.4S, v1.4S, v25.4S                // ........................*..............................................................................................................
        mls v4.4S, v19.4S, v7.S[0]               // ..........................*............................................................................................................
        add v28.4S, v1.4S, v25.4S                // ..........................*............................................................................................................
        sqrdmulh v19.4S, v28.4S, v23.S[3]        // ............................*..........................................................................................................
        sqrdmulh v9.4S, v22.4S, v14.S[1]         // ..............................*........................................................................................................
        add v2.4S, v6.4S, v4.4S                  // ...............................*.......................................................................................................
        mul v0.4S, v28.4S, v23.S[2]              // ................................*......................................................................................................
        sqrdmulh v27.4S, v2.4S, v23.S[3]         // ..................................*....................................................................................................
        sub v17.4S, v6.4S, v4.4S                 // ..................................*....................................................................................................
        mul v3.4S, v2.4S, v23.S[2]               // ....................................*..................................................................................................
        sqrdmulh v20.4S, v17.4S, v14.S[1]        // ......................................*................................................................................................
        ldr q1, [x0, #16]                        // ......................................*................................................................................................
        mls v3.4S, v27.4S, v7.S[0]               // ........................................*..............................................................................................
        mls v0.4S, v19.4S, v7.S[0]               // ..........................................*............................................................................................
        ldr q16, [x1, #-32]                      // ..........................................*............................................................................................
        add v31.4S, v1.4S, v10.4S                // ...........................................*...........................................................................................
        mul v30.4S, v17.4S, v14.S[0]             // ............................................*..........................................................................................
        mls v30.4S, v20.4S, v7.S[0]              // ..............................................*........................................................................................
        add v27.4S, v31.4S, v3.4S                // ..............................................*........................................................................................
        sub v23.4S, v1.4S, v10.4S                // ...............................................*.......................................................................................
        sub v24.4S, v31.4S, v3.4S                // ................................................*......................................................................................
        sqrdmulh v4.4S, v27.4S, v14.S[3]         // ................................................*......................................................................................
        sqrdmulh v10.4S, v24.4S, v16.S[1]        // ..................................................*....................................................................................
        mul v18.4S, v24.4S, v16.S[0]             // ....................................................*..................................................................................
        add v15.4S, v23.4S, v30.4S               // ....................................................*..................................................................................
        sub v23.4S, v23.4S, v30.4S               // .....................................................*.................................................................................
        mul v29.4S, v27.4S, v14.S[2]             // ......................................................*................................................................................
        sub v2.4S, v5.4S, v0.4S                  // ......................................................*................................................................................
        add v12.4S, v5.4S, v0.4S                 // .......................................................*...............................................................................
        mls v18.4S, v10.4S, v7.S[0]              // ........................................................*..............................................................................
        ldr q3, [x1, #-16]                       // ........................................................*..............................................................................
        mls v29.4S, v4.4S, v7.S[0]               // ..........................................................*............................................................................
        mul v4.4S, v22.4S, v14.S[0]              // ............................................................*..........................................................................
        add v1.4S, v2.4S, v18.4S                 // .............................................................*.........................................................................
        sub v24.4S, v2.4S, v18.4S                // ..............................................................*........................................................................
        mls v4.4S, v9.4S, v7.S[0]                // ..............................................................*........................................................................
        ldr q20, [x2, #16]                       // ...............................................................*.......................................................................
        add v25.4S, v12.4S, v29.4S               // ...............................................................*.......................................................................
        mul v9.4S, v23.4S, v3.S[0]               // ................................................................*......................................................................
        sub v5.4S, v12.4S, v29.4S                // ................................................................*......................................................................
        sqrdmulh v31.4S, v23.4S, v3.S[1]         // ..................................................................*....................................................................
        trn2 v6.4S, v1.4S, v24.4S                // ..................................................................*....................................................................
        trn2 v10.4S, v25.4S, v5.4S               // ...................................................................*...................................................................
        sqrdmulh v13.4S, v15.4S, v16.S[3]        // ....................................................................*..................................................................
        trn2 v30.2D, v10.2D, v6.2D               // .....................................................................*.................................................................
        ldr q3, [x2], #(12*16)                   // .....................................................................*.................................................................
        mul v12.4S, v15.4S, v16.S[2]             // ......................................................................*................................................................
        trn1 v27.2D, v10.2D, v6.2D               // ......................................................................*................................................................
        mls v9.4S, v31.4S, v7.S[0]               // ........................................................................*..............................................................
        trn1 v22.4S, v25.4S, v5.4S               // ........................................................................*..............................................................
        sub v6.4S, v26.4S, v4.4S                 // .........................................................................*.............................................................
        mls v12.4S, v13.4S, v7.S[0]              // ..........................................................................*............................................................
        trn1 v1.4S, v1.4S, v24.4S                // ..........................................................................*............................................................
        add v13.4S, v26.4S, v4.4S                // ...........................................................................*...........................................................
        trn2 v10.2D, v22.2D, v1.2D               // ............................................................................*..........................................................
        mul v28.4S, v30.4S, v3.4S                // ............................................................................*..........................................................
        sub v31.4S, v6.4S, v9.4S                 // .............................................................................*.........................................................
        sub count, count, #1
ntt_layer45678_start:
                                                 // Instructions:    164
                                                 // Expected cycles: 119
                                                 // Expected IPC:    1.38

                                                 // -------------------------------------------------- cycle (expected) -------------------------------------------------->
                                                 // 0                        25                       50                       75                       100
                                                 // |------------------------|------------------------|------------------------|------------------------|------------------
        add v2.4S, v13.4S, v12.4S                // *......................................................................................................................
        sqrdmulh v5.4S, v30.4S, v20.4S           // *......................................................................................................................
        sub v25.4S, v13.4S, v12.4S               // .*.....................................................................................................................
        add v17.4S, v6.4S, v9.4S                 // ..*....................................................................................................................
        mul v19.4S, v10.4S, v3.4S                // ..*....................................................................................................................
        trn2 v4.4S, v2.4S, v25.4S                // ...*...................................................................................................................
        ldr q24, [x2, #-80]                      // ...*...................................................................................................................
        trn2 v29.4S, v17.4S, v31.4S              // ....*..................................................................................................................
        sqrdmulh v15.4S, v10.4S, v20.4S          // ....*..................................................................................................................
        mls v28.4S, v5.4S, v7.S[0]               // ......*................................................................................................................
        trn2 v3.2D, v4.2D, v29.2D                // ......*................................................................................................................
        sqrdmulh v12.4S, v3.4S, v24.4S           // ........*..............................................................................................................
        mul v16.4S, v3.4S, v21.4S                // ..........*............................................................................................................
        mls v19.4S, v15.4S, v7.S[0]              // ............*..........................................................................................................
        ldr q10, [x2, #-160]                     // ............*..........................................................................................................
        add v13.4S, v27.4S, v28.4S               // .............*.........................................................................................................
        mls v16.4S, v12.4S, v7.S[0]              // ..............*........................................................................................................
        sqrdmulh v9.4S, v13.4S, v8.4S            // ................*......................................................................................................
        sub v30.4S, v27.4S, v28.4S               // ................*......................................................................................................
        ldr q18, [x1], #(4*16)                   // .................*.....................................................................................................
        mul v8.4S, v13.4S, v10.4S                // ..................*....................................................................................................
        ldr q10, [x0, #208]                      // ..................*....................................................................................................
        sqrdmulh v14.4S, v30.4S, v11.4S          // ....................*..................................................................................................
        ldr q23, [x0, #224]                      // ....................*..................................................................................................
        sqrdmulh v13.4S, v10.4S, v18.S[1]        // ......................*................................................................................................
        sqrdmulh v12.4S, v23.4S, v18.S[1]        // ........................*..............................................................................................
        ldr q6, [x2, #-128]                      // ........................*..............................................................................................
        mul v3.4S, v10.4S, v18.S[0]              // ..........................*............................................................................................
        mls v3.4S, v13.4S, v7.S[0]               // ............................*..........................................................................................
        ldr q13, [x0, #240]                      // ............................*..........................................................................................
        trn1 v27.4S, v2.4S, v25.4S               // .............................*.........................................................................................
        mul v2.4S, v30.4S, v6.4S                 // ..............................*........................................................................................
        trn1 v20.4S, v17.4S, v31.4S              // ..............................*........................................................................................
        trn1 v25.2D, v4.2D, v29.2D               // ...............................*.......................................................................................
        sqrdmulh v10.4S, v13.4S, v18.S[1]        // ................................*......................................................................................
        trn2 v5.2D, v27.2D, v20.2D               // ................................*......................................................................................
        ldr q6, [x2, #-16]                       // .................................*.....................................................................................
        mls v8.4S, v9.4S, v7.S[0]                // ..................................*....................................................................................
        sub v15.4S, v25.4S, v16.4S               // ..................................*....................................................................................
        sqrdmulh v31.4S, v5.4S, v24.4S           // ....................................*..................................................................................
        sqrdmulh v30.4S, v15.4S, v6.4S           // ......................................*................................................................................
        ldr q9, [x2, #-48]                       // ......................................*................................................................................
        mul v4.4S, v5.4S, v21.4S                 // ........................................*..............................................................................
        ldr q21, [x2, #-64]                      // ........................................*..............................................................................
        ldr q6, [x2, #-32]                       // .........................................*.............................................................................
        add v5.4S, v25.4S, v16.4S                // .........................................*.............................................................................
        mls v4.4S, v31.4S, v7.S[0]               // ..........................................*............................................................................
        mul v0.4S, v5.4S, v21.4S                 // ............................................*..........................................................................
        mul v17.4S, v13.4S, v18.S[0]             // ..............................................*........................................................................
        mls v17.4S, v10.4S, v7.S[0]              // ................................................*......................................................................
        ldr q28, [x0, #176]                      // ................................................*......................................................................
        sqrdmulh v26.4S, v5.4S, v9.4S            // ..................................................*....................................................................
        mul v9.4S, v15.4S, v6.4S                 // ....................................................*..................................................................
        trn1 v6.2D, v22.2D, v1.2D                // ....................................................*..................................................................
        mls v9.4S, v30.4S, v7.S[0]               // ......................................................*................................................................
        add v25.4S, v28.4S, v17.4S               // ......................................................*................................................................
        mls v2.4S, v14.4S, v7.S[0]               // ........................................................*..............................................................
        trn1 v5.2D, v27.2D, v20.2D               // ........................................................*..............................................................
        ldr q20, [x2, #16]                       // .........................................................*.............................................................
        mul v29.4S, v25.4S, v18.S[2]             // ..........................................................*............................................................
        add v15.4S, v6.4S, v19.4S                // ..........................................................*............................................................
        ldr q30, [x0, #192]                      // ...........................................................*...........................................................
        sub v19.4S, v6.4S, v19.4S                // ...........................................................*...........................................................
        add v31.4S, v15.4S, v8.4S                // ............................................................*..........................................................
        mls v0.4S, v26.4S, v7.S[0]               // ............................................................*..........................................................
        ldr q14, [x1, #-48]                      // .............................................................*.........................................................
        add v21.4S, v19.4S, v2.4S                // .............................................................*.........................................................
        sub v24.4S, v19.4S, v2.4S                // ..............................................................*........................................................
        sqrdmulh v27.4S, v25.4S, v18.S[3]        // ..............................................................*........................................................
        sub v26.4S, v15.4S, v8.4S                // ...............................................................*.......................................................
        ldr q2, [x0, #144]                       // ...............................................................*.......................................................
        mul v16.4S, v30.4S, v18.S[0]             // ................................................................*......................................................
        sub v25.4S, v28.4S, v17.4S               // ................................................................*......................................................
        trn1 v11.4S, v31.4S, v26.4S              // .................................................................*.....................................................
        ldr q1, [x0, #160]                       // .................................................................*.....................................................
        trn1 v6.4S, v21.4S, v24.4S               // ..................................................................*....................................................
        sqrdmulh v13.4S, v25.4S, v14.S[1]        // ..................................................................*....................................................
        add v8.4S, v2.4S, v3.4S                  // ...................................................................*...................................................
        trn2 v28.2D, v11.2D, v6.2D               // ....................................................................*..................................................
        sqrdmulh v19.4S, v30.4S, v18.S[1]        // ....................................................................*..................................................
        sub v10.4S, v5.4S, v4.4S                 // .....................................................................*.................................................
        ldr q22, [x1, #-32]                      // .....................................................................*.................................................
        str q28, [x0, #32]                       // ......................................................................*................................................
        mls v29.4S, v27.4S, v7.S[0]              // ......................................................................*................................................
        add v15.4S, v10.4S, v9.4S                // .......................................................................*...............................................
        mul v25.4S, v25.4S, v14.S[0]             // ........................................................................*..............................................
        ldr q27, [x1, #-16]                      // ........................................................................*..............................................
        trn2 v17.4S, v31.4S, v26.4S              // .........................................................................*.............................................
        trn2 v21.4S, v21.4S, v24.4S              // ..........................................................................*............................................
        mls v16.4S, v19.4S, v7.S[0]              // ..........................................................................*............................................
        sub v24.4S, v8.4S, v29.4S                // ...........................................................................*...........................................
        sub v10.4S, v10.4S, v9.4S                // ............................................................................*..........................................
        mls v25.4S, v13.4S, v7.S[0]              // ............................................................................*..........................................
        trn1 v13.2D, v11.2D, v6.2D               // .............................................................................*.........................................
        ldr q28, [x0, #128]                      // .............................................................................*.........................................
        sqrdmulh v30.4S, v24.4S, v22.S[1]        // ..............................................................................*........................................
        trn2 v19.2D, v17.2D, v21.2D              // ..............................................................................*........................................
        trn1 v6.2D, v17.2D, v21.2D               // ...............................................................................*.......................................
        mul v31.4S, v23.4S, v18.S[0]             // ................................................................................*......................................
        str q13, [x0], #(16*8)                   // ................................................................................*......................................
        str q6, [x0, #-112]                      // .................................................................................*.....................................
        str q19, [x0, #-80]                      // .................................................................................*.....................................
        ldr q11, [x2, #80]                       // ..................................................................................*....................................
        mls v31.4S, v12.4S, v7.S[0]              // ..................................................................................*....................................
        ldr q21, [x2, #96]                       // ...................................................................................*...................................
        trn1 v9.4S, v15.4S, v10.4S               // ...................................................................................*...................................
        trn2 v6.4S, v15.4S, v10.4S               // ....................................................................................*..................................
        mul v24.4S, v24.4S, v22.S[0]             // ....................................................................................*..................................
        sub v10.4S, v2.4S, v3.4S                 // .....................................................................................*.................................
        ldr q3, [x2], #(12*16)                   // .....................................................................................*.................................
        mls v24.4S, v30.4S, v7.S[0]              // ......................................................................................*................................
        add v26.4S, v8.4S, v29.4S                // ......................................................................................*................................
        ldr q8, [x2, #-144]                      // .......................................................................................*...............................
        add v17.4S, v5.4S, v4.4S                 // .......................................................................................*...............................
        sqrdmulh v2.4S, v26.4S, v14.S[3]         // ........................................................................................*..............................
        sub v13.4S, v1.4S, v31.4S                // ........................................................................................*..............................
        add v30.4S, v1.4S, v31.4S                // .........................................................................................*.............................
        add v15.4S, v10.4S, v25.4S               // ..........................................................................................*............................
        sqrdmulh v19.4S, v13.4S, v14.S[1]        // ..........................................................................................*............................
        sub v25.4S, v10.4S, v25.4S               // ...........................................................................................*...........................
        mul v29.4S, v13.4S, v14.S[0]             // ............................................................................................*..........................
        sub v5.4S, v28.4S, v16.4S                // ............................................................................................*..........................
        sqrdmulh v4.4S, v30.4S, v18.S[3]         // ..............................................................................................*........................
        sub v23.4S, v17.4S, v0.4S                // ..............................................................................................*........................
        add v31.4S, v17.4S, v0.4S                // ...............................................................................................*.......................
        mul v18.4S, v30.4S, v18.S[2]             // ................................................................................................*......................
        add v1.4S, v28.4S, v16.4S                // ................................................................................................*......................
        trn2 v12.4S, v31.4S, v23.4S              // .................................................................................................*.....................
        mls v29.4S, v19.4S, v7.S[0]              // ..................................................................................................*....................
        trn1 v13.4S, v31.4S, v23.4S              // ..................................................................................................*....................
        trn2 v30.2D, v12.2D, v6.2D               // ...................................................................................................*...................
        mls v18.4S, v4.4S, v7.S[0]               // ....................................................................................................*..................
        trn2 v10.2D, v13.2D, v9.2D               // ....................................................................................................*..................
        trn1 v31.2D, v13.2D, v9.2D               // .....................................................................................................*.................
        mul v19.4S, v26.4S, v14.S[2]             // ......................................................................................................*................
        trn1 v12.2D, v12.2D, v6.2D               // ......................................................................................................*................
        sub v6.4S, v5.4S, v29.4S                 // .......................................................................................................*...............
        mls v19.4S, v2.4S, v7.S[0]               // ........................................................................................................*..............
        add v13.4S, v5.4S, v29.4S                // ........................................................................................................*..............
        str q10, [x0, #-32]                      // .........................................................................................................*.............
        sub v10.4S, v1.4S, v18.4S                // .........................................................................................................*.............
        add v28.4S, v1.4S, v18.4S                // ..........................................................................................................*............
        sqrdmulh v5.4S, v25.4S, v27.S[1]         // ..........................................................................................................*............
        str q31, [x0, #-64]                      // ...........................................................................................................*...........
        add v26.4S, v10.4S, v24.4S               // ...........................................................................................................*...........
        sub v31.4S, v10.4S, v24.4S               // ............................................................................................................*..........
        mul v9.4S, v25.4S, v27.S[0]              // ............................................................................................................*..........
        str q12, [x0, #-48]                      // .............................................................................................................*.........
        sub v24.4S, v28.4S, v19.4S               // .............................................................................................................*.........
        sqrdmulh v10.4S, v15.4S, v22.S[3]        // ..............................................................................................................*........
        trn1 v1.4S, v26.4S, v31.4S               // ..............................................................................................................*........
        str q30, [x0, #-16]                      // ...............................................................................................................*.......
        add v30.4S, v28.4S, v19.4S               // ...............................................................................................................*.......
        mls v9.4S, v5.4S, v7.S[0]                // ................................................................................................................*......
        trn2 v25.4S, v26.4S, v31.4S              // ................................................................................................................*......
        trn2 v14.4S, v30.4S, v24.4S              // .................................................................................................................*.....
        mul v12.4S, v15.4S, v22.S[2]             // ..................................................................................................................*....
        trn1 v22.4S, v30.4S, v24.4S              // ..................................................................................................................*....
        trn1 v27.2D, v14.2D, v25.2D              // ...................................................................................................................*...
        mls v12.4S, v10.4S, v7.S[0]              // ....................................................................................................................*..
        trn2 v30.2D, v14.2D, v25.2D              // ....................................................................................................................*..
        sub v31.4S, v6.4S, v9.4S                 // .....................................................................................................................*.
        trn2 v10.2D, v22.2D, v1.2D               // ......................................................................................................................*
        mul v28.4S, v30.4S, v3.4S                // ......................................................................................................................*

                                                  // -------------------------------------------------- cycle (expected) -------------------------------------------------->
                                                  // 0                        25                       50                       75                       100
                                                  // |------------------------|------------------------|------------------------|------------------------|------------------
        // add v9.4S, v6.4S, v9.4S                // ..*....................................................................................................................
        // sqrdmulh v6.4S, v30.4S, v20.4S         // *......................................................................................................................
        // ldr q24, [x2, #-160]                   // ............*..........................................................................................................
        // add v25.4S, v13.4S, v12.4S             // *......................................................................................................................
        // sub v15.4S, v13.4S, v12.4S             // .*.....................................................................................................................
        // mul v19.4S, v10.4S, v3.4S              // ..*....................................................................................................................
        // trn2 v5.4S, v9.4S, v31.4S              // ....*..................................................................................................................
        // sqrdmulh v3.4S, v10.4S, v20.4S         // ....*..................................................................................................................
        // trn2 v10.4S, v25.4S, v15.4S            // ...*...................................................................................................................
        // mls v28.4S, v6.4S, v7.S[0]             // ......*................................................................................................................
        // trn2 v13.2D, v10.2D, v5.2D             // ......*................................................................................................................
        // ldr q30, [x2, #-80]                    // ...*...................................................................................................................
        // mul v12.4S, v13.4S, v21.4S             // ..........*............................................................................................................
        // mls v19.4S, v3.4S, v7.S[0]             // ............*..........................................................................................................
        // add v20.4S, v27.4S, v28.4S             // .............*.........................................................................................................
        // sqrdmulh v13.4S, v13.4S, v30.4S        // ........*..............................................................................................................
        // sub v3.4S, v27.4S, v28.4S              // ................*......................................................................................................
        // mul v24.4S, v20.4S, v24.4S             // ..................*....................................................................................................
        // sqrdmulh v6.4S, v3.4S, v11.4S          // ....................*..................................................................................................
        // ldr q27, [x2, #-128]                   // ........................*..............................................................................................
        // mls v12.4S, v13.4S, v7.S[0]            // ..............*........................................................................................................
        // trn1 v25.4S, v25.4S, v15.4S            // .............................*.........................................................................................
        // mul v27.4S, v3.4S, v27.4S              // ..............................*........................................................................................
        // trn1 v31.4S, v9.4S, v31.4S             // ..............................*........................................................................................
        // trn1 v3.2D, v10.2D, v5.2D              // ...............................*.......................................................................................
        // ldr q13, [x2, #-48]                    // ......................................*................................................................................
        // ldr q15, [x2, #-64]                    // ........................................*..............................................................................
        // sqrdmulh v9.4S, v20.4S, v8.4S          // ................*......................................................................................................
        // trn2 v20.2D, v25.2D, v31.2D            // ................................*......................................................................................
        // ldr q10, [x2, #-16]                    // .................................*.....................................................................................
        // mls v27.4S, v6.4S, v7.S[0]             // ........................................................*..............................................................
        // add v5.4S, v3.4S, v12.4S               // .........................................*.............................................................................
        // sub v6.4S, v3.4S, v12.4S               // ..................................*....................................................................................
        // sqrdmulh v3.4S, v20.4S, v30.4S         // ....................................*..................................................................................
        // trn1 v12.2D, v22.2D, v1.2D             // ....................................................*..................................................................
        // sqrdmulh v10.4S, v6.4S, v10.4S         // ......................................*................................................................................
        // mls v24.4S, v9.4S, v7.S[0]             // ..................................*....................................................................................
        // sub v9.4S, v12.4S, v19.4S              // ...........................................................*...........................................................
        // trn1 v25.2D, v25.2D, v31.2D            // ........................................................*..............................................................
        // sqrdmulh v31.4S, v5.4S, v13.4S         // ..................................................*....................................................................
        // add v30.4S, v9.4S, v27.4S              // .............................................................*.........................................................
        // add v13.4S, v12.4S, v19.4S             // ..........................................................*............................................................
        // mul v1.4S, v20.4S, v21.4S              // ........................................*..............................................................................
        // ldr q12, [x2, #-32]                    // .........................................*.............................................................................
        // add v21.4S, v13.4S, v24.4S             // ............................................................*..........................................................
        // sub v13.4S, v13.4S, v24.4S             // ...............................................................*.......................................................
        // mls v1.4S, v3.4S, v7.S[0]              // ..........................................*............................................................................
        // sub v3.4S, v9.4S, v27.4S               // ..............................................................*........................................................
        // mul v9.4S, v6.4S, v12.4S               // ....................................................*..................................................................
        // trn2 v12.4S, v21.4S, v13.4S            // .........................................................................*.............................................
        // trn1 v6.4S, v30.4S, v3.4S              // ..................................................................*....................................................
        // trn2 v30.4S, v30.4S, v3.4S             // ..........................................................................*............................................
        // mls v9.4S, v10.4S, v7.S[0]             // ......................................................*................................................................
        // trn1 v13.4S, v21.4S, v13.4S            // .................................................................*.....................................................
        // mul v15.4S, v5.4S, v15.4S              // ............................................*..........................................................................
        // sub v3.4S, v25.4S, v1.4S               // .....................................................................*.................................................
        // add v5.4S, v25.4S, v1.4S               // .......................................................................................*...............................
        // mls v15.4S, v31.4S, v7.S[0]            // ............................................................*..........................................................
        // trn1 v21.2D, v13.2D, v6.2D             // .............................................................................*.........................................
        // trn2 v6.2D, v13.2D, v6.2D              // ....................................................................*..................................................
        // add v10.4S, v3.4S, v9.4S               // .......................................................................*...............................................
        // sub v13.4S, v3.4S, v9.4S               // ............................................................................*..........................................
        // str q21, [x0], #(16*8)                 // ................................................................................*......................................
        // trn1 v3.2D, v12.2D, v30.2D             // ...............................................................................*.......................................
        // trn2 v31.2D, v12.2D, v30.2D            // ..............................................................................*........................................
        // trn1 v21.4S, v10.4S, v13.4S            // ...................................................................................*...................................
        // sub v30.4S, v5.4S, v15.4S              // ..............................................................................................*........................
        // add v12.4S, v5.4S, v15.4S              // ...............................................................................................*.......................
        // str q3, [x0, #-112]                    // .................................................................................*.....................................
        // trn2 v13.4S, v10.4S, v13.4S            // ....................................................................................*..................................
        // trn1 v19.4S, v12.4S, v30.4S            // ..................................................................................................*....................
        // trn2 v12.4S, v12.4S, v30.4S            // .................................................................................................*.....................
        // str q6, [x0, #-96]                     // ......................................................................*................................................
        // str q31, [x0, #-80]                    // .................................................................................*.....................................
        // trn1 v10.2D, v19.2D, v21.2D            // .....................................................................................................*.................
        // trn2 v3.2D, v19.2D, v21.2D             // ....................................................................................................*..................
        // trn1 v21.2D, v12.2D, v13.2D            // ......................................................................................................*................
        // trn2 v13.2D, v12.2D, v13.2D            // ...................................................................................................*...................
        // str q10, [x0, #-64]                    // ...........................................................................................................*...........
        // str q3, [x0, #-32]                     // .........................................................................................................*.............
        // str q13, [x0, #-16]                    // ...............................................................................................................*.......
        // str q21, [x0, #-48]                    // .............................................................................................................*.........
        // ldr q9, [x0, #64]                      // ...........................................................*...........................................................
        // ldr q23, [x1], #(4*16)                 // .................*.....................................................................................................
        // ldr q21, [x2, #96]                     // ...................................................................................*...................................
        // ldr q1, [x0, #32]                      // .................................................................*.....................................................
        // ldr q14, [x1, #-48]                    // .............................................................*.........................................................
        // ldr q13, [x0, #0]                      // .............................................................................*.........................................
        // ldr q11, [x2, #80]                     // ..................................................................................*....................................
        // sqrdmulh v16.4S, v9.4S, v23.S[1]       // ....................................................................*..................................................
        // ldr q17, [x0, #80]                     // ..................*....................................................................................................
        // mul v15.4S, v9.4S, v23.S[0]            // ................................................................*......................................................
        // ldr q30, [x0, #112]                    // ............................*..........................................................................................
        // ldr q27, [x0, #96]                     // ....................*..................................................................................................
        // ldr q8, [x2, #48]                      // .......................................................................................*...............................
        // sqrdmulh v12.4S, v17.4S, v23.S[1]      // ......................*................................................................................................
        // ldr q6, [x0, #48]                      // ................................................*......................................................................
        // mls v15.4S, v16.4S, v7.S[0]            // ..........................................................................*............................................
        // sqrdmulh v18.4S, v27.4S, v23.S[1]      // ........................*..............................................................................................
        // sqrdmulh v19.4S, v30.4S, v23.S[1]      // ................................*......................................................................................
        // add v5.4S, v13.4S, v15.4S              // ................................................................................................*......................
        // mul v25.4S, v27.4S, v23.S[0]           // ................................................................................*......................................
        // sub v26.4S, v13.4S, v15.4S             // ............................................................................................*..........................
        // mls v25.4S, v18.4S, v7.S[0]            // ..................................................................................*....................................
        // mul v10.4S, v17.4S, v23.S[0]           // ..........................*............................................................................................
        // mls v10.4S, v12.4S, v7.S[0]            // ............................*..........................................................................................
        // mul v4.4S, v30.4S, v23.S[0]            // ..............................................*........................................................................
        // sub v22.4S, v1.4S, v25.4S              // ........................................................................................*..............................
        // mls v4.4S, v19.4S, v7.S[0]             // ................................................*......................................................................
        // add v28.4S, v1.4S, v25.4S              // .........................................................................................*.............................
        // sqrdmulh v19.4S, v28.4S, v23.S[3]      // ..............................................................................................*........................
        // sqrdmulh v9.4S, v22.4S, v14.S[1]       // ..........................................................................................*............................
        // add v2.4S, v6.4S, v4.4S                // ......................................................*................................................................
        // mul v0.4S, v28.4S, v23.S[2]            // ................................................................................................*......................
        // sqrdmulh v27.4S, v2.4S, v23.S[3]       // ..............................................................*........................................................
        // sub v17.4S, v6.4S, v4.4S               // ................................................................*......................................................
        // mul v3.4S, v2.4S, v23.S[2]             // ..........................................................*............................................................
        // sqrdmulh v20.4S, v17.4S, v14.S[1]      // ..................................................................*....................................................
        // ldr q1, [x0, #16]                      // ...............................................................*.......................................................
        // mls v3.4S, v27.4S, v7.S[0]             // ......................................................................*................................................
        // mls v0.4S, v19.4S, v7.S[0]             // ....................................................................................................*..................
        // ldr q16, [x1, #-32]                    // .....................................................................*.................................................
        // add v31.4S, v1.4S, v10.4S              // ...................................................................*...................................................
        // mul v30.4S, v17.4S, v14.S[0]           // ........................................................................*..............................................
        // mls v30.4S, v20.4S, v7.S[0]            // ............................................................................*..........................................
        // add v27.4S, v31.4S, v3.4S              // ......................................................................................*................................
        // sub v23.4S, v1.4S, v10.4S              // .....................................................................................*.................................
        // sub v24.4S, v31.4S, v3.4S              // ...........................................................................*...........................................
        // sqrdmulh v4.4S, v27.4S, v14.S[3]       // ........................................................................................*..............................
        // sqrdmulh v10.4S, v24.4S, v16.S[1]      // ..............................................................................*........................................
        // mul v18.4S, v24.4S, v16.S[0]           // ....................................................................................*..................................
        // add v15.4S, v23.4S, v30.4S             // ..........................................................................................*............................
        // sub v23.4S, v23.4S, v30.4S             // ...........................................................................................*...........................
        // mul v29.4S, v27.4S, v14.S[2]           // ......................................................................................................*................
        // sub v2.4S, v5.4S, v0.4S                // .........................................................................................................*.............
        // add v12.4S, v5.4S, v0.4S               // ..........................................................................................................*............
        // mls v18.4S, v10.4S, v7.S[0]            // ......................................................................................*................................
        // ldr q3, [x1, #-16]                     // ........................................................................*..............................................
        // mls v29.4S, v4.4S, v7.S[0]             // ........................................................................................................*..............
        // mul v4.4S, v22.4S, v14.S[0]            // ............................................................................................*..........................
        // add v1.4S, v2.4S, v18.4S               // ...........................................................................................................*...........
        // sub v24.4S, v2.4S, v18.4S              // ............................................................................................................*..........
        // mls v4.4S, v9.4S, v7.S[0]              // ..................................................................................................*....................
        // ldr q20, [x2, #16]                     // .........................................................*.............................................................
        // add v25.4S, v12.4S, v29.4S             // ...............................................................................................................*.......
        // mul v9.4S, v23.4S, v3.S[0]             // ............................................................................................................*..........
        // sub v5.4S, v12.4S, v29.4S              // .............................................................................................................*.........
        // sqrdmulh v31.4S, v23.4S, v3.S[1]       // ..........................................................................................................*............
        // trn2 v6.4S, v1.4S, v24.4S              // ................................................................................................................*......
        // trn2 v10.4S, v25.4S, v5.4S             // .................................................................................................................*.....
        // sqrdmulh v13.4S, v15.4S, v16.S[3]      // ..............................................................................................................*........
        // trn2 v30.2D, v10.2D, v6.2D             // ....................................................................................................................*..
        // ldr q3, [x2], #(12*16)                 // .....................................................................................*.................................
        // mul v12.4S, v15.4S, v16.S[2]           // ..................................................................................................................*....
        // trn1 v27.2D, v10.2D, v6.2D             // ...................................................................................................................*...
        // mls v9.4S, v31.4S, v7.S[0]             // ................................................................................................................*......
        // trn1 v22.4S, v25.4S, v5.4S             // ..................................................................................................................*....
        // sub v6.4S, v26.4S, v4.4S               // .......................................................................................................*...............
        // mls v12.4S, v13.4S, v7.S[0]            // ....................................................................................................................*..
        // trn1 v1.4S, v1.4S, v24.4S              // ..............................................................................................................*........
        // add v13.4S, v26.4S, v4.4S              // ........................................................................................................*..............
        // trn2 v10.2D, v22.2D, v1.2D             // ......................................................................................................................*
        // mul v28.4S, v30.4S, v3.4S              // ......................................................................................................................*
        // sub v31.4S, v6.4S, v9.4S               // .....................................................................................................................*.

        subs count, count, 1
        cbnz count, ntt_layer45678_start
        add v9.4S, v6.4S, v9.4S                  // ..............................................................................*........................................................
        sqrdmulh v6.4S, v30.4S, v20.4S           // ..............................................................................*........................................................
        ldr q24, [x2, #-160]                     // ...............................................................................*.......................................................
        add v25.4S, v13.4S, v12.4S               // ...............................................................................*.......................................................
        sub v15.4S, v13.4S, v12.4S               // ................................................................................*......................................................
        mul v19.4S, v10.4S, v3.4S                // ................................................................................*......................................................
        trn2 v5.4S, v9.4S, v31.4S                // .................................................................................*.....................................................
        sqrdmulh v3.4S, v10.4S, v20.4S           // ..................................................................................*....................................................
        trn2 v10.4S, v25.4S, v15.4S              // ..................................................................................*....................................................
        mls v28.4S, v6.4S, v7.S[0]               // ....................................................................................*..................................................
        trn2 v13.2D, v10.2D, v5.2D               // ....................................................................................*..................................................
        ldr q30, [x2, #-80]                      // .....................................................................................*.................................................
        mul v12.4S, v13.4S, v21.4S               // ......................................................................................*................................................
        mls v19.4S, v3.4S, v7.S[0]               // ........................................................................................*..............................................
        add v20.4S, v27.4S, v28.4S               // .........................................................................................*.............................................
        sqrdmulh v13.4S, v13.4S, v30.4S          // ..........................................................................................*............................................
        sub v3.4S, v27.4S, v28.4S                // ..........................................................................................*............................................
        mul v24.4S, v20.4S, v24.4S               // ............................................................................................*..........................................
        sqrdmulh v6.4S, v3.4S, v11.4S            // ..............................................................................................*........................................
        ldr q27, [x2, #-128]                     // ..............................................................................................*........................................
        mls v12.4S, v13.4S, v7.S[0]              // ................................................................................................*......................................
        trn1 v25.4S, v25.4S, v15.4S              // ................................................................................................*......................................
        mul v27.4S, v3.4S, v27.4S                // ..................................................................................................*....................................
        trn1 v31.4S, v9.4S, v31.4S               // ..................................................................................................*....................................
        trn1 v3.2D, v10.2D, v5.2D                // ...................................................................................................*...................................
        ldr q13, [x2, #-48]                      // ...................................................................................................*...................................
        ldr q15, [x2, #-64]                      // ....................................................................................................*..................................
        sqrdmulh v9.4S, v20.4S, v8.4S            // ....................................................................................................*..................................
        trn2 v20.2D, v25.2D, v31.2D              // .....................................................................................................*.................................
        ldr q10, [x2, #-16]                      // .....................................................................................................*.................................
        mls v27.4S, v6.4S, v7.S[0]               // ......................................................................................................*................................
        add v5.4S, v3.4S, v12.4S                 // ......................................................................................................*................................
        sub v6.4S, v3.4S, v12.4S                 // .......................................................................................................*...............................
        sqrdmulh v3.4S, v20.4S, v30.4S           // ........................................................................................................*..............................
        trn1 v12.2D, v22.2D, v1.2D               // ........................................................................................................*..............................
        sqrdmulh v10.4S, v6.4S, v10.4S           // ..........................................................................................................*............................
        mls v24.4S, v9.4S, v7.S[0]               // ............................................................................................................*..........................
        sub v9.4S, v12.4S, v19.4S                // ............................................................................................................*..........................
        trn1 v25.2D, v25.2D, v31.2D              // .............................................................................................................*.........................
        sqrdmulh v31.4S, v5.4S, v13.4S           // ..............................................................................................................*........................
        add v30.4S, v9.4S, v27.4S                // ..............................................................................................................*........................
        add v13.4S, v12.4S, v19.4S               // ...............................................................................................................*.......................
        mul v1.4S, v20.4S, v21.4S                // ................................................................................................................*......................
        ldr q12, [x2, #-32]                      // ................................................................................................................*......................
        add v21.4S, v13.4S, v24.4S               // .................................................................................................................*.....................
        sub v13.4S, v13.4S, v24.4S               // ..................................................................................................................*....................
        mls v1.4S, v3.4S, v7.S[0]                // ..................................................................................................................*....................
        sub v3.4S, v9.4S, v27.4S                 // ...................................................................................................................*...................
        mul v9.4S, v6.4S, v12.4S                 // ....................................................................................................................*..................
        trn2 v12.4S, v21.4S, v13.4S              // ....................................................................................................................*..................
        trn1 v6.4S, v30.4S, v3.4S                // .....................................................................................................................*.................
        trn2 v30.4S, v30.4S, v3.4S               // ......................................................................................................................*................
        mls v9.4S, v10.4S, v7.S[0]               // ......................................................................................................................*................
        trn1 v13.4S, v21.4S, v13.4S              // .......................................................................................................................*...............
        mul v15.4S, v5.4S, v15.4S                // ........................................................................................................................*..............
        sub v3.4S, v25.4S, v1.4S                 // ........................................................................................................................*..............
        add v5.4S, v25.4S, v1.4S                 // .........................................................................................................................*.............
        mls v15.4S, v31.4S, v7.S[0]              // ..........................................................................................................................*............
        trn1 v21.2D, v13.2D, v6.2D               // ..........................................................................................................................*............
        trn2 v6.2D, v13.2D, v6.2D                // ...........................................................................................................................*...........
        add v10.4S, v3.4S, v9.4S                 // ............................................................................................................................*..........
        sub v13.4S, v3.4S, v9.4S                 // ............................................................................................................................*..........
        str q21, [x0], #(16*8)                   // .............................................................................................................................*.........
        trn1 v3.2D, v12.2D, v30.2D               // .............................................................................................................................*.........
        trn2 v31.2D, v12.2D, v30.2D              // ..............................................................................................................................*........
        trn1 v21.4S, v10.4S, v13.4S              // ..............................................................................................................................*........
        sub v30.4S, v5.4S, v15.4S                // ...............................................................................................................................*.......
        add v12.4S, v5.4S, v15.4S                // ...............................................................................................................................*.......
        str q3, [x0, #-112]                      // ................................................................................................................................*......
        trn2 v13.4S, v10.4S, v13.4S              // ................................................................................................................................*......
        trn1 v19.4S, v12.4S, v30.4S              // .................................................................................................................................*.....
        trn2 v12.4S, v12.4S, v30.4S              // .................................................................................................................................*.....
        str q6, [x0, #-96]                       // ..................................................................................................................................*....
        str q31, [x0, #-80]                      // ..................................................................................................................................*....
        trn1 v10.2D, v19.2D, v21.2D              // ...................................................................................................................................*...
        trn2 v3.2D, v19.2D, v21.2D               // ...................................................................................................................................*...
        trn1 v21.2D, v12.2D, v13.2D              // ....................................................................................................................................*..
        trn2 v13.2D, v12.2D, v13.2D              // ....................................................................................................................................*..
        str q10, [x0, #-64]                      // .....................................................................................................................................*.
        str q3, [x0, #-32]                       // .....................................................................................................................................*.
        str q13, [x0, #-16]                      // ......................................................................................................................................*
        str q21, [x0, #-48]                      // ......................................................................................................................................*

        pop_stack
        ret

/* ***************** REGISTER DEALLOCATIONS ****************** */
    .unreq in
    .unreq r012345_ptr
    .unreq r67_ptr
    .unreq inp
    .unreq count
    .unreq wtmp
    .unreq data0
    .unreq data1
    .unreq data2
    .unreq data3
    .unreq data4
    .unreq data5
    .unreq data6
    .unreq data7
    .unreq q_data0
    .unreq q_data1
    .unreq q_data2
    .unreq q_data3
    .unreq q_data4
    .unreq q_data5
    .unreq q_data6
    .unreq q_data7
    .unreq root0
    .unreq root1
    .unreq root2
    .unreq root3
    .unreq root0_tw
    .unreq root1_tw
    .unreq root2_tw
    .unreq q_root0
    .unreq q_root1
    .unreq q_root2
    .unreq q_root3
    .unreq q_root0_tw
    .unreq q_root1_tw
    .unreq q_root2_tw
    .unreq consts
    .unreq tmp
    .unreq t0
    .unreq t1
    .unreq t2
    .unreq t3

/* simpasm: footer-start */
#endif /* MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */