intt.S

/// Copyright (c) 2024-2025 The mlkem-native project authors
/// Copyright (c) 2022 Arm Limited
/// Copyright (c) 2022 Hanno Becker
/// Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
/// SPDX-License-Identifier: MIT
///
/// Permission is hereby granted, free of charge, to any person obtaining a copy
/// of this software and associated documentation files (the "Software"), to deal
/// in the Software without restriction, including without limitation the rights
/// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
/// copies of the Software, and to permit persons to whom the Software is
/// furnished to do so, subject to the following conditions:
///
/// The above copyright notice and this permission notice shall be included in all
/// copies or substantial portions of the Software.
///
/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
/// SOFTWARE.
///

#include "../../../common.h"
#if defined(MLK_ARITH_BACKEND_AARCH64_OPT) && \
    !defined(MLK_MULTILEVEL_BUILD_NO_SHARED)
/* simpasm: header-end */

// Bounds:
// If C is chosen so that |src| < q * C, then |dst| < q * (0.0508 * C + 1/2)
//
// See mlken/reduce.c and test/test_bounds.py for more details.
.macro mulmodq dst, src, const, idx0, idx1
        // Signed barrett multiplication using
        // round-to-nearest-even-integer approximation.
        // Following https://eprint.iacr.org/2021/986.pdf, this
        // is functionally the same as a signed Montgomery multiplication
        // with a suitable constant of absolute value < q.
        sqrdmulh t2.8h,      \src\().8h, \const\().h[\idx1\()]
        mul      \dst\().8h, \src\().8h, \const\().h[\idx0\()]
        mls      \dst\().8h, t2.8h,      consts.h[0]
.endm

.macro mulmod dst, src, const, const_twisted
        sqrdmulh t2.8h,      \src\().8h, \const_twisted\().8h
        mul      \dst\().8h, \src\().8h, \const\().8h
        mls      \dst\().8h, t2.8h,      consts.h[0]
.endm

.macro gs_butterfly a, b, root, idx0, idx1
        sub tmp.8h,   \a\().8h, \b\().8h
        add \a\().8h, \a\().8h, \b\().8h
        mulmodq  \b, tmp, \root, \idx0, \idx1
.endm

.macro gs_butterfly_v a, b, root, root_twisted
        sub tmp.8h,   \a\().8h, \b\().8h
        add \a\().8h, \a\().8h, \b\().8h
        mulmod  \b, tmp, \root, \root_twisted
.endm

.macro mul_ninv dst0, dst1, dst2, dst3, src0, src1, src2, src3
        mulmod \dst0, \src0, ninv, ninv_tw
        mulmod \dst1, \src1, ninv, ninv_tw
        mulmod \dst2, \src2, ninv, ninv_tw
        mulmod \dst3, \src3, ninv, ninv_tw
.endm

.macro barrett_reduce a
        sqdmulh t0.8h,    \a\().8h, consts.h[1]
        srshr   t0.8h,    t0.8h,    #11
        mls     \a\().8h, t0.8h,    consts.h[0]
.endm

.macro load_roots_123
        ldr q_root0, [r12345_ptr], #32
        ldr q_root1, [r12345_ptr, #-16]
.endm

.macro load_next_roots_45
        ldr q_root0, [r12345_ptr], #16
.endm

.macro load_next_roots_67
        ldr q_root0,    [r67_ptr], #(6*16)
        ldr q_root0_tw, [r67_ptr, #(-6*16 + 1*16)]
        ldr q_root1,    [r67_ptr, #(-6*16 + 2*16)]
        ldr q_root1_tw, [r67_ptr, #(-6*16 + 3*16)]
        ldr q_root2,    [r67_ptr, #(-6*16 + 4*16)]
        ldr q_root2_tw, [r67_ptr, #(-6*16 + 5*16)]
.endm

.macro transpose4 data
        trn1 t0.4s, \data\()0.4s, \data\()1.4s
        trn2 t1.4s, \data\()0.4s, \data\()1.4s
        trn1 t2.4s, \data\()2.4s, \data\()3.4s
        trn2 t3.4s, \data\()2.4s, \data\()3.4s

        trn2 \data\()2.2d, t0.2d, t2.2d
        trn2 \data\()3.2d, t1.2d, t3.2d
        trn1 \data\()0.2d, t0.2d, t2.2d
        trn1 \data\()1.2d, t1.2d, t3.2d
.endm

.macro transpose_single data_out, data_in
        trn1 \data_out\()0.4s, \data_in\()0.4s, \data_in\()1.4s
        trn2 \data_out\()1.4s, \data_in\()0.4s, \data_in\()1.4s
        trn1 \data_out\()2.4s, \data_in\()2.4s, \data_in\()3.4s
        trn2 \data_out\()3.4s, \data_in\()2.4s, \data_in\()3.4s
.endm

.macro save_vregs
        sub sp, sp, #(16*4)
        stp  d8,  d9, [sp, #16*0]
        stp d10, d11, [sp, #16*1]
        stp d12, d13, [sp, #16*2]
        stp d14, d15, [sp, #16*3]
.endm

.macro restore_vregs
        ldp  d8,  d9, [sp, #16*0]
        ldp d10, d11, [sp, #16*1]
        ldp d12, d13, [sp, #16*2]
        ldp d14, d15, [sp, #16*3]
        add sp, sp, #(16*4)
.endm

.macro push_stack
        save_vregs
.endm

.macro pop_stack
        restore_vregs
.endm

// For comparability reasons, the output range for the coefficients of this
// invNTT code is supposed to match the implementation from PQClean on commit
// ee71d2c823982bfcf54686f3cf1d666f396dc9aa. After the invNTT, the coefficients
// are NOT canonically reduced. The ordering of the coefficients is canonical,
// also matching PQClean.

        in         .req x0
        r12345_ptr .req x1
        r67_ptr    .req x2

        inp     .req x3
        count   .req x4
        wtmp    .req w5

        data0  .req v8
        data1  .req v9
        data2  .req v10
        data3  .req v11
        data4  .req v12
        data5  .req v13
        data6  .req v14
        data7  .req v15

        q_data0  .req q8
        q_data1  .req q9
        q_data2  .req q10
        q_data3  .req q11
        q_data4  .req q12
        q_data5  .req q13
        q_data6  .req q14
        q_data7  .req q15

        root0    .req v0
        root1    .req v1
        root2    .req v2
        root0_tw .req v4
        root1_tw .req v5
        root2_tw .req v6

        consts     .req v7
        q_consts   .req q7

        q_root0    .req q0
        q_root1    .req q1
        q_root2    .req q2
        q_root0_tw .req q4
        q_root1_tw .req q5
        q_root2_tw .req q6

        tmp .req v24
        t0  .req v25
        t1  .req v26
        t2  .req v27
        t3  .req v28

        ninv             .req v29
        ninv_tw          .req v30

        .text
        .global MLK_ASM_NAMESPACE(intt_asm)
        .balign 4
MLK_ASM_FN_SYMBOL(intt_asm)
        push_stack

        // Setup constants
        mov wtmp, #3329
        mov consts.h[0], wtmp
        mov wtmp, #20159
        mov consts.h[1], wtmp
        mov wtmp, #512
        dup ninv.8h, wtmp
        mov wtmp, #5040
        dup ninv_tw.8h, wtmp

        mov inp, in
        mov count, #8

scale_start:

        ldr q_data0, [inp, #(16*0)]
        ldr q_data1, [inp, #(16*1)]
        ldr q_data2, [inp, #(16*2)]
        ldr q_data3, [inp, #(16*3)]

        mul_ninv data0, data1, data2, data3, data0, data1, data2, data3
        // Bounds: Absolute value < q

        str q_data0, [inp], #64
        str q_data1, [inp, #(-64 + 16*1)]
        str q_data2, [inp, #(-64 + 16*2)]
        str q_data3, [inp, #(-64 + 16*3)]

        subs count, count, #1
        cbnz count, scale_start

        mov inp, in
        mov count, #8

        .p2align 2
                                          // Instructions:    11
                                          // Expected cycles: 20
                                          // Expected IPC:    0.55
                                          //
                                          // Cycle bound:     20.0
                                          // IPC bound:       0.55
                                          //
                                          // Wall time:     0.01s
                                          // User time:     0.01s
                                          //
                                          // ----- cycle (expected) ------>
                                          // 0                        25
                                          // |------------------------|----
        ldr q26, [x3, #0]                 // *.............................
        ldr q8, [x3, #16]                 // ..*...........................
        ldr q24, [x3, #32]                // ....*.........................
        ldr q16, [x3, #48]                // ......*.......................
        ldr q9, [x2], #(6*16)             // ........*.....................
        trn1 v0.4S, v24.4S, v16.4S        // ..........*...................
        ldr q6, [x2, #-80]                // ...........*..................
        ldr q3, [x2, #-64]                // .............*................
        ldr q15, [x2, #-48]               // ...............*..............
        ldr q4, [x2, #-32]                // .................*............
        ldr q28, [x2, #-16]               // ...................*..........

                                           // ------ cycle (expected) ------>
                                           // 0                        25
                                           // |------------------------|-----
        // ldr q26, [x3, #0]               // *..............................
        // ldr q8, [x3, #16]               // ..*............................
        // ldr q24, [x3, #32]              // ....*..........................
        // ldr q16, [x3, #48]              // ......*........................
        // trn1 v0.4S, v24.4S, v16.4S      // ..........*....................
        // ldr q9, [x2], #(6*16)           // ........*......................
        // ldr q6, [x2, #-80]              // ...........*...................
        // ldr q3, [x2, #-64]              // .............*.................
        // ldr q15, [x2, #-48]             // ...............*...............
        // ldr q4, [x2, #-32]              // .................*.............
        // ldr q28, [x2, #-16]             // ...................*...........

        sub count, count, #1
layer4567_start:
                                                // Instructions:    83
                                                // Expected cycles: 94
                                                // Expected IPC:    0.88
                                                //
                                                // Cycle bound:     94.0
                                                // IPC bound:       0.88
                                                //
                                                // Wall time:     3.34s
                                                // User time:     3.34s
                                                //
                                                // ------------------------------------- cycle (expected) -------------------------------------->
                                                // 0                        25                       50                       75
                                                // |------------------------|------------------------|------------------------|------------------
        trn1 v12.4S, v26.4S, v8.4S              // *.............................................................................................
        trn2 v26.4S, v26.4S, v8.4S              // .*............................................................................................
        trn2 v8.4S, v24.4S, v16.4S              // ..*...........................................................................................
        trn2 v11.2D, v12.2D, v0.2D              // ...*..........................................................................................
        trn1 v12.2D, v12.2D, v0.2D              // ....*.........................................................................................
        trn2 v16.2D, v26.2D, v8.2D              // .....*........................................................................................
        trn1 v26.2D, v26.2D, v8.2D              // ......*.......................................................................................
        sub v8.8H, v11.8H, v16.8H               // .......*......................................................................................
        add v11.8H, v11.8H, v16.8H              // ........*.....................................................................................
        sub v16.8H, v12.8H, v26.8H              // .........*....................................................................................
        add v12.8H, v12.8H, v26.8H              // ..........*...................................................................................
        sqrdmulh v26.8H, v8.8H, v28.8H          // ...........*..................................................................................
        sqrdmulh v15.8H, v16.8H, v15.8H         // ............*.................................................................................
        mul v16.8H, v16.8H, v3.8H               // .............*................................................................................
        mul v8.8H, v8.8H, v4.8H                 // ..............*...............................................................................
        sub v0.8H, v12.8H, v11.8H               // ...............*..............................................................................
        add v12.8H, v12.8H, v11.8H              // ................*.............................................................................
        mls v16.8H, v15.8H, v7.H[0]             // .................*............................................................................
        mls v8.8H, v26.8H, v7.H[0]              // ..................*...........................................................................
        sqrdmulh v26.8H, v0.8H, v6.8H           // ...................*..........................................................................
        mul v11.8H, v0.8H, v9.8H                // ....................*.........................................................................
        ldr q15, [x1], #16                      // .....................*........................................................................
        sub v0.8H, v16.8H, v8.8H                // .......................*......................................................................
        mls v11.8H, v26.8H, v7.H[0]             // ........................*.....................................................................
        add v26.8H, v16.8H, v8.8H               // .........................*....................................................................
        sqrdmulh v8.8H, v0.8H, v6.8H            // ..........................*...................................................................
        mul v16.8H, v0.8H, v9.8H                // ...........................*..................................................................
        trn1 v0.4S, v12.4S, v26.4S              // ............................*.................................................................
        trn2 v12.4S, v12.4S, v26.4S             // .............................*................................................................
        ldr q26, [x3, #64]                      // ..............................e...............................................................
        mls v16.8H, v8.8H, v7.H[0]              // ................................*.............................................................
        ldr q8, [x3, #80]                       // .................................e............................................................
        ldr q24, [x3, #96]                      // ...................................e..........................................................
        trn1 v9.4S, v11.4S, v16.4S              // .....................................*........................................................
        trn2 v11.4S, v11.4S, v16.4S             // ......................................*.......................................................
        ldr q16, [x3, #112]                     // .......................................e......................................................
        trn2 v6.2D, v0.2D, v9.2D                // .........................................*....................................................
        trn2 v3.2D, v12.2D, v11.2D              // ..........................................*...................................................
        trn1 v0.2D, v0.2D, v9.2D                // ...........................................*..................................................
        trn1 v12.2D, v12.2D, v11.2D             // ............................................*.................................................
        sub v11.8H, v6.8H, v3.8H                // .............................................*................................................
        sub v9.8H, v0.8H, v12.8H                // ..............................................*...............................................
        add v12.8H, v0.8H, v12.8H               // ...............................................*..............................................
        sqrdmulh v0.8H, v11.8H, v15.H[5]        // ................................................*.............................................
        sqrdmulh v4.8H, v9.8H, v15.H[3]         // .................................................*............................................
        mul v9.8H, v9.8H, v15.H[2]              // ..................................................*...........................................
        mul v11.8H, v11.8H, v15.H[4]            // ...................................................*..........................................
        add v6.8H, v6.8H, v3.8H                 // ....................................................*.........................................
        sqdmulh v3.8H, v12.8H, v7.H[1]          // .....................................................*........................................
        mls v9.8H, v4.8H, v7.H[0]               // ......................................................*.......................................
        mls v11.8H, v0.8H, v7.H[0]              // .......................................................*......................................
        sqdmulh v0.8H, v6.8H, v7.H[1]           // ........................................................*.....................................
        srshr v3.8H, v3.8H, #11                 // .........................................................*....................................
        sqdmulh v4.8H, v9.8H, v7.H[1]           // ..........................................................*...................................
        sqdmulh v28.8H, v11.8H, v7.H[1]         // ...........................................................*..................................
        mls v12.8H, v3.8H, v7.H[0]              // ............................................................*.................................
        srshr v0.8H, v0.8H, #11                 // .............................................................*................................
        srshr v3.8H, v4.8H, #11                 // ..............................................................*...............................
        srshr v4.8H, v28.8H, #11                // ...............................................................*..............................
        mls v6.8H, v0.8H, v7.H[0]               // ................................................................*.............................
        mls v9.8H, v3.8H, v7.H[0]               // .................................................................*............................
        mls v11.8H, v4.8H, v7.H[0]              // ..................................................................*...........................
        trn1 v0.4S, v24.4S, v16.4S              // ...................................................................e..........................
        sub v3.8H, v12.8H, v6.8H                // ....................................................................*.........................
        add v12.8H, v12.8H, v6.8H               // .....................................................................*........................
        sub v6.8H, v9.8H, v11.8H                // ......................................................................*.......................
        sqrdmulh v4.8H, v3.8H, v15.H[1]         // .......................................................................*......................
        mul v3.8H, v3.8H, v15.H[0]              // ........................................................................*.....................
        sqrdmulh v28.8H, v6.8H, v15.H[1]        // .........................................................................*....................
        mul v15.8H, v6.8H, v15.H[0]             // ..........................................................................*...................
        add v11.8H, v9.8H, v11.8H               // ...........................................................................*..................
        mls v3.8H, v4.8H, v7.H[0]               // ............................................................................*.................
        str q12, [x3], #(64)                    // .............................................................................*................
        mls v15.8H, v28.8H, v7.H[0]             // ..............................................................................*...............
        str q11, [x3, #-48]                     // ...............................................................................*..............
        ldr q9, [x2], #(6*16)                   // ................................................................................e.............
        str q3, [x3, #-32]                      // ..................................................................................*...........
        ldr q6, [x2, #-80]                      // ...................................................................................e..........
        str q15, [x3, #-16]                     // .....................................................................................*........
        ldr q3, [x2, #-64]                      // ......................................................................................e.......
        ldr q15, [x2, #-48]                     // ........................................................................................e.....
        ldr q4, [x2, #-32]                      // ..........................................................................................e...
        ldr q28, [x2, #-16]                     // ............................................................................................e.

                                                      // ----------------------------------------------------------------- cycle (expected) ------------------------------------------------------------------>
                                                      // 0                        25                       50                       75                       100                      125
                                                      // |------------------------|------------------------|------------------------|------------------------|------------------------|------------------------
        // ldr q8, [x3, #(16*0)]                      // e...............................................................'.............................~.......................................................
        // ldr q9, [x3, #(16*1)]                      // ...e............................................................'................................~....................................................
        // ldr q10, [x3, #(16*2)]                     // .....e..........................................................'..................................~..................................................
        // ldr q11, [x3, #(16*3)]                     // .........e......................................................'......................................~..............................................
        // trn1 v25.4s, v8.4s, v9.4s                  // ................................................................*.....................................................................................
        // trn2 v26.4s, v8.4s, v9.4s                  // ................................................................'*....................................................................................
        // trn1 v27.4s, v10.4s, v11.4s                // .....................................e..........................'..................................................................~..................
        // trn2 v28.4s, v10.4s, v11.4s                // ................................................................'.*...................................................................................
        // trn2 v10.2d, v25.2d, v27.2d                // ................................................................'..*..................................................................................
        // trn2 v11.2d, v26.2d, v28.2d                // ................................................................'....*................................................................................
        // trn1 v8.2d, v25.2d, v27.2d                 // ................................................................'...*.................................................................................
        // trn1 v9.2d, v26.2d, v28.2d                 // ................................................................'.....*...............................................................................
        // ldr q0,    [x2], #(6*16)                   // ..................................................e.............'...............................................................................~.....
        // ldr q4, [x2, #(-6*16 + 1*16)]              // .....................................................e..........'..................................................................................~..
        // ldr q1,    [x2, #(-6*16 + 2*16)]           // ........................................................e.......'.....................................................................................
        // ldr q5, [x2, #(-6*16 + 3*16)]              // ..........................................................e.....'.....................................................................................
        // ldr q2,    [x2, #(-6*16 + 4*16)]           // ............................................................e...'.....................................................................................
        // ldr q6, [x2, #(-6*16 + 5*16)]              // ..............................................................e.'.....................................................................................
        // sub v24.8h,   v8.8h, v9.8h                 // ................................................................'........*............................................................................
        // add v8.8h, v8.8h, v9.8h                    // ................................................................'.........*...........................................................................
        // sqrdmulh v27.8h,      v24.8h, v5.8h        // ................................................................'...........*.........................................................................
        // mul      v9.8h, v24.8h, v1.8h              // ................................................................'............*........................................................................
        // mls      v9.8h, v27.8h,      v7.h[0]       // ................................................................'................*....................................................................
        // sub v24.8h,   v10.8h, v11.8h               // ................................................................'......*..............................................................................
        // add v10.8h, v10.8h, v11.8h                 // ................................................................'.......*.............................................................................
        // sqrdmulh v27.8h,      v24.8h, v6.8h        // ................................................................'..........*..........................................................................
        // mul      v11.8h, v24.8h, v2.8h             // ................................................................'.............*.......................................................................
        // mls      v11.8h, v27.8h,      v7.h[0]      // ................................................................'.................*...................................................................
        // sub v24.8h,   v8.8h, v10.8h                // ................................................................'..............*......................................................................
        // add v8.8h, v8.8h, v10.8h                   // ................................................................'...............*.....................................................................
        // sqrdmulh v27.8h,      v24.8h, v4.8h        // ................................................................'..................*..................................................................
        // mul      v10.8h, v24.8h, v0.8h             // ................................................................'...................*.................................................................
        // mls      v10.8h, v27.8h,      v7.h[0]      // ................................................................'.......................*.............................................................
        // sub v24.8h,   v9.8h, v11.8h                // ................................................................'......................*..............................................................
        // add v9.8h, v9.8h, v11.8h                   // ................................................................'........................*............................................................
        // sqrdmulh v27.8h,      v24.8h, v4.8h        // ................................................................'.........................*...........................................................
        // mul      v11.8h, v24.8h, v0.8h             // ................................................................'..........................*..........................................................
        // mls      v11.8h, v27.8h,      v7.h[0]      // ..~.............................................................'...............................*.....................................................
        // trn1 v25.4s, v8.4s, v9.4s                  // ................................................................'...........................*.........................................................
        // trn2 v26.4s, v8.4s, v9.4s                  // ................................................................'............................*........................................................
        // trn1 v27.4s, v10.4s, v11.4s                // .......~........................................................'....................................*................................................
        // trn2 v28.4s, v10.4s, v11.4s                // ........~.......................................................'.....................................*...............................................
        // trn2 v10.2d, v25.2d, v27.2d                // ...........~....................................................'........................................*............................................
        // trn2 v11.2d, v26.2d, v28.2d                // ............~...................................................'.........................................*...........................................
        // trn1 v8.2d, v25.2d, v27.2d                 // .............~..................................................'..........................................*..........................................
        // trn1 v9.2d, v26.2d, v28.2d                 // ..............~.................................................'...........................................*.........................................
        // ldr q0, [x1], #16                          // ................................................................'....................*................................................................
        // sub v24.8h,   v8.8h, v9.8h                 // ................~...............................................'.............................................*.......................................
        // add v8.8h, v8.8h, v9.8h                    // .................~..............................................'..............................................*......................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[3]      // ...................~............................................'................................................*....................................
        // mul      v9.8h, v24.8h, v0.h[2]            // ....................~...........................................'.................................................*...................................
        // mls      v9.8h, v27.8h,      v7.h[0]       // ........................~.......................................'.....................................................*...............................
        // sub v24.8h,   v10.8h, v11.8h               // ...............~................................................'............................................*........................................
        // add v10.8h, v10.8h, v11.8h                 // ......................~.........................................'...................................................*.................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[5]      // ..................~.............................................'...............................................*.....................................
        // mul      v11.8h, v24.8h, v0.h[4]           // .....................~..........................................'..................................................*..................................
        // mls      v11.8h, v27.8h,      v7.h[0]      // .........................~......................................'......................................................*..............................
        // sqdmulh v25.8h,    v8.8h, v7.h[1]          // .......................~........................................'....................................................*................................
        // srshr   v25.8h,    v25.8h,    #11          // ...........................~....................................'........................................................*............................
        // mls     v8.8h, v25.8h,    v7.h[0]          // ..............................~.................................'...........................................................*.........................
        // sqdmulh v25.8h,    v10.8h, v7.h[1]         // ..........................~.....................................'.......................................................*.............................
        // srshr   v25.8h,    v25.8h,    #11          // ...............................~................................'............................................................*........................
        // mls     v10.8h, v25.8h,    v7.h[0]         // ..................................~.............................'...............................................................*.....................
        // sqdmulh v25.8h,    v9.8h, v7.h[1]          // ............................~...................................'.........................................................*...........................
        // srshr   v25.8h,    v25.8h,    #11          // ................................~...............................'.............................................................*.......................
        // mls     v9.8h, v25.8h,    v7.h[0]          // ...................................~............................'................................................................*....................
        // sqdmulh v25.8h,    v11.8h, v7.h[1]         // .............................~..................................'..........................................................*..........................
        // srshr   v25.8h,    v25.8h,    #11          // .................................~..............................'..............................................................*......................
        // mls     v11.8h, v25.8h,    v7.h[0]         // ....................................~...........................'.................................................................*...................
        // sub v24.8h,   v8.8h, v10.8h                // ......................................~.........................'...................................................................*.................
        // add v8.8h, v8.8h, v10.8h                   // .......................................~........................'....................................................................*................
        // sqrdmulh v27.8h,      v24.8h, v0.h[1]      // .........................................~......................'......................................................................*..............
        // mul      v10.8h, v24.8h, v0.h[0]           // ..........................................~.....................'.......................................................................*.............
        // mls      v10.8h, v27.8h,      v7.h[0]      // ..............................................~.................'...........................................................................*.........
        // sub v24.8h,   v9.8h, v11.8h                // ........................................~.......................'.....................................................................*...............
        // add v9.8h, v9.8h, v11.8h                   // .............................................~..................'..........................................................................*..........
        // sqrdmulh v27.8h,      v24.8h, v0.h[1]      // ...........................................~....................'........................................................................*............
        // mul      v11.8h, v24.8h, v0.h[0]           // ............................................~...................'.........................................................................*...........
        // mls      v11.8h, v27.8h,      v7.h[0]      // ................................................~...............'.............................................................................*.......
        // str q8, [x3], #(64)                        // ...............................................~................'............................................................................*........
        // str q9, [x3, #(-64 + 16*1)]                // .................................................~..............'..............................................................................*......
        // str q10, [x3, #(-64 + 16*2)]               // ....................................................~...........'.................................................................................*...
        // str q11, [x3, #(-64 + 16*3)]               // .......................................................~........'....................................................................................*

        sub count, count, #1
        cbnz count, layer4567_start
                                                // Instructions:    72
                                                // Expected cycles: 79
                                                // Expected IPC:    0.91
                                                //
                                                // Cycle bound:     79.0
                                                // IPC bound:       0.91
                                                //
                                                // Wall time:     9.28s
                                                // User time:     9.28s
                                                //
                                                // ------------------------------ cycle (expected) ------------------------------>
                                                // 0                        25                       50                       75
                                                // |------------------------|------------------------|------------------------|---
        trn1 v11.4S, v26.4S, v8.4S              // *..............................................................................
        trn2 v24.4S, v24.4S, v16.4S             // .*.............................................................................
        trn2 v26.4S, v26.4S, v8.4S              // ..*............................................................................
        trn1 v18.2D, v11.2D, v0.2D              // ...*...........................................................................
        trn2 v11.2D, v11.2D, v0.2D              // ....*..........................................................................
        trn2 v12.2D, v26.2D, v24.2D             // .....*.........................................................................
        trn1 v8.2D, v26.2D, v24.2D              // ......*........................................................................
        sub v26.8H, v11.8H, v12.8H              // .......*.......................................................................
        sub v13.8H, v18.8H, v8.8H               // ........*......................................................................
        add v24.8H, v18.8H, v8.8H               // .........*.....................................................................
        mul v16.8H, v26.8H, v4.8H               // ..........*....................................................................
        sqrdmulh v17.8H, v13.8H, v15.8H         // ...........*...................................................................
        mul v3.8H, v13.8H, v3.8H                // ............*..................................................................
        sqrdmulh v26.8H, v26.8H, v28.8H         // .............*.................................................................
        add v10.8H, v11.8H, v12.8H              // ..............*................................................................
        mls v3.8H, v17.8H, v7.H[0]              // ................*..............................................................
        mls v16.8H, v26.8H, v7.H[0]             // .................*.............................................................
        sub v26.8H, v24.8H, v10.8H              // ..................*............................................................
        ldr q4, [x1], #16                       // ...................*...........................................................
        sub v12.8H, v3.8H, v16.8H               // .....................*.........................................................
        sqrdmulh v15.8H, v26.8H, v6.8H          // ......................*........................................................
        mul v11.8H, v26.8H, v9.8H               // .......................*.......................................................
        mul v8.8H, v12.8H, v9.8H                // ........................*......................................................
        sqrdmulh v12.8H, v12.8H, v6.8H          // .........................*.....................................................
        add v0.8H, v24.8H, v10.8H               // ..........................*....................................................
        mls v11.8H, v15.8H, v7.H[0]             // ...........................*...................................................
        add v6.8H, v3.8H, v16.8H                // ............................*..................................................
        mls v8.8H, v12.8H, v7.H[0]              // .............................*.................................................
        trn2 v26.4S, v0.4S, v6.4S               // ...............................*...............................................
        trn2 v12.4S, v11.4S, v8.4S              // .................................*.............................................
        trn1 v3.4S, v11.4S, v8.4S               // ..................................*............................................
        trn1 v17.4S, v0.4S, v6.4S               // ...................................*...........................................
        trn1 v8.2D, v26.2D, v12.2D              // ....................................*..........................................
        trn2 v13.2D, v26.2D, v12.2D             // .....................................*.........................................
        trn1 v11.2D, v17.2D, v3.2D              // ......................................*........................................
        trn2 v15.2D, v17.2D, v3.2D              // .......................................*.......................................
        sub v12.8H, v11.8H, v8.8H               // ........................................*......................................
        add v16.8H, v15.8H, v13.8H              // .........................................*.....................................
        sub v26.8H, v15.8H, v13.8H              // ..........................................*....................................
        mul v0.8H, v12.8H, v4.H[2]              // ...........................................*...................................
        sqrdmulh v9.8H, v12.8H, v4.H[3]         // ............................................*..................................
        mul v13.8H, v26.8H, v4.H[4]             // .............................................*.................................
        sqrdmulh v26.8H, v26.8H, v4.H[5]        // ..............................................*................................
        add v24.8H, v11.8H, v8.8H               // ...............................................*...............................
        mls v0.8H, v9.8H, v7.H[0]               // ................................................*..............................
        sqdmulh v12.8H, v16.8H, v7.H[1]         // .................................................*.............................
        mls v13.8H, v26.8H, v7.H[0]             // ..................................................*............................
        sqdmulh v11.8H, v24.8H, v7.H[1]         // ...................................................*...........................
        sqdmulh v8.8H, v0.8H, v7.H[1]           // ....................................................*..........................
        srshr v12.8H, v12.8H, #11               // .....................................................*.........................
        sqdmulh v26.8H, v13.8H, v7.H[1]         // ......................................................*........................
        srshr v11.8H, v11.8H, #11               // .......................................................*.......................
        mls v16.8H, v12.8H, v7.H[0]             // ........................................................*......................
        srshr v8.8H, v8.8H, #11                 // .........................................................*.....................
        srshr v26.8H, v26.8H, #11               // ..........................................................*....................
        mls v24.8H, v11.8H, v7.H[0]             // ...........................................................*...................
        mls v0.8H, v8.8H, v7.H[0]               // ............................................................*..................
        mls v13.8H, v26.8H, v7.H[0]             // .............................................................*.................
        sub v26.8H, v24.8H, v16.8H              // ...............................................................*...............
        add v15.8H, v24.8H, v16.8H              // ................................................................*..............
        sub v12.8H, v0.8H, v13.8H               // .................................................................*.............
        mul v11.8H, v26.8H, v4.H[0]             // ..................................................................*............
        sqrdmulh v16.8H, v26.8H, v4.H[1]        // ...................................................................*...........
        mul v26.8H, v12.8H, v4.H[0]             // ....................................................................*..........
        sqrdmulh v8.8H, v12.8H, v4.H[1]         // .....................................................................*.........
        add v12.8H, v0.8H, v13.8H               // ......................................................................*........
        mls v11.8H, v16.8H, v7.H[0]             // .......................................................................*.......
        str q15, [x3], #(64)                    // ........................................................................*......
        mls v26.8H, v8.8H, v7.H[0]              // .........................................................................*.....
        str q12, [x3, #-48]                     // ..........................................................................*....
        str q11, [x3, #-32]                     // ............................................................................*..
        str q26, [x3, #-16]                     // ..............................................................................*

                                                 // ------------------------------ cycle (expected) ------------------------------>
                                                 // 0                        25                       50                       75
                                                 // |------------------------|------------------------|------------------------|---
        // trn1 v12.4S, v26.4S, v8.4S            // *..............................................................................
        // trn2 v26.4S, v26.4S, v8.4S            // ..*............................................................................
        // trn2 v8.4S, v24.4S, v16.4S            // .*.............................................................................
        // trn2 v11.2D, v12.2D, v0.2D            // ....*..........................................................................
        // trn1 v12.2D, v12.2D, v0.2D            // ...*...........................................................................
        // trn2 v16.2D, v26.2D, v8.2D            // .....*.........................................................................
        // trn1 v26.2D, v26.2D, v8.2D            // ......*........................................................................
        // sub v8.8H, v11.8H, v16.8H             // .......*.......................................................................
        // add v11.8H, v11.8H, v16.8H            // ..............*................................................................
        // sub v16.8H, v12.8H, v26.8H            // ........*......................................................................
        // add v12.8H, v12.8H, v26.8H            // .........*.....................................................................
        // sqrdmulh v26.8H, v8.8H, v28.8H        // .............*.................................................................
        // sqrdmulh v15.8H, v16.8H, v15.8H       // ...........*...................................................................
        // mul v16.8H, v16.8H, v3.8H             // ............*..................................................................
        // mul v8.8H, v8.8H, v4.8H               // ..........*....................................................................
        // sub v0.8H, v12.8H, v11.8H             // ..................*............................................................
        // add v12.8H, v12.8H, v11.8H            // ..........................*....................................................
        // mls v16.8H, v15.8H, v7.H[0]           // ................*..............................................................
        // mls v8.8H, v26.8H, v7.H[0]            // .................*.............................................................
        // sqrdmulh v26.8H, v0.8H, v6.8H         // ......................*........................................................
        // mul v11.8H, v0.8H, v9.8H              // .......................*.......................................................
        // ldr q15, [x1], #16                    // ...................*...........................................................
        // sub v0.8H, v16.8H, v8.8H              // .....................*.........................................................
        // mls v11.8H, v26.8H, v7.H[0]           // ...........................*...................................................
        // add v26.8H, v16.8H, v8.8H             // ............................*..................................................
        // sqrdmulh v8.8H, v0.8H, v6.8H          // .........................*.....................................................
        // mul v16.8H, v0.8H, v9.8H              // ........................*......................................................
        // trn1 v0.4S, v12.4S, v26.4S            // ...................................*...........................................
        // trn2 v12.4S, v12.4S, v26.4S           // ...............................*...............................................
        // mls v16.8H, v8.8H, v7.H[0]            // .............................*.................................................
        // trn1 v9.4S, v11.4S, v16.4S            // ..................................*............................................
        // trn2 v11.4S, v11.4S, v16.4S           // .................................*.............................................
        // trn2 v6.2D, v0.2D, v9.2D              // .......................................*.......................................
        // trn2 v3.2D, v12.2D, v11.2D            // .....................................*.........................................
        // trn1 v0.2D, v0.2D, v9.2D              // ......................................*........................................
        // trn1 v12.2D, v12.2D, v11.2D           // ....................................*..........................................
        // sub v11.8H, v6.8H, v3.8H              // ..........................................*....................................
        // sub v9.8H, v0.8H, v12.8H              // ........................................*......................................
        // add v12.8H, v0.8H, v12.8H             // ...............................................*...............................
        // sqrdmulh v0.8H, v11.8H, v15.H[5]      // ..............................................*................................
        // sqrdmulh v4.8H, v9.8H, v15.H[3]       // ............................................*..................................
        // mul v9.8H, v9.8H, v15.H[2]            // ...........................................*...................................
        // mul v11.8H, v11.8H, v15.H[4]          // .............................................*.................................
        // add v6.8H, v6.8H, v3.8H               // .........................................*.....................................
        // sqdmulh v3.8H, v12.8H, v7.H[1]        // ...................................................*...........................
        // mls v9.8H, v4.8H, v7.H[0]             // ................................................*..............................
        // mls v11.8H, v0.8H, v7.H[0]            // ..................................................*............................
        // sqdmulh v0.8H, v6.8H, v7.H[1]         // .................................................*.............................
        // srshr v3.8H, v3.8H, #11               // .......................................................*.......................
        // sqdmulh v4.8H, v9.8H, v7.H[1]         // ....................................................*..........................
        // sqdmulh v28.8H, v11.8H, v7.H[1]       // ......................................................*........................
        // mls v12.8H, v3.8H, v7.H[0]            // ...........................................................*...................
        // srshr v0.8H, v0.8H, #11               // .....................................................*.........................
        // srshr v3.8H, v4.8H, #11               // .........................................................*.....................
        // srshr v4.8H, v28.8H, #11              // ..........................................................*....................
        // mls v6.8H, v0.8H, v7.H[0]             // ........................................................*......................
        // mls v9.8H, v3.8H, v7.H[0]             // ............................................................*..................
        // mls v11.8H, v4.8H, v7.H[0]            // .............................................................*.................
        // sub v3.8H, v12.8H, v6.8H              // ...............................................................*...............
        // add v12.8H, v12.8H, v6.8H             // ................................................................*..............
        // sub v6.8H, v9.8H, v11.8H              // .................................................................*.............
        // sqrdmulh v4.8H, v3.8H, v15.H[1]       // ...................................................................*...........
        // mul v3.8H, v3.8H, v15.H[0]            // ..................................................................*............
        // sqrdmulh v28.8H, v6.8H, v15.H[1]      // .....................................................................*.........
        // mul v15.8H, v6.8H, v15.H[0]           // ....................................................................*..........
        // add v11.8H, v9.8H, v11.8H             // ......................................................................*........
        // mls v3.8H, v4.8H, v7.H[0]             // .......................................................................*.......
        // str q12, [x3], #(64)                  // ........................................................................*......
        // mls v15.8H, v28.8H, v7.H[0]           // .........................................................................*.....
        // str q11, [x3, #-48]                   // ..........................................................................*....
        // str q3, [x3, #-32]                    // ............................................................................*..
        // str q15, [x3, #-16]                   // ..............................................................................*


        // ---------------------------------------------------------------------

        mov count, #4
        load_roots_123

        .p2align 2

                                          // Instructions:    12
                                          // Expected cycles: 19
                                          // Expected IPC:    0.63
                                          //
                                          // Cycle bound:     19.0
                                          // IPC bound:       0.63
                                          //
                                          // Wall time:     0.01s
                                          // User time:     0.01s
                                          //
                                          // ----- cycle (expected) ------>
                                          // 0                        25
                                          // |------------------------|----
        ldr q24, [x0, #128]               // *.............................
        ldr q16, [x0, #192]               // ..*...........................
        ldr q9, [x0, #256]                // ....*.........................
        ldr q6, [x0, #320]                // ......*.......................
        ldr q3, [x0, #384]                // ........*.....................
        ldr q4, [x0, #448]                // ..........*...................
        add v28.8H, v9.8H, v6.8H          // ............*.................
        add v19.8H, v24.8H, v16.8H        // .............*................
        add v13.8H, v3.8H, v4.8H          // ..............*...............
        ldr q11, [x0, #0]                 // ...............*..............
        add v23.8H, v28.8H, v13.8H        // .................*............
        ldr q15, [x0, #64]                // ..................*...........

                                           // ------ cycle (expected) ------>
                                           // 0                        25
                                           // |------------------------|-----
        // ldr q11, [x0, #0]               // ...............*...............
        // ldr q15, [x0, #64]              // ..................*............
        // ldr q24, [x0, #128]             // *..............................
        // ldr q16, [x0, #192]             // ..*............................
        // ldr q9, [x0, #256]              // ....*..........................
        // ldr q6, [x0, #320]              // ......*........................
        // ldr q3, [x0, #384]              // ........*......................
        // ldr q4, [x0, #448]              // ..........*....................
        // add v28.8H, v9.8H, v6.8H        // ............*..................
        // add v13.8H, v3.8H, v4.8H        // ..............*................
        // add v19.8H, v24.8H, v16.8H      // .............*.................
        // add v23.8H, v28.8H, v13.8H      // .................*.............

        sub count, count, #1
layer123_start:
                                                // Instructions:    76
                                                // Expected cycles: 84
                                                // Expected IPC:    0.90
                                                //
                                                // Cycle bound:     84.0
                                                // IPC bound:       0.90
                                                //
                                                // Wall time:     2.81s
                                                // User time:     2.81s
                                                //
                                                // -------------------------------- cycle (expected) --------------------------------->
                                                // 0                        25                       50                       75
                                                // |------------------------|------------------------|------------------------|--------
        sub v12.8H, v11.8H, v15.8H              // *...................................................................................
        add v26.8H, v11.8H, v15.8H              // .*..................................................................................
        sub v8.8H, v24.8H, v16.8H               // ..*.................................................................................
        sqrdmulh v11.8H, v12.8H, v0.H[7]        // ...*................................................................................
        mul v12.8H, v12.8H, v0.H[6]             // ....*...............................................................................
        sub v16.8H, v26.8H, v19.8H              // .....*..............................................................................
        add v26.8H, v26.8H, v19.8H              // ......*.............................................................................
        sqrdmulh v15.8H, v8.8H, v1.H[1]         // .......*............................................................................
        mul v8.8H, v8.8H, v1.H[0]               // ........*...........................................................................
        mls v12.8H, v11.8H, v7.H[0]             // .........*..........................................................................
        sub v11.8H, v9.8H, v6.8H                // ..........*.........................................................................
        sqrdmulh v24.8H, v16.8H, v0.H[3]        // ...........*........................................................................
        mul v16.8H, v16.8H, v0.H[2]             // ............*.......................................................................
        sub v9.8H, v26.8H, v23.8H               // .............*......................................................................
        add v26.8H, v26.8H, v23.8H              // ..............*.....................................................................
        mls v8.8H, v15.8H, v7.H[0]              // ...............*....................................................................
        sqrdmulh v15.8H, v11.8H, v1.H[3]        // ................*...................................................................
        mul v11.8H, v11.8H, v1.H[2]             // .................*..................................................................
        sub v6.8H, v3.8H, v4.8H                 // ..................*.................................................................
        sub v3.8H, v12.8H, v8.8H                // ...................*................................................................
        add v12.8H, v12.8H, v8.8H               // ....................*...............................................................
        mls v11.8H, v15.8H, v7.H[0]             // .....................*..............................................................
        sqrdmulh v8.8H, v6.8H, v1.H[5]          // ......................*.............................................................
        mls v16.8H, v24.8H, v7.H[0]             // .......................*............................................................
        mul v15.8H, v6.8H, v1.H[4]              // ........................*...........................................................
        sqrdmulh v24.8H, v3.8H, v0.H[3]         // .........................*..........................................................
        mul v6.8H, v3.8H, v0.H[2]               // ..........................*.........................................................
        sqrdmulh v3.8H, v9.8H, v0.H[1]          // ...........................*........................................................
        mul v9.8H, v9.8H, v0.H[0]               // ............................*.......................................................
        str q26, [x0], #(16)                    // .............................*......................................................
        mls v15.8H, v8.8H, v7.H[0]              // ..............................*.....................................................
        mls v6.8H, v24.8H, v7.H[0]              // ...............................*....................................................
        sub v26.8H, v28.8H, v13.8H              // ................................*...................................................
        mls v9.8H, v3.8H, v7.H[0]               // .................................*..................................................
        sub v8.8H, v11.8H, v15.8H               // ..................................*.................................................
        sqrdmulh v24.8H, v26.8H, v0.H[5]        // ...................................*................................................
        mul v26.8H, v26.8H, v0.H[4]             // ....................................*...............................................
        add v11.8H, v11.8H, v15.8H              // .....................................*..............................................
        sqrdmulh v15.8H, v8.8H, v0.H[5]         // ......................................*.............................................
        mul v8.8H, v8.8H, v0.H[4]               // .......................................*............................................
        mls v26.8H, v24.8H, v7.H[0]             // ........................................*...........................................
        sub v24.8H, v12.8H, v11.8H              // .........................................*..........................................
        add v12.8H, v12.8H, v11.8H              // ..........................................*.........................................
        mls v8.8H, v15.8H, v7.H[0]              // ...........................................*........................................
        sqrdmulh v11.8H, v24.8H, v0.H[1]        // ............................................*.......................................
        mul v15.8H, v24.8H, v0.H[0]             // .............................................*......................................
        sub v24.8H, v16.8H, v26.8H              // ..............................................*.....................................
        add v26.8H, v16.8H, v26.8H              // ...............................................*....................................
        sub v16.8H, v6.8H, v8.8H                // ................................................*...................................
        mls v15.8H, v11.8H, v7.H[0]             // .................................................*..................................
        sqrdmulh v11.8H, v24.8H, v0.H[1]        // ..................................................*.................................
        mul v24.8H, v24.8H, v0.H[0]             // ...................................................*................................
        add v8.8H, v6.8H, v8.8H                 // ....................................................*...............................
        sqrdmulh v6.8H, v16.8H, v0.H[1]         // .....................................................*..............................
        mul v16.8H, v16.8H, v0.H[0]             // ......................................................*.............................
        mls v24.8H, v11.8H, v7.H[0]             // .......................................................*............................
        str q9, [x0, #240]                      // ........................................................*...........................
        ldr q11, [x0, #0]                       // .........................................................e..........................
        mls v16.8H, v6.8H, v7.H[0]              // ...........................................................*........................
        str q15, [x0, #304]                     // ............................................................*.......................
        ldr q15, [x0, #64]                      // .............................................................e......................
        str q24, [x0, #368]                     // ...............................................................*....................
        ldr q24, [x0, #128]                     // ................................................................e...................
        str q16, [x0, #432]                     // ..................................................................*.................
        ldr q16, [x0, #192]                     // ...................................................................e................
        str q12, [x0, #48]                      // .....................................................................*..............
        ldr q9, [x0, #256]                      // ......................................................................e.............
        ldr q6, [x0, #320]                      // ........................................................................e...........
        ldr q3, [x0, #384]                      // ..........................................................................e.........
        ldr q4, [x0, #448]                      // ............................................................................e.......
        str q26, [x0, #112]                     // ..............................................................................*.....
        add v28.8H, v9.8H, v6.8H                // ...............................................................................e....
        add v13.8H, v3.8H, v4.8H                // ................................................................................e...
        str q8, [x0, #176]                      // .................................................................................*..
        add v19.8H, v24.8H, v16.8H              // ..................................................................................e.
        add v23.8H, v28.8H, v13.8H              // ...................................................................................e

                                                      // --------------------------------------------- cycle (expected) --------------------------------------------->
                                                      // 0                        25                       50                       75                       100
                                                      // |------------------------|------------------------|------------------------|------------------------|--------
        // ldr q8, [x0, #0]                           // e..........................'........................................................~........................
        // ldr q9, [x0, #(1*(512/8))]                 // ....e......................'............................................................~....................
        // ldr q10, [x0, #(2*(512/8))]                // .......e...................'...............................................................~.................
        // ldr q11, [x0, #(3*(512/8))]                // ..........e................'..................................................................~..............
        // ldr q12, [x0, #(4*(512/8))]                // .............e.............'.....................................................................~...........
        // ldr q13, [x0, #(5*(512/8))]                // ...............e...........'.......................................................................~.........
        // ldr q14, [x0, #(6*(512/8))]                // .................e.........'.........................................................................~.......
        // ldr q15, [x0, #(7*(512/8))]                // ...................e.......'...........................................................................~.....
        // sub v24.8h,   v8.8h, v9.8h                 // ...........................*.................................................................................
        // add v8.8h, v8.8h, v9.8h                    // ...........................'*................................................................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[7]      // ...........................'..*..............................................................................
        // mul      v9.8h, v24.8h, v0.h[6]            // ...........................'...*.............................................................................
        // mls      v9.8h, v27.8h,      v7.h[0]       // ...........................'........*........................................................................
        // sub v24.8h,   v10.8h, v11.8h               // ...........................'.*...............................................................................
        // add v10.8h, v10.8h, v11.8h                 // .........................e.'.................................................................................
        // sqrdmulh v27.8h,      v24.8h, v1.h[1]      // ...........................'......*..........................................................................
        // mul      v11.8h, v24.8h, v1.h[0]           // ...........................'.......*.........................................................................
        // mls      v11.8h, v27.8h,      v7.h[0]      // ...........................'..............*..................................................................
        // sub v24.8h,   v12.8h, v13.8h               // ...........................'.........*.......................................................................
        // add v12.8h, v12.8h, v13.8h                 // ......................e....'..............................................................................~..
        // sqrdmulh v27.8h,      v24.8h, v1.h[3]      // ...........................'...............*.................................................................
        // mul      v13.8h, v24.8h, v1.h[2]           // ...........................'................*................................................................
        // mls      v13.8h, v27.8h,      v7.h[0]      // ...........................'....................*............................................................
        // sub v24.8h,   v14.8h, v15.8h               // ...........................'.................*...............................................................
        // add v14.8h, v14.8h, v15.8h                 // .......................e...'...............................................................................~.
        // sqrdmulh v27.8h,      v24.8h, v1.h[5]      // ...........................'.....................*...........................................................
        // mul      v15.8h, v24.8h, v1.h[4]           // ...........................'.......................*.........................................................
        // mls      v15.8h, v27.8h,      v7.h[0]      // ...........................'.............................*...................................................
        // sub v24.8h,   v8.8h, v10.8h                // ...........................'....*............................................................................
        // add v8.8h, v8.8h, v10.8h                   // ...........................'.....*...........................................................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[3]      // ...........................'..........*......................................................................
        // mul      v10.8h, v24.8h, v0.h[2]           // ...........................'...........*.....................................................................
        // mls      v10.8h, v27.8h,      v7.h[0]      // ...........................'......................*..........................................................
        // sub v24.8h,   v9.8h, v11.8h                // ...........................'..................*..............................................................
        // add v9.8h, v9.8h, v11.8h                   // ...........................'...................*.............................................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[3]      // ...........................'........................*........................................................
        // mul      v11.8h, v24.8h, v0.h[2]           // ...........................'.........................*.......................................................
        // mls      v11.8h, v27.8h,      v7.h[0]      // ...........................'..............................*..................................................
        // sub v24.8h,   v12.8h, v14.8h               // ...........................'...............................*.................................................
        // add v12.8h, v12.8h, v14.8h                 // ..........................e'.................................................................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[5]      // ...........................'..................................*..............................................
        // mul      v14.8h, v24.8h, v0.h[4]           // ...........................'...................................*.............................................
        // mls      v14.8h, v27.8h,      v7.h[0]      // ...........................'.......................................*.........................................
        // sub v24.8h,   v13.8h, v15.8h               // ...........................'.................................*...............................................
        // add v13.8h, v13.8h, v15.8h                 // ...........................'....................................*............................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[5]      // ...........................'.....................................*...........................................
        // mul      v15.8h, v24.8h, v0.h[4]           // ...........................'......................................*..........................................
        // mls      v15.8h, v27.8h,      v7.h[0]      // ...........................'..........................................*......................................
        // sub v24.8h,   v8.8h, v12.8h                // ...........................'............*....................................................................
        // add v8.8h, v8.8h, v12.8h                   // ...........................'.............*...................................................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[1]      // ...........................'..........................*......................................................
        // mul      v12.8h, v24.8h, v0.h[0]           // ...........................'...........................*.....................................................
        // mls      v12.8h, v27.8h,      v7.h[0]      // ...........................'................................*................................................
        // sub v24.8h,   v9.8h, v13.8h                // ...........................'........................................*........................................
        // add v9.8h, v9.8h, v13.8h                   // ...........................'.........................................*.......................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[1]      // ...........................'...........................................*.....................................
        // mul      v13.8h, v24.8h, v0.h[0]           // ...........................'............................................*....................................
        // mls      v13.8h, v27.8h,      v7.h[0]      // ...........................'................................................*................................
        // sub v24.8h,   v10.8h, v14.8h               // ...........................'.............................................*...................................
        // add v10.8h, v10.8h, v14.8h                 // ...........................'..............................................*..................................
        // sqrdmulh v27.8h,      v24.8h, v0.h[1]      // ...........................'.................................................*...............................
        // mul      v14.8h, v24.8h, v0.h[0]           // ...........................'..................................................*..............................
        // mls      v14.8h, v27.8h,      v7.h[0]      // ...........................'......................................................*..........................
        // sub v24.8h,   v11.8h, v15.8h               // ...........................'...............................................*.................................
        // add v11.8h, v11.8h, v15.8h                 // ...........................'...................................................*.............................
        // sqrdmulh v27.8h,      v24.8h, v0.h[1]      // ...........................'....................................................*............................
        // mul      v15.8h, v24.8h, v0.h[0]           // ...........................'.....................................................*...........................
        // mls      v15.8h, v27.8h,      v7.h[0]      // ..~........................'..........................................................*......................
        // str q12, [x0, #(4*(512/8))]                // ...........................'.......................................................*.........................
        // str q13, [x0, #(5*(512/8))]                // ...~.......................'...........................................................*.....................
        // str q14, [x0, #(6*(512/8))]                // ......~....................'..............................................................*..................
        // str q15, [x0, #(7*(512/8))]                // .........~.................'.................................................................*...............
        // str q8, [x0], #(16)                        // ...........................'............................*....................................................
        // str q9, [x0, #(-16 + 1*(512/8))]           // ............~..............'....................................................................*............
        // str q10, [x0, #(-16 + 2*(512/8))]          // .....................~.....'.............................................................................*...
        // str q11, [x0, #(-16 + 3*(512/8))]          // ........................~..'................................................................................*

        sub count, count, #1
        cbnz count, layer123_start
                                                // Instructions:    64
                                                // Expected cycles: 66
                                                // Expected IPC:    0.97
                                                //
                                                // Cycle bound:     66.0
                                                // IPC bound:       0.97
                                                //
                                                // Wall time:     8.33s
                                                // User time:     8.33s
                                                //
                                                // ----------------------- cycle (expected) ------------------------>
                                                // 0                        25                       50
                                                // |------------------------|------------------------|---------------
        add v10.8H, v11.8H, v15.8H              // *.................................................................
        sub v12.8H, v28.8H, v13.8H              // .*................................................................
        sub v11.8H, v11.8H, v15.8H              // ..*...............................................................
        sub v22.8H, v10.8H, v19.8H              // ...*..............................................................
        mul v18.8H, v12.8H, v0.H[4]             // ....*.............................................................
        sqrdmulh v26.8H, v12.8H, v0.H[5]        // .....*............................................................
        sqrdmulh v12.8H, v22.8H, v0.H[3]        // ......*...........................................................
        mul v13.8H, v22.8H, v0.H[2]             // .......*..........................................................
        sub v31.8H, v24.8H, v16.8H              // ........*.........................................................
        sqrdmulh v22.8H, v11.8H, v0.H[7]        // .........*........................................................
        mls v18.8H, v26.8H, v7.H[0]             // ..........*.......................................................
        mls v13.8H, v12.8H, v7.H[0]             // ...........*......................................................
        sqrdmulh v2.8H, v31.8H, v1.H[1]         // ............*.....................................................
        mul v5.8H, v31.8H, v1.H[0]              // .............*....................................................
        mul v15.8H, v11.8H, v0.H[6]             // ..............*...................................................
        sub v12.8H, v13.8H, v18.8H              // ...............*..................................................
        sub v4.8H, v3.8H, v4.8H                 // ................*.................................................
        mls v5.8H, v2.8H, v7.H[0]               // .................*................................................
        sqrdmulh v26.8H, v12.8H, v0.H[1]        // ..................*...............................................
        mul v12.8H, v12.8H, v0.H[0]             // ...................*..............................................
        mls v15.8H, v22.8H, v7.H[0]             // ....................*.............................................
        sqrdmulh v8.8H, v4.8H, v1.H[5]          // .....................*............................................
        mul v4.8H, v4.8H, v1.H[4]               // ......................*...........................................
        mls v12.8H, v26.8H, v7.H[0]             // .......................*..........................................
        sub v21.8H, v15.8H, v5.8H               // ........................*.........................................
        sub v28.8H, v9.8H, v6.8H                // .........................*........................................
        mls v4.8H, v8.8H, v7.H[0]               // ..........................*.......................................
        mul v24.8H, v21.8H, v0.H[2]             // ...........................*......................................
        sqrdmulh v8.8H, v21.8H, v0.H[3]         // ............................*.....................................
        sqrdmulh v6.8H, v28.8H, v1.H[3]         // .............................*....................................
        add v19.8H, v10.8H, v19.8H              // ..............................*...................................
        mul v28.8H, v28.8H, v1.H[2]             // ...............................*..................................
        mls v24.8H, v8.8H, v7.H[0]              // ................................*.................................
        sub v11.8H, v19.8H, v23.8H              // .................................*................................
        str q12, [x0, #384]                     // ..................................*...............................
        mls v28.8H, v6.8H, v7.H[0]              // ...................................*..............................
        sqrdmulh v16.8H, v11.8H, v0.H[1]        // ....................................*.............................
        mul v9.8H, v11.8H, v0.H[0]              // .....................................*............................
        add v6.8H, v15.8H, v5.8H                // ......................................*...........................
        add v26.8H, v28.8H, v4.8H               // .......................................*..........................
        sub v15.8H, v28.8H, v4.8H               // ........................................*.........................
        mls v9.8H, v16.8H, v7.H[0]              // .........................................*........................
        add v3.8H, v6.8H, v26.8H                // ..........................................*.......................
        mul v8.8H, v15.8H, v0.H[4]              // ...........................................*......................
        sqrdmulh v15.8H, v15.8H, v0.H[5]        // ............................................*.....................
        str q9, [x0, #256]                      // .............................................*....................
        sub v2.8H, v6.8H, v26.8H                // ..............................................*...................
        str q3, [x0, #64]                       // ...............................................*..................
        mls v8.8H, v15.8H, v7.H[0]              // ................................................*.................
        sqrdmulh v15.8H, v2.8H, v0.H[1]         // .................................................*................
        mul v11.8H, v2.8H, v0.H[0]              // ..................................................*...............
        add v16.8H, v13.8H, v18.8H              // ...................................................*..............
        sub v12.8H, v24.8H, v8.8H               // ....................................................*.............
        add v8.8H, v24.8H, v8.8H                // .....................................................*............
        mls v11.8H, v15.8H, v7.H[0]             // ......................................................*...........
        sqrdmulh v26.8H, v12.8H, v0.H[1]        // .......................................................*..........
        mul v12.8H, v12.8H, v0.H[0]             // ........................................................*.........
        str q8, [x0, #192]                      // .........................................................*........
        add v15.8H, v19.8H, v23.8H              // ..........................................................*.......
        str q11, [x0, #320]                     // ...........................................................*......
        mls v12.8H, v26.8H, v7.H[0]             // ............................................................*.....
        str q15, [x0], #(16)                    // .............................................................*....
        str q16, [x0, #112]                     // ...............................................................*..
        str q12, [x0, #432]                     // .................................................................*

                                                 // ----------------------- cycle (expected) ------------------------>
                                                 // 0                        25                       50
                                                 // |------------------------|------------------------|---------------
        // sub v12.8H, v11.8H, v15.8H            // ..*...............................................................
        // add v26.8H, v11.8H, v15.8H            // *.................................................................
        // sub v8.8H, v24.8H, v16.8H             // ........*.........................................................
        // sqrdmulh v11.8H, v12.8H, v0.H[7]      // .........*........................................................
        // mul v12.8H, v12.8H, v0.H[6]           // ..............*...................................................
        // sub v16.8H, v26.8H, v19.8H            // ...*..............................................................
        // add v26.8H, v26.8H, v19.8H            // ..............................*...................................
        // sqrdmulh v15.8H, v8.8H, v1.H[1]       // ............*.....................................................
        // mul v8.8H, v8.8H, v1.H[0]             // .............*....................................................
        // mls v12.8H, v11.8H, v7.H[0]           // ....................*.............................................
        // sub v11.8H, v9.8H, v6.8H              // .........................*........................................
        // sqrdmulh v24.8H, v16.8H, v0.H[3]      // ......*...........................................................
        // mul v16.8H, v16.8H, v0.H[2]           // .......*..........................................................
        // sub v9.8H, v26.8H, v23.8H             // .................................*................................
        // add v26.8H, v26.8H, v23.8H            // ..........................................................*.......
        // mls v8.8H, v15.8H, v7.H[0]            // .................*................................................
        // sqrdmulh v15.8H, v11.8H, v1.H[3]      // .............................*....................................
        // mul v11.8H, v11.8H, v1.H[2]           // ...............................*..................................
        // sub v6.8H, v3.8H, v4.8H               // ................*.................................................
        // sub v3.8H, v12.8H, v8.8H              // ........................*.........................................
        // add v12.8H, v12.8H, v8.8H             // ......................................*...........................
        // mls v11.8H, v15.8H, v7.H[0]           // ...................................*..............................
        // sqrdmulh v8.8H, v6.8H, v1.H[5]        // .....................*............................................
        // mls v16.8H, v24.8H, v7.H[0]           // ...........*......................................................
        // mul v15.8H, v6.8H, v1.H[4]            // ......................*...........................................
        // sqrdmulh v24.8H, v3.8H, v0.H[3]       // ............................*.....................................
        // mul v6.8H, v3.8H, v0.H[2]             // ...........................*......................................
        // sqrdmulh v3.8H, v9.8H, v0.H[1]        // ....................................*.............................
        // mul v9.8H, v9.8H, v0.H[0]             // .....................................*............................
        // str q26, [x0], #(16)                  // .............................................................*....
        // mls v15.8H, v8.8H, v7.H[0]            // ..........................*.......................................
        // mls v6.8H, v24.8H, v7.H[0]            // ................................*.................................
        // sub v26.8H, v28.8H, v13.8H            // .*................................................................
        // mls v9.8H, v3.8H, v7.H[0]             // .........................................*........................
        // sub v8.8H, v11.8H, v15.8H             // ........................................*.........................
        // sqrdmulh v24.8H, v26.8H, v0.H[5]      // .....*............................................................
        // mul v26.8H, v26.8H, v0.H[4]           // ....*.............................................................
        // add v11.8H, v11.8H, v15.8H            // .......................................*..........................
        // sqrdmulh v15.8H, v8.8H, v0.H[5]       // ............................................*.....................
        // mul v8.8H, v8.8H, v0.H[4]             // ...........................................*......................
        // mls v26.8H, v24.8H, v7.H[0]           // ..........*.......................................................
        // sub v24.8H, v12.8H, v11.8H            // ..............................................*...................
        // add v12.8H, v12.8H, v11.8H            // ..........................................*.......................
        // mls v8.8H, v15.8H, v7.H[0]            // ................................................*.................
        // sqrdmulh v11.8H, v24.8H, v0.H[1]      // .................................................*................
        // mul v15.8H, v24.8H, v0.H[0]           // ..................................................*...............
        // sub v24.8H, v16.8H, v26.8H            // ...............*..................................................
        // add v26.8H, v16.8H, v26.8H            // ...................................................*..............
        // sub v16.8H, v6.8H, v8.8H              // ....................................................*.............
        // mls v15.8H, v11.8H, v7.H[0]           // ......................................................*...........
        // sqrdmulh v11.8H, v24.8H, v0.H[1]      // ..................*...............................................
        // mul v24.8H, v24.8H, v0.H[0]           // ...................*..............................................
        // add v8.8H, v6.8H, v8.8H               // .....................................................*............
        // sqrdmulh v6.8H, v16.8H, v0.H[1]       // .......................................................*..........
        // mul v16.8H, v16.8H, v0.H[0]           // ........................................................*.........
        // mls v24.8H, v11.8H, v7.H[0]           // .......................*..........................................
        // str q9, [x0, #240]                    // .............................................*....................
        // mls v16.8H, v6.8H, v7.H[0]            // ............................................................*.....
        // str q15, [x0, #304]                   // ...........................................................*......
        // str q24, [x0, #368]                   // ..................................*...............................
        // str q16, [x0, #432]                   // .................................................................*
        // str q12, [x0, #48]                    // ...............................................*..................
        // str q26, [x0, #112]                   // ...............................................................*..
        // str q8, [x0, #176]                    // .........................................................*........


        pop_stack
        ret

/****************** REGISTER DEALLOCATIONS *******************/
    .unreq in
    .unreq r12345_ptr
    .unreq r67_ptr
    .unreq inp
    .unreq count
    .unreq wtmp
    .unreq data0
    .unreq data1
    .unreq data2
    .unreq data3
    .unreq data4
    .unreq data5
    .unreq data6
    .unreq data7
    .unreq q_data0
    .unreq q_data1
    .unreq q_data2
    .unreq q_data3
    .unreq q_data4
    .unreq q_data5
    .unreq q_data6
    .unreq q_data7
    .unreq root0
    .unreq root1
    .unreq root2
    .unreq root0_tw
    .unreq root1_tw
    .unreq root2_tw
    .unreq consts
    .unreq q_consts
    .unreq q_root0
    .unreq q_root1
    .unreq q_root2
    .unreq q_root0_tw
    .unreq q_root1_tw
    .unreq q_root2_tw
    .unreq tmp
    .unreq t0
    .unreq t1
    .unreq t2
    .unreq t3
    .unreq ninv
    .unreq ninv_tw

/* simpasm: footer-start */
#endif /* MLK_ARITH_BACKEND_AARCH64_OPT &&
          !MLK_MULTILEVEL_BUILD_NO_SHARED */