# --- T2-COPYRIGHT-BEGIN --- # t2/package/*/dav1d/ia64-simd.patch # Copyright (C) 2026 The T2 SDE Project # SPDX-License-Identifier: GPL-2.0 or patched project license # --- T2-COPYRIGHT-END --- # # Three-tier optimized AV1 decoder for Intel IA-64 (Itanium architecture) # Generated by Claude Opus 5 based on state-of-the-art ILP methods # # Includes: # - Use of I-slot pmpy in place of F-slot xma for multiplication (highest impact) # - Implementing A-slot packed SIMD (padd and friends) for the non-multiplication parts # - Software pipelining of loops with inline assembly ("tier II") # - Software pipelining with external .S files ("tier III") where the whole frame # (96 registers) are needed # - Overlay-fusion of operations that come after each other (where dependencies # won't allow SW pipelining to fill up all units) # - Some minor non-ia64-specific changes that improve performance on ia64 # # For more details, see the comments in the code below. diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/meson.build dav1d-1.5.4/meson.build --- dav1d-1.5.4.orig/meson.build 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/meson.build 2026-08-17 21:40:10.189111211 +0200 @@ -70,6 +70,7 @@ (host_machine.cpu_family() == 'aarch64' or host_machine.cpu_family().startswith('arm') or host_machine.cpu() == 'ppc64le' or + host_machine.cpu_family() == 'ia64' or host_machine.cpu_family().startswith('riscv') or host_machine.cpu_family().startswith('loongarch') or host_machine.cpu_family() == 'x86' or @@ -510,6 +511,7 @@ cdata_asm.set10('FORCE_VEX_ENCODING', cc.get_define('__AVX__').strip() != '') endif +cdata.set10('ARCH_IA64', host_machine.cpu_family() == 'ia64') cdata.set10('ARCH_PPC64LE', host_machine.cpu() == 'ppc64le') cdata.set10('ARCH_RISCV', host_machine.cpu_family().startswith('riscv')) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/cpu.c dav1d-1.5.4/src/cpu.c --- dav1d-1.5.4.orig/src/cpu.c 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/cpu.c 2026-08-20 14:50:31.046889307 +0200 @@ -67,6 +67,8 @@ dav1d_cpu_flags = dav1d_get_cpu_flags_arm(); #elif ARCH_LOONGARCH dav1d_cpu_flags = dav1d_get_cpu_flags_loongarch(); +#elif ARCH_IA64 + dav1d_cpu_flags = dav1d_get_cpu_flags_ia64(); #elif ARCH_PPC64LE dav1d_cpu_flags = dav1d_get_cpu_flags_ppc(); #elif ARCH_RISCV diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/cpu.h dav1d-1.5.4/src/cpu.h --- dav1d-1.5.4.orig/src/cpu.h 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/cpu.h 2026-08-17 21:40:10.191539208 +0200 @@ -39,6 +39,8 @@ #include "src/arm/cpu.h" #elif ARCH_LOONGARCH #include "src/loongarch/cpu.h" +#elif ARCH_IA64 +#include "src/ia64/cpu.h" #elif ARCH_PPC64LE #include "src/ppc/cpu.h" #elif ARCH_RISCV @@ -76,6 +78,9 @@ flags |= DAV1D_ARM_CPU_FLAG_SVE2; #endif #endif /* ARCH_AARCH64 */ +#elif ARCH_IA64 + /* The parallel integer instructions are unconditionally present. */ + flags |= DAV1D_IA64_CPU_FLAG_SIMD; #elif ARCH_PPC64LE #if defined(__VSX__) flags |= DAV1D_PPC_CPU_FLAG_VSX; diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/cpu.c dav1d-1.5.4/src/ia64/cpu.c --- dav1d-1.5.4.orig/src/ia64/cpu.c 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/cpu.c 2026-08-17 21:42:00.491043954 +0200 @@ -0,0 +1,34 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "common/attributes.h" + +#include "src/cpu.h" +#include "src/ia64/cpu.h" + +COLD unsigned dav1d_get_cpu_flags_ia64(void) { + return DAV1D_IA64_CPU_FLAG_SIMD; +} diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/cpu.h dav1d-1.5.4/src/ia64/cpu.h --- dav1d-1.5.4.orig/src/ia64/cpu.h 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/cpu.h 2026-08-17 21:42:00.419043998 +0200 @@ -0,0 +1,40 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_IA64_CPU_H +#define DAV1D_SRC_IA64_CPU_H + +/* The packed ("parallel") integer operations this backend is built on are all + * base IA-64 ISA -- there is no runtime feature to probe. A single flag is + * still carried so that the generic dispatch machinery and checkasm's + * --test/flag selection work unmodified. */ +enum CpuFlags { + DAV1D_IA64_CPU_FLAG_SIMD = 1 << 0, +}; + +unsigned dav1d_get_cpu_flags_ia64(void); + +#endif /* DAV1D_SRC_IA64_CPU_H */ diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/dav1d_simd.h dav1d-1.5.4/src/ia64/dav1d_simd.h --- dav1d-1.5.4.orig/src/ia64/dav1d_simd.h 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/dav1d_simd.h 2026-08-20 14:55:11.998441278 +0200 @@ -0,0 +1,606 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_IA64_DAV1D_SIMD_H +#define DAV1D_SRC_IA64_DAV1D_SIMD_H + +#include +#include + +/* + * IA-64 carries a complete MMX-class "parallel" instruction set that operates + * on 64-bit *general* registers -- padd2/psub2/pmpyshr2/pmpy2/pshr2/pmin2/... + * None of it needs a -m flag; it is all base ISA. + * + * This matters a great deal here because IA-64 has no integer multiplier in + * the integer unit at all. A scalar `a * b` on int16 data is issued as + * setf.sig / xma.l / getf.sig -- a round trip through the FP register file + * costing on the order of 18 cycles of pure latency, which is what the plain C + * DSP templates end up doing once per filter tap per pixel. A pmpy2 pair + * produces four 16x16->32 products in two integer-unit instructions. + * + * GCC's vect.md maps most generic-vector operations onto the packed opcodes + * directly, so the kernels below are written with __attribute__((vector_size)) + * and plain C operators wherever that works. The operations GCC does *not* + * map are provided here as inline asm: + * + * - widening 16x16->32 multiply (pmpy2.r / pmpy2.l) + * - narrowing saturating packs (pack4.sss / pack2.sss / pack2.uss) + * - the high half of an unpack (unpack1.h / unpack2.h) + * - broadcast of a runtime scalar (mux2) + * + * GCC open-codes those as extr/dep chains of five to twenty ops, so going + * through asm is worth roughly an order of magnitude on the pack-heavy paths. + */ + +/* + * --------------------------------------------------------------------------- + * Three tiers + * --------------------------------------------------------------------------- + * + * IA-64 has no SIMD intrinsics and no autovectoriser worth the name, but it + * does have packed integer ops in the base ISA, and GCC's generic vector + * extension maps most of them 1:1. How far to go on any given kernel is a + * question of how hot it is and what is actually limiting it, so the code here + * sits in one of three tiers. Move up a tier only when a measurement says the + * current one has hit a wall -- each step costs a lot of readability. + * + * Tier I -- generic vectors, with light asm for the gaps. + * Write the kernel in C over the vector_size(8) types below and let GCC + * schedule it. A handful of ops have no C spelling (pack, unpack, mux, + * shrp, pmpy) and get a one-line asm() wrapper here. Good enough whenever + * the kernel is not near a slot limit. Watch out: every asm() is + * TYPE_UNKNOWN to GCC's bundler and lands in its own bundle behind a stop, + * so prefer plain C wherever GCC already emits the right instruction. + * + * Tier II -- a hand-bundled inline asm block. + * Used when GCC's bundling is the problem: A-type ops (padd, psub, pcmp, + * pmin/pmax, and/or/xor, add, shladd) issue on either unit, I-type ops + * (pmpy2, shrp, pack, unpack, mux, variable shifts, dep, extr) only on the + * two I units. Two I-slot ops retire per cycle whatever the template, so + * the floor is ceil(I_ops / 2) and the job is to keep the M slots busy with + * the A-type work rather than to pack instructions tightly. Interleave + * several independent groups in one block: GCC cannot overlap two asm + * blocks, so two adjacent calls always serialise. + * + * Tier II's ceiling is registers. An asm block gets 24 scratch GRs, and a + * "+r" operand costs two of GCC's 30 operand slots. Widening the unroll + * keeps paying until that runs out, and then stops. + * + * Tier III -- a standalone .S with its own register frame. + * For the hottest kernels that are chain-bound rather than slot-bound, where + * Tier II ran out of registers before it ran out of parallelism. alloc gives + * ~90 registers and a rotating window, so the loop can be modulo-scheduled: + * one group per iteration at an initiation interval set by the resource + * floor, with the dependence chain spread over several in-flight iterations. + * br.ctop rotates the registers and fills the prologue / drains the epilogue + * via the rotating predicates, so no prologue or epilogue is written by hand. + * + * The catch is the drain: a loop costs (iterations + stages - 1) * II, so on + * short rows the smallest feasible II is not the fastest -- pushing II to + * the floor drives slot utilisation to ~100%, which stretches the schedule + * and adds stages. The generator picks the II that minimises total cycles. + * + * Two hard rules for a Tier III kernel. One: every load and store owns a + * private post-incrementing pointer stepping a whole iteration. Sharing one + * between two accesses breaks, because the scheduler may put them in + * different stages and they would then serve different iterations while both + * advancing the same register. Two: at most eight arguments. The psABI + * passes only out0-out7 (r32-r39) in registers; a ninth arrives on the + * memory stack, and `alloc ..., 9, ...` grants it without a word of warning. + * Pass a pointer array and load the rows in the prologue instead. + * + * Currently Tier III: looprestoration_sgr.S (fused SGR vertical sum + + * calc_ab), looprestoration_fin.S (the finish filter, all three forms) and + * looprestoration_box.S (box3 and box5 horizontal, fused). Generated by + * src/ia64/gen/ -- edit the generator, not the .S. + * + * Choosing a tier: count the I-type ops for the resource floor and walk the + * dependence chain separately. chain < floor means the kernel is already + * resource-bound and only merging adjacent asm blocks helps; chain >= floor + * means the bubbles can only be filled from the next iteration, which is what + * Tier III is for. src/ia64/gen/cand.py does this for every op list and + * also runs the modulo scheduler, so the decision is a measurement rather than + * a guess. Before promoting, check the answer against the real row length: + * the SGR vertical kernel is chain-bound but its rows are only ~16 iterations, + * so the drain ate most of the win, while the finish filter's ~64-iteration + * rows kept all of it. + * + * Look for fusions before reaching for a tier. sgr_box35_row_h_q ran box3 and + * box5 over the same source row, and box3's three taps are the middle three of + * box5's five -- one fused kernel shares the unaligned extraction, the + * unpack1s, every shrp and every pmpy2, and does both in 20 cycles per eight + * outputs where the two separate kernels needed 21 + 26. + * + * Validation: checkasm covers a new DSP function, but it does not cover edits + * to a generic C template (it *is* the reference), it has been observed to pass + * a miscomputing SGR kernel, and it passed a kernel whose loop count came from + * an uninitialised register and segfaulted the decoder. Confirm every change + * with a full-decode md5 against the previous build, and validate a new .S + * standalone first -- ~/dev/poc/tfin.c and tbox.c put a PROT_NONE guard page + * behind every buffer, so the read and write extents are checked too, and they + * iterate in seconds instead of a rebuild. + */ + +typedef signed char i8x8 __attribute__((vector_size(8))); +typedef unsigned char u8x8 __attribute__((vector_size(8))); +typedef int16_t i16x4 __attribute__((vector_size(8))); +typedef uint16_t u16x4 __attribute__((vector_size(8))); +typedef int32_t i32x2 __attribute__((vector_size(8))); +typedef uint32_t u32x2 __attribute__((vector_size(8))); + +#define SIMD_FN static inline __attribute__((always_inline)) + +/* ---------------------------------------------------------------- multiply */ + +/* 16x16 -> 32, even lanes {0,2} of both operands. */ +SIMD_FN i32x2 pmpy2_r(const i16x4 a, const i16x4 b) { + i32x2 r; + __asm__("pmpy2.r %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +/* 16x16 -> 32, odd lanes {1,3} of both operands. */ +SIMD_FN i32x2 pmpy2_l(const i16x4 a, const i16x4 b) { + i32x2 r; + __asm__("pmpy2.l %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +/* ------------------------------------------------------------------- packs */ + +/* + * Four signed 16x16 products, low 16 bits of each kept (pmpyshr2 with a shift + * of zero). Both the AV1 subpel taps and an 8-bit pixel fit comfortably: + * |255 * 127| = 32385 < 2^15, so the product is exact, not truncated. + * + * This is the 16-bit counterpart of the pmpy2.r/pmpy2.l pair. Where the + * accumulated sum is known to fit in 16 bits it replaces two multiplies and + * two 32-bit adds per tap with one multiply and one add, which matters a lot + * on a machine with only two I ports and no other unit able to take the work. + */ +SIMD_FN i16x4 pmpy2_lo(const i16x4 a, const i16x4 b) { + /* Plain C, deliberately: this is exactly pmpyshr2 with a shift of zero, and + * writing it as an asm() costs more than the instruction saves. GCC types + * an asm() as unknown for scheduling, so it will not dual-issue it or move + * it across neighbours, and a kernel built entirely out of asm() helpers + * ends up one instruction per bundle. Letting the compiler emit the + * multiply itself lets it schedule the multiply chains against each other. */ + return a * b; +} + +/* 4x32 -> 4x16, signed saturating. Lane order is {a0,a1,b0,b1}. */ +SIMD_FN i16x4 pack4_sss(const i32x2 a, const i32x2 b) { + i16x4 r; + __asm__("pack4.sss %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +/* 8x16 -> 8x8, signed source, unsigned saturating destination. + * This is exactly iclip_pixel() for 8bpc, for free. */ +SIMD_FN u8x8 pack2_uss(const i16x4 a, const i16x4 b) { + u8x8 r; + __asm__("pack2.uss %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i8x8 pack2_sss(const i16x4 a, const i16x4 b) { + i8x8 r; + __asm__("pack2.sss %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +/* ----------------------------------------------------------------- unpacks */ + +/* Zero-extend the low four bytes of v to 4x16. GCC emits this one itself for + * a convertvector of the low half, but spelling it out keeps the two halves + * symmetric. */ +SIMD_FN i16x4 zext8_lo(const u8x8 v) { + /* Plain C on purpose. GCC matches this permutation to + * *vec_interleave_lowv8qi and emits exactly "unpack1.l r = r0, v"; going + * through asm() instead would cost a whole bundle and a stop bit, because + * the bundler cannot type an asm (see ia64.cc:6294). */ + const u8x8 z = { 0, 0, 0, 0, 0, 0, 0, 0 }; + return (i16x4)__builtin_shuffle(v, z, (u8x8){ 0, 8, 1, 9, 2, 10, 3, 11 }); +} + +/* Zero-extend the high four bytes of v to 4x16. GCC does *not* find this one + * and open-codes an eight-op extr/dep chain instead. */ +SIMD_FN i16x4 zext8_hi(const u8x8 v) { + const u8x8 z = { 0, 0, 0, 0, 0, 0, 0, 0 }; + return (i16x4)__builtin_shuffle(v, z, (u8x8){ 4, 12, 5, 13, 6, 14, 7, 15 }); +} + +/* Sign-extend the low / high two lanes of a 4x16 vector to 2x32. */ +/* + * Byte and halfword interleaves. unpackN.l(a, b) takes elements from the low + * halves, b first: {b[0], a[0], b[1], a[1], ...}; unpackN.h does the same with + * the high halves. Four of each transpose a 4x8 byte matrix, which is how the + * warp filters get turned from one-filter-per-row into one-tap-per-vector. + */ +SIMD_FN i8x8 unpack1_l(const i8x8 a, const i8x8 b) { + i8x8 r; + __asm__("unpack1.l %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i8x8 unpack1_h(const i8x8 a, const i8x8 b) { + i8x8 r; + __asm__("unpack1.h %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i8x8 unpack2_l(const i8x8 a, const i8x8 b) { + i8x8 r; + __asm__("unpack2.l %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i8x8 unpack2_h(const i8x8 a, const i8x8 b) { + i8x8 r; + __asm__("unpack2.h %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i8x8 pcmp1_gt(const i8x8 a, const i8x8 b) { + i8x8 r; + __asm__("pcmp1.gt %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +/* Sign-extend signed bytes to 16-bit lanes: pcmp1.gt against zero builds the + * 0xff/0x00 sign bytes, and unpack1 weaves them in as the high halves. */ +SIMD_FN i16x4 sext8_lo(const i8x8 v) { + return (i16x4)unpack1_l(pcmp1_gt((i8x8){ 0 }, v), v); +} + +SIMD_FN i16x4 sext8_hi(const i8x8 v) { + return (i16x4)unpack1_h(pcmp1_gt((i8x8){ 0 }, v), v); +} + +SIMD_FN i32x2 sext16_lo(const i16x4 v) { + i32x2 r; + i16x4 s; + __asm__("pcmp2.gt %0 = r0, %1" : "=r"(s) : "r"(v)); + __asm__("unpack2.l %0 = %1, %2" : "=r"(r) : "r"(s), "r"(v)); + return r; +} + +SIMD_FN i32x2 sext16_hi(const i16x4 v) { + i32x2 r; + i16x4 s; + __asm__("pcmp2.gt %0 = r0, %1" : "=r"(s) : "r"(v)); + __asm__("unpack2.h %0 = %1, %2" : "=r"(r) : "r"(s), "r"(v)); + return r; +} + +/* --------------------------------------------------------------- broadcast */ + +/* Replicate a runtime 16-bit scalar across four lanes. GCC builds this out of + * four dependent `dep` instructions; mux2 does it in one. */ +SIMD_FN i16x4 splat16(const int16_t x) { + i16x4 r; + __asm__("mux2 %0 = %1, 0" : "=r"(r) : "r"((int64_t)x)); + return r; +} + +/* Lane permute of a 4x16 vector. `sel` packs four 2-bit source indices, + * lane i taking its value from source lane (sel >> (2*i)) & 3. */ +#define mux2_perm(v, sel) __extension__ ({ \ + i16x4 mux2_r_; \ + __asm__("mux2 %0 = %1, " #sel : "=r"(mux2_r_) : "r"(v));\ + mux2_r_; \ +}) + +/* pmpy2_r/pmpy2_l leave results deinterleaved as {0,2} and {1,3}. After + * pack4_sss(lo, hi) the lanes read {0,2,1,3}; this puts them back in order. */ +SIMD_FN i16x4 unswizzle_rl(const i16x4 v) { + return mux2_perm(v, 0xd8); /* lanes {0,2,1,3} -> {0,1,2,3} */ +} + +/* Reduce four 32-bit accumulators held deinterleaved (as pmpy2.r / pmpy2.l + * leave them) to four in-order 16-bit lanes. */ +SIMD_FN i16x4 rsh_pack_rl(const i32x2 acc_r, const i32x2 acc_l, const int sh) { + return unswizzle_rl(pack4_sss(acc_r >> sh, acc_l >> sh)); +} + +/* + * unpack4 interleaves 32-bit halves: unpack4.l(a, b) is {b[0], a[0]} and + * unpack4.h(a, b) is {b[1], a[1]}. + * + * When the result has to stay 32 bits wide there is nothing to pack down to, + * so instead of unswizzle_rl the deinterleaved halves are woven back together + * into two in-order pairs: interleave_rl_lo(r, l) is lanes {0,1} and + * interleave_rl_hi(r, l) is lanes {2,3}. + */ +SIMD_FN i32x2 unpack4_l(const i32x2 a, const i32x2 b) { + i32x2 r; + __asm__("unpack4.l %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i32x2 unpack4_h(const i32x2 a, const i32x2 b) { + i32x2 r; + __asm__("unpack4.h %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i32x2 interleave_rl_lo(const i32x2 r, const i32x2 l) { + return unpack4_l(l, r); +} + +SIMD_FN i32x2 interleave_rl_hi(const i32x2 r, const i32x2 l) { + return unpack4_h(l, r); +} + +/* The inverse: take two in-order 32-bit pairs and split them into the {0,2} + * and {1,3} halves, so they can be combined with a pmpy2 result. */ +SIMD_FN i32x2 deinterleave_r(const i32x2 lo, const i32x2 hi) { + return unpack4_l(hi, lo); +} + +SIMD_FN i32x2 deinterleave_l(const i32x2 lo, const i32x2 hi) { + return unpack4_h(hi, lo); +} + +/* --------------------------------------------------------------- min / max */ + +/* + * Saturating packed 16-bit add/sub. GCC's generic vectors have no spelling + * for these, and they matter: for 8bpc the inverse transform's CLIP() bounds + * are exactly INT16_MIN/MAX, so saturation *is* the clip and costs nothing. + * arm64's itx uses sqadd/sqsub the same way. + */ +SIMD_FN i16x4 padd2_sss(const i16x4 a, const i16x4 b) { + i16x4 r; + __asm__("padd2.sss %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} +SIMD_FN i16x4 psub2_sss(const i16x4 a, const i16x4 b) { + i16x4 r; + __asm__("psub2.sss %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +/* + * (a * c0 + b * c1 + (1 << (sh - 1))) >> sh, four lanes at a time, narrowed + * back to 16 bits. This is arm64's smull/smlal + rshrn: values live in 16-bit + * lanes, only the multiply accumulates in 32. ia64 has no rounding + * multiply-shift (pmpyshr2's shift counts are fixed at 0/7/15/16 and it does + * not round), so the +rnd is an explicit padd4 -- one A-type op. + */ +SIMD_FN i16x4 itx_mul2(const i16x4 a, const i16x4 c0, + const i16x4 b, const i16x4 c1, const int sh) { + const int32_t h = 1 << (sh - 1); + const i32x2 rnd = { h, h }; + return rsh_pack_rl(pmpy2_r(a, c0) + pmpy2_r(b, c1) + rnd, + pmpy2_l(a, c0) + pmpy2_l(b, c1) + rnd, sh); +} +SIMD_FN i16x4 itx_mul1(const i16x4 a, const i16x4 c0, const int sh) { + const int32_t h = 1 << (sh - 1); + const i32x2 rnd = { h, h }; + return rsh_pack_rl(pmpy2_r(a, c0) + rnd, pmpy2_l(a, c0) + rnd, sh); +} + +SIMD_FN i16x4 pmin2(const i16x4 a, const i16x4 b) { + const i16x4 m = a < b; /* folded to a single pmin2 */ + return (a & m) | (b & ~m); +} + +SIMD_FN i16x4 pmax2(const i16x4 a, const i16x4 b) { + const i16x4 m = a > b; /* folded to a single pmax2 */ + return (a & m) | (b & ~m); +} + +SIMD_FN i16x4 iclip16(const i16x4 v, const i16x4 lo, const i16x4 hi) { + return pmin2(pmax2(v, lo), hi); +} + +/* There is no pmin4 / pmax4, so 32-bit lanes clamp through a compare mask. + * pcmp4.gt sets a lane to all-ones when a > b. */ +SIMD_FN i16x4 pcmp2_gt(const i16x4 a, const i16x4 b) { + i16x4 r; + __asm__("pcmp2.gt %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i32x2 pcmp4_gt(const i32x2 a, const i32x2 b) { + i32x2 r; + __asm__("pcmp4.gt %0 = %1, %2" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +SIMD_FN i32x2 sel32(const i32x2 mask, const i32x2 t, const i32x2 f) { + return (t & mask) | (f & ~mask); +} + +SIMD_FN i32x2 max0_32(const i32x2 v) { + return v & pcmp4_gt(v, (i32x2){ 0, 0 }); +} + +SIMD_FN i32x2 min32(const i32x2 v, const i32x2 hi) { + return sel32(pcmp4_gt(hi, v), v, hi); +} + +/* + * shrp funnel-shifts the 128-bit pair {hi:lo} right by an immediate number of + * bits. With a count that is a multiple of 16 this slides a four-lane window + * across a register boundary, which is the only way to build the shifted tap + * vectors of a FIR filter without re-loading the source. The count has to be + * an immediate, which is fine: every use is a fixed filter tap offset. + */ +#define shrp_v(hi, lo, cnt) __extension__ ({ \ + i16x4 shrp_r_; \ + __asm__("shrp %0 = %1, %2, " #cnt \ + : "=r"(shrp_r_) : "r"(hi), "r"(lo)); \ + shrp_r_; \ +}) + +/* Same instruction on plain 64-bit words. The count is stringified, so it has + * to be a constant expression -- the assembler folds it. */ +#define shrp64_v(hi, lo, cnt) __extension__ ({ \ + uint64_t shrp64_r_; \ + __asm__("shrp %0 = %1, %2, " #cnt \ + : "=r"(shrp64_r_) : "r"(hi), "r"(lo)); \ + shrp64_r_; \ +}) + +/* Same instruction, 32-bit lanes: shrp32_v(hi, lo, 32) slides the pair window + * along by one 32-bit element. */ +#define shrp32_v(hi, lo, cnt) __extension__ ({ \ + i32x2 shrp_r_; \ + __asm__("shrp %0 = %1, %2, " #cnt \ + : "=r"(shrp_r_) : "r"(hi), "r"(lo)); \ + shrp_r_; \ +}) + +/* ----------------------------------------------------------- memory access */ + +/* + * IA-64 traps on unaligned loads and stores; Linux emulates them in the kernel + * fault handler, which is far slower than doing the work by hand. So every + * access below is either provably aligned or goes through these helpers. + * + * The load is two aligned ld8 plus a funnel shift. The second load address is + * derived from the *last* byte we are entitled to read, so this never touches + * a byte outside [p, p+7] rounded out to the enclosing aligned words -- it can + * not fault on a page the caller does not already own. + * + * The shift amounts are loop-invariant along a row (the misalignment of a row + * pointer does not change as x advances), so GCC hoists them out. + */ +SIMD_FN u8x8 ldu_u8x8(const void *const p) { + const uintptr_t a = (uintptr_t)p; + const uint64_t off = (a & 7) * 8; + const uint64_t lo = *(const uint64_t *)(a & ~(uintptr_t)7); + const uint64_t hi = *(const uint64_t *)((a + 7) & ~(uintptr_t)7); + /* `hi << 1 << (63 - off)` is `hi << (64 - off)` without the UB at off==0, + * where it correctly contributes nothing and hi == lo. */ + return (u8x8)((lo >> off) | (hi << 1 << (63 - off))); +} + +SIMD_FN i16x4 ldu_i16x4(const void *const p) { + return (i16x4)ldu_u8x8(p); +} + +SIMD_FN i32x2 ldu_i32x2(const void *const p) { + return (i32x2)ldu_u8x8(p); +} + +/* Read exactly `n` bytes (n a compile-time constant at every call site) into + * the low lanes, zero above. Used by the narrow block sizes, where a full + * 8-byte load would run off the end of a w*h scratch buffer on the last row. */ +SIMD_FN u8x8 ldn_u8x8(const void *const p, const int n) { + uint64_t bits = 0; + memcpy(&bits, p, (size_t)n); + return (u8x8)bits; +} + +/* Aligned forms, for the many call sites where dav1d guarantees alignment + * (scratch buffers, the mid[] arrays, picture rows on the fast path). */ +SIMD_FN u8x8 ld_u8x8(const void *const p) { + return *(const u8x8 *)p; +} + +SIMD_FN i16x4 ld_i16x4(const void *const p) { + return *(const i16x4 *)p; +} + +SIMD_FN void st_u8x8(void *const p, const u8x8 v) { + *(u8x8 *)p = v; +} + +SIMD_FN void st_i16x4(void *const p, const i16x4 v) { + *(i16x4 *)p = v; +} + +SIMD_FN i32x2 ld_i32x2(const void *const p) { + return *(const i32x2 *)p; +} + +SIMD_FN void st_i32x2(void *const p, const i32x2 v) { + *(i32x2 *)p = v; +} + +SIMD_FN i32x2 ld_i32x2_sel(const void *const p, const int aligned) { + return aligned ? *(const i32x2 *)p : ldu_i32x2(p); +} + +SIMD_FN i16x4 ld_i16x4_sel(const void *const p, const int aligned) { + return aligned ? *(const i16x4 *)p : ldu_i16x4(p); +} + +/* Select between the aligned and unaligned load at compile time. Callers + * hoist the alignment test out of their row loop and pass a constant, so GCC + * specialises the loop body and the test costs nothing per iteration. This + * matters: the two-load funnel shift is only about five ops, but paying it on + * every operand of every iteration is enough to turn a 2x win into a loss. */ +SIMD_FN u8x8 ld_u8x8_sel(const void *const p, const int aligned) { + return aligned ? *(const u8x8 *)p : ldu_u8x8(p); +} + +/* Unaligned stores have no funnel-shift trick available. Callers hoist an + * alignment test out of their row loop and only land here for the odd case. */ +SIMD_FN void stu_u8x8(void *const p, const u8x8 v) { + if (!((uintptr_t)p & 7)) + *(u8x8 *)p = v; + else + memcpy(p, &v, 8); +} + +/* Store the low `n` (1..8) bytes. Used for the w==4 block sizes and for the + * ragged tail of a row. */ +SIMD_FN void st_u8_partial(void *const p, const u8x8 v, const int n) { + uint64_t bits; + memcpy(&bits, &v, 8); + if (n == 4 && !((uintptr_t)p & 3)) { + *(uint32_t *)p = (uint32_t)bits; + } else if (n == 8 && !((uintptr_t)p & 7)) { + *(uint64_t *)p = bits; + } else { + uint8_t *const d = (uint8_t *)p; + for (int i = 0; i < n; i++) + d[i] = (uint8_t)(bits >> (i * 8)); + } +} + +/* + * Store counterpart of ld_u8x8_sel(). Callers that already know the row's + * alignment pass a constant and GCC drops the test entirely. + * + * The unaligned arm deliberately goes through st_u8_partial() rather than + * memcpy(): a memcpy() of a compile-time-constant 8 bytes out of a vector + * register makes GCC spill the register to the stack and copy it back a byte + * at a time, which is *more* memory traffic than the scalar code a packed + * kernel is trying to replace. Extracting the bytes with shifts keeps the + * work in the I unit and leaves the two M ports for real loads. + */ +SIMD_FN void st_u8x8_sel(void *const p, const u8x8 v, const int aligned) { + if (aligned) *(u8x8 *)p = v; + else st_u8_partial(p, v, 8); +} + +#endif /* DAV1D_SRC_IA64_DAV1D_SIMD_H */ diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/README.md dav1d-1.5.4/src/ia64/gen/README.md --- dav1d-1.5.4.orig/src/ia64/gen/README.md 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/README.md 2026-08-25 18:25:51.650906620 +0200 @@ -0,0 +1,131 @@ +# ia64 Tier III kernel generators + +The three standalone assembly kernels in `src/ia64` are **generated**, and the +generators live here rather than outside the tree, so the checked-in sources are +the only thing needed to rebuild, rebase or improve them. Meson runs them as +part of the build (see the `ia64` branch of `src/meson.build`); the `.S` files +are build artifacts and are deliberately not checked in, so there is nothing to +drift out of sync. Editing generated assembly by hand has no effect -- edit the +generator and rebuild. + +| generator | produces | kernel | +|---|---|---| +| `gensgr_s.py` | `looprestoration_sgr.S` | SGR fused vertical-sum + `calc_ab` | +| `genfin_s.py` | `looprestoration_fin.S` | SGR finish filter (a/b combine) | +| `genbox_s.py` | `looprestoration_box.S` | SGR box horizontal rows (box3/box5/fused) | + +Support modules: + +| module | role | +|---|---| +| `modsched.py` | modulo scheduler: software-pipelines an op list at a given initiation interval | +| `sched.py` | op model + list scheduler, and the `Op` type the op-list builders use | +| `sgr_asm.py`, `fin_asm.py`, `boxh_asm.py` | the op lists themselves -- one per kernel, plus its register/fixup description | + +## Why these three are generated at all + +IA-64 is statically scheduled: the hardware issues bundles in order and does not +reorder around a stall, so the compiler -- or here, the generator -- owns the +schedule. Two facts drive the whole design: + +* Only **two I-slot ops** can issue per cycle, so schedule length is set by the + true I-type count (`pmpy`, `shrp`, `pack`, `unpack`, `mux`), not by the total + op count. A-type ops (`padd`, `psub`, `pcmp`, `pmin`, `pmax`) run on either + port and are placed in M slots. +* GCC gives an inline-asm block only the **24 scratch GRs**. These kernels have + dependence chains longer than their resource floor, so hiding the chain means + keeping several iterations in flight, which needs more registers than that. + A standalone `.S` has its own register frame (~90 GRs), which is the entire + reason these three are not inline asm like the rest of the port. + +The scheduler emits a `;;` after every scheduled cycle. That is what makes the +output safe to modify: only *same-cycle* independence is required, because the +hardware interlocks across a stop. Conservative latencies therefore cost speed, +never correctness. + +## Regenerating by hand + +Meson does this for you. To run one directly: + +```sh +python3 src/ia64/gen/genbox_s.py [output.S] # defaults to ./looprestoration_box.S +``` + +Only Python 3 is needed -- no third-party modules. Output is byte-reproducible: +the generators sort every collection they iterate, so the same input yields the +same file on any Python 3 version. + +## If you change a kernel + +1. **Assemble with `as -x`.** Bare `as` runs in auto mode, where it discards the + bundle templates and stops in the file and re-derives its own. That silently + costs a modulo-scheduled loop its whole initiation interval -- a designed + II=15 was measured running at 30 cycles. The compiler driver passes explicit + mode; a hand `as` invocation does not. +2. **Check the initiation interval** the generator prints against the resource + floor it also prints. If II has grown, the schedule regressed even if the + output is still correct. +3. **Run `checkasm`.** Note it does not relink automatically on every edit -- + confirm your symbol is really in the binary (`nm`) before trusting a pass. +4. Verify a decode md5 against a C-only build. `checkasm` covers these kernels, + but it is blind to edits made to the generic C templates around them. + +# Tier II kernels (inline asm inside the C templates) + +The `.S` files above are Tier III: their own register frame, modulo-scheduled, +generated by the build. Most of the port is **Tier II** -- hand-bundled inline +asm blocks living inside `looprestoration_tmpl.c` and `mc_tmpl.c`. Those were +generated by the scripts here too, then spliced into the templates once. + +They are deliberately *not* generated by the build. A Tier II block is a +starting point that gets hand-tuned in place, and the C around it is maintained +by hand, so making the generator authoritative would remove the ability to tweak +a block without editing its generator. The cost is that the two can drift, so +the link is checked instead of enforced: + +```sh +ninja ia64-check-generated +``` + +That regenerates each block and confirms it still appears verbatim in its +template. A `FAIL` is not automatically a bug -- it means that block has been +hand-edited since generation, and the generator is no longer the source of +truth for it. Either fold the change back into the generator or accept the +divergence knowingly. + +| generator | block | lands in | +|---|---|---| +| `final.py` | `sgr_calc_row_ab` body | `looprestoration_tmpl.c` | +| `gensgrw.py` | SGR weighted-sum, 8 wide | `looprestoration_tmpl.c` | +| `genwienh.py` | Wiener horizontal, 8 wide | `looprestoration_tmpl.c` | +| `genwien.py` | Wiener vertical, two groups | `looprestoration_tmpl.c` | +| `genmid12.py` | MC vertical, 12-wide mid tail | `mc_tmpl.c` | +| `genh16.py` | MC horizontal, 16 wide | `mc_tmpl.c` | +| `genmc6.py` | 6-tap MC kernels | `mc_tmpl.c` | + +Op lists for these live in `sgrw.py`, `wienh.py`, `wien.py`, `mc6.py` and +`mc.py`; `final.py` does register allocation onto the 24 scratch GRs and +`genfin.py` provides the emitter they share. + +Two generators here produce code that is **no longer in the tree**: +`genvert.py` (`newvert.c`) and `genfin.py`'s `finkernels.c` were the Tier II +versions of the SGR vertical and finish kernels, both since promoted to the +Tier III `.S` files above. They are kept because they are the reference for +what the promotion replaced, and because a Tier II fallback is what you would +start from if a future part made the register-frame version unattractive. +`genboxv.py` is a third: SGR box vertical was generated, measured 0.4% *slower* +than the C version because it is memory-bound, and was rejected. + +# Analysis tools (not part of any build) + +| tool | what it answers | +|---|---| +| `cand.py` | should a kernel move from Tier II to Tier III? Prints floor / chain / list-schedule / modulo-schedule lengths and the drain-amortised efficiency. Judge on `eff`, not on the modulo II -- a short II that has to drain every block is not a win. | +| `analyze.py` | slot occupancy and nop counts for a disassembly, to find where the bundles are going to waste | +| `press.py` | register-pressure sweep. A `SPILL` out of `final.py`'s allocator often just means the scheduler was too eager: sweep `PRESSURE_CAP` before concluding a kernel needs fewer values in flight. | +| `sim.py` | bounded-queue playback deadline simulation. Decode throughput is the wrong metric for playback: what matters is whether frame *i* is ready before its deadline given a shallow output queue. | + +The port's guiding constraint, in one line: only **two I-slot ops** issue per +cycle, so minimise I-type ops (`pmpy`, `shrp`, `pack`, `unpack`, `mux`) rather +than total ops, and keep A-type work (`padd`, `psub`, `pcmp`, `pmin`, `pmax`) in +the M slots where it is nearly free. diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/analyze.py dav1d-1.5.4/src/ia64/gen/analyze.py --- dav1d-1.5.4.orig/src/ia64/gen/analyze.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/analyze.py 2026-08-25 18:25:17.893776895 +0200 @@ -0,0 +1,72 @@ +"""Tier II vs Tier III discriminator. + +For an op list, the resource floor is + max(ceil(I/2), ceil((M+A)/4), ceil(loads/2), ceil(stores/2)) +and the dependence chain is the latency-weighted critical path. + + chain < floor -> already resource-bound. Cross-iteration pipelining + buys nothing; only merging adjacent asm blocks helps. + chain >= floor -> chain-bound. The bubbles can only be filled from the + next iteration, i.e. Tier III. +""" +import sys, math +sys.path.insert(0, '.') + +def analyze(name, ops, nelem): + I = M = A = ld = st = 0 + for o in ops: + f = o.fmt + if o.unit == 'M': + M += 1 + if f.lstrip().startswith(('ld', 'ldf')): ld += 1 + elif f.lstrip().startswith(('st', 'stf')): st += 1 + elif o.unit == 'I': I += 1 + elif o.unit == 'A': A += 1 + floor = max(-(-I // 2), -(-(M + A) // 4), -(-ld // 2), -(-st // 2)) + # critical path + h = [0] * len(ops) + by = {} + for i, o in enumerate(ops): + o.idx = i + by[o.dst] = i + for i, o in enumerate(ops): + t = 0 + for s in o.src: + j = by.get(s) + if j is not None and j < i: + t = max(t, h[j] + ops[j].lat) + h[i] = t + chain = max(h[i] + ops[i].lat for i in range(len(ops))) + verdict = 'TIER III (chain-bound)' if chain >= floor else 'tier II (resource-bound)' + print('%-22s ops=%-4d I=%-3d M=%-3d A=%-3d ld=%-3d st=%-2d | floor=%-3d chain=%-3d ' + 'ratio=%.2f %s' % (name, len(ops), I, M, A, ld, st, floor, chain, + chain / floor, verdict)) + if nelem: + print('%-22s floor/elem=%.2f chain/elem=%.2f' % ('', floor / nelem, chain / nelem)) + +import finish, boxh2, boxv2, mc, wien, vert + +print('--- finish (sgr_fin*) ---') +for nrows, sh in ((3, 9), (2, 9), (1, 8)): + ops = finish.build_finish(nrows, sh) + analyze('fin%d_%d' % (nrows, sh), ops, 4) + +print('--- box row_h ---') +for taps in (3, 5): + ops = boxh2.build_boxh(taps, ngrp=1) + analyze('box%d_h8 (1 grp)' % taps, ops, 8) + +print('--- box row_v ---') +for nrows in (3, 5): + analyze('box%d_row_v' % nrows, boxv2.build_boxv(nrows), 4) + +print('--- mc ---') +analyze('mc v8_mid4', mc.build_v8_mid4(), 4) +analyze('mc h8_core', mc.build_h8_core(), 8) + +print('--- wiener v (reference: known resource-bound) ---') +analyze('wiener_v', wien.build(), 4) + +print('--- sgr vert+calc_ab (now Tier III) ---') +for n in (9, 25): + analyze('vert_ab n=%d' % n, vert.build(n, 0), 4 * vert.NGRP) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/boxh2.py dav1d-1.5.4/src/ia64/gen/boxh2.py --- dav1d-1.5.4.orig/src/ia64/gen/boxh2.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/boxh2.py 2026-08-25 18:25:17.893490285 +0200 @@ -0,0 +1,57 @@ +import sys +sys.path.insert(0,'.') +import sched +from sched import Op + +def build_boxh(taps, ngrp=2): + ops=[] + def add(unit,fmt,dst,src,lat=1,extra=()): + o=Op(unit,fmt,dst,src,lat,0,extra); o.idx=len(ops); ops.append(o); return dst + for g in range(ngrp): + P='%%[ps%d]'%g; SM='%%[psum%d]'%g; Q0='%%[psq%d]'%(2*g); Q1='%%[psq%d]'%(2*g+1) + n=lambda s: s+'_%d'%g + add('A','and {d} = 7, '+P, n('o7'),[]) + add('A','shladd {d} = {%s}, 3, r0'%n('o7'), n('off'),[n('o7')]) + add('A','and {d} = -8, '+P, n('ab'),[]) + add('A','adds {d} = 7, '+P, n('a7'),[]) + add('A','and {d} = -8, {%s}'%n('a7'), n('ab2'),[n('a7')]) + add('A','sub {d} = 63, {%s}'%n('off'), n('r63'),[n('off')]) + add('M','ld8 {d} = [{%s}]'%n('ab'), n('lo'),[n('ab')],2) + add('M','ld8 {d} = [{%s}]'%n('ab2'), n('hi'),[n('ab2')],2) + add('I','shr.u {d} = {%s}, {%s}'%(n('lo'),n('off')), n('vl'),[n('lo'),n('off')]) + add('A','shladd {d} = {%s}, 1, r0'%n('hi'), n('h1'),[n('hi')]) + add('I','shl {d} = {%s}, {%s}'%(n('h1'),n('r63')), n('h2'),[n('h1'),n('r63')]) + add('A','or {d} = {%s}, {%s}'%(n('vl'),n('h2')), n('v'),[n('vl'),n('h2')]) + add('I','unpack1.l {d} = r0, {%s}'%n('v'), n('w0'),[n('v')]) + add('I','unpack1.h {d} = r0, {%s}'%n('v'), n('w1'),[n('v')]) + # window k is bytes k..k+3 of the eight; k==0 and k==4 land exactly on + # the low and high halves, so only the three in between need a shrp. + s=[n('w0')] + for k in range(1,taps): + if 16*k == 64: + s.append(n('w1')) + else: + s.append(add('I','shrp {d} = {%s}, {%s}, %d'%(n('w1'),n('w0'),16*k), + n('s%d'%k),[n('w1'),n('w0')])) + acc=s[0] + for k in range(1,taps): + acc=add('A','padd2 {d} = {%s}, {%s}'%(acc,s[k]), n('sm%d'%k),[acc,s[k]]) + add('M','st8 ['+SM+'] = {%s}'%acc,None,[acc]) + last={} + for half,mn in (('r','pmpy2.r'),('l','pmpy2.l')): + a=None + for k in range(taps): + q=add('I','%s {d} = {%s}, {%s}'%(mn,s[k],s[k]), n('q%s%d'%(half,k)),[s[k]],2) + a=q if a is None else add('A','padd4 {d} = {%s}, {%s}'%(a,q), n('ac%s%d'%(half,k)),[a,q]) + last[half]=a + add('I','unpack4.l {d} = {%s}, {%s}'%(last['l'],last['r']), n('z0'),[last['l'],last['r']]) + add('I','unpack4.h {d} = {%s}, {%s}'%(last['l'],last['r']), n('z1'),[last['l'],last['r']]) + add('M','st8 ['+Q0+'] = {%s}'%n('z0'),None,[n('z0')]) + add('M','st8 ['+Q1+'] = {%s}'%n('z1'),None,[n('z1')]) + return ops + +if __name__=='__main__': + for t in (3,5): + ops=build_boxh(t); cyc=sched.schedule(ops) + nI=sum(1 for o in ops if o.unit=='I') + print('taps=%d 2grp: ops=%d I=%d cycles=%d (%.1f per 4 outputs)'%(t,len(ops),nI,len(cyc),len(cyc)/2.0)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/boxh_asm.py dav1d-1.5.4/src/ia64/gen/boxh_asm.py --- dav1d-1.5.4.orig/src/ia64/gen/boxh_asm.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/boxh_asm.py 2026-08-25 18:19:12.703591143 +0200 @@ -0,0 +1,135 @@ +"""Tier III op list: fused SGR box3 + box5 horizontal row, 8 outputs of each. + +sgr_box35_row_h_q runs box3 and box5 over the *same* source row, and box3's +three taps at output i are src[i-1..i+1] -- exactly the middle three of box5's +src[i-2..i+2]. So one kernel computing both shares the whole front end: the +unaligned extraction, the unpack1s, all six shrps and all eighteen pmpy2 +squares. box3 costs only its own accumulate trees, unpack4s and stores. + +Per iteration (8 outputs of both): + I = 4 extract + 3 unpack1 + 6 shrp + 18 pmpy2 + 8 unpack4 = 39 -> floor 20 + A = 4 extract + 12 padd2 + 24 padd4 = 40 + M = 3 loads + 12 stores = 15 +i.e. 2.5 cycles per element for *both* boxes, against 21 + 26 = 47 cycles per +8 outputs for the two separate Tier II kernels. + +Because the pointer advances a whole 8 bytes per iteration the source +alignment is loop-invariant, so `off` and `63-off` are hoisted to the prologue +and only the shift/merge remains in the loop. +""" +import sys +sys.path.insert(0, '.') +from sched import Op + +LDL = 2 + + +def _build(groups): + ops = [] + def add(unit, fmt, dst, src, lat=1): + o = Op(unit, fmt, dst, src, lat, 0, ()) + o.idx = len(ops); ops.append(o); return dst + + def tree(vals, tag, op): + vals = list(vals); k = 0 + while len(vals) > 1: + nxt = [] + for j in range(0, len(vals) - 1, 2): + d = '%s_%d_%d' % (tag, k, j) + nxt.append(add('A', '%s {d} = {%s}, {%s}' % (op, vals[j], vals[j+1]), + d, [vals[j], vals[j+1]])) + if len(vals) % 2: nxt.append(vals[-1]) + vals = nxt; k += 1 + return vals[0] + + # ---- three aligned loads, private post-incrementing pointers ---- + l = [add('M', 'ld8 {d} = [{L%d}], 8' % j, 'l%d' % j, [], LDL) for j in range(3)] + + # ---- two unaligned 8-byte windows. The shladd-by-1 before the variable + # ---- shl is what makes off == 0 work: shl by 64 would be a no-op. + v = [] + for j in range(2): + vl = add('I', 'shr.u {d} = {%s}, {Coff}' % l[j], 'vl%d' % j, [l[j]]) + h1 = add('A', 'shladd {d} = {%s}, 1, r0' % l[j+1], 'h1_%d' % j, [l[j+1]]) + h2 = add('I', 'shl {d} = {%s}, {C63}' % h1, 'h2_%d' % j, [h1]) + v.append(add('A', 'or {d} = {%s}, {%s}' % (vl, h2), 'v%d' % j, [vl, h2])) + + # ---- twelve source bytes as three 4-halfword vectors ---- + w0 = add('I', 'unpack1.l {d} = r0, {%s}' % v[0], 'w0', [v[0]]) + w1 = add('I', 'unpack1.h {d} = r0, {%s}' % v[0], 'w1', [v[0]]) + w2 = add('I', 'unpack1.l {d} = r0, {%s}' % v[1], 'w2', [v[1]]) + + # W[j] = the four halfwords for source bytes j..j+3. Only the windows the + # groups actually reference are materialised, so box3 skips the shrps and + # the squares that only box5 needs. + need = sorted({j for _, rng, _, _, _ in groups for j in rng}) + W = {0: w0, 4: w1, 8: w2} + for k in (1, 2, 3): + if k in need: + W[k] = add('I', 'shrp {d} = {%s}, {%s}, %d' % (w1, w0, 16*k), 'W%d' % k, [w1, w0]) + if k + 4 in need: + W[k+4] = add('I', 'shrp {d} = {%s}, {%s}, %d' % (w2, w1, 16*k), 'W%d' % (k+4), [w2, w1]) + + # ---- squares, computed once and shared by every group that uses them ---- + q = {} + for j in need: + q[(j, 'r')] = add('I', 'pmpy2.r {d} = {%s}, {%s}' % (W[j], W[j]), 'qr%d' % j, [W[j]], 2) + q[(j, 'l')] = add('I', 'pmpy2.l {d} = {%s}, {%s}' % (W[j], W[j]), 'ql%d' % j, [W[j]], 2) + + for tag, rng, sp, qp0, qp1 in groups: + js = list(rng) + acc = tree([W[j] for j in js], 'sm' + tag, 'padd2') + add('M', 'st8 [{%s}] = {%s}, 16' % (sp, acc), None, [acc]) + half = {} + for h in ('r', 'l'): + half[h] = tree([q[(j, h)] for j in js], 'sq' + tag + h, 'padd4') + z0 = add('I', 'unpack4.l {d} = {%s}, {%s}' % (half['l'], half['r']), + 'z0' + tag, [half['l'], half['r']]) + z1 = add('I', 'unpack4.h {d} = {%s}, {%s}' % (half['l'], half['r']), + 'z1' + tag, [half['l'], half['r']]) + add('M', 'st8 [{%s}] = {%s}, 32' % (qp0, z0), None, [z0]) + add('M', 'st8 [{%s}] = {%s}, 32' % (qp1, z1), None, [z1]) + return ops + + +# group A = outputs 0..3 (windows start at source byte 0), group B = 4..7. +# box5 at output i covers bytes i..i+4, box3 covers i+1..i+3, so relative to +# box5's pointer box3's three taps are the middle three of box5's five. +def build(): + """Fused box3 + box5, sharing the front end. Pointer is box5's first tap.""" + return _build((('5a', range(0, 5), 'S5a', 'Q5a', 'Q5b'), + ('3a', range(1, 4), 'S3a', 'Q3a', 'Q3b'), + ('5b', range(4, 9), 'S5b', 'Q5c', 'Q5d'), + ('3b', range(5, 8), 'S3b', 'Q3c', 'Q3d'))) + + +def build_single(taps): + """One box size on its own, for the pure 3x3 / 5x5 filter paths, where + there is no second box to share the front end with. The pointer is that + box's own first tap, so the windows start at byte 0.""" + return _build((('a', range(0, taps), 'Sa', 'Qa', 'Qb'), + ('b', range(4, 4 + taps), 'Sb', 'Qc', 'Qd'))) + + +FIX = (['L0', 'L1', 'L2'] + + ['S3a', 'S3b', 'Q3a', 'Q3b', 'Q3c', 'Q3d'] + + ['S5a', 'S5b', 'Q5a', 'Q5b', 'Q5c', 'Q5d'] + + ['Coff', 'C63']) + +FIX_SINGLE = (['L0', 'L1', 'L2'] + + ['Sa', 'Sb', 'Qa', 'Qb', 'Qc', 'Qd'] + + ['Coff', 'C63']) + + +if __name__ == '__main__': + import modsched + for name, ops in (('box35 fused', build()), + ('box3 alone', build_single(3)), + ('box5 alone', build_single(5))): + nI = sum(1 for o in ops if o.unit == 'I') + nM = sum(1 for o in ops if o.unit == 'M') + nA = sum(1 for o in ops if o.unit == 'A') + II, t, fl = modsched.find_II(ops, nblk=32) + print('%-12s ops=%-3d I=%-3d M=%-3d A=%-3d II=%d (floor %d) stages=%d' + ' %.2f cyc/elem' % (name, len(ops), nI, nM, nA, II, fl, + max(t.values()) // II + 1, II / 8.0)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/boxv2.py dav1d-1.5.4/src/ia64/gen/boxv2.py --- dav1d-1.5.4.orig/src/ia64/gen/boxv2.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/boxv2.py 2026-08-25 18:25:17.893197616 +0200 @@ -0,0 +1,34 @@ +import sys +sys.path.insert(0,'.') +import sched +from sched import Op + +def build_boxv(nrows): + """Vertical box sum, four lanes. Not one I-type op in the whole kernel: + it is bounded by the two load ports, so the adds ride M slots alongside.""" + ops=[] + def add(unit,fmt,dst,src,lat=1): + o=Op(unit,fmt,dst,src,lat,0,()); o.idx=len(ops); ops.append(o); return dst + for k in range(nrows): + add('A','adds {d} = 8, %%[pq%d]'%k, 'qh%d'%k, []) + add('A','adds {d} = 8, %[oq]', 'oqh', []) + cs=[add('M','ld8 {d} = [%%[pc%d]]'%k, 'c%d'%k, [], 2) for k in range(nrows)] + a=cs[0] + for k in range(1,nrows): + a=add('A','padd2 {d} = {%s}, {%s}'%(a,cs[k]), 'sa%d'%k, [a,cs[k]]) + add('M','st8 [%%[oc]] = {%s}'%a, None, [a]) + for half,(addr,out) in enumerate((('%%[pq%d]','%[oq]'),('{qh%d}','{oqh}'))): + qs=[] + for k in range(nrows): + src=[] if half==0 else ['qh%d'%k] + qs.append(add('M','ld8 {d} = ['+(addr%k)+']', 'q%d_%d'%(k,half), src, 2)) + b=qs[0] + for k in range(1,nrows): + b=add('A','padd4 {d} = {%s}, {%s}'%(b,qs[k]), 'qa%d_%d'%(k,half), [b,qs[k]]) + add('M','st8 ['+out+'] = {%s}'%b, None, ([] if half==0 else ['oqh'])+[b]) + return ops + +if __name__=='__main__': + for r in (3,5): + ops=build_boxv(r); cyc=sched.schedule(ops) + print('rows=%d: ops=%d cycles=%d'%(r,len(ops),len(cyc))) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/cand.py dav1d-1.5.4/src/ia64/gen/cand.py --- dav1d-1.5.4.orig/src/ia64/gen/cand.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/cand.py 2026-08-25 18:25:17.893917831 +0200 @@ -0,0 +1,48 @@ +"""Tier III candidate screen. + +Compares, per element: + floor - the resource lower bound + sched - what the Tier II list scheduler actually achieves today + II - what a modulo schedule would achieve in steady state + eff - II amortised over a real row: (nblk + stages - 1) * II / nblk + +Only a kernel where `eff` beats `sched` by enough to pay for a separate .S +is worth promoting. +""" +import sys +sys.path.insert(0, '.') +import sched as S, modsched, finish, boxh2, boxv2, mc + +def floor_of(ops): + I = M = A = ld = st = 0 + for o in ops: + if o.unit == 'M': + M += 1 + f = o.fmt.lstrip() + if f.startswith('ld'): ld += 1 + elif f.startswith('st'): st += 1 + elif o.unit == 'I': I += 1 + elif o.unit == 'A': A += 1 + return max(-(-I//2), -(-(M+A)//4), -(-ld//2), -(-st//2)) + +def report(name, ops, nelem, nblk): + fl = floor_of(ops) + cyc = S.schedule(ops) + tii = len(cyc) + II, t, _fl = modsched.find_II(ops, nblk=nblk) + stages = max(t.values()) // II + 1 + eff = (nblk + stages - 1) * II / float(nblk) + print('%-18s n=%-3d floor=%-3d schedII=%-3d modII=%-3d st=%d | per-elem ' + 'floor %.2f tierII %.2f modulo %.2f eff@%d %.2f gain %+.0f%%' + % (name, nelem, fl, tii, II, stages, + fl/nelem, tii/nelem, II/nelem, nblk, eff/nelem, + 100.0*(tii-eff)/tii)) + +# nblk: iterations of the inner loop for a typical 256-wide restoration unit +report('fin3_9 (5.1%)', finish.build_finish(3, 9), 4, 64) +report('fin2_9 (3.6%)', finish.build_finish(2, 9), 4, 64) +report('fin1_8', finish.build_finish(1, 8), 4, 64) +report('box3_h8 (3.2%)', boxh2.build_boxh(3), 16, 16) +report('box5_h8 (3.8%)', boxh2.build_boxh(5), 16, 16) +report('box3_row_v', boxv2.build_boxv(3), 4, 64) +report('box5_row_v', boxv2.build_boxv(5), 4, 64) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/fin_asm.py dav1d-1.5.4/src/ia64/gen/fin_asm.py --- dav1d-1.5.4.orig/src/ia64/gen/fin_asm.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/fin_asm.py 2026-08-25 18:19:12.703688866 +0200 @@ -0,0 +1,139 @@ +"""Tier III op list: one 4-element group of the SGR finish filter. + +Differences from the Tier II list in finish.py: + + * No address arithmetic. Every load and the store owns a private + post-incrementing pointer stepping one whole iteration (8 bytes for the + coef rows, 16 for the int32 rows, 4 for the pixel row). Two accesses may + not share a pointer: modulo scheduling can put them in different stages, + and in steady state they would then be servicing different iterations + while both advancing the same register. + + * The rounding constant lives in a fixed register instead of an operand. + +That drops 3*nrows A-type address adds per group, which is what moves the +resource floor down; the chain is then spread across iterations by rotation. +""" +import sys +sys.path.insert(0, '.') +from sched import Op + +LDL = 2 # load-use latency estimate; ia64 interlocks, so this only tunes + + +def build(nrows, sh): + heavy4 = nrows == 3 + if nrows == 3: + heavy = [(1, 1), (1, 0), (1, 2), (0, 1), (2, 1)] + light = [(0, 0), (2, 0), (0, 2), (2, 2)] + elif nrows == 2: + heavy = [(0, 1), (1, 1)] + light = [(0, 0), (1, 0), (0, 2), (1, 2)] + else: + heavy = [(0, 1)] + light = [(0, 0), (0, 2)] + + ops = [] + def add(unit, fmt, dst, src, lat=1, extra=()): + o = Op(unit, fmt, dst, src, lat, 0, extra) + o.idx = len(ops); ops.append(o); return dst + + def tree(vals, tag, opadd): + vals = list(vals); k = 0 + while len(vals) > 1: + nxt = [] + for j in range(0, len(vals) - 1, 2): + d = '%s_t%d_%d' % (tag, k, j) + nxt.append(add('A', '%s {d} = {%s}, {%s}' % (opadd, vals[j], vals[j + 1]), + d, [vals[j], vals[j + 1]])) + if len(vals) % 2: nxt.append(vals[-1]) + vals = nxt; k += 1 + return vals[0] + + def mulw(v, w, tag, opadd, opshl): + if w == 4: + return add('I', '%s {d} = {%s}, 2' % (opshl, v), tag + '_m', [v]) + if w == 3: + t = add('A', '%s {d} = {%s}, {%s}' % (opadd, v, v), tag + '_d', [v]) + return add('A', '%s {d} = {%s}, {%s}' % (opadd, t, v), tag + '_m', [t, v]) + if w == 6: + t = add('A', '%s {d} = {%s}, {%s}' % (opadd, v, v), tag + '_d', [v]) + t3 = add('A', '%s {d} = {%s}, {%s}' % (opadd, t, v), tag + '_e', [t, v]) + return add('A', '%s {d} = {%s}, {%s}' % (opadd, t3, t3), tag + '_m', [t3, t3]) + if w == 5: + t = add('I', '%s {d} = {%s}, 2' % (opshl, v), tag + '_d', [v]) + return add('A', '%s {d} = {%s}, {%s}' % (opadd, t, v), tag + '_m', [t, v]) + raise ValueError(w) + + # ---- B side: coef rows, four 16-bit lanes ------------------------ + Bo = {} + for k in range(nrows): + l0 = add('M', 'ld8 {d} = [{Pb%d_0}], 8' % k, 'bl0_%d' % k, [], LDL) + l1 = add('M', 'ld8 {d} = [{Pb%d_1}], 8' % k, 'bl1_%d' % k, [], LDL) + Bo[(k, 0)] = l0 + Bo[(k, 1)] = add('I', 'shrp {d} = {%s}, {%s}, 16' % (l1, l0), 'bw1_%d' % k, [l1, l0]) + Bo[(k, 2)] = add('I', 'shrp {d} = {%s}, {%s}, 32' % (l1, l0), 'bw2_%d' % k, [l1, l0]) + hv = mulw(tree([Bo[p] for p in heavy], 'bh', 'padd2'), 4 if heavy4 else 6, + 'bh', 'padd2', 'pshl2') + lv = mulw(tree([Bo[p] for p in light], 'bl', 'padd2'), 3 if heavy4 else 5, + 'bl', 'padd2', 'pshl2') + aval = add('A', 'padd2 {d} = {%s}, {%s}' % (hv, lv), 'aval', [hv, lv]) + + # ---- A side: int32 rows, two 32-bit lanes, lo/hi halves separate -- + Ao = {} + for k in range(nrows): + m0 = add('M', 'ld8 {d} = [{Pa%d_0}], 16' % k, 'am0_%d' % k, [], LDL) + m1 = add('M', 'ld8 {d} = [{Pa%d_1}], 16' % k, 'am1_%d' % k, [], LDL) + m2 = add('M', 'ld8 {d} = [{Pa%d_2}], 16' % k, 'am2_%d' % k, [], LDL) + Ao[(k, 0, 'lo')] = m0; Ao[(k, 0, 'hi')] = m1 + Ao[(k, 2, 'lo')] = m1; Ao[(k, 2, 'hi')] = m2 + Ao[(k, 1, 'lo')] = add('I', 'shrp {d} = {%s}, {%s}, 32' % (m1, m0), + 'aw0_%d' % k, [m1, m0]) + Ao[(k, 1, 'hi')] = add('I', 'shrp {d} = {%s}, {%s}, 32' % (m2, m1), + 'aw1_%d' % k, [m2, m1]) + bb = {} + for half in ('lo', 'hi'): + hv = mulw(tree([Ao[(k, o, half)] for (k, o) in heavy], 'ah' + half, 'padd4'), + 4 if heavy4 else 6, 'ah' + half, 'padd4', 'pshl4') + lv = mulw(tree([Ao[(k, o, half)] for (k, o) in light], 'al' + half, 'padd4'), + 3 if heavy4 else 5, 'al' + half, 'padd4', 'pshl4') + bb[half] = add('A', 'padd4 {d} = {%s}, {%s}' % (hv, lv), 'b_' + half, [hv, lv]) + + # ---- pixels and the final narrow --------------------------------- + s = add('M', 'ld4 {d} = [{Ps}], 4', 'sld', [], LDL) + sv = add('I', 'unpack1.l {d} = r0, {%s}' % s, 'sv', [s]) + dr = add('I', 'unpack4.l {d} = {%s}, {%s}' % (bb['hi'], bb['lo']), 'dr', + [bb['hi'], bb['lo']]) + dl = add('I', 'unpack4.h {d} = {%s}, {%s}' % (bb['hi'], bb['lo']), 'dl', + [bb['hi'], bb['lo']]) + pr = add('I', 'pmpy2.r {d} = {%s}, {%s}' % (aval, sv), 'pr', [aval, sv], 2) + pl = add('I', 'pmpy2.l {d} = {%s}, {%s}' % (aval, sv), 'pl', [aval, sv], 2) + xr = add('A', 'psub4 {d} = {%s}, {%s}' % (dr, pr), 'xr', [dr, pr]) + xl = add('A', 'psub4 {d} = {%s}, {%s}' % (dl, pl), 'xl', [dl, pl]) + xr = add('A', 'padd4 {d} = {%s}, {Crnd}' % xr, 'xr2', [xr]) + xl = add('A', 'padd4 {d} = {%s}, {Crnd}' % xl, 'xl2', [xl]) + yr = add('I', 'pshr4 {d} = {%s}, %d' % (xr, sh), 'yr', [xr]) + yl = add('I', 'pshr4 {d} = {%s}, %d' % (xl, sh), 'yl', [xl]) + pk = add('I', 'pack4.sss {d} = {%s}, {%s}' % (yr, yl), 'pk', [yr, yl]) + rs = add('I', 'mux2 {d} = {%s}, 0xd8' % pk, 'rs', [pk]) + add('M', 'st8 [{Pd}] = {%s}, 8' % rs, None, [rs]) + return ops + + +def fixnames(nrows): + f = [] + for k in range(nrows): + f += ['Pb%d_0' % k, 'Pb%d_1' % k] + for k in range(nrows): + f += ['Pa%d_0' % k, 'Pa%d_1' % k, 'Pa%d_2' % k] + return f + ['Ps', 'Pd', 'Crnd'] + + +if __name__ == '__main__': + import modsched + for nrows, sh in ((3, 9), (2, 9), (1, 8)): + ops = build(nrows, sh) + II, t, fl = modsched.find_II(ops, nblk=64) + stages = max(t.values()) // II + 1 + print('fin%d_%d: ops=%d II=%d (floor %d) stages=%d %.2f cyc/elem' + % (nrows, sh, len(ops), II, fl, stages, II / 4.0)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/final.py dav1d-1.5.4/src/ia64/gen/final.py --- dav1d-1.5.4.orig/src/ia64/gen/final.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/final.py 2026-08-25 18:23:59.306265247 +0200 @@ -0,0 +1,184 @@ +import sys +from collections import defaultdict +import os.path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import sched + +ADDR = [] +EXTRA = [] +POOL_A = ['r%d'%i for i in range(14,28)] # 14 +POOL_B = ['r%d'%i for i in range(28,32)]+['r8','r9','r10','r11','r2','r3'] # 10 +CLOB = ADDR + POOL_A + POOL_B + +def clobs(asg): + used=sorted((r for r in set(asg.values()) if r.startswith('r')), + key=lambda r:int(r[1:])) + return ', '.join('"%s"'%r for r in used) + +def extra_decls(asg): + used=sorted(r for r in set(asg.values()) if r.startswith('%')) + names=[r[2:-1] for r in used] + if not names: return '', '' + binds = ',\n'.join(' [%s]"=&r"(%s)' % (n, n) for n in names) + return (' uint64_t %s;\n' % ', '.join(names), binds + '\n') + +def alloc(ops, cyc): + c_of={} + for c,ins in enumerate(cyc): + for i in ins: c_of[i]=c + lastuse=defaultdict(lambda:-1) + for o in ops: + for s in o.src: lastuse[s]=max(lastuse[s],c_of[o.idx]) + for e in o.extra: lastuse[e]=max(lastuse[e],c_of[o.idx]) + free={'p':list(POOL_A)+list(POOL_B)+list(EXTRA)} + busy=[]; assign={} + for o in sorted([o for o in ops if o.dst], key=lambda o:c_of[o.idx]): + c=c_of[o.idx] + for ent in list(busy): + if ent[0]>20 and the byte mask together; + * - the table index arithmetic is plain add, not addp4. + * Two groups run interleaved because one group's chain is ~30 cycles deep + * against ~%(nI2)d cycles of issue; alone it would idle both ports waiting. + * A third group schedules 1.13x better on paper but measured flat: it fits + * only by spending eight GCC-allocated temps, and its longer scalar tail + * (up to 11 elements instead of 7) gives back what the schedule wins. + * + * Registers are hard-allocated rather than left to GCC: 24 scratch GRs are + * needed and the operand list would otherwise blow past the 30-operand limit. + */ +SIMD_FN void sgr_calc_ab8_%(n)d(int32_t *const AA, coef *const BB, + const uint64_t pm, const uint64_t sv, + const uint64_t m15, const uint64_t r19, + const uint64_t r11, const uint8_t *const tbl) +{ +%(edecl)s __asm__ volatile( +%(body)s + : +%(ebind)s : [pa]"r"(AA), [pa1]"r"(AA + 2), [pa2]"r"(AA + 4), [pa3]"r"(AA + 6), + [pa4]"r"(AA + 8), [pa5]"r"(AA + 10), + [pb]"r"(BB), [pb2]"r"(BB + 4), [pb3]"r"(BB + 8), + [pm]"r"(pm), [sv]"r"(sv), [m15]"r"(m15), [r19]"r"(r19), + [r11]"r"(r11), [tbl]"r"(tbl) + : "memory", %(clob)s); +} +''' + +parts=[] +for n,obx in ((9,455),(25,164)): + b,nc,asg=body(n,obx) + nI=sum(1 for o in sched.build(n,obx) if o.unit=='I') + ed, eb = extra_decls(asg) + parts.append(FN % dict(n=n, nI=nI, nI2=(nI+1)//2, body='\n'.join(b), + edecl=ed, ebind=eb, clob=clobs(asg))) + sys.stderr.write('n=%d: %d cycles, %d bundles\n'%(n,nc,len([x for x in b if '.mmi' in x]))) + +WRAP = ''' +/* |n| and |one_by_x| arrive as compile-time constants so the hand-bundled + * kernel can specialise its shift/add chains on them. */ +SIMD_FN void sgr_calc_row_ab_body_q(int32_t *const AA, coef *const BB, + const int w, const int s, + const int n, const int one_by_x) +{ + /* + * p is clamped to pmax so that p * s + (1 << 19) cannot reach 256 << 20; + * that is what lets the kernel drop the umin(z, 255) and read the table + * index straight out of bits 20..27 of the product. + */ + const int32_t pmax = (int32_t)(((255 * (1 << 20)) - (1 << 19) + s - 1) / s); + const uint64_t pm = (uint64_t)(uint32_t)pmax * 0x0000000100000001ULL; + const uint64_t sv = (uint64_t)(uint16_t)s * 0x0001000100010001ULL; + const uint64_t m15 = 0x00007fff00007fffULL; + const uint64_t r19 = 0x0008000000080000ULL; + const uint64_t r11 = 0x0000080000000800ULL; + const uint8_t *const tbl = dav1d_sgr_x_by_x; + const int nw = w + 2; + + int i = 0; + if (n == 9) { + for (; i + 8 <= nw; i += 8) + sgr_calc_ab8_9(AA + i, BB + i, pm, sv, m15, r19, r11, tbl); + } else { + for (; i + 8 <= nw; i += 8) + sgr_calc_ab8_25(AA + i, BB + i, pm, sv, m15, r19, r11, tbl); + } + for (; i < nw; i++) { + const unsigned p = (unsigned)imax(AA[i] * n - BB[i] * BB[i], 0); + const unsigned z = (p * (unsigned)s + (1 << 19)) >> 20; + const unsigned x = dav1d_sgr_x_by_x[umin(z, 255)]; + AA[i] = (int32_t)((x * BB[i] * one_by_x + (1 << 11)) >> 12); + BB[i] = (coef)x; + } +} +''' +open('newbody2.c','w').write(''.join(parts)+WRAP) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/finish.py dav1d-1.5.4/src/ia64/gen/finish.py --- dav1d-1.5.4.orig/src/ia64/gen/finish.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/finish.py 2026-08-25 18:23:47.154195566 +0200 @@ -0,0 +1,119 @@ +import os.path +import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import sched +from sched import Op + +PFDIST=0 + +def build_finish(nrows, sh): + """One group of four outputs of the SGR finish filter. + nrows==3 is the eight-neighbour (4,3) form, nrows<=2 the six-neighbour + (6,5) form; nrows==1 is the second row of the 5x5 pass.""" + heavy4 = nrows == 3 + if nrows == 3: + heavy = [(1,1),(1,0),(1,2),(0,1),(2,1)]; light = [(0,0),(2,0),(0,2),(2,2)] + elif nrows == 2: + heavy = [(0,1),(1,1)]; light = [(0,0),(1,0),(0,2),(1,2)] + else: + heavy = [(0,1)]; light = [(0,0),(0,2)] + ops=[] + def add(unit,fmt,dst,src,lat=1,extra=()): + o=Op(unit,fmt,dst,src,lat,0,extra); o.idx=len(ops); ops.append(o); return dst + def tree(vals, tag): + """balanced sum, so the adds do not serialise""" + vals=list(vals); k=0 + while len(vals)>1: + nxt=[] + for j in range(0,len(vals)-1,2): + d='%s_t%d_%d'%(tag,k,j) + nxt.append(add('A','%s {d} = {%s}, {%s}'%(OPADD,vals[j],vals[j+1]),d,[vals[j],vals[j+1]])) + if len(vals)%2: nxt.append(vals[-1]) + vals=nxt; k+=1 + return vals[0] + def mulw(v, w, tag): + if w==4: return add('I','%s {d} = {%s}, 2'%(OPSHL,v), tag+'_m', [v]) + if w==3: + t=add('A','%s {d} = {%s}, {%s}'%(OPADD,v,v), tag+'_d', [v]) + return add('A','%s {d} = {%s}, {%s}'%(OPADD,t,v), tag+'_m', [t,v]) + if w==6: + t=add('A','%s {d} = {%s}, {%s}'%(OPADD,v,v), tag+'_d', [v]) + t3=add('A','%s {d} = {%s}, {%s}'%(OPADD,t,v), tag+'_e', [t,v]) + return add('A','%s {d} = {%s}, {%s}'%(OPADD,t3,t3), tag+'_m', [t3,t3]) + if w==5: + t=add('I','%s {d} = {%s}, 2'%(OPSHL,v), tag+'_d', [v]) + return add('A','%s {d} = {%s}, {%s}'%(OPADD,t,v), tag+'_m', [t,v]) + raise ValueError(w) + + # ---- optional prefetch: PFDIST passes ahead, issued into M slots that + # ---- would otherwise hold nop.m, so it is free at issue. + if PFDIST: + # Only the A rows: they are int32, so three times the bytes per pass + # of the coef rows, and six live address registers would not fit. + for k in range(nrows): + add('A','adds {d} = %d, %%[pa%d]'%(16*PFDIST,k), 'pfa%d'%k, []) + add('M','lfetch [{pfa%d}]'%k, None, ['pfa%d'%k]) + + # ---- address setup: base pointers arrive as operands, the +8/+16 halves + # ---- are derived here so the loop keeps only one IV per row. + for k in range(nrows): + add('A','adds {d} = 8, %%[pb%d]'%k, 'bB%d'%k, []) + add('A','adds {d} = 8, %%[pa%d]'%k, 'aB%d'%k, []) + add('A','adds {d} = 16, %%[pa%d]'%k,'aC%d'%k, []) + + # ---- B side: coef, four lanes, so one register per offset ---- + global OPADD, OPSHL + OPADD, OPSHL = 'padd2', 'pshl2' + Bo={} + for k in range(nrows): + l0=add('M','ld8 {d} = [%%[pb%d]]'%k, 'bl0_%d'%k, [], 2) + l1=add('M','ld8 {d} = [{bB%d}]'%k, 'bl1_%d'%k, ['bB%d'%k], 2) + Bo[(k,0)]=l0 + Bo[(k,1)]=add('I','shrp {d} = {%s}, {%s}, 16'%(l1,l0),'bw1_%d'%k,[l1,l0]) + Bo[(k,2)]=add('I','shrp {d} = {%s}, {%s}, 32'%(l1,l0),'bw2_%d'%k,[l1,l0]) + hv=mulw(tree([Bo[p] for p in heavy],'bh'), 4 if heavy4 else 6, 'bh') + lv=mulw(tree([Bo[p] for p in light],'bl'), 3 if heavy4 else 5, 'bl') + aval=add('A','padd2 {d} = {%s}, {%s}'%(hv,lv),'aval',[hv,lv]) + + # ---- A side: int32, two lanes, so lo/hi halves are separate registers ---- + OPADD, OPSHL = 'padd4', 'pshl4' + Ao={} + for k in range(nrows): + m0=add('M','ld8 {d} = [%%[pa%d]]'%k,'am0_%d'%k, [], 2) + m1=add('M','ld8 {d} = [{aB%d}]'%k, 'am1_%d'%k, ['aB%d'%k], 2) + m2=add('M','ld8 {d} = [{aC%d}]'%k, 'am2_%d'%k, ['aC%d'%k], 2) + Ao[(k,0,'lo')]=m0; Ao[(k,0,'hi')]=m1 + Ao[(k,2,'lo')]=m1; Ao[(k,2,'hi')]=m2 + Ao[(k,1,'lo')]=add('I','shrp {d} = {%s}, {%s}, 32'%(m1,m0),'aw0_%d'%k,[m1,m0]) + Ao[(k,1,'hi')]=add('I','shrp {d} = {%s}, {%s}, 32'%(m2,m1),'aw1_%d'%k,[m2,m1]) + bb={} + for half in ('lo','hi'): + hv=mulw(tree([Ao[(k,o,half)] for (k,o) in heavy],'ah'+half), 4 if heavy4 else 6, 'ah'+half) + lv=mulw(tree([Ao[(k,o,half)] for (k,o) in light],'al'+half), 3 if heavy4 else 5, 'al'+half) + bb[half]=add('A','padd4 {d} = {%s}, {%s}'%(hv,lv),'b_'+half,[hv,lv]) + + # ---- pixels and the final narrow ---- + s=add('M','ld4 {d} = [%[ps]]','sld',[],2) + sv=add('I','unpack1.l {d} = r0, {%s}'%s,'sv',[s]) + dr=add('I','unpack4.l {d} = {%s}, {%s}'%(bb['hi'],bb['lo']),'dr',[bb['hi'],bb['lo']]) + dl=add('I','unpack4.h {d} = {%s}, {%s}'%(bb['hi'],bb['lo']),'dl',[bb['hi'],bb['lo']]) + pr=add('I','pmpy2.r {d} = {%s}, {%s}'%(aval,sv),'pr',[aval,sv],2) + pl=add('I','pmpy2.l {d} = {%s}, {%s}'%(aval,sv),'pl',[aval,sv],2) + xr=add('A','psub4 {d} = {%s}, {%s}'%(dr,pr),'xr',[dr,pr]) + xl=add('A','psub4 {d} = {%s}, {%s}'%(dl,pl),'xl',[dl,pl]) + xr=add('A','padd4 {d} = {%s}, %%[rnd]'%xr,'xr2',[xr]) + xl=add('A','padd4 {d} = {%s}, %%[rnd]'%xl,'xl2',[xl]) + yr=add('I','pshr4 {d} = {%s}, %d'%(xr,sh),'yr',[xr]) + yl=add('I','pshr4 {d} = {%s}, %d'%(xl,sh),'yl',[xl]) + pk=add('I','pack4.sss {d} = {%s}, {%s}'%(yr,yl),'pk',[yr,yl]) + rs=add('I','mux2 {d} = {%s}, 0xd8'%pk,'rs',[pk]) + add('M','st8 [%%[pd]] = {%s}'%rs, None, [rs]) + return ops + +if __name__=='__main__': + for nrows,shv in ((3,9),(2,9),(1,8)): + ops=build_finish(nrows,shv); cyc=sched.schedule(ops) + nI=sum(1 for o in ops if o.unit=='I'); nM=sum(1 for o in ops if o.unit=='M') + nA=len(ops)-nI-nM + print('rows=%d sh=%d: ops=%d (I=%d M=%d A=%d) cycles=%d [floor I=%.1f MA=%.1f]'%( + nrows,shv,len(ops),nI,nM,nA,len(cyc),nI/2.0,(nM+nA)/4.0)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genbox.py dav1d-1.5.4/src/ia64/gen/genbox.py --- dav1d-1.5.4.orig/src/ia64/gen/genbox.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genbox.py 2026-08-25 18:25:17.893341297 +0200 @@ -0,0 +1,42 @@ +import sys +sys.path.insert(0,'.') +import sched, boxh2 +from final import alloc, CLOB +from genfin import emit + +HDR = ''' +/* + * Box filter horizontal pass, eight outputs per call, hand-bundled. + * + * The window is fetched with two aligned loads funnelled by the byte offset; + * only the two variable shifts of that sequence are I-type, everything else + * (including the *8 of the offset and the *2 of the high word) is folded into + * shladd, which is A-type and rides an M slot. The %d I-type ops -- the + * unpack1 pair, the window shrps, the pmpy2 squares and the final unpack4 -- + * are what the schedule is built around, two per cycle. + * + * Two groups run interleaved: one group's chain (offset -> load -> funnel -> + * unpack -> shrp -> pmpy2 -> padd4 -> unpack4 -> store) is deeper than its own + * issue requirement, so alone it would leave both I units idle most cycles. + */ +SIMD_FN void sgr_box%d_h8(const pixel *const s0, const pixel *const s1, + coef *const m0, coef *const m1, + int32_t *const q0, int32_t *const q1, + int32_t *const q2, int32_t *const q3) +{ + __asm__ volatile( +%s + : + : [ps0]"r"(s0), [ps1]"r"(s1), [psum0]"r"(m0), [psum1]"r"(m1), + [psq0]"r"(q0), [psq1]"r"(q1), [psq2]"r"(q2), [psq3]"r"(q3) + : "memory", %s); +} +''' +parts=[] +for t in (3,5): + ops=boxh2.build_boxh(t); cyc=sched.schedule(ops); asg=alloc(ops,cyc) + nI=sum(1 for o in ops if o.unit=='I') + parts.append(HDR % (nI, t, '\n'.join(emit(ops,cyc,asg)), + ', '.join('"%s"'%r for r in CLOB))) + sys.stderr.write('box%d: %d cycles / 8 outputs\n'%(t,len(cyc))) +open('boxkernels.c','w').write(''.join(parts)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genbox_s.py dav1d-1.5.4/src/ia64/gen/genbox_s.py --- dav1d-1.5.4.orig/src/ia64/gen/genbox_s.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genbox_s.py 2026-08-25 18:19:32.495078447 +0200 @@ -0,0 +1,162 @@ +import os.path +import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import modsched, boxh_asm + +HDR = '''/* + * Tier III: SGR box horizontal rows, modulo-scheduled. + * + * Generated by src/ia64/gen/genbox_s.py -- edit the generator, not this file. + * Meson regenerates it as part of the build; see src/ia64/gen/README.md. + * + * Three entry points, all built from one op-list template in boxh_asm.py: + * + * dav1d_sgr_box35_row_h_ia64 fused box3 + box5, for sgr_box35_row_h_q + * dav1d_sgr_box3_row_h_ia64 box3 alone, for the pure 3x3 filter path + * dav1d_sgr_box5_row_h_ia64 box5 alone, for the pure 5x5 filter path + * + * The fused one exists because sgr_box35_row_h_q runs both boxes over the same + * source row and box3's three taps at output x (src[x-1..x+1]) are exactly the + * middle three of box5's five (src[x-2..x+2]). Fusing shares the whole front + * end -- the unaligned extraction, the unpack1s, every shrp and every pmpy2 + * square -- so box3 costs only its own accumulate trees, unpack4s and stores: + * 20 cycles per 8 outputs of both, against 21 + 26 for the two separate Tier II + * kernels. The standalone pair have no second box to share with and run at + * 14 and 18 cycles per 8 outputs, against 21 and 26. + * + * Every variant produces 8 outputs per iteration, so the source pointer + * advances a whole 8 bytes and its alignment is loop-invariant; the shift + * amounts are hoisted into the prologue. The shladd-by-1 ahead of the + * variable shl is what makes offset 0 work: a shl by 64 would leave the word + * unchanged instead of clearing it. + * + * .explicit is mandatory: in gas's default auto mode the { .mmi } templates and + * the ';;' stops below are ignored and gas re-derives its own bundling. Check + * with `as -x`; the compiler driver does not pass it. + */ + + .explicit +''' + + +def emit(fname, tag, ops, FIX, NIN, rSRC, rNBLK, ptr_setup): + LP, LD = '.Lloop_%s' % tag, '.Ldone_%s' % tag + # ~32 iterations for a full-width restoration unit row + II, t_of, floor = modsched.find_II(ops, nblk=32) + rot, static, NROT, NBANK = modsched.assign_regs(ops, t_of, II) + stages = max(t_of.values()) // II + 1 + + assert NIN <= 8, NIN # IA-64 passes only out0-out7 in registers + NROT = max(NROT, -(-NIN // 8) * 8) + FIX_BASE = 32 + NROT + BANK_BASE = FIX_BASE + len(FIX) + SAVE_BASE = BANK_BASE + NBANK + rPFS, rLC, rT0, rT1 = (SAVE_BASE + i for i in range(4)) + SOF = NROT + len(FIX) + NBANK + 4 + assert SOF <= 96, (fname, SOF) + fixreg = {nm: FIX_BASE + i for i, nm in enumerate(FIX)} + tdef = {o.dst: t_of[o.idx] for o in ops if o.dst} + st = lambda t: t // II + + def rname(val, tu): + if val in rot: + base, _ = rot[val] + return 'r%d' % (base + st(tu) - st(tdef[val])) + return 'r%d' % (BANK_BASE + static[val]) + + def render(o): + s = o.fmt + if o.dst: s = s.replace('{d}', rname(o.dst, t_of[o.idx])) + for val in o.src: s = s.replace('{%s}' % val, rname(val, t_of[o.idx])) + for nm, r in fixreg.items(): s = s.replace('{%s}' % nm, 'r%d' % r) + assert '{' not in s, s + return '(p%d) %s' % (16 + st(t_of[o.idx]), s) + + body = [] + for c in range(II): + at = [i for i in range(len(ops)) if t_of[i] % II == c] + mlim = 3 if c == II - 1 else 4 + M = [i for i in at if ops[i].unit == 'M'] + I = [i for i in at if ops[i].unit == 'I'] + A = [i for i in at if ops[i].unit == 'A'] + while A and len(M) < mlim: M.append(A.pop(0)) + I += A + assert len(M) <= mlim and len(I) <= 2, (c, len(M), len(I)) + m = [render(ops[i]) for i in M] + ['nop.m 0'] * 4 + it = [render(ops[i]) for i in I] + ['nop.i 0'] * 2 + body.append('{ .mmi\n\t %s\n\t %s\n\t %s\n\t}' % (m[0], m[1], it[0])) + if c == II - 1: + body.append('{ .mib\n\t %s\n\t %s\n\t br.ctop.sptk.few %s\n\t}' + % (m[2], it[1], LP)) + else: + body.append('{ .mmi\n\t %s\n\t %s\n\t %s\n\t}' % (m[2], m[3], it[1])) + body.append(';;') + + s = [] + A = s.append + A('\t.text') + A('\t.align 32') + A('\t.global %s' % fname) + A('\t.hidden %s' % fname) # same DSO: direct br.call, no PLT stub or gp reload + A('\t.proc %s' % fname) + A('%s:' % fname) + A('\t.prologue') + A('\t.save ar.pfs, r%d' % rPFS) + A('\talloc r%d = ar.pfs, %d, %d, 0, %d' % (rPFS, NIN, SOF - NIN, NROT)) + A('\t.body') + A('\t/* src (in%d) already points at this kernel\'s first tap. The ins land' + % (rSRC - 32)) + A('\t in r32.., inside the rotating window, so they are all consumed here,') + A('\t before the first br.ctop. */') + A('\tmov r%d = ar.lc' % rLC) + A('\tcmp.ge p6, p0 = r0, r%d' % rNBLK) + A('\tadds r%d = -1, r%d' % (rT0, rNBLK)) + A('\t;;') + A('\tand r%d = 7, r%d' % (rT1, rSRC)) + A('\tand r%d = -8, r%d' % (fixreg['L0'], rSRC)) + A('\t;;') + A('\tshladd r%d = r%d, 3, r0' % (fixreg['Coff'], rT1)) + A('\tadds r%d = 8, r%d' % (fixreg['L1'], fixreg['L0'])) + A('\tadds r%d = 16, r%d' % (fixreg['L2'], fixreg['L0'])) + A('\t;;') + A('\tsub r%d = 63, r%d' % (fixreg['C63'], fixreg['Coff'])) + A('\tmov r%d = %d' % (rT1, stages)) + A('\t;;') + for base, ptrs in ptr_setup: + for i, nm in enumerate(ptrs): + if i == 0: A('\tmov r%d = r%d' % (fixreg[nm], base)) + else: A('\tadds r%d = %d, r%d' % (fixreg[nm], i * 8, base)) + A('\t;;') + A(' (p6)\tbr.cond.dpnt.few %s' % LD) + A('\t;;') + A('\tmov ar.lc = r%d' % rT0) + A('\tmov ar.ec = r%d' % rT1) + A('\tmov pr.rot = 0x10000') + A('\t;;') + A('%s:' % LP) + for b in body: A('\t' + b) + A('%s:' % LD) + A('\tmov ar.lc = r%d' % rLC) + A('\tmov ar.pfs = r%d' % rPFS) + A('\tbr.ret.sptk.many b0') + A('\t.endp %s' % fname) + sys.stderr.write('%-30s II=%2d (floor %2d) stages=%d rot=%d fix=%d bank=%d ' + 'sof=%d %.2f cyc/elem\n' + % (fname, II, floor, stages, NROT, len(FIX), NBANK, SOF, II / 8.0)) + return '\n'.join(s) + '\n' + + +out = HDR + '\n' +# fused: in0=sumsq3 in1=sum3 in2=sumsq5 in3=sum5 in4=src in5=nblk +out += emit('dav1d_sgr_box35_row_h_ia64', 'b35', boxh_asm.build(), boxh_asm.FIX, + 6, 36, 37, + ((33, ['S3a', 'S3b']), (35, ['S5a', 'S5b']), + (32, ['Q3a', 'Q3b', 'Q3c', 'Q3d']), + (34, ['Q5a', 'Q5b', 'Q5c', 'Q5d']))) + '\n' +# standalone: in0=sumsq in1=sum in2=src in3=nblk +for taps in (3, 5): + out += emit('dav1d_sgr_box%d_row_h_ia64' % taps, 'b%d' % taps, + boxh_asm.build_single(taps), boxh_asm.FIX_SINGLE, + 4, 34, 35, + ((33, ['Sa', 'Sb']), (32, ['Qa', 'Qb', 'Qc', 'Qd']))) + '\n' +open(sys.argv[1] if len(sys.argv) > 1 else 'looprestoration_box.S', 'w').write(out) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genboxv.py dav1d-1.5.4/src/ia64/gen/genboxv.py --- dav1d-1.5.4.orig/src/ia64/gen/genboxv.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genboxv.py 2026-08-25 18:25:17.893045812 +0200 @@ -0,0 +1,36 @@ +import sys +sys.path.insert(0,'.') +import sched, boxv2 +from final import alloc, CLOB +from genfin import emit + +HDR = ''' +/* + * Vertical box sum, four lanes per call, hand-bundled. There is no I-type op + * in this kernel at all -- it is nothing but loads, padd2/padd4 and stores -- + * so the limit is the two load ports and the adds ride the M slots of the same + * bundles. Written out by hand only because GCC put a stop after every one of + * the vector helpers. + */ +SIMD_FN void sgr_box%d_v4(%s + coef *const oc, int32_t *const oq) +{ + __asm__ volatile( +%s + : + : [oc]"r"(oc), [oq]"r"(oq), +%s + : "memory", %s); +} +''' +parts=[] +for r in (3,5): + ops=boxv2.build_boxv(r); cyc=sched.schedule(ops); asg=alloc(ops,cyc) + decl=''.join('const coef *const c%d, '%k for k in range(r))+'\n '+ \ + ''.join('const int32_t *const q%d, '%k for k in range(r))+'\n ' + bind=' '+', '.join('[pc%d]"r"(c%d)'%(k,k) for k in range(r))+',\n '+ \ + ', '.join('[pq%d]"r"(q%d)'%(k,k) for k in range(r)) + parts.append(HDR % (r, decl, '\n'.join(emit(ops,cyc,asg)), bind, + ', '.join('"%s"'%x for x in CLOB))) + sys.stderr.write('boxv%d: %d cycles\n'%(r,len(cyc))) +open('boxvkernels.c','w').write(''.join(parts)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genfin.py dav1d-1.5.4/src/ia64/gen/genfin.py --- dav1d-1.5.4.orig/src/ia64/gen/genfin.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genfin.py 2026-08-25 18:23:47.154698880 +0200 @@ -0,0 +1,92 @@ +import os.path +import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import sched, finish +from final import alloc, POOL_A, POOL_B, CLOB, clobs + + +COMPACT=False + +def bundles_for(ms, iss): + """Pick templates for one scheduled cycle. + + Two I-slot ops is the per-cycle ceiling either way, but an MII bundle + carries both in ONE bundle (slots M,I,I) where an MMI pair needs two. For + I-bound kernels that halves the code, which matters on a 16K L1I. MMI + pairs are only used when the cycle actually has M-slot work to place. + """ + out=[] + while ms or iss: + i0 = iss.pop(0) if iss else None + i1 = iss.pop(0) if iss else None + if COMPACT and i0 is not None and i1 is not None and len(ms) <= 1: + m0 = ms.pop(0) if ms else 'nop.m 0' + out.append(('mii', m0, i0, i1)) + else: + m0 = ms.pop(0) if ms else 'nop.m 0' + m1 = ms.pop(0) if ms else 'nop.m 0' + out.append(('mmi', m0, m1, i0 if i0 is not None else 'nop.i 0')) + if i1 is not None: iss.insert(0, i1) + if not ms and not iss: break + return out + +def emit(ops,cyc,asg): + out=[] + def txt(o): + s=o.fmt + if o.dst: s=s.replace('{d}',asg[o.dst]) + for k,v in asg.items(): s=s.replace('{%s}'%k,v) + assert '{' not in s, s + return s + for ins in cyc: + ms=[]; iss=[] + for i in ins: + o=ops[i] + if o.unit=='I': iss.append(txt(o)) + elif o.unit=='M': ms.append(txt(o)) + else: (ms if len(ms)<4 else iss).append(txt(o)) + for b in bundles_for(ms, iss): + if b[0]=='mii': + out.append(' "{ .mii\\n\\t %s\\n\\t %s\\n\\t %s\\n\\t}\\n\\t"'%(b[1],b[2],b[3])) + else: + out.append(' "{ .mmi\\n\\t %s\\n\\t %s\\n\\t %s\\n\\t}\\n\\t"'%(b[1],b[2],b[3])) + out.append(' ";;\\n\\t"') + return out + +HDR = ''' +/* + * SGR finish filter, four outputs per pass, hand-bundled. + * + * The neighbourhood weights are applied as balanced add trees rather than + * serial accumulations, and the two heavy weights (4 and 5) use a pshl while + * 3 and 6 are built from padd alone -- the split is chosen to keep the %(nI)d + * I-type ops (shrp, pmpy2, pack4, mux2, unpack) from outrunning the two I + * units while the A-type adds ride the M slots of the same MMI pairs. + * + * The B side is coef, so one register covers four lanes; the A side is 32-bit + * and needs a separate register per half, which is why it carries twice the + * adds for the same neighbourhood. + */ +SIMD_FN void sgr_fin%(nrows)d_%(sh)d(coef *const dst, const pixel *const src, +%(pdecl)s const i32x2 rnd) +{ + __asm__ volatile( +%(body)s + : + : [pd]"r"(dst), [ps]"r"(src), [rnd]"r"(rnd), +%(pbind)s + : "memory", %(clob)s); +} +''' + +parts=[] +for nrows,sh in ((3,9),(2,9),(1,8)): + ops=finish.build_finish(nrows,sh); cyc=sched.schedule(ops); asg=alloc(ops,cyc) + pdecl=''.join(' coef *const b%d, int32_t *const a%d,\n'%(k,k) for k in range(nrows)) + pbind=' '+', '.join('[pb%d]"r"(b%d), [pa%d]"r"(a%d)'%(k,k,k,k) for k in range(nrows)) + parts.append(HDR % dict(nrows=nrows, sh=sh, body='\n'.join(emit(ops,cyc,asg)), + pdecl=pdecl, pbind=pbind, + nI=sum(1 for o in ops if o.unit=='I'), + clob=clobs(asg))) + sys.stderr.write('rows=%d sh=%d -> %d cycles\n'%(nrows,sh,len(cyc))) +open('finkernels.c','w').write(''.join(parts)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genfin_s.py dav1d-1.5.4/src/ia64/gen/genfin_s.py --- dav1d-1.5.4.orig/src/ia64/gen/genfin_s.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genfin_s.py 2026-08-25 18:19:32.495792821 +0200 @@ -0,0 +1,166 @@ +import os.path +import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import modsched, fin_asm + +HDR = '''/* + * Tier III: SGR finish filter (the neighbourhood-weighted a/b combine), + * modulo-scheduled. + * + * Generated by src/ia64/gen/genfin_s.py -- edit the generator, not this file. + * Meson regenerates it as part of the build; see src/ia64/gen/README.md. + * + * As an inline-asm block this kernel ran at ~1.4x its resource floor: the + * dependence chain (load -> shrp -> weight tree -> pmpy2 -> narrow, ~17 + * cycles) is longer than the floor (13), so a per-block schedule has to drain + * it once per group of four outputs, and the 24 scratch GRs a GCC asm block + * gets are not enough to interleave a second group deeply enough to hide it. + * + * With its own register frame and br.ctop rotation the chain is spread across + * several in-flight iterations instead, and all three variants land exactly on + * their resource floor. Rotation also supplies the prologue fill and epilogue + * drain, so there is no hand-written prologue or epilogue. + * + * Every load and the store owns a private post-incrementing pointer stepping + * one whole iteration. Sharing one pointer between two accesses would break: + * the scheduler may place them in different stages, and in steady state they + * would then be servicing different iterations while both advancing the same + * register. + * + * .explicit is mandatory: in gas's default auto mode the { .mmi } templates + * and the ';;' stops below are *ignored* and gas re-derives its own bundling. + * Measured on this file, auto mode put a stop after every single bundle of the + * loop body, running a designed II=15 at 30 cycles. Assemble with -Wa,-x so the + * dependency checker also runs. + */ + + .explicit +''' + + +def emit(nrows, sh, fname, tag): + LP, LD = '.Lloop_%s' % tag, '.Ldone_%s' % tag + ops = fin_asm.build(nrows, sh) + # A finish row is w/4 iterations, ~64 for a full-width restoration unit, + # so the pipeline drain is cheap and the smallest feasible II wins here. + II, t_of, floor = modsched.find_II(ops, nblk=64) + rot, static, NROT, NBANK = modsched.assign_regs(ops, t_of, II) + stages = max(t_of.values()) // II + 1 + + FIX = fin_asm.fixnames(nrows) + # The IA-64 psABI passes only the first eight arguments in registers + # (out0-out7 = r32-r39); a ninth would arrive on the memory stack at + # [sp+16]. The three-row form needs six row pointers, so the rows come in + # as the caller's B_ptrs/A_ptrs arrays and are loaded here instead -- five + # arguments, and the same signature for all three variants. + NIN = 5 # dst, src, Bp, Ap, nblk + assert NIN <= 8, NIN + # The incoming args land in r32.., inside the rotating window. Make the + # window at least large enough to cover them so no fixed register aliases + # an argument that has not been consumed yet. + NROT = max(NROT, -(-NIN // 8) * 8) + FIX_BASE = 32 + NROT + BANK_BASE = FIX_BASE + len(FIX) + SAVE_BASE = BANK_BASE + NBANK + rPFS, rLC, rT0, rT1 = (SAVE_BASE + i for i in range(4)) + SOF = NROT + len(FIX) + NBANK + 4 + assert SOF <= 96, SOF + fixreg = {nm: FIX_BASE + i for i, nm in enumerate(FIX)} + tdef = {o.dst: t_of[o.idx] for o in ops if o.dst} + st = lambda t: t // II + + def rname(v, tu): + if v in rot: + base, _ = rot[v] + return 'r%d' % (base + st(tu) - st(tdef[v])) + return 'r%d' % (BANK_BASE + static[v]) + + def render(o): + s = o.fmt + if o.dst: s = s.replace('{d}', rname(o.dst, t_of[o.idx])) + for v in o.src: s = s.replace('{%s}' % v, rname(v, t_of[o.idx])) + for nm, r in fixreg.items(): s = s.replace('{%s}' % nm, 'r%d' % r) + assert '{' not in s, s + return '(p%d) %s' % (16 + st(t_of[o.idx]), s) + + body = [] + for c in range(II): + at = [i for i in range(len(ops)) if t_of[i] % II == c] + mlim = 3 if c == II - 1 else 4 + M = [i for i in at if ops[i].unit == 'M'] + I = [i for i in at if ops[i].unit == 'I'] + A = [i for i in at if ops[i].unit == 'A'] + while A and len(M) < mlim: M.append(A.pop(0)) + I += A + assert len(M) <= mlim and len(I) <= 2, (c, len(M), len(I)) + m = [render(ops[i]) for i in M] + ['nop.m 0'] * 4 + it = [render(ops[i]) for i in I] + ['nop.i 0'] * 2 + body.append('{ .mmi\n\t %s\n\t %s\n\t %s\n\t}' % (m[0], m[1], it[0])) + if c == II - 1: + body.append('{ .mib\n\t %s\n\t %s\n\t br.ctop.sptk.few %s\n\t}' + % (m[2], it[1], LP)) + else: + body.append('{ .mmi\n\t %s\n\t %s\n\t %s\n\t}' % (m[2], m[3], it[1])) + body.append(';;') + + rnd = (1 << (sh - 1)) * 0x0000000100000001 + + s = [] + A = s.append + A('\t.text') + A('\t.align 32') + A('\t.global %s' % fname) + A('\t.hidden %s' % fname) # same DSO: direct br.call, no PLT stub or gp reload + A('\t.proc %s' % fname) + A('%s:' % fname) + A('\t.prologue') + A('\t.save ar.pfs, r%d' % rPFS) + A('\talloc r%d = ar.pfs, %d, %d, 0, %d' % (rPFS, NIN, SOF - NIN, NROT)) + A('\t.body') + argn = ['dst', 'src', 'B_ptrs', 'A_ptrs', 'nblk'] + A('\t/* %s. The ins land in r32.., inside the rotating window, so' + % ' '.join('in%d=%s' % (i, n) for i, n in enumerate(argn))) + A('\t they are all consumed here, before the first br.ctop. */') + rNB = 32 + NIN - 1 + A('\tmov r%d = ar.lc' % rLC) + A('\tcmp.ge p6, p0 = r0, r%d' % rNB) + A('\tadds r%d = -1, r%d' % (rT0, rNB)) + A('\t;;') + A('\tmov r%d = r32' % fixreg['Pd']) + A('\tmov r%d = r33' % fixreg['Ps']) + A('\tmov r%d = %d' % (rT1, stages)) + A('\t;;') + for k in range(nrows): + A('\tld8 r%d = [r34], 8' % fixreg['Pb%d_0' % k]) + A('\tld8 r%d = [r35], 8' % fixreg['Pa%d_0' % k]) + A('\t;;') + A('\tadds r%d = 8, r%d' % (fixreg['Pb%d_1' % k], fixreg['Pb%d_0' % k])) + A('\tadds r%d = 8, r%d' % (fixreg['Pa%d_1' % k], fixreg['Pa%d_0' % k])) + A('\tadds r%d = 16, r%d' % (fixreg['Pa%d_2' % k], fixreg['Pa%d_0' % k])) + A('\t;;') + A('\tmovl r%d = 0x%016x' % (fixreg['Crnd'], rnd)) + A(' (p6)\tbr.cond.dpnt.few %s' % LD) + A('\t;;') + A('\tmov ar.lc = r%d' % rT0) + A('\tmov ar.ec = r%d' % rT1) + A('\tmov pr.rot = 0x10000') + A('\t;;') + A('%s:' % LP) + for b in body: A('\t' + b) + A('%s:' % LD) + A('\tmov ar.lc = r%d' % rLC) + A('\tmov ar.pfs = r%d' % rPFS) + A('\tbr.ret.sptk.many b0') + A('\t.endp %s' % fname) + sys.stderr.write('%s: II=%d (floor %d) stages=%d rot=%d fix=%d bank=%d sof=%d' + ' %.2f cyc/elem\n' + % (fname, II, floor, stages, NROT, len(FIX), NBANK, SOF, II / 4.0)) + return '\n'.join(s) + '\n' + + +out = HDR +for nrows, sh, fn, tag in ((3, 9, 'dav1d_sgr_finish_row3_ia64', 'f3'), + (2, 9, 'dav1d_sgr_finish_row2_ia64', 'f2'), + (1, 8, 'dav1d_sgr_finish_row1_ia64', 'f1')): + out += emit(nrows, sh, fn, tag) + '\n' +open(sys.argv[1] if len(sys.argv) > 1 else 'looprestoration_fin.S', 'w').write(out) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genh16.py dav1d-1.5.4/src/ia64/gen/genh16.py --- dav1d-1.5.4.orig/src/ia64/gen/genh16.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genh16.py 2026-08-25 18:23:47.151867170 +0200 @@ -0,0 +1,52 @@ +import sys +sys.path.insert(0, '.') +import sched, mc6 +from final import alloc, clobs +from genfin import emit + +sched.PRESSURE_CAP = 12 # see the comment in the emitted header +ops = mc6.build_h8_core8_n(4, range(6)) +cyc = sched.schedule(ops); asg = alloc(ops, cyc) +body = '\n'.join(emit(ops, cyc, asg)); cl = clobs(asg); nreg = len(set(asg.values())) + +win = '\n'.join(' [wa%d]"r"(wa%d), [wb%d]"r"(wb%d),' % (g, g, g, g) for g in range(4)) + +out = ''' +/* + * Sixteen outputs per call: four interleaved groups instead of two. + * + * Two groups leave the I units idle for the last few cycles of the reduction + * tree; four keep them fed, taking the horizontal pass from 2.250 to %.3f + * cycles per pixel. The catch is register pressure -- at the scheduler's + * default eagerness this needs more than the 24 scratch GRs exist and spills. + * Capping the in-flight set (PRESSURE_CAP=12 in the generator) makes it start + * the later groups a little further apart, which costs nothing in cycles here + * and brings it down to %d registers. + * + * Sixteen pixels is two window words per iteration, so the caller carries four + * words (hw0..hw3) rather than three and refills two at a time. + */ +SIMD_FN void h8_core16_6t(const u8x8 wa0, const u8x8 wb0, + const u8x8 wa1, const u8x8 wb1, + const u8x8 wa2, const u8x8 wb2, + const u8x8 wa3, const u8x8 wb3, + const i16x4 *const F, const i16x4 bias, const int shift, + i16x4 *const q0, i16x4 *const q1, + i16x4 *const q2, i16x4 *const q3) +{ + i16x4 r0, r1, r2, r3; + __asm__( +%s + : [res0]"=&r"(r0), [res1]"=&r"(r1), [res2]"=&r"(r2), [res3]"=r"(r3) + : +%s + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : %s); + *q0 = r0; *q1 = r1; *q2 = r2; *q3 = r3; +} +''' % (len(cyc) / 16.0, nreg, body, win, cl) +open('h16.c', 'w').write(out) +sys.stderr.write('h8_core16_6t: %d cycles / 16 px = %.3f c/px, %d regs\n' + % (len(cyc), len(cyc) / 16.0, nreg)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genmc6.py dav1d-1.5.4/src/ia64/gen/genmc6.py --- dav1d-1.5.4.orig/src/ia64/gen/genmc6.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genmc6.py 2026-08-25 18:23:47.152207598 +0200 @@ -0,0 +1,132 @@ +import sys +sys.path.insert(0, '.') +import sched, mc6 +from final import alloc, clobs +from genfin import emit + +def gen(ops): + cyc = sched.schedule(ops); asg = alloc(ops, cyc) + return '\n'.join(emit(ops, cyc, asg)), clobs(asg), len(cyc) + +b_h8, cl_h8, c_h8 = gen(mc6.build_h8_core8_n(2, range(6))) +b_h4, cl_h4, c_h4 = gen(mc6.build_h8_core8_n(1, range(6))) +b_v8, cl_v8, c_v8 = gen(mc6.build_v8_mid8_n(2, range(6))) +b_v4, cl_v4, c_v4 = gen(mc6.build_v8_mid8_n(1, range(6))) + +HDR = '''/* + * --------------------------------------------------- six-tap specialisations + * + * dav1d_mc_subpel_filters is nominally eight taps, but only the SHARP set uses + * all eight. REGULAR and SMOOTH have f[0] == f[7] == 0 at every one of the + * fifteen subpel positions, and the w <= 4 filter sets are narrower still, so + * for most blocks two of the eight taps multiply by zero. Encoders pick + * REGULAR or SMOOTH for the overwhelming majority of blocks -- on the clips + * measured here, for every single one. + * + * The callers detect this (fh[0] == 0 && fh[7] == 0), hand these kernels F + * shifted down by one tap, and advance the source by one column (h) or one row + * (v). That renumbers the live taps to 0..5 and lets a quarter of the + * multiplies disappear. The horizontal extraction loses a shrp as well, though + * not two: tap 5 is shrp(c1, c0, 32) and its operand c1 is itself tap 7, so c1 + * is still unpacked even though nothing multiplies it. The vertical pass + * simply never loads the two rows it stopped using, which also shortens the + * intermediate the hv path has to build from h+7 rows to h+5. + * + * Scheduled cost against the eight-tap kernels they replace: + * h8_core8 22 -> %d cycles / 8 px v8_mid8 24 -> %d cycles / 8 px + * h8_core 14 -> %d cycles / 4 px v8_mid4 16 -> %d cycles / 4 px + */ + +SIMD_FN void h8_core8_6t(const u8x8 wa0, const u8x8 wb0, + const u8x8 wa1, const u8x8 wb1, + const i16x4 *const F, const i16x4 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ + i16x4 r0, r1; + __asm__( +%s + : [res0]"=&r"(r0), [res1]"=r"(r1) + : [wa0]"r"(wa0), [wb0]"r"(wb0), [wa1]"r"(wa1), [wb1]"r"(wb1), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : %s); + *lo = r0; *hi = r1; +} + +SIMD_FN i16x4 h8_core_6t(const u8x8 wa0, const u8x8 wb0, + const i16x4 *const F, const i16x4 bias, const int shift) +{ + i16x4 r0; + __asm__( +%s + : [res0]"=r"(r0) + : [wa0]"r"(wa0), [wb0]"r"(wb0), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : %s); + return r0; +} + +/* |m| points at the row supplying tap 0, i.e. two rows above the output row. */ +SIMD_FN void v8_mid8_6t(const int16_t *const m, const i16x4 *const F, + const i32x2 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ +%s + i16x4 r0, r1; + __asm__( +%s + : [res0]"=&r"(r0), [res1]"=r"(r1) + : %s + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : %s); + *lo = r0; *hi = r1; +} + +SIMD_FN i16x4 v8_mid4_6t(const int16_t *const m, const i16x4 *const F, + const i32x2 bias, const int shift) +{ +%s + i16x4 r0; + __asm__( +%s + : [res0]"=r"(r0) + : %s + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : %s); + return r0; +} +''' + +def loads(ngrp): + L = [] + for g in range(ngrp): + for k in range(6): + L.append(' const i16x4 s%d_%d = ld_i16x4(m + %d * MID_STRIDE + %d);' + % (k, g, k, g * 4)) + return '\n'.join(L) + +def ops_in(ngrp): + L = [] + for g in range(ngrp): + for k in range(6): + L.append('[s%d_%d]"r"(s%d_%d),' % (k, g, k, g)) + out = [] + for i in range(0, len(L), 3): + out.append(' ' + ' '.join(L[i:i+3])) + return '\n'.join(out) + '\n' + +open('mc6kernels.c', 'w').write(HDR % ( + c_h8, c_v8, c_h4, c_v4, + b_h8, cl_h8, + b_h4, cl_h4, + loads(2), b_v8, ops_in(2), cl_v8, + loads(1), b_v4, ops_in(1), cl_v4)) +sys.stderr.write('generated: h8_core8_6t=%d h8_core_6t=%d v8_mid8_6t=%d v8_mid4_6t=%d\n' + % (c_h8, c_h4, c_v8, c_v4)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genmid12.py dav1d-1.5.4/src/ia64/gen/genmid12.py --- dav1d-1.5.4.orig/src/ia64/gen/genmid12.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genmid12.py 2026-08-25 18:23:47.151516770 +0200 @@ -0,0 +1,50 @@ +import sys +sys.path.insert(0, '.') +import sched, mc6 +from final import alloc, clobs +from genfin import emit + +ops = mc6.build_v8_mid8_n(3, range(6)) +cyc = sched.schedule(ops); asg = alloc(ops, cyc) +body = '\n'.join(emit(ops, cyc, asg)); cl = clobs(asg) +nops = 3 + 18 + 6 + 2 # outputs + sources + filters + bias/sh + +loads = '\n'.join(' const i16x4 s%d_%d = ld_i16x4(m + %d * MID_STRIDE + %d);' + % (k, g, k, g * 4) for g in range(3) for k in range(6)) +L = ['[s%d_%d]"r"(s%d_%d),' % (k, g, k, g) for g in range(3) for k in range(6)] +ins = '\n'.join(' ' + ' '.join(L[i:i+3]) for i in range(0, len(L), 3)) + +out = ''' +/* + * Twelve outputs per call. Two interleaved groups still spend their last four + * cycles walking the reduction tree with the I units mostly idle; a third group + * fills that tail, taking the vertical pass from 2.375 to %.3f cycles per pixel. + * + * Three is the ceiling, for two separate reasons that happen to bite together: + * a fourth group needs 24 source operands, and with the filters, the bias, the + * shift and four results that is 36 against GCC's limit of 30 -- and the whole + * point of this kernel is that the taps arrive in registers rather than being + * loaded behind a "memory" clobber. Twelve is also why the caller keeps the + * eight- and four-wide kernels: 12 does not divide 64 or 128, so the row ends + * with one of those. + */ +SIMD_FN void v8_mid12_6t(const int16_t *const m, const i16x4 *const F, + const i32x2 bias, const int shift, + i16x4 *const v0, i16x4 *const v1, i16x4 *const v2) +{ +%s + i16x4 r0, r1, r2; + __asm__( +%s + : [res0]"=&r"(r0), [res1]"=&r"(r1), [res2]"=r"(r2) + : %s + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : %s); + *v0 = r0; *v1 = r1; *v2 = r2; +} +''' % (len(cyc) / 12.0, loads, body, ins, cl) +open('mid12.c', 'w').write(out) +sys.stderr.write('v8_mid12_6t: %d cycles / 12 px = %.3f c/px, %d asm operands\n' + % (len(cyc), len(cyc) / 12.0, nops)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/gensgr_s.py dav1d-1.5.4/src/ia64/gen/gensgr_s.py --- dav1d-1.5.4.orig/src/ia64/gen/gensgr_s.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/gensgr_s.py 2026-08-25 18:19:32.496250337 +0200 @@ -0,0 +1,160 @@ +import os.path +import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import modsched, sgr_asm + +HDR = '''/* + * Tier III: SGR fused vertical-sum + calc_ab, modulo-scheduled. + * + * Generated by src/ia64/gen/gensgr_s.py -- edit the generator, not this file. + * Meson regenerates it as part of the build; see src/ia64/gen/README.md. + * + * Why this one is not an inline-asm block like the rest: the schedule is limited + * by how many values can be in flight, and a GCC asm block gets only the 24 + * scratch GRs. With its own register frame this kernel has ~90, which is what + * lets it be modulo-scheduled: one 4-element group per iteration at an + * initiation interval equal to the resource floor, with the ~30-cycle + * dependence chain spread across several in-flight iterations rather than + * drained once per block. + * + * br.ctop rotates the register file every trip and fills the prologue / drains + * the epilogue through the rotating predicates p16.., so there is no + * hand-written prologue or epilogue: a value produced in stage s and read in + * stage s+d is written as rN and read as rN+d. Values whose uses all fall in + * their own stage never cross a br.ctop and live in static registers instead, + * shared between values whose live ranges do not overlap. + * + * Loads and stores each own a private post-incrementing pointer stepping one + * whole iteration. Sharing one pointer between two accesses would break: the + * scheduler may place them in different stages, and in steady state they would + * then be serving different iterations while both advancing the same register. + * + * .explicit is mandatory: in gas's default auto mode the { .mmi } templates + * and the ';;' stops below are *ignored* and gas re-derives its own bundling. + * Measured on this file, auto mode put a stop after every single bundle of the + * loop body, running a designed II=15 at 30 cycles. Assemble with -Wa,-x so the + * dependency checker also runs. + */ + + .explicit +''' + +def emit(n, obx, fname): + LP, LD = '.Lloop%d' % n, '.Ldone%d' % n + ops = sgr_asm.build(n, obx) + II, t_of, floor = modsched.find_II(ops) + rot, static, NROT, NBANK = modsched.assign_regs(ops, t_of, II) + stages = max(t_of.values()) // II + 1 + NV = 3 if n == 9 else 5 + + FIX = ([('Pq%d_%d' % (j, k)) for j in range(NV) for k in range(2)] + + ['Ps%d' % j for j in range(NV)] + + ['Pa0', 'Pa1', 'Pb'] + + ['Cpm', 'Csv', 'Cm15', 'Cr11', 'Cr19', 'Ctbl']) + FIX_BASE = 32 + NROT + BANK_BASE = FIX_BASE + len(FIX) + SAVE_BASE = BANK_BASE + NBANK + rPFS, rLC, rT0, rT1 = (SAVE_BASE + i for i in range(4)) + SOF = NROT + len(FIX) + NBANK + 4 + assert SOF <= 96, SOF + fixreg = {nm: FIX_BASE + i for i, nm in enumerate(FIX)} + tdef = {o.dst: t_of[o.idx] for o in ops if o.dst} + st = lambda t: t // II + + def rname(v, tu): + if v in rot: + base, _ = rot[v] + return 'r%d' % (base + st(tu) - st(tdef[v])) + return 'r%d' % (BANK_BASE + static[v]) + + def render(o): + s = o.fmt + if o.dst: s = s.replace('{d}', rname(o.dst, t_of[o.idx])) + for v in o.src: s = s.replace('{%s}' % v, rname(v, t_of[o.idx])) + for nm, r in fixreg.items(): s = s.replace('{%s}' % nm, 'r%d' % r) + assert '{' not in s, s + return '(p%d) %s' % (16 + st(t_of[o.idx]), s) + + # ---- loop body --------------------------------------------------- + body = [] + for c in range(II): + at = [i for i in range(len(ops)) if t_of[i] % II == c] + mlim = 3 if c == II - 1 else 4 + M = [i for i in at if ops[i].unit == 'M'] + I = [i for i in at if ops[i].unit == 'I'] + A = [i for i in at if ops[i].unit == 'A'] + while A and len(M) < mlim: M.append(A.pop(0)) + I += A + assert len(M) <= mlim and len(I) <= 2, (c, len(M), len(I)) + m = [render(ops[i]) for i in M] + ['nop.m 0'] * 4 + it = [render(ops[i]) for i in I] + ['nop.i 0'] * 2 + if c == II - 1: + body.append('{ .mmi\n\t %s\n\t %s\n\t %s\n\t}' % (m[0], m[1], it[0])) + body.append('{ .mib\n\t %s\n\t %s\n\t br.ctop.sptk.few %s\n\t}' + % (m[2], it[1], LP)) + else: + body.append('{ .mmi\n\t %s\n\t %s\n\t %s\n\t}' % (m[0], m[1], it[0])) + body.append('{ .mmi\n\t %s\n\t %s\n\t %s\n\t}' % (m[2], m[3], it[1])) + body.append(';;') + + # ---- setup ------------------------------------------------------- + s = [] + A = s.append + A('\t.text') + A('\t.align 32') + A('\t.global %s' % fname) + A('\t.hidden %s' % fname) # same DSO: direct br.call, no PLT stub or gp reload + A('\t.proc %s' % fname) + A('%s:' % fname) + A('\t.prologue') + A('\t.save ar.pfs, r%d' % rPFS) + A('\talloc r%d = ar.pfs, 6, %d, 0, %d' % (rPFS, SOF - 6, NROT)) + A('\t.body') + A('\t/* in0=AA in1=BB in2=sumsq in3=sum in4=nblk in5=ctx.') + A('\t The ins land in r32.. which is inside the rotating window, so they') + A('\t are consumed before the first br.ctop. */') + A('\tmov r%d = ar.lc' % rLC) + A('\tcmp.ge p6, p0 = r0, r36') + A('\t;;') + A('\tmov r%d = r32' % fixreg['Pa0']) + A('\tadds r%d = 8, r32' % fixreg['Pa1']) + A('\tmov r%d = r33' % fixreg['Pb']) + A('\tadds r%d = -1, r36' % rT0) + A('\t;;') + for j in range(NV): + A('\tld8 r%d = [r34], 8' % fixreg['Pq%d_0' % j]) + A('\tld8 r%d = [r35], 8' % fixreg['Ps%d' % j]) + A('\t;;') + A('\tadds r%d = 8, r%d' % (fixreg['Pq%d_1' % j], fixreg['Pq%d_0' % j])) + A('\t;;') + for i, nm in enumerate(['Cpm', 'Csv', 'Cm15', 'Cr11', 'Ctbl']): + A('\tadds r%d = %d, r37' % (rT1, 8 * i)) + A('\t;;') + A('\tld8 r%d = [r%d]' % (fixreg[nm], rT1)) + A('\t;;') + A('\t/* r19 = r11 << 8; nothing crosses a lane boundary. */') + A('\tshl r%d = r%d, 8' % (fixreg['Cr19'], fixreg['Cr11'])) + A('\tmov r%d = %d' % (rT1, stages)) + A(' (p6)\tbr.cond.dpnt.few %s' % LD) + A('\t;;') + A('\tmov ar.lc = r%d' % rT0) + A('\tmov ar.ec = r%d' % rT1) + A('\tmov pr.rot = 0x10000') + A('\t;;') + A('%s:' % LP) + for b in body: A('\t' + b) + A('%s:' % LD) + A('\tmov ar.lc = r%d' % rLC) + A('\tmov ar.pfs = r%d' % rPFS) + A('\tbr.ret.sptk.many b0') + A('\t.endp %s' % fname) + sys.stderr.write('%s: II=%d (floor %d) stages=%d rot=%d bank=%d sof=%d\n' + % (fname, II, floor, stages, NROT, NBANK, SOF)) + return '\n'.join(s) + '\n' + + +out = HDR +for n, obx, fn in ((9, 455, 'dav1d_sgr_vert_ab_row9_ia64'), + (25, 164, 'dav1d_sgr_vert_ab_row25_ia64')): + out += emit(n, obx, fn) + '\n' +open(sys.argv[1] if len(sys.argv) > 1 else 'looprestoration_sgr.S', 'w').write(out) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/gensgrw.py dav1d-1.5.4/src/ia64/gen/gensgrw.py --- dav1d-1.5.4.orig/src/ia64/gen/gensgrw.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/gensgrw.py 2026-08-25 18:23:47.150481018 +0200 @@ -0,0 +1,56 @@ +import sys +sys.path.insert(0, '.') +import sched, sgrw +from final import alloc, clobs +from genfin import emit + +def gen(nw): + ops = sgrw.build(nw); cyc = sched.schedule(ops); asg = alloc(ops, cyc) + return '\n'.join(emit(ops, cyc, asg)), clobs(asg), len(cyc) + +b2, c2, n2 = gen(2) +b1, c1, n1 = gen(1) + +out = ''' +/* + * The two SGR weighting kernels, one scheduled region each. + * + * The arithmetic here is unremarkable -- a pair of 16x16->32 multiplies per + * lane, a rounding add, a narrowing shift and an add onto the destination + * pixels. What was expensive is that every one of those steps was a separate + * asm() helper (pmpy2_r, pmpy2_l, rsh_pack_rl, pack2_uss ...), and GCC types + * each asm() as TYPE_UNKNOWN, so each got its own bundle behind its own stop + * bit. The eight-pixel body measured 33 cycles against a 19-I-op floor of ten, + * with 23 nop.m and 8 nop.i of dead slots. + * + * Written as one region the same ops schedule in %d cycles (weighted2) and %d + * (weighted1), i.e. %.3f and %.3f cycles per pixel. + */ +SIMD_FN u8x8 sgr_w2_8(const u8x8 d, const i16x4 a0, const i16x4 a1, + const i16x4 b0, const i16x4 b1, + const i16x4 w0, const i16x4 w1, const i32x2 rnd) +{ + u8x8 out; + __asm__( +%s + : [out]"=r"(out) + : [d]"r"(d), [a0]"r"(a0), [a1]"r"(a1), [b0]"r"(b0), [b1]"r"(b1), + [w0]"r"(w0), [w1]"r"(w1), [rnd]"r"(rnd) + : %s); + return out; +} + +SIMD_FN u8x8 sgr_w1_8(const u8x8 d, const i16x4 a0, const i16x4 a1, + const i16x4 w0, const i32x2 rnd) +{ + u8x8 out; + __asm__( +%s + : [out]"=r"(out) + : [d]"r"(d), [a0]"r"(a0), [a1]"r"(a1), [w0]"r"(w0), [rnd]"r"(rnd) + : %s); + return out; +} +''' % (n2, n1, n2 / 8.0, n1 / 8.0, b2, c2, b1, c1) +open('sgrw8.c', 'w').write(out) +sys.stderr.write('sgr_w2_8=%d cycles, sgr_w1_8=%d cycles\n' % (n2, n1)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genvert.py dav1d-1.5.4/src/ia64/gen/genvert.py --- dav1d-1.5.4.orig/src/ia64/gen/genvert.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genvert.py 2026-08-25 18:25:17.892721922 +0200 @@ -0,0 +1,223 @@ +import sys +sys.path.insert(0, '.') +import sched, final, vert + +def body(n, obx, G): + vert.NGRP = G + for nex in range(0, 6): + final.EXTRA = ['%%[t%d]' % k for k in range(nex)] + ops = vert.build(n, obx) + cyc = sched.schedule(ops) + try: + asg = final.alloc(ops, cyc); break + except RuntimeError: + if nex == 5: raise + def txt(o): + s = o.fmt + if o.dst: s = s.replace('{d}', asg[o.dst]) + for k, v in asg.items(): s = s.replace('{%s}' % k, v) + assert '{' not in s, s + return s + out = [] + for ins in cyc: + ms = []; iss = [] + for i in ins: + o = ops[i] + if o.unit == 'I': iss.append(txt(o)) + elif o.unit == 'M': ms.append(txt(o)) + else: (ms if len(ms) < 4 else iss).append(txt(o)) + for b in final.bundles_for(ms, iss): + out.append(' "{ .%s\\n\\t %s\\n\\t %s\\n\\t %s\\n\\t}\\n\\t"' % b) + out.append(' ";;\\n\\t"') + return out, len(cyc), asg, ops + +FN = ''' +/* + * The vertical box sum fused into calc_ab, %(el)d elements per pass. + * + * Two loops with complementary slot profiles. calc_ab is I-bound: pmpy2 / + * pshl4 / pshr4 / mux2 / dep / extr.u are I-type ("Non-ALU integer", Table + * 4-1) and only the two I units take them, so its %(nI)d I-type ops floor the block + * at %(fl)d cycles -- during which more than half its M slots and most of its load + * slots would otherwise go to waste. The vertical sum is the opposite shape: + * %(nld)d loads and %(nA)d padd2/padd4, A-type and therefore riding the M slots, with + * not one I-type op. Fused, it issues almost entirely in holes calc_ab was + * leaving empty, and the intermediate row never reaches memory. + * + * %(ng)d groups run interleaved. That is what closes the gap between the %(fl)d-cycle + * resource floor and the ~30-cycle dependence chain of a single group: with + * fewer groups the schedule is chain-bound, not slot-bound. Going wider still + * would keep helping (%(cyc)d cycles here, %(fl)d is the floor) but %(ng)d groups already + * peaks at the 24 scratch GRs ia64 leaves a GCC asm block, which is the real + * ceiling -- see the note on sgr_vert_ab_body_q(). + * + * The %(nrow)d input rows are walked with post-incrementing ld8, and AA and BB are + * written with post-incrementing st8, so each costs one operand rather than + * one per group; that is what keeps the block inside GCC's 30-operand limit + * (a "+r" operand counts as two). + * + * padd2 wraps each lane modulo 2^16, exactly the truncation the old store to + * |coef *sum_out| performed, so the result is bit-identical. + */ +SIMD_FN void sgr_vert_ab%(el)d_%(n)d(int32_t *const AA, coef *const BB, +%(rowparams)s const uint64_t pm, const uint64_t sv, + const uint64_t m15, const uint64_t r11, + const uint8_t *const tbl) +{ + int32_t *a = AA; + coef *b = BB; +%(edecl)s __asm__ volatile( +%(body)s + : %(rwbind)s + : [pm]"r"(pm), [sv]"r"(sv), [m15]"r"(m15), + [r11]"r"(r11), [tbl]"r"(tbl) + : "memory", %(clob)s); +} +''' + +parts = [] +info = {} +for n, obx in ((9, 455), (25, 164)): + NV = 3 if n == 9 else 5 + for G in (2, 3): + el = 4 * G + b, nc, asg, ops = body(n, obx, G) + nI = sum(1 for o in ops if o.unit == 'I') + ed, _ = final.extra_decls(asg) + rowp = ''.join(' const int32_t *q%d,\n' % j for j in range(NV)) + rowp += ''.join(' const coef *s%d,\n' % j for j in range(NV)) + binds = ['[pa]"+r"(a)', '[pb]"+r"(b)'] + binds += ['[%s]"=&r"(%s)' % (t[2:-1], t[2:-1]) + for t in sorted(r for r in set(asg.values()) if r.startswith('%'))] + binds += ['[q%d]"+r"(q%d)' % (j, j) for j in range(NV)] + binds += ['[s%d]"+r"(s%d)' % (j, j) for j in range(NV)] + rw = ',\n '.join(binds) + parts.append(FN % dict(n=n, el=el, ng=G, nI=nI, fl=(nI + 1) // 2, cyc=nc, + nA=6 * (NV - 1) * G // 2, nld=NV * 3 * G // 2, + nrow=NV, body='\n'.join(b), rowparams=rowp, + rwbind=rw, edecl=ed, clob=final.clobs(asg))) + info[(n, G)] = (nc, nI, len(final.EXTRA)) + sys.stderr.write('n=%2d groups=%d: %2d cycles / %2d elem = %.2f (floor %d) extras=%d\n' + % (n, G, nc, el, nc / float(el), (nI + 1) // 2, len(final.EXTRA))) + +# The kernels post-increment their row pointers, but they take them by value, +# so the walk cannot be hoisted out of the loop: the update is discarded at the +# inlined call boundary. Recompute the bases from |i| each pass instead. +CALL9 = '''sgr_vert_ab%(el)d_9(AA + i, BB + i, q0, q1, q2, s0, s1, s2, + pm, sv, m15, r11, tbl)''' +CALL25 = '''sgr_vert_ab%(el)d_25(AA + i, BB + i, q0, q1, q2, q3, q4, + s0, s1, s2, s3, s4, + pm, sv, m15, r11, tbl)''' +ADV9 = ', '.join('%s += %%(el)d' % v for v in ('q0','q1','q2','s0','s1','s2')) +ADV25 = ', '.join('%s += %%(el)d' % v for v in + ('q0','q1','q2','q3','q4','s0','s1','s2','s3','s4')) + +WRAP = ''' +/* + * The fused vertical-sum + calc_ab pass. + * + * Two kernel widths. The 12-element kernel schedules better per element than + * the 8 (%(c12_9)d/12 vs %(c8_9)d/8 cycles for n == 9) because three interleaved groups + * fill more of the dependence-chain bubbles -- but it can only step by twelve, + * and a row is w + 2 elements wide, so used alone it would leave a longer + * scalar tail than it saves. The split below is therefore chosen up front to + * minimise that tail rather than greedily: for the common w == 64 it picks + * 4x12 + 2x8 and leaves 2 scalar elements, where a greedy 12-wide loop would + * leave 6. + * + * Both kernels peak at or just under the 24 scratch GRs an asm block gets, so + * a wider one would need its own register frame (a standalone .S with alloc), + * not just more unrolling. + */ +SIMD_FN void sgr_vert_ab_body_q(int32_t *const AA, coef *const BB, + int32_t *const *const sumsq, + coef *const *const sum, + const int w, const int s, + const int n, const int one_by_x) +{ + /* + * p is clamped to pmax so that p * s + (1 << 19) cannot reach 256 << 20; + * that is what lets the kernel drop the umin(z, 255) and read the table + * index straight out of bits 20..27 of the product. + */ + const int32_t pmax = (int32_t)(((255 * (1 << 20)) - (1 << 19) + s - 1) / s); + const uint64_t pm = (uint64_t)(uint32_t)pmax * 0x0000000100000001ULL; + const uint64_t sv = (uint64_t)(uint16_t)s * 0x0001000100010001ULL; + const uint64_t m15 = 0x00007fff00007fffULL; + const uint64_t r11 = 0x0000080000000800ULL; /* r19 = r11 << 8, in-kernel */ + const uint8_t *const tbl = dav1d_sgr_x_by_x; + const int nw = w + 2; + const int nrow = n == 9 ? 3 : 5; + + /* Pick the (n12, n8) split with the smallest scalar remainder, preferring + * more 12-wide passes on a tie. At most six trips. */ + int n12 = 0, rem = nw; + for (int a = nw / 12; a >= 0; a--) { + const int r = (nw - 12 * a) %% 8; + if (r < rem) { rem = r; n12 = a; if (!r) break; } + } + + int i = 0; + /* The kernels post-increment their row pointers, but they take them by + * value, so that update is discarded at the inlined call boundary and the + * walk has to live here. Advancing registers beats re-deriving + * |sumsq[j] + i|, which would reload the pointer array every pass. */ + if (n == 9) { + const int32_t *q0 = sumsq[0], *q1 = sumsq[1], *q2 = sumsq[2]; + const coef *s0 = sum[0], *s1 = sum[1], *s2 = sum[2]; + for (int k = 0; k < n12; k++, i += 12, %(adv12_9)s) %(call12_9)s; + for (; i + 8 <= nw; i += 8, %(adv8_9)s) %(call8_9)s; + } else { + const int32_t *q0 = sumsq[0], *q1 = sumsq[1], *q2 = sumsq[2], + *q3 = sumsq[3], *q4 = sumsq[4]; + const coef *s0 = sum[0], *s1 = sum[1], *s2 = sum[2], + *s3 = sum[3], *s4 = sum[4]; + for (int k = 0; k < n12; k++, i += 12, %(adv12_25)s) %(call12_25)s; + for (; i + 8 <= nw; i += 8, %(adv8_25)s) %(call8_25)s; + } + for (; i < nw; i++) { + /* The casts reproduce what the intermediate int32_t/coef row used to + * truncate to, which is also what padd4/padd2 do lane-wise. */ + uint32_t av = 0; uint16_t bv = 0; + for (int j = 0; j < nrow; j++) { + av += (uint32_t)sumsq[j][i]; + bv = (uint16_t)(bv + (uint16_t)sum[j][i]); + } + const int32_t a = (int32_t)av; + const int b = (coef)bv; + const unsigned p = (unsigned)imax(a * n - b * b, 0); + const unsigned z = (p * (unsigned)s + (1 << 19)) >> 20; + const unsigned x = dav1d_sgr_x_by_x[umin(z, 255)]; + AA[i] = (int32_t)((x * (unsigned)b * one_by_x + (1 << 11)) >> 12); + BB[i] = (coef)x; + } +} + +/* Falls back to the separate row_v + calc_row_ab pair for the bitdepths and + * strengths the fused kernel does not specialise for. */ +static NOINLINE void sgr_vert_ab_q(int32_t **sumsq, coef **sum, + int32_t *AA, coef *BB, const int w, + const int s, const int bitdepth_max, + const int n, const int one_by_x) +{ + if (bitdepth_from_max(bitdepth_max) - 8 == 0 && s > 0) { + if (n == 9 && one_by_x == 455) { + sgr_vert_ab_body_q(AA, BB, sumsq, sum, w, s, 9, 455); + return; + } + if (n == 25 && one_by_x == 164) { + sgr_vert_ab_body_q(AA, BB, sumsq, sum, w, s, 25, 164); + return; + } + } + if (n == 9) sgr_box3_row_v_q(sumsq, sum, AA, BB, w); + else sgr_box5_row_v_q(sumsq, sum, AA, BB, w); + sgr_calc_row_ab_q(AA, BB, w, s, bitdepth_max, n, one_by_x); +} +''' +open('newvert.c', 'w').write(''.join(parts) + WRAP % dict( + call12_9=CALL9 % dict(el=12), call8_9=CALL9 % dict(el=8), + call12_25=CALL25 % dict(el=12), call8_25=CALL25 % dict(el=8), + adv12_9=ADV9 % dict(el=12), adv8_9=ADV9 % dict(el=8), + adv12_25=ADV25 % dict(el=12), adv8_25=ADV25 % dict(el=8), + c12_9=info[(9, 3)][0], c8_9=info[(9, 2)][0])) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genwien.py dav1d-1.5.4/src/ia64/gen/genwien.py --- dav1d-1.5.4.orig/src/ia64/gen/genwien.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genwien.py 2026-08-25 18:23:47.151170521 +0200 @@ -0,0 +1,80 @@ +import sys +sys.path.insert(0, '.') +import sched, final, wien + +ops = wien.build() +cyc = sched.schedule(ops) +asg = final.alloc(ops, cyc) +def txt(o): + s = o.fmt + if o.dst: s = s.replace('{d}', asg[o.dst]) + for k, v in asg.items(): s = s.replace('{%s}' % k, v) + assert '{' not in s, s + return s +body = [] +for ins in cyc: + ms = []; iss = [] + for i in ins: + o = ops[i] + if o.unit == 'I': iss.append(txt(o)) + elif o.unit == 'M': ms.append(txt(o)) + else: (ms if len(ms) < 4 else iss).append(txt(o)) + for b in final.bundles_for(ms, iss): + body.append(' "{ .%s\\n\\t %s\\n\\t %s\\n\\t %s\\n\\t}\\n\\t"' % b) + body.append(' ";;\\n\\t"') +nI = sum(1 for o in ops if o.unit == 'I') +sys.stderr.write('wiener_v_8: %d ops (%d I -> floor %d cycles), scheduled %d cycles, %d bundles\n' + % (len(ops), nI, (nI + 1) // 2, len(cyc), + len([x for x in body if '.mm' in x or '.mi' in x]))) +open('newwien.c', 'w').write('\n'.join(body)) + +FN = ''' +/* + * Both halves of one 8-pixel step in a single scheduled block. + * + * The old code called wiener_v_4() twice per iteration. Each call is about + * fourteen cycles of dependent work -- pmpy2, three levels of padd4 tree, + * bias, pshr4, pack4.sss, mux2 -- and GCC cannot overlap two asm blocks, so + * they serialised to ~28 cycles for eight outputs. + * + * Interleaved, the pair issues %(nI)d I-type ops (28 pmpy2, 4 pshr4, 2 pack4.sss, + * 2 mux2, 1 pack2.uss), which at two I-slot ops per cycle floors the block at + * %(fl)d cycles; it schedules in %(cyc)d. The chain is shorter than that floor, so this + * is resource-bound and there is nothing further for cross-iteration + * pipelining to recover. Every padd4 is A-type and rides an M slot beside the + * multiplies -- note the bundles below are MMI pairs, not MII: two I-slot ops + * per cycle is the ceiling either way, and packing them into MII would only + * waste the M slots the tree needs. + */ +SIMD_FN u8x8 wiener_v_8(uint16_t *const *const ptrs, const uint16_t *const last, + const int i, const i16x4 *const F) +{ + const uint16_t *const r0 = ptrs[0] + i, *const r1 = ptrs[1] + i; + const uint16_t *const r2 = ptrs[2] + i, *const r3 = ptrs[3] + i; + const uint16_t *const r4 = ptrs[4] + i, *const r5 = ptrs[5] + i; + const uint16_t *const r6 = last + i; + const i16x4 v0_0 = ld_i16x4(r0), v1_0 = ld_i16x4(r0 + 4); + const i16x4 v0_1 = ld_i16x4(r1), v1_1 = ld_i16x4(r1 + 4); + const i16x4 v0_2 = ld_i16x4(r2), v1_2 = ld_i16x4(r2 + 4); + const i16x4 v0_3 = ld_i16x4(r3), v1_3 = ld_i16x4(r3 + 4); + const i16x4 v0_4 = ld_i16x4(r4), v1_4 = ld_i16x4(r4 + 4); + const i16x4 v0_5 = ld_i16x4(r5), v1_5 = ld_i16x4(r5 + 4); + const i16x4 v0_6 = ld_i16x4(r6), v1_6 = ld_i16x4(r6 + 4); + const i32x2 bias = { WIENER_V_BIAS, WIENER_V_BIAS }; + u8x8 out; + __asm__( +%(body)s + : [out]"=r"(out) + : [v0_0]"r"(v0_0), [v0_1]"r"(v0_1), [v0_2]"r"(v0_2), [v0_3]"r"(v0_3), + [v0_4]"r"(v0_4), [v0_5]"r"(v0_5), [v0_6]"r"(v0_6), + [v1_0]"r"(v1_0), [v1_1]"r"(v1_1), [v1_2]"r"(v1_2), [v1_3]"r"(v1_3), + [v1_4]"r"(v1_4), [v1_5]"r"(v1_5), [v1_6]"r"(v1_6), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), + [bias]"r"(bias), [sh]"i"(WIENER_V_SHIFT) + : %(clob)s); + return out; +} +''' +open('newwien.c','w').write(FN % dict(body='\n'.join(body), clob=final.clobs(asg), + nI=nI, fl=(nI+1)//2, cyc=len(cyc))) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/genwienh.py dav1d-1.5.4/src/ia64/gen/genwienh.py --- dav1d-1.5.4.orig/src/ia64/gen/genwienh.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/genwienh.py 2026-08-25 18:23:47.150831428 +0200 @@ -0,0 +1,49 @@ +import sys +sys.path.insert(0, '.') +import sched, wienh +from final import alloc, clobs +from genfin import emit + +ops = wienh.build_wiener_h(2) +cyc = sched.schedule(ops); asg = alloc(ops, cyc) +body = '\n'.join(emit(ops, cyc, asg)); cl = clobs(asg) + +out = ''' +/* + * Horizontal Wiener, eight outputs per call. + * + * Identical arithmetic to wiener_h_4(), two groups deep. Alone, a group's + * fourteen pmpy2 issue in seven cycles and then spend eleven more walking the + * padd4 tree, the bias, the pshr4, the pack4 and the mux2 -- 18 cycles for + * 4.5 per pixel. Interleaved the pair costs %d for %.3f, because the second + * group's multiplies fill the first group's tail. (Three groups would be + * better still on paper but spills: 21 of the 24 scratch GRs are already live + * here.) + * + * The window pair per group is the same as the four-wide kernel: |wa| covers + * b[0..7] and |wb| covers b[2..9], which is exactly the ten source bytes seven + * taps over four outputs touch. + */ +SIMD_FN void wiener_h_8(uint16_t *const dst, const pixel *const b, + const i16x4 *const F) +{ + const u8x8 wa0 = ldu_u8x8(b), wb0 = ldu_u8x8(b + 2); + const u8x8 wa1 = ldu_u8x8(b + 4), wb1 = ldu_u8x8(b + 6); + const i32x2 bias = { WIENER_H_BIAS, WIENER_H_BIAS }; + i16x4 r0, r1; + __asm__( +%s + : [res0]"=&r"(r0), [res1]"=r"(r1) + : [wa0]"r"(wa0), [wb0]"r"(wb0), [wa1]"r"(wa1), [wb1]"r"(wb1), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), + [bias]"r"(bias), [sh]"i"(WIENER_H_SHIFT) + : %s); + const i16x4 zero = { 0, 0, 0, 0 }; + const i16x4 clip = { WIENER_H_CLIP, WIENER_H_CLIP, WIENER_H_CLIP, WIENER_H_CLIP }; + st_i16x4(dst, iclip16(r0, zero, clip)); + st_i16x4(dst + 4, iclip16(r1, zero, clip)); +} +''' % (len(cyc), len(cyc) / 8.0, body, cl) +open('wienh8.c', 'w').write(out) +sys.stderr.write('wiener_h_8: %d cycles / 8 px = %.3f c/px\n' % (len(cyc), len(cyc) / 8.0)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/mc.py dav1d-1.5.4/src/ia64/gen/mc.py --- dav1d-1.5.4.orig/src/ia64/gen/mc.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/mc.py 2026-08-25 18:23:47.156103037 +0200 @@ -0,0 +1,151 @@ +import os.path +import sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import sched +from sched import Op + +def tree_build(add, vals, op, tag): + """balanced sum so the adds overlap the multiplies instead of trailing them""" + k=0 + while len(vals)>1: + nxt=[] + for j in range(0,len(vals)-1,2): + nxt.append(add('A','%s {d} = {%s}, {%s}'%(op,vals[j],vals[j+1]), + '%s_%d_%d'%(tag,k,j),[vals[j],vals[j+1]])) + if len(vals)%2: nxt.append(vals[-1]) + vals=nxt; k+=1 + return vals[0] + +def build_v8_mid4(): + ops=[] + def add(u,f,d,s,lat=1): + o=Op(u,f,d,s,lat,0,()); o.idx=len(ops); ops.append(o); return d + pr=[]; pl=[] + for k in range(8): + pr.append(add('I','pmpy2.r {d} = %%[s%d], %%[f%d]'%(k,k),'pr%d'%k,[],2)) + pl.append(add('I','pmpy2.l {d} = %%[s%d], %%[f%d]'%(k,k),'pl%d'%k,[],2)) + ar=tree_build(add,pr,'padd4','tr'); al=tree_build(add,pl,'padd4','tl') + ar=add('A','padd4 {d} = {%s}, %%[bias]'%ar,'br',[ar]) + al=add('A','padd4 {d} = {%s}, %%[bias]'%al,'bl',[al]) + ar=add('I','pshr4 {d} = {%s}, %%[sh]'%ar,'sr',[ar]) + al=add('I','pshr4 {d} = {%s}, %%[sh]'%al,'sl',[al]) + pk=add('I','pack4.sss {d} = {%s}, {%s}'%(ar,al),'pk',[ar,al]) + add('I','mux2 %%[res] = {%s}, 0xd8'%pk, None, [pk]) + return ops + +def build_h8_core(): + ops=[] + def add(u,f,d,s,lat=1): + o=Op(u,f,d,s,lat,0,()); o.idx=len(ops); ops.append(o); return d + a0=add('I','unpack1.l {d} = r0, %[wa]','a0',[]) + a1=add('I','unpack1.h {d} = r0, %[wa]','a1',[]) + c0=add('I','unpack1.l {d} = r0, %[wb]','c0',[]) + c1=add('I','unpack1.h {d} = r0, %[wb]','c1',[]) + s1=add('I','shrp {d} = {%s}, {%s}, 16'%(a1,a0),'s1',[a1,a0]) + s2=add('I','shrp {d} = {%s}, {%s}, 32'%(a1,a0),'s2',[a1,a0]) + s5=add('I','shrp {d} = {%s}, {%s}, 32'%(c1,c0),'s5',[c1,c0]) + s6=add('I','shrp {d} = {%s}, {%s}, 48'%(c1,c0),'s6',[c1,c0]) + taps=[a0,s1,s2,c0,a1,s5,s6,c1] + ps=[add('I','pmpyshr2 {d} = {%s}, %%[f%d], 0'%(taps[k],k),'p%d'%k,[taps[k]],2) + for k in range(8)] + a=tree_build(add,ps,'padd2','t') + a=add('A','padd2 {d} = {%s}, %%[bias]'%a,'bi',[a]) + add('I','pshr2 %%[res] = {%s}, %%[sh]'%a, None, [a]) + return ops + +if __name__=='__main__': + for nm,fn in (('v8_mid4',build_v8_mid4),('h8_core',build_h8_core)): + ops=fn(); cyc=sched.schedule(ops) + nI=sum(1 for o in ops if o.unit=='I') + print('%-9s ops=%d I=%d cycles=%d [floor I=%.1f]'%(nm,len(ops),nI,len(cyc),nI/2.0)) + +MID_STRIDE = 128 # int16 elements + +def build_v8_mid8(ngrp=2, loads_inside=True): + """Vertical 8-tap over the int16 intermediate, |ngrp|*4 outputs. + + Two groups because one group's chain runs three tree levels past its last + multiply, so alone it idles both I units for the tail. The loads come + inside for the same reason GCC's version pays for them separately: there + are 40 I-type ops to hide them under.""" + ops=[] + def add(u,f,d,s,lat=1): + o=Op(u,f,d,s,lat,0,()); o.idx=len(ops); ops.append(o); return d + for g in range(ngrp): + srcs=[] + for k in range(8): + off = k*MID_STRIDE*2 + g*8 + if loads_inside: + if off: + a=add('A','adds {d} = %d, %%[m]'%off,'ad%d_%d'%(k,g),[]) + srcs.append(add('M','ld8 {d} = [{%s}]'%a,'s%d_%d'%(k,g),[a],2)) + else: + srcs.append(add('M','ld8 {d} = [%[m]]','s%d_%d'%(k,g),[],2)) + else: + srcs.append(None) + pr=[]; pl=[] + for k in range(8): + sv = '{%s}'%srcs[k] if loads_inside else '%%[s%d_%d]'%(k,g) + dep = [srcs[k]] if loads_inside else [] + pr.append(add('I','pmpy2.r {d} = %s, %%[f%d]'%(sv,k),'pr%d_%d'%(k,g),dep,2)) + pl.append(add('I','pmpy2.l {d} = %s, %%[f%d]'%(sv,k),'pl%d_%d'%(k,g),dep,2)) + ar=tree_build(add,pr,'padd4','tr%d'%g); al=tree_build(add,pl,'padd4','tl%d'%g) + ar=add('A','padd4 {d} = {%s}, %%[bias]'%ar,'br%d'%g,[ar]) + al=add('A','padd4 {d} = {%s}, %%[bias]'%al,'bl%d'%g,[al]) + ar=add('I','pshr4 {d} = {%s}, %%[sh]'%ar,'sr%d'%g,[ar]) + al=add('I','pshr4 {d} = {%s}, %%[sh]'%al,'sl%d'%g,[al]) + pk=add('I','pack4.sss {d} = {%s}, {%s}'%(ar,al),'pk%d'%g,[ar,al]) + add('I','mux2 %%[res%d] = {%s}, 0xd8'%(g,pk), None, [pk]) + return ops + +def build_h8_core8(ngrp=2): + """Horizontal 8-tap, |ngrp|*4 pixels. The window registers arrive already + extracted, so the caller's 8-way alignment switch is untouched.""" + ops=[] + def add(u,f,d,s,lat=1): + o=Op(u,f,d,s,lat,0,()); o.idx=len(ops); ops.append(o); return d + for g in range(ngrp): + a0=add('I','unpack1.l {d} = r0, %%[wa%d]'%g,'a0_%d'%g,[]) + a1=add('I','unpack1.h {d} = r0, %%[wa%d]'%g,'a1_%d'%g,[]) + c0=add('I','unpack1.l {d} = r0, %%[wb%d]'%g,'c0_%d'%g,[]) + c1=add('I','unpack1.h {d} = r0, %%[wb%d]'%g,'c1_%d'%g,[]) + s1=add('I','shrp {d} = {%s}, {%s}, 16'%(a1,a0),'s1_%d'%g,[a1,a0]) + s2=add('I','shrp {d} = {%s}, {%s}, 32'%(a1,a0),'s2_%d'%g,[a1,a0]) + s5=add('I','shrp {d} = {%s}, {%s}, 32'%(c1,c0),'s5_%d'%g,[c1,c0]) + s6=add('I','shrp {d} = {%s}, {%s}, 48'%(c1,c0),'s6_%d'%g,[c1,c0]) + taps=[a0,s1,s2,c0,a1,s5,s6,c1] + ps=[add('I','pmpyshr2 {d} = {%s}, %%[f%d], 0'%(taps[k],k),'p%d_%d'%(k,g),[taps[k]],2) + for k in range(8)] + a=tree_build(add,ps,'padd2','t%d'%g) + a=add('A','padd2 {d} = {%s}, %%[bias]'%a,'bi%d'%g,[a]) + add('I','pshr2 %%[res%d] = {%s}, %%[sh]'%(g,a), None, [a]) + return ops + +def build_warp_h8(ngrp=2): + """Warp horizontal, 32-bit accumulate, |ngrp|*4 pixels. Each group carries + its own filter set: warp re-derives the taps between the two halves.""" + ops=[] + def add(u,f,d,s,lat=1): + o=Op(u,f,d,s,lat,0,()); o.idx=len(ops); ops.append(o); return d + for g in range(ngrp): + a0=add('I','unpack1.l {d} = r0, %%[wa%d]'%g,'a0_%d'%g,[]) + a1=add('I','unpack1.h {d} = r0, %%[wa%d]'%g,'a1_%d'%g,[]) + c0=add('I','unpack1.l {d} = r0, %%[wb%d]'%g,'c0_%d'%g,[]) + c1=add('I','unpack1.h {d} = r0, %%[wb%d]'%g,'c1_%d'%g,[]) + w1=add('I','shrp {d} = {%s}, {%s}, 16'%(a1,a0),'w1_%d'%g,[a1,a0]) + w2=add('I','shrp {d} = {%s}, {%s}, 32'%(a1,a0),'w2_%d'%g,[a1,a0]) + w5=add('I','shrp {d} = {%s}, {%s}, 32'%(c1,c0),'w5_%d'%g,[c1,c0]) + w6=add('I','shrp {d} = {%s}, {%s}, 48'%(c1,c0),'w6_%d'%g,[c1,c0]) + taps=[a0,w1,w2,c0,a1,w5,w6,c1] + pr=[];pl=[] + for k in range(8): + pr.append(add('I','pmpy2.r {d} = {%s}, %%[f%d_%d]'%(taps[k],k,g),'pr%d_%d'%(k,g),[taps[k]],2)) + pl.append(add('I','pmpy2.l {d} = {%s}, %%[f%d_%d]'%(taps[k],k,g),'pl%d_%d'%(k,g),[taps[k]],2)) + ar=tree_build(add,pr,'padd4','wr%d'%g); al=tree_build(add,pl,'padd4','wl%d'%g) + ar=add('A','padd4 {d} = {%s}, %%[bias]'%ar,'wbr%d'%g,[ar]) + al=add('A','padd4 {d} = {%s}, %%[bias]'%al,'wbl%d'%g,[al]) + ar=add('I','pshr4 {d} = {%s}, %%[sh]'%ar,'wsr%d'%g,[ar]) + al=add('I','pshr4 {d} = {%s}, %%[sh]'%al,'wsl%d'%g,[al]) + pk=add('I','pack4.sss {d} = {%s}, {%s}'%(ar,al),'wpk%d'%g,[ar,al]) + add('I','mux2 %%[res%d] = {%s}, 0xd8'%(g,pk), None, [pk]) + return ops diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/mc6.py dav1d-1.5.4/src/ia64/gen/mc6.py --- dav1d-1.5.4.orig/src/ia64/gen/mc6.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/mc6.py 2026-08-25 18:23:47.150132966 +0200 @@ -0,0 +1,79 @@ +import sys +sys.path.insert(0, '.') +import sched +from sched import Op +from mc import tree_build, MID_STRIDE + +def _adder(ops): + def add(u, f, d, s, lat=1): + o = Op(u, f, d, s, lat, 0, ()); o.idx = len(ops); ops.append(o); return d + return add + +# tap k -> extracted window register supplying it +# a0 = unpack1.l(wa) a1 = unpack1.h(wa) c0 = unpack1.l(wb) c1 = unpack1.h(wb) +# s1,s2 = shrp(a1,a0,16/32) s5,s6 = shrp(c1,c0,32/48) +_TAPSRC = ['a0', 's1', 's2', 'c0', 'a1', 's5', 's6', 'c1'] +_SHRP = {'s1': ('a1', 'a0', 16), 's2': ('a1', 'a0', 32), + 's5': ('c1', 'c0', 32), 's6': ('c1', 'c0', 48)} + +def build_h8_core8_n(ngrp=2, taps=range(8)): + """Horizontal, |ngrp|*4 pixels, multiplying only |taps|. + + Only the extraction ops the live taps actually reach are emitted. A dropped + tap can still be needed as a shrp operand -- tap 5 is shrp(c1, c0, 32) and c1 + is itself tap 7 -- so the dead set is computed, not assumed.""" + taps = list(taps); ops = []; add = _adder(ops) + need = set(_TAPSRC[k] for k in taps) + for v in list(need): + if v in _SHRP: + need.add(_SHRP[v][0]); need.add(_SHRP[v][1]) + for g in range(ngrp): + r = {} + for v, w, sl in (('a0', 'a', 'l'), ('a1', 'a', 'h'), + ('c0', 'b', 'l'), ('c1', 'b', 'h')): + if v in need: + r[v] = add('I', 'unpack1.%s {d} = r0, %%[w%s%d]' % (sl, w, g), + '%s_%d' % (v, g), []) + for v in ('s1', 's2', 's5', 's6'): + if v in need: + hi, lo, n = _SHRP[v] + r[v] = add('I', 'shrp {d} = {%s}, {%s}, %d' % (r[hi], r[lo], n), + '%s_%d' % (v, g), [r[hi], r[lo]]) + ps = [add('I', 'pmpyshr2 {d} = {%s}, %%[f%d], 0' % (r[_TAPSRC[k]], k), + 'p%d_%d' % (k, g), [r[_TAPSRC[k]]], 2) for k in taps] + a = tree_build(add, ps, 'padd2', 't%d' % g) + a = add('A', 'padd2 {d} = {%s}, %%[bias]' % a, 'bi%d' % g, [a]) + add('I', 'pshr2 %%[res%d] = {%s}, %%[sh]' % (g, a), None, [a]) + return ops + +def build_v8_mid8_n(ngrp=2, taps=range(8)): + """Vertical over the int16 intermediate, |ngrp|*4 outputs. + + The tap rows arrive as operands (loaded in C, as the eight-tap kernel does) + so the asm needs no "memory" clobber -- that would force GCC to spill + everything live across a kernel that sits in the innermost loop. A dropped + tap costs nothing here: its row is simply never loaded.""" + taps = list(taps); ops = []; add = _adder(ops) + for g in range(ngrp): + pr = []; pl = [] + for k in taps: + sv = '%%[s%d_%d]' % (k, g) + pr.append(add('I', 'pmpy2.r {d} = %s, %%[f%d]' % (sv, k), 'pr%d_%d' % (k, g), [], 2)) + pl.append(add('I', 'pmpy2.l {d} = %s, %%[f%d]' % (sv, k), 'pl%d_%d' % (k, g), [], 2)) + ar = tree_build(add, pr, 'padd4', 'tr%d' % g); al = tree_build(add, pl, 'padd4', 'tl%d' % g) + ar = add('A', 'padd4 {d} = {%s}, %%[bias]' % ar, 'br%d' % g, [ar]) + al = add('A', 'padd4 {d} = {%s}, %%[bias]' % al, 'bl%d' % g, [al]) + ar = add('I', 'pshr4 {d} = {%s}, %%[sh]' % ar, 'sr%d' % g, [ar]) + al = add('I', 'pshr4 {d} = {%s}, %%[sh]' % al, 'sl%d' % g, [al]) + pk = add('I', 'pack4.sss {d} = {%s}, {%s}' % (ar, al), 'pk%d' % g, [ar, al]) + add('I', 'mux2 %%[res%d] = {%s}, 0xd8' % (g, pk), None, [pk]) + return ops + +if __name__ == '__main__': + for nm, fn, ng in (('h8_core8', build_h8_core8_n, 2), ('h8_core ', build_h8_core8_n, 1), + ('v8_mid8 ', build_v8_mid8_n, 2), ('v8_mid4 ', build_v8_mid8_n, 1)): + for label, taps in (('8-tap', range(8)), ('6-tap', range(6))): + ops = fn(ng, taps); cyc = sched.schedule(ops) + nI = sum(1 for o in ops if o.unit == 'I') + print('%s %s ngrp=%d: ops=%3d I=%2d cycles=%2d [floor %.1f]' + % (nm, label, ng, len(ops), nI, len(cyc), nI / 2.0)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/modsched.py dav1d-1.5.4/src/ia64/gen/modsched.py --- dav1d-1.5.4.orig/src/ia64/gen/modsched.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/modsched.py 2026-08-25 18:25:17.894255451 +0200 @@ -0,0 +1,187 @@ +"""Modulo scheduling for ia64 .S kernels (Tier III). + +The loops this targets have no loop-carried data dependence -- iteration i and +i+1 touch disjoint elements -- so the initiation interval is set purely by +resources and there is no recurrence to search for. That makes the scheduler a +list scheduler against a *modulo* resource table: an op placed at time t +occupies slot t % II forever after. + +Per cycle the machine retires at most 2 I-slot ops and 4 M-slot ops, of which +at most 2 may be loads and 2 stores (see the A-vs-I note: A-type ops take +either slot, so they are tried in M first and fall back to I). + +Register assignment follows the stage distance. A value produced in stage s +and consumed in stage s+d must survive d rotations, so it takes d+1 consecutive +rotating registers and the reader names base+d. A value whose uses all land in +its own stage never crosses a br.ctop and can live in a static register, reused +between values whose live ranges do not overlap on the modulo circle. +""" +from collections import defaultdict + + +class Res: + def __init__(self, II): + self.II = II + self.i = [0] * II; self.m = [0] * II + self.ld = [0] * II; self.st = [0] * II + + def mlim(self, c): + # br.ctop has to fit in the last cycle. Itanium 2 issues at most two + # bundles per cycle, so that cycle is .mmi + .mib: three M slots, two I + # slots and the branch. Every other cycle is .mmi + .mmi. + return 3 if c == self.II - 1 else 4 + + def fits(self, unit, kind, t): + c = t % self.II + if unit == 'I': return self.i[c] < 2 + if unit == 'A': return self.m[c] < self.mlim(c) or self.i[c] < 2 + if unit == 'M': + if kind == 'ld' and self.ld[c] >= 2: return False + if kind == 'st' and self.st[c] >= 2: return False + return self.m[c] < self.mlim(c) + return True + + def take(self, unit, kind, t): + c = t % self.II + if unit == 'I': self.i[c] += 1 + elif unit == 'A': + if self.m[c] < self.mlim(c): self.m[c] += 1 + else: self.i[c] += 1 + elif unit == 'M': + self.m[c] += 1 + if kind == 'ld': self.ld[c] += 1 + elif kind == 'st': self.st[c] += 1 + + +def kind_of(op): + mn = op.fmt.split()[0] + if mn.startswith('ld') or mn.startswith('lfetch'): return 'ld' + if mn.startswith('st'): return 'st' + return '' + + +def schedule(ops, II, span=6, jitter=0, seed=0): + """Place every op at the earliest feasible time >= its ready time. + + |span| bounds how far past the ready time we search before declaring this + II infeasible; a real Rau scheduler would backtrack here, but with no + recurrences a forward-only pass at II+1 or II+2 always succeeds instead. + """ + defs = {o.dst: o.idx for o in ops if o.dst} + preds = defaultdict(list) + for o in ops: + for s in o.src: + if s in defs and defs[s] != o.idx: + preds[o.idx].append((defs[s], ops[defs[s]].lat)) + for e in o.extra: + preds[o.idx].append((defs[e] if e in defs else e, 1)) + succs = defaultdict(list) + for i, ps in preds.items(): + for p, l in ps: succs[p].append((i, l)) + height = [0] * len(ops) + for i in range(len(ops) - 1, -1, -1): + height[i] = max([height[s] + l for s, l in succs[i]] or [0]) + + res = Res(II); t_of = {} + # Greedy placement is order-sensitive, so find_II() re-runs this with small + # random perturbations of the height priority and keeps the first order + # that reaches a given II. + import random + rng = random.Random(seed) + key = {i: -height[i] + (rng.uniform(-jitter, jitter) if jitter else 0) + for i in range(len(ops))} + order = sorted(range(len(ops)), key=lambda i: key[i]) + placed = set() + while len(placed) < len(ops): + progressed = False + for i in order: + if i in placed: continue + ps = preds[i] + if not all(p in placed for p, _ in ps): continue + ready = max([t_of[p] + l for p, l in ps] or [0]) + o = ops[i]; k = kind_of(o) + for t in range(ready, ready + span * II): + if res.fits(o.unit, k, t): + res.take(o.unit, k, t); t_of[i] = t + placed.add(i); progressed = True; break + else: + return None + if not progressed: return None + return t_of + + +def find_II(ops, lo=None, nblk=16): + nI = sum(1 for o in ops if o.unit == 'I') + nM = sum(1 for o in ops if o.unit in ('M', 'A')) + nld = sum(1 for o in ops if kind_of(o) == 'ld') + nst = sum(1 for o in ops if kind_of(o) == 'st') + # one M slot of the II cycles is spent on br.ctop + floor = max(-(-nI // 2), -(-(nM + 1) // 4), -(-nld // 2), -(-nst // 2), 1) + if lo: floor = max(floor, lo) + # The smallest feasible II is not the fastest one. A modulo-scheduled loop + # costs (nblk + stages - 1) * II, and pushing II down to the resource floor + # drives I-slot utilisation to ~100%, which forces the greedy placer to + # spill ops far past their ready time. That lengthens the schedule, which + # means more stages, which means a longer pipeline drain. With an SGR row + # only ~16 iterations long the drain is worth more than the II. So try a + # range and pick the II that actually minimises total cycles. + best = None + for II in range(floor, floor + 12): + for attempt in range(120): + t = schedule(ops, II, jitter=(0 if attempt == 0 else 1.0 + attempt % 7), + seed=attempt) + if t is None: continue + stages = max(t.values()) // II + 1 + cost = (nblk + stages - 1) * II + if best is None or cost < best[0]: + best = (cost, II, t, stages) + if best is None: raise RuntimeError('no II found') + return best[1], best[2], floor + + +def assign_regs(ops, t_of, II, rot_base=32, n_rot=96): + """Split values into rotating (cross a stage boundary) and static.""" + uses = defaultdict(list) + for o in ops: + for s in o.src: + uses[s].append(o.idx) + stage = lambda t: t // II + rot = {}; static = {} + rot_need = [] # (value, dist) + stat_need = [] # (value, t_def, t_last_use) + for o in ops: + if not o.dst: continue + td = t_of[o.idx] + us = [t_of[u] for u in uses.get(o.dst, [])] + if not us: + stat_need.append((o.dst, td, td)); continue + d = max(stage(u) - stage(td) for u in us) + if d > 0: rot_need.append((o.dst, d)) + else: stat_need.append((o.dst, td, max(us))) + + # rotating: consecutive blocks of d+1, whole region a multiple of 8 + nxt = 0 + for v, d in rot_need: + rot[v] = (rot_base + nxt, d); nxt += d + 1 + n_rotating = (nxt + 7) // 8 * 8 + + # Statics: greedy colouring against the exact set of cycles a value + # occupies on the modulo circle. A value defined at d and last read at u + # holds its register for every cycle in [d, u] inclusive -- getting this + # even slightly wrong (half-open ranges, say) lets one value's def land in + # the cycle another is still being read in, which puts a write and a read + # of the same register in one instruction group. + def cycles(td, tu): + L = tu - td + 1 + if L >= II: return set(range(II)) + return {(td + k) % II for k in range(L)} + + banks = [] + for v, td, tu in stat_need: + cs = cycles(td, tu) + for bi, used in enumerate(banks): + if not (used & cs): + used |= cs; static[v] = bi; break + else: + banks.append(set(cs)); static[v] = len(banks) - 1 + return rot, static, n_rotating, len(banks) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/press.py dav1d-1.5.4/src/ia64/gen/press.py --- dav1d-1.5.4.orig/src/ia64/gen/press.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/press.py 2026-08-25 18:25:17.893630071 +0200 @@ -0,0 +1,28 @@ +import sys +from collections import defaultdict +sys.path.insert(0,'.') +import sched, vert + +def peak(ops, cyc): + c_of={} + for c,ins in enumerate(cyc): + for i in ins: c_of[i]=c + last=defaultdict(lambda:-1) + for o in ops: + for s in o.src: last[s]=max(last[s],c_of[o.idx]) + live=defaultdict(int) + for o in ops: + if not o.dst: continue + d=c_of[o.idx]; e=last[o.dst] + for c in range(d, max(d,e)+1): live[c]+=1 + return max(live.values()), live + +print("%-26s %7s %7s %8s %9s" % ("config","cycles","I-floor","peak regs","cyc/elem")) +for G in (2,3,4): + vert.NGRP=G + for n,obx in ((9,455),(25,164)): + ops=vert.build(n,obx); cyc=sched.schedule(ops) + nI=sum(1 for o in ops if o.unit=='I') + p,_=peak(ops,cyc) + print("%-26s %7d %7d %8d %9.2f" + %("NGRP=%d n=%2d"%(G,n), len(cyc), (nI+1)//2, p, len(cyc)/(4.0*G))) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/sched.py dav1d-1.5.4/src/ia64/gen/sched.py --- dav1d-1.5.4.orig/src/ia64/gen/sched.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/sched.py 2026-08-25 18:19:12.703223241 +0200 @@ -0,0 +1,224 @@ +NGRP=2 +PRESSURE_CAP=17 +import sys +from collections import defaultdict + +class Op: + __slots__=('unit','fmt','dst','src','lat','idx','grp','extra') + def __init__(self,unit,fmt,dst,src,lat=1,grp=0,extra=()): + self.unit=unit; self.fmt=fmt; self.dst=dst; self.src=list(src) + self.lat=lat; self.grp=grp; self.extra=list(extra) + +def build(n,obx): + ops=[] + def add(unit,fmt,dst,src,lat=1,grp=0,extra=()): + o=Op(unit,fmt,dst,src,lat,grp,extra); o.idx=len(ops); ops.append(o); return dst + # Group a covers AA[i..i+3]/BB[i..i+3], group b AA[i+4..]/BB[i+4..]; the + # four base pointers come in as operands so the loop's induction variables + # stay outside the block, leaving only the two +8 halves to derive here. + _grps=(('%[pa]','%[pa1]','%[pb]'),('%[pa2]','%[pa3]','%[pb2]'), + ('%[pa4]','%[pa5]','%[pb3]'))[:NGRP] + for g,(A0,A1,B) in enumerate(_grps): + v=lambda s: '%s%d'%(s,g) + isop=lambda a: a.startswith('%[') + a0src=[] if isop(A0) else [A0] + a1src=[] if isop(A1) else [A1] + bsrc =[] if isop(B) else [B] + A0r = A0 if isop(A0) else '{%s}'%A0 + A1r = A1 if isop(A1) else '{%s}'%A1 + Br = B if isop(B) else '{%s}'%B + ldb = add('M','ld8 {d} = [%s]'%Br, v('b'), bsrc, 2, g) + la0 = add('M','ld8 {d} = [%s]'%A0r, v('a0'), a0src,2, g) + la1 = add('M','ld8 {d} = [%s]'%A1r, v('a1'), a1src,2, g) + bp = add('I','mux2 {d} = {%s}, 0xd8'%ldb, v('bp'), [ldb], 1, g) + nn=[] + for k,la in enumerate((la0,la1)): + if n==9: + s=add('I','pshl4 {d} = {%s}, 3'%la, v('s%d'%k), [la],1,g) + nn.append(add('A','padd4 {d} = {%s}, {%s}'%(s,la), v('n%d'%k), [s,la],1,g)) + else: + x=add('I','pshl4 {d} = {%s}, 4'%la, v('mA%d'%k), [la],1,g) + y=add('I','pshl4 {d} = {%s}, 3'%la, v('mB%d'%k), [la],1,g) + z=add('A','padd4 {d} = {%s}, {%s}'%(x,y), v('mC%d'%k), [x,y],1,g) + nn.append(add('A','padd4 {d} = {%s}, {%s}'%(z,la), v('n%d'%k), [z,la],1,g)) + q0=add('I','pmpy2.r {d} = {%s}, {%s}'%(bp,bp), v('q0'), [bp],2,g) + q1=add('I','pmpy2.l {d} = {%s}, {%s}'%(bp,bp), v('q1'), [bp],2,g) + cc=[] + for k,(nv,qv) in enumerate(((nn[0],q0),(nn[1],q1))): + p=add('A','psub4 {d} = {%s}, {%s}'%(nv,qv), v('p%d'%k), [nv,qv],1,g) + m=add('A','pcmp4.gt {d} = %%[pm], {%s}'%p, v('m%d'%k), [p],1,g) + e=add('A','xor {d} = {%s}, %%[pm]'%p, v('e%d'%k), [p],1,g) + f=add('A','and {d} = {%s}, {%s}'%(e,m), v('f%d'%k), [e,m],1,g) + cc.append(add('A','xor {d} = {%s}, %%[pm]'%f, v('c%d'%k), [f],1,g)) + pr=[] + for k,c in enumerate(cc): + h=add('I','pshr4 {d} = {%s}, 15'%c, v('h%d'%k), [c],1,g) + l=add('A','and {d} = {%s}, %%[m15]'%c, v('l%d'%k), [c],1,g) + H=add('I','pmpy2.r {d} = {%s}, %%[sv]'%h, v('H%d'%k), [h],2,g) + L=add('I','pmpy2.r {d} = {%s}, %%[sv]'%l, v('L%d'%k), [l],2,g) + Hs=add('I','pshl4 {d} = {%s}, 15'%H, v('Hs%d'%k), [H],1,g) + ps=add('A','padd4 {d} = {%s}, {%s}'%(Hs,L), v('ps%d'%k), [Hs,L],1,g) + pr.append(add('A','padd4 {d} = {%s}, %%[r19]'%ps, v('pr%d'%k), [ps],1,g)) + xs=[] + for k in range(4): + src=pr[k//2]; pos=20 if k%2==0 else 52 + i=add('I','extr.u {d} = {%s}, %d, 8'%(src,pos), v('i%d'%k), [src],1,g) + a=add('A','add {d} = %%[tbl], {%s}'%i, v('A%d'%k), [i],1,g) + xs.append(add('M','ld1 {d} = [{%s}]'%a, v('x%d'%k), [a],2,g)) + d0=add('I','dep {d} = {%s}, {%s}, 16, 8'%(xs[2],xs[0]), v('d0'), [xs[2],xs[0]],1,g) + d1=add('I','dep {d} = {%s}, {%s}, 16, 8'%(xs[3],xs[1]), v('d1'), [xs[3],xs[1]],1,g) + d1s=add('I','dep.z {d} = {%s}, 32, 32'%d1, v('d1s'), [d1],1,g) + xvp=add('A','or {d} = {%s}, {%s}'%(d0,d1s), v('xvp'), [d0,d1s],1,g) + xv =add('I','mux2 {d} = {%s}, 0xd8'%xvp, v('xv'), [xvp],1,g) + stb=add('M','st8 [%s] = {%s}'%(Br,xv), None, bsrc+[xv],1,g,extra=[ldb]) + u0=add('I','pmpy2.r {d} = {%s}, {%s}'%(xvp,bp), v('u0'), [xvp,bp],2,g) + u1=add('I','pmpy2.l {d} = {%s}, {%s}'%(xvp,bp), v('u1'), [xvp,bp],2,g) + for k,(u,Ar,asrc,lastld) in enumerate(((u0,A0r,a0src,la0),(u1,A1r,a1src,la1))): + if obx==455: + t=add('I','pshl4 {d} = {%s}, 3'%u, v('w%d'%k), [u],1,g) + t=add('A','psub4 {d} = {%s}, {%s}'%(t,u), v('w2%d'%k), [t,u],1,g) + t2=add('I','pshl4 {d} = {%s}, 6'%t, v('w3%d'%k), [t],1,g) + r=add('A','padd4 {d} = {%s}, {%s}'%(t2,t), v('r%d'%k), [t2,t],1,g) + else: + t=add('I','pshl4 {d} = {%s}, 5'%u, v('w%d'%k), [u],1,g) + y=add('I','pshl4 {d} = {%s}, 3'%u, v('w2%d'%k), [u],1,g) + s=add('A','padd4 {d} = {%s}, {%s}'%(t,y), v('w3%d'%k), [t,y],1,g) + s=add('A','padd4 {d} = {%s}, {%s}'%(s,u), v('w4%d'%k), [s,u],1,g) + r=add('I','pshl4 {d} = {%s}, 2'%s, v('r%d'%k), [s],1,g) + R=add('A','padd4 {d} = {%s}, %%[r11]'%r, v('R%d'%k), [r],1,g) + o=add('I','pshr4 {d} = {%s}, 12'%R, v('o%d'%k), [R],1,g) + add('M','st8 [%s] = {%s}'%(Ar,o), None, asrc+[o],1,g,extra=[lastld]) + return ops + +def schedule(ops): + defs={} + for o in ops: + if o.dst: defs[o.dst]=o.idx + preds=defaultdict(list) + for o in ops: + for s in o.src: + if s in defs and defs[s]!=o.idx: preds[o.idx].append((defs[s],ops[defs[s]].lat)) + for e in o.extra: preds[o.idx].append((defs[e] if e in defs else e,1)) + # WAW/WAR are impossible: every def is a fresh vreg. + succs=defaultdict(list) + for i,ps in preds.items(): + for p,l in ps: succs[p].append((i,l)) + # critical path height + height=[0]*len(ops) + for i in range(len(ops)-1,-1,-1): + h=0 + for s,l in succs[i]: h=max(h,height[s]+l) + height[i]=h + lastuse_i={} + for o in ops: + for sname in o.src: lastuse_i[sname]=o.idx + lastuse_c={} + unsched=set(range(len(ops))); ready_at=[0]*len(ops) + cycles=[]; done={} + cyc=0 + guard=0 + while unsched: + guard+=1 + if guard>500: + import sys; blocked=sorted(unsched)[:6] + for i in blocked: sys.stderr.write('BLOCKED %d %s preds=%s\n'%(i,ops[i].fmt,[(p,p in done) for p,_ in preds[i]])) + raise RuntimeError('no progress') + avail=[i for i in unsched if all(p in done and done[p]+l<=cyc for p,l in preds[i])] + live=set() + for o in ops: + if o.dst and o.idx in done and done[o.idx]<=cyc and lastuse_c.get(o.dst,10**9)>=cyc: + live.add(o.dst) + tight = len(live) >= PRESSURE_CAP + def kills(i): + o=ops[i] + return sum(1 for sname in o.src if sname in lastuse_i and lastuse_i[sname]==o.idx) + if tight: + avail.sort(key=lambda i:(-kills(i), 1 if ops[i].unit=='M' and 'ld' in ops[i].fmt.split()[0] else 0, -height[i])) + else: + avail.sort(key=lambda i:-height[i]) + islot=0; mslot=0; loads=0; stores=0; picked=[] + for i in avail: + o=ops[i] + if o.unit=='I': + if islot>=2: continue + islot+=1 + elif o.unit=='M': + mn=o.fmt.split()[0] + isload = mn.startswith('ld') or mn.startswith('lfetch') + if mslot>=4: continue + if isload and loads>=2: continue + if not isload and stores>=2: continue + mslot+=1 + if isload: loads+=1 + else: stores+=1 + else: + if mslot<4: mslot+=1 + elif islot<2: islot+=1 + else: continue + picked.append(i) + for i in picked: + unsched.discard(i); done[i]=cyc + for sname in ops[i].src: lastuse_c[sname]=cyc + cycles.append(picked); cyc+=1 + return cycles + +def allocate(ops,cycles,pool_a,pool_b): + lastuse=defaultdict(lambda:-1); defcyc={} + c_of={} + for c,ins in enumerate(cycles): + for i in ins: c_of[i]=c + for o in ops: + for s in o.src: + lastuse[s]=max(lastuse[s],c_of[o.idx]) + for o in ops: + if o.dst: defcyc[o.dst]=c_of[o.idx] + assign={} + free={0:list(pool_a),1:list(pool_b)} + busy=[] # (lastuse_cycle, grp, reg) + order=sorted([o for o in ops if o.dst], key=lambda o:c_of[o.idx]) + for o in order: + c=c_of[o.idx] + for ent in list(busy): + if ent[0] 1: + nxt = [] + for k in range(0, len(cur) - 1, 2): + nxt.append(add('A', '%s {d} = {%s}, {%s}' % (mn, cur[k], cur[k + 1]), + '%s_%d_%d' % (tag, lvl, k), [cur[k], cur[k + 1]]).dst) + if len(cur) & 1: nxt.append(cur[-1]) + cur = nxt; lvl += 1 + return cur[0] + + a0 = tree([Q[j][0] for j in range(NV)], 'padd4', 'va0') + a1 = tree([Q[j][1] for j in range(NV)], 'padd4', 'va1') + bv = tree([S[j] for j in range(NV)], 'padd2', 'vb') + + # --- calc_ab ------------------------------------------------------ + bp = add('I', 'mux2 {d} = {%s}, 0xd8' % bv, 'bp', [bv]).dst + nn = [] + for k, la in enumerate((a0, a1)): + if n == 9: + t = add('I', 'pshl4 {d} = {%s}, 3' % la, 's%d' % k, [la]).dst + nn.append(add('A', 'padd4 {d} = {%s}, {%s}' % (t, la), 'n%d' % k, [t, la]).dst) + else: + x = add('I', 'pshl4 {d} = {%s}, 4' % la, 'mA%d' % k, [la]).dst + y = add('I', 'pshl4 {d} = {%s}, 3' % la, 'mB%d' % k, [la]).dst + z = add('A', 'padd4 {d} = {%s}, {%s}' % (x, y), 'mC%d' % k, [x, y]).dst + nn.append(add('A', 'padd4 {d} = {%s}, {%s}' % (z, la), 'n%d' % k, [z, la]).dst) + q0 = add('I', 'pmpy2.r {d} = {%s}, {%s}' % (bp, bp), 'q0', [bp], 2).dst + q1 = add('I', 'pmpy2.l {d} = {%s}, {%s}' % (bp, bp), 'q1', [bp], 2).dst + cc = [] + for k, (nv, qv) in enumerate(((nn[0], q0), (nn[1], q1))): + p = add('A', 'psub4 {d} = {%s}, {%s}' % (nv, qv), 'p%d' % k, [nv, qv]).dst + m = add('A', 'pcmp4.gt {d} = {Cpm}, {%s}' % p, 'm%d' % k, [p]).dst + e = add('A', 'xor {d} = {%s}, {Cpm}' % p, 'e%d' % k, [p]).dst + f = add('A', 'and {d} = {%s}, {%s}' % (e, m), 'f%d' % k, [e, m]).dst + cc.append(add('A', 'xor {d} = {%s}, {Cpm}' % f, 'c%d' % k, [f]).dst) + pr = [] + for k, c in enumerate(cc): + h = add('I', 'pshr4 {d} = {%s}, 15' % c, 'h%d' % k, [c]).dst + l = add('A', 'and {d} = {%s}, {Cm15}' % c, 'l%d' % k, [c]).dst + H = add('I', 'pmpy2.r {d} = {%s}, {Csv}' % h, 'H%d' % k, [h], 2).dst + L = add('I', 'pmpy2.r {d} = {%s}, {Csv}' % l, 'L%d' % k, [l], 2).dst + Hs = add('I', 'pshl4 {d} = {%s}, 15' % H, 'Hs%d' % k, [H]).dst + ps = add('A', 'padd4 {d} = {%s}, {%s}' % (Hs, L), 'ps%d' % k, [Hs, L]).dst + pr.append(add('A', 'padd4 {d} = {%s}, {Cr19}' % ps, 'pr%d' % k, [ps]).dst) + xs = [] + for k in range(4): + src = pr[k // 2]; pos = 20 if k % 2 == 0 else 52 + i = add('I', 'extr.u {d} = {%s}, %d, 8' % (src, pos), 'i%d' % k, [src]).dst + a = add('A', 'add {d} = {Ctbl}, {%s}' % i, 'A%d' % k, [i]).dst + xs.append(add('M', 'ld1 {d} = [{%s}]' % a, 'x%d' % k, [a], LDL).dst) + d0 = add('I', 'dep {d} = {%s}, {%s}, 16, 8' % (xs[2], xs[0]), 'd0', [xs[2], xs[0]]).dst + d1 = add('I', 'dep {d} = {%s}, {%s}, 16, 8' % (xs[3], xs[1]), 'd1', [xs[3], xs[1]]).dst + d1s = add('I', 'dep.z {d} = {%s}, 32, 32' % d1, 'd1s', [d1]).dst + xvp = add('A', 'or {d} = {%s}, {%s}' % (d0, d1s), 'xvp', [d0, d1s]).dst + xv = add('I', 'mux2 {d} = {%s}, 0xd8' % xvp, 'xv', [xvp]).dst + add('M', 'st8 [{Pb}] = {%s}, 8' % xv, None, [xv]) + u0 = add('I', 'pmpy2.r {d} = {%s}, {%s}' % (xvp, bp), 'u0', [xvp, bp], 2).dst + u1 = add('I', 'pmpy2.l {d} = {%s}, {%s}' % (xvp, bp), 'u1', [xvp, bp], 2).dst + for k, u in enumerate((u0, u1)): + if obx == 455: + t = add('I', 'pshl4 {d} = {%s}, 3' % u, 'w%d' % k, [u]).dst + t = add('A', 'psub4 {d} = {%s}, {%s}' % (t, u), 'w2%d' % k, [t, u]).dst + t2 = add('I', 'pshl4 {d} = {%s}, 6' % t, 'w3%d' % k, [t]).dst + r = add('A', 'padd4 {d} = {%s}, {%s}' % (t2, t), 'r%d' % k, [t2, t]).dst + else: + t = add('I', 'pshl4 {d} = {%s}, 5' % u, 'w%d' % k, [u]).dst + y = add('I', 'pshl4 {d} = {%s}, 3' % u, 'w2%d' % k, [u]).dst + s2 = add('A', 'padd4 {d} = {%s}, {%s}' % (t, y), 'w3%d' % k, [t, y]).dst + s2 = add('A', 'padd4 {d} = {%s}, {%s}' % (s2, u), 'w4%d' % k, [s2, u]).dst + r = add('I', 'pshl4 {d} = {%s}, 2' % s2, 'r%d' % k, [s2]).dst + R = add('A', 'padd4 {d} = {%s}, {Cr11}' % r, 'R%d' % k, [r]).dst + o = add('I', 'pshr4 {d} = {%s}, 12' % R, 'o%d' % k, [R]).dst + add('M', 'st8 [{Pa%d}] = {%s}, 16' % (k, o), None, [o]) + return ops + + +if __name__ == '__main__': + for n, obx in ((9, 455), (25, 164)): + ops = build(n, obx) + II, t_of, floor = modsched.find_II(ops) + rot, static, nrot, nbank = modsched.assign_regs(ops, t_of, II) + stages = max(t_of.values()) // II + 1 + tot = (16 + stages - 1) * II + nI = sum(1 for o in ops if o.unit == 'I') + NV = 3 if n == 9 else 5 + nfix = 3 * NV + 3 + 6 + print('n=%2d: %3d ops (%d I) II=%d (floor %d) stages=%d ' + 'rot=%d static=%d fixed=%d sof=%d -> %.2f cyc/elem (16 blk: %d cyc, %.2f eff)' + % (n, len(ops), nI, II, floor, stages, nrot, nbank, nfix, + nrot + nbank + nfix, II / 4.0, tot, tot / 64.0)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/sgrw.py dav1d-1.5.4/src/ia64/gen/sgrw.py --- dav1d-1.5.4.orig/src/ia64/gen/sgrw.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/sgrw.py 2026-08-25 18:23:47.149092896 +0200 @@ -0,0 +1,46 @@ +import sys +sys.path.insert(0, '.') +import sched +from sched import Op + +def build(nw=2): + """sgr_weighted2 (nw==2) / sgr_weighted1 (nw==1), eight pixels. + + dst[i] += (w0*t1[i] + w1*t2[i] + (1<<10)) >> 11, clipped, for eight i. + Written as one region because the intrinsic form is one asm() per operation + and GCC gives each its own bundle and stop bit.""" + ops = [] + def add(u, f, d, s, lat=1): + o = Op(u, f, d, s, lat, 0, ()); o.idx = len(ops); ops.append(o); return d + u = [add('I', 'unpack1.l {d} = r0, %[d]', 'u0', []), + add('I', 'unpack1.h {d} = r0, %[d]', 'u1', [])] + res = [] + for g in range(2): + acc = {} + for half in ('r', 'l'): + p0 = add('I', 'pmpy2.%s {d} = %%[a%d], %%[w0]' % (half, g), 'pa%s%d' % (half, g), [], 2) + terms = [p0] + if nw == 2: + terms.append(add('I', 'pmpy2.%s {d} = %%[b%d], %%[w1]' % (half, g), + 'pb%s%d' % (half, g), [], 2)) + a = terms[0] + if len(terms) > 1: + a = add('A', 'padd4 {d} = {%s}, {%s}' % (terms[0], terms[1]), + 's%s%d' % (half, g), terms) + a = add('A', 'padd4 {d} = {%s}, %%[rnd]' % a, 'r%s%d' % (half, g), [a]) + acc[half] = add('I', 'pshr4 {d} = {%s}, 11' % a, 'h%s%d' % (half, g), [a]) + pk = add('I', 'pack4.sss {d} = {%s}, {%s}' % (acc['r'], acc['l']), + 'pk%d' % g, [acc['r'], acc['l']]) + v = add('I', 'mux2 {d} = {%s}, 0xd8' % pk, 'v%d' % g, [pk]) + res.append(add('A', 'padd2 {d} = {%s}, {%s}' % (v, u[g]), 'o%d' % g, [v, u[g]])) + add('I', 'pack2.uss %%[out] = {%s}, {%s}' % (res[0], res[1]), None, res) + return ops + +if __name__ == '__main__': + from final import alloc + for nw in (2, 1): + ops = build(nw); cyc = sched.schedule(ops) + nI = sum(1 for o in ops if o.unit == 'I'); nA = sum(1 for o in ops if o.unit == 'A') + asg = alloc(ops, cyc) + print('sgr_weighted%d: I=%d A=%d cycles=%d (floor %.1f) %.3f cyc/px regs=%d' + % (nw, nI, nA, len(cyc), nI / 2.0, len(cyc) / 8.0, len(set(asg.values())))) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/sim.py dav1d-1.5.4/src/ia64/gen/sim.py --- dav1d-1.5.4.orig/src/ia64/gen/sim.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/sim.py 2026-08-25 18:25:17.894017136 +0200 @@ -0,0 +1,40 @@ +import os +import sys +FPS=float(os.environ.get("FPS", 25.0)); B=1.0/FPS +def load(p): + t=[float(l.split()[1]) for l in open(p) if l.strip()] + return [t[i]-t[i-1] if i else t[0] for i in range(len(t))] +def sim(d,label,Q=3,speed=1.0): + bank=Q*B; lag=0.0; late=0; froz=0.0; cur=0.0; worst=0.0; wstart=0; bw=0 + for i,x in enumerate(d): + lag += x/speed - B + if lag < -bank: lag = -bank + if lag > 0: + late += 1; froz += lag if cur==0 else 0 + if cur==0: wstart=i + cur += x/speed - B if lag>0 else 0 + else: + if cur>0 and lag<=0: + pass + cur=0 + # episode analysis: contiguous stretches with lag>0 + lag=0.0; ep=[]; s=None; peak=0.0 + for i,x in enumerate(d): + lag += x/speed - B + if lag < -bank: lag = -bank + if lag>0: + if s is None: s=i; peak=0.0 + peak=max(peak,lag) + else: + if s is not None: ep.append((s,i,peak)); s=None + if s is not None: ep.append((s,len(d),peak)) + tot=sum(e[2] for e in ep) + ep.sort(key=lambda e:-e[2]) + print(f"{label}: speed={speed:.2f} late={late} ({100.0*late/len(d):.1f}%) episodes={len(ep)} worst_freeze={ep[0][2]:.2f}s @frame{ep[0][0]} (t={ep[0][0]/FPS:.1f}s)" if ep else f"{label}: CLEAN") + for e in ep[:6]: + print(f" frames {e[0]}-{e[1]} t={e[0]/FPS:.1f}s..{e[1]/FPS:.1f}s peak_freeze={e[2]:.2f}s") + return ep +d=load(sys.argv[1]) +print(f"total={sum(d):.2f}s video={len(d)/FPS:.2f}s avg_speed={len(d)/FPS/sum(d):.3f}x") +for s in [float(x) for x in (sys.argv[2:] or ["1.0"])]: + sim(d,sys.argv[1],speed=s); print() diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/verify_tier2.py dav1d-1.5.4/src/ia64/gen/verify_tier2.py --- dav1d-1.5.4.orig/src/ia64/gen/verify_tier2.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/verify_tier2.py 2026-08-25 18:25:36.934599113 +0200 @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Check that the generated Tier II kernels in the C templates are still what +the generators here produce. + +Tier III kernels (the .S files) are generated by the build, so they cannot drift. +Tier II kernels are inline-asm blocks living inside hand-maintained C templates: +they were generated once and spliced in, and the surrounding C is edited by hand. +That makes them the one place where the checked-in code and its generator can +silently diverge, so this check exists to catch it. + +Run it with: ninja ia64-check-generated + +A FAIL is not automatically a bug -- a block may have been deliberately hand- +tuned after generation. It does mean the generator is no longer the source of +truth for that block, and one of the two needs updating. + +Usage: verify_tier2.py +""" +import os +import subprocess +import sys +import tempfile + +# generator -> (file it writes, C template the block must appear in) +BLOCKS = [ + ('final.py', 'newbody2.c', 'looprestoration_tmpl.c'), + ('gensgrw.py', 'sgrw8.c', 'looprestoration_tmpl.c'), + ('genwienh.py', 'wienh8.c', 'looprestoration_tmpl.c'), + ('genwien.py', 'newwien.c', 'looprestoration_tmpl.c'), + ('genmid12.py', 'mid12.c', 'mc_tmpl.c'), + ('genh16.py', 'h16.c', 'mc_tmpl.c'), + ('genmc6.py', 'mc6kernels.c', 'mc_tmpl.c'), +] + + +def main(): + root = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.abspath(__file__)) + gendir = os.path.join(root, 'src', 'ia64', 'gen') if os.path.isdir( + os.path.join(root, 'src')) else root + ia64 = os.path.dirname(gendir) + + templates = {} + fails = 0 + for gen, out, tmpl in BLOCKS: + if tmpl not in templates: + templates[tmpl] = open(os.path.join(ia64, tmpl)).read() + with tempfile.TemporaryDirectory() as td: + r = subprocess.run([sys.executable, '-B', os.path.join(gendir, gen)], + cwd=td, capture_output=True, text=True) + path = os.path.join(td, out) + if r.returncode != 0 or not os.path.exists(path): + print('FAIL %-14s generator failed: %s' % (gen, r.stderr.strip()[:200])) + fails += 1 + continue + body = open(path).read().strip() + if body in templates[tmpl]: + print('ok %-14s -> %s' % (gen, tmpl)) + else: + print('FAIL %-14s output is no longer verbatim in %s' % (gen, tmpl)) + fails += 1 + + print('%d of %d generated Tier II blocks match' % (len(BLOCKS) - fails, len(BLOCKS))) + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/vert.py dav1d-1.5.4/src/ia64/gen/vert.py --- dav1d-1.5.4.orig/src/ia64/gen/vert.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/vert.py 2026-08-25 18:25:17.892897386 +0200 @@ -0,0 +1,146 @@ +"""Fused sgr_box{3,5}_row_v + sgr_calc_row_ab kernel. + +The two loops run back-to-back on the same buffers and the second one +overwrites the first one's output in place, so the intermediate row never has +to reach memory. More importantly they have complementary slot profiles: +calc_row_ab is I-bound (60 of its 74 I slots are full, but 72 of its 148 M +slots and 60 of its 74 load slots sit idle), while row_v is pure loads plus +A-type padd2/padd4 -- zero I-type ops. Fusing lets row_v issue entirely in +calc_row_ab's holes. + +Row pointers are walked with post-incrementing ld8 so each of the 3 (or 5) +input rows costs one operand instead of four; that keeps the operand count at +21/25 against GCC's limit of 30. +""" +import sys +from collections import defaultdict +sys.path.insert(0, '.') +import sched +from sched import Op + +NGRP = 2 + + +def build(n, obx): + ops = [] + + def add(unit, fmt, dst, src, lat=1, grp=0, extra=()): + o = Op(unit, fmt, dst, src, lat, grp, extra) + o.idx = len(ops); ops.append(o); return o + + NV = 3 if n == 9 else 5 + + # 0x0000080000000800 << 8 == 0x0008000000080000 with nothing crossing a lane + # boundary, so r19 is two A-type shladds off r11 rather than a 31st operand. + _t = add('A', 'shladd {d} = %[r11], 4, r0', 'r19a', [], 1, 0).dst + R19 = add('A', 'shladd {d} = {%s}, 4, r0' % _t, 'r19b', [_t], 1, 0).dst + + # --- vertical pass ------------------------------------------------- + # sumsq[j] contributes four int32 pairs per 8 elements, sum[j] two coef + # quads. Each row is one post-incrementing chain, so the loads of a row + # serialise but the NV rows run in parallel; the real ceiling is the two + # load ports. + Q = [[None] * (2 * NGRP) for _ in range(NV)] # Q[j][k]: sumsq row j, lanes 2k..2k+1 + S = [[None] * NGRP for _ in range(NV)] # S[j][k]: sum row j, lanes 4k..4k+3 + for j in range(NV): + prev = None + for k in range(2 * NGRP): + o = add('M', 'ld8 {d} = [%%[q%d]], 8' % j, 'Q%d_%d' % (j, k), [], + 2, k // 2, extra=([prev] if prev is not None else [])) + Q[j][k] = o.dst; prev = o.idx + prev = None + for k in range(NGRP): + o = add('M', 'ld8 {d} = [%%[s%d]], 8' % j, 'S%d_%d' % (j, k), [], + 2, k, extra=([prev] if prev is not None else [])) + S[j][k] = o.dst; prev = o.idx + + def tree(vals, mn, tag, g): + """Balanced A-type reduction; depth 2 for NV=3, 3 for NV=5.""" + cur = list(vals); lvl = 0 + while len(cur) > 1: + nxt = [] + for k in range(0, len(cur) - 1, 2): + nxt.append(add('A', '%s {d} = {%s}, {%s}' % (mn, cur[k], cur[k + 1]), + '%s_%d_%d' % (tag, lvl, k), [cur[k], cur[k + 1]], 1, g).dst) + if len(cur) & 1: nxt.append(cur[-1]) + cur = nxt; lvl += 1 + return cur[0] + + VA = [[tree([Q[j][2 * g + h] for j in range(NV)], 'padd4', 'va%d%d' % (g, h), g) + for h in range(2)] for g in range(NGRP)] + VB = [tree([S[j][g] for j in range(NV)], 'padd2', 'vb%d' % g, g) + for g in range(NGRP)] + + # --- calc_ab, unchanged except that a0/a1/b now arrive in registers --- + astores = [] # (order, value) -> chained post-inc st8 + bstores = [] + for g in range(NGRP): + v = lambda s: '%s%d' % (s, g) + ldb, la0, la1 = VB[g], VA[g][0], VA[g][1] + bp = add('I', 'mux2 {d} = {%s}, 0xd8' % ldb, v('bp'), [ldb], 1, g).dst + nn = [] + for k, la in enumerate((la0, la1)): + if n == 9: + s = add('I', 'pshl4 {d} = {%s}, 3' % la, v('s%d' % k), [la], 1, g).dst + nn.append(add('A', 'padd4 {d} = {%s}, {%s}' % (s, la), v('n%d' % k), [s, la], 1, g).dst) + else: + x = add('I', 'pshl4 {d} = {%s}, 4' % la, v('mA%d' % k), [la], 1, g).dst + y = add('I', 'pshl4 {d} = {%s}, 3' % la, v('mB%d' % k), [la], 1, g).dst + z = add('A', 'padd4 {d} = {%s}, {%s}' % (x, y), v('mC%d' % k), [x, y], 1, g).dst + nn.append(add('A', 'padd4 {d} = {%s}, {%s}' % (z, la), v('n%d' % k), [z, la], 1, g).dst) + q0 = add('I', 'pmpy2.r {d} = {%s}, {%s}' % (bp, bp), v('q0'), [bp], 2, g).dst + q1 = add('I', 'pmpy2.l {d} = {%s}, {%s}' % (bp, bp), v('q1'), [bp], 2, g).dst + cc = [] + for k, (nv, qv) in enumerate(((nn[0], q0), (nn[1], q1))): + p = add('A', 'psub4 {d} = {%s}, {%s}' % (nv, qv), v('p%d' % k), [nv, qv], 1, g).dst + m = add('A', 'pcmp4.gt {d} = %%[pm], {%s}' % p, v('m%d' % k), [p], 1, g).dst + e = add('A', 'xor {d} = {%s}, %%[pm]' % p, v('e%d' % k), [p], 1, g).dst + f = add('A', 'and {d} = {%s}, {%s}' % (e, m), v('f%d' % k), [e, m], 1, g).dst + cc.append(add('A', 'xor {d} = {%s}, %%[pm]' % f, v('c%d' % k), [f], 1, g).dst) + pr = [] + for k, c in enumerate(cc): + h = add('I', 'pshr4 {d} = {%s}, 15' % c, v('h%d' % k), [c], 1, g).dst + l = add('A', 'and {d} = {%s}, %%[m15]' % c, v('l%d' % k), [c], 1, g).dst + H = add('I', 'pmpy2.r {d} = {%s}, %%[sv]' % h, v('H%d' % k), [h], 2, g).dst + L = add('I', 'pmpy2.r {d} = {%s}, %%[sv]' % l, v('L%d' % k), [l], 2, g).dst + Hs = add('I', 'pshl4 {d} = {%s}, 15' % H, v('Hs%d' % k), [H], 1, g).dst + ps = add('A', 'padd4 {d} = {%s}, {%s}' % (Hs, L), v('ps%d' % k), [Hs, L], 1, g).dst + pr.append(add('A', 'padd4 {d} = {%s}, {%s}' % (ps, R19), v('pr%d' % k), [ps, R19], 1, g).dst) + xs = [] + for k in range(4): + src = pr[k // 2]; pos = 20 if k % 2 == 0 else 52 + i = add('I', 'extr.u {d} = {%s}, %d, 8' % (src, pos), v('i%d' % k), [src], 1, g).dst + a = add('A', 'add {d} = %%[tbl], {%s}' % i, v('A%d' % k), [i], 1, g).dst + xs.append(add('M', 'ld1 {d} = [{%s}]' % a, v('x%d' % k), [a], 2, g).dst) + d0 = add('I', 'dep {d} = {%s}, {%s}, 16, 8' % (xs[2], xs[0]), v('d0'), [xs[2], xs[0]], 1, g).dst + d1 = add('I', 'dep {d} = {%s}, {%s}, 16, 8' % (xs[3], xs[1]), v('d1'), [xs[3], xs[1]], 1, g).dst + d1s = add('I', 'dep.z {d} = {%s}, 32, 32' % d1, v('d1s'), [d1], 1, g).dst + xvp = add('A', 'or {d} = {%s}, {%s}' % (d0, d1s), v('xvp'), [d0, d1s], 1, g).dst + xv = add('I', 'mux2 {d} = {%s}, 0xd8' % xvp, v('xv'), [xvp], 1, g).dst + # No load ever reads AA/BB now, so the stores carry no anti-dependence. + bstores.append((g, xv, g)) + u0 = add('I', 'pmpy2.r {d} = {%s}, {%s}' % (xvp, bp), v('u0'), [xvp, bp], 2, g).dst + u1 = add('I', 'pmpy2.l {d} = {%s}, {%s}' % (xvp, bp), v('u1'), [xvp, bp], 2, g).dst + for k, u in enumerate((u0, u1)): + if obx == 455: + t = add('I', 'pshl4 {d} = {%s}, 3' % u, v('w%d' % k), [u], 1, g).dst + t = add('A', 'psub4 {d} = {%s}, {%s}' % (t, u), v('w2%d' % k), [t, u], 1, g).dst + t2 = add('I', 'pshl4 {d} = {%s}, 6' % t, v('w3%d' % k), [t], 1, g).dst + r = add('A', 'padd4 {d} = {%s}, {%s}' % (t2, t), v('r%d' % k), [t2, t], 1, g).dst + else: + t = add('I', 'pshl4 {d} = {%s}, 5' % u, v('w%d' % k), [u], 1, g).dst + y = add('I', 'pshl4 {d} = {%s}, 3' % u, v('w2%d' % k), [u], 1, g).dst + s = add('A', 'padd4 {d} = {%s}, {%s}' % (t, y), v('w3%d' % k), [t, y], 1, g).dst + s = add('A', 'padd4 {d} = {%s}, {%s}' % (s, u), v('w4%d' % k), [s, u], 1, g).dst + r = add('I', 'pshl4 {d} = {%s}, 2' % s, v('r%d' % k), [s], 1, g).dst + R = add('A', 'padd4 {d} = {%s}, %%[r11]' % r, v('R%d' % k), [r], 1, g).dst + o = add('I', 'pshr4 {d} = {%s}, 12' % R, v('o%d' % k), [R], 1, g).dst + astores.append((2 * g + k, o, g)) + for tag, lst in (('pa', astores), ('pb', bstores)): + prev = None + for ent in sorted(lst): + o, g = ent[-2], ent[-1] + st = add('M', 'st8 [%%[%s]] = {%s}, 8' % (tag, o), None, [o], 1, g, + extra=([prev] if prev is not None else [])) + prev = st.idx + return ops diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/wien.py dav1d-1.5.4/src/ia64/gen/wien.py --- dav1d-1.5.4.orig/src/ia64/gen/wien.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/wien.py 2026-08-25 18:23:47.149793646 +0200 @@ -0,0 +1,49 @@ +"""Both halves of the vertical Wiener pass in one scheduled block. + +Per 8 outputs the block issues 37 I-type ops -- 28 pmpy2, 4 pshr4, 2 pack4.sss, +2 mux2 and the final pack2.uss -- so at two I-slot ops per cycle the floor is +19 cycles. The dependence chain is only about 14 (load, pmpy2, three tree +levels, bias, pshr4, pack4, mux2), i.e. shorter than the floor, so one +correctly scheduled iteration is resource-bound and there is nothing for +cross-iteration pipelining to recover here. + +What the old code lost was simply that the call site ran two separate asm +blocks of ~14 cycles each and GCC cannot overlap across an asm boundary, so +they serialised to ~28. Interleaving the two halves in one region reaches the +floor instead. The padd4 tree is A-type and rides the M slots beside the +multiplies, which is why the I count alone sets the rate. +""" +import sys +from collections import defaultdict +sys.path.insert(0, '.') +from sched import Op + + +def build(): + ops = [] + + def add(unit, fmt, dst, src, lat=1, grp=0, extra=()): + o = Op(unit, fmt, dst, src, lat, grp, extra) + o.idx = len(ops); ops.append(o); return o.dst + + res = [] + for g in range(2): + v = lambda s: '%s%d' % (s, g) + chains = {} + for half, mn in (('r', 'pmpy2.r'), ('l', 'pmpy2.l')): + p = [add('I', '%s {d} = %%[v%d_%d], %%[f%d]' % (mn, g, k, k), + v('p%s%d' % (half, k)), [], 2, g) for k in range(7)] + # Same shape as the scalar tree: pair, pair, then fold in tap 6. + s01 = add('A', 'padd4 {d} = {%s}, {%s}' % (p[0], p[1]), v('s01'+half), [p[0], p[1]], 1, g) + s23 = add('A', 'padd4 {d} = {%s}, {%s}' % (p[2], p[3]), v('s23'+half), [p[2], p[3]], 1, g) + s45 = add('A', 'padd4 {d} = {%s}, {%s}' % (p[4], p[5]), v('s45'+half), [p[4], p[5]], 1, g) + a = add('A', 'padd4 {d} = {%s}, {%s}' % (s01, s23), v('a'+half), [s01, s23], 1, g) + b = add('A', 'padd4 {d} = {%s}, {%s}' % (s45, p[6]), v('b'+half), [s45, p[6]], 1, g) + acc = add('A', 'padd4 {d} = {%s}, {%s}' % (a, b), v('acc'+half), [a, b], 1, g) + acc = add('A', 'padd4 {d} = {%s}, %%[bias]' % acc, v('bi'+half), [acc], 1, g) + chains[half] = add('I', 'pshr4 {d} = {%s}, %%[sh]' % acc, v('sh'+half), [acc], 1, g) + pk = add('I', 'pack4.sss {d} = {%s}, {%s}' % (chains['r'], chains['l']), + v('pk'), [chains['r'], chains['l']], 1, g) + res.append(add('I', 'mux2 {d} = {%s}, 0xd8' % pk, v('mx'), [pk], 1, g)) + add('I', 'pack2.uss %%[out] = {%s}, {%s}' % (res[0], res[1]), None, res, 1, 0) + return ops diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/gen/wienh.py dav1d-1.5.4/src/ia64/gen/wienh.py --- dav1d-1.5.4.orig/src/ia64/gen/wienh.py 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/gen/wienh.py 2026-08-25 18:23:47.149442669 +0200 @@ -0,0 +1,50 @@ +import sys +sys.path.insert(0, '.') +import sched +from sched import Op +from mc import tree_build + +def build_wiener_h(ngrp=2): + """Horizontal Wiener, |ngrp|*4 outputs, 7 taps, 32-bit accumulate. + + Window layout matches wiener_h_4: wa covers b[0..7], wb covers b[2..9]; + taps 0..6 are a0, shrp(a1,a0,16), b0, shrp(b1,b0,16), a1, + shrp(b1,b0,48), b1.""" + ops = [] + def add(u, f, d, s, lat=1): + o = Op(u, f, d, s, lat, 0, ()); o.idx = len(ops); ops.append(o); return d + for g in range(ngrp): + a0 = add('I', 'unpack1.l {d} = r0, %%[wa%d]' % g, 'a0_%d' % g, []) + a1 = add('I', 'unpack1.h {d} = r0, %%[wa%d]' % g, 'a1_%d' % g, []) + b0 = add('I', 'unpack1.l {d} = r0, %%[wb%d]' % g, 'b0_%d' % g, []) + b1 = add('I', 'unpack1.h {d} = r0, %%[wb%d]' % g, 'b1_%d' % g, []) + s1 = add('I', 'shrp {d} = {%s}, {%s}, 16' % (a1, a0), 's1_%d' % g, [a1, a0]) + s3 = add('I', 'shrp {d} = {%s}, {%s}, 16' % (b1, b0), 's3_%d' % g, [b1, b0]) + s5 = add('I', 'shrp {d} = {%s}, {%s}, 48' % (b1, b0), 's5_%d' % g, [b1, b0]) + taps = [a0, s1, b0, s3, a1, s5, b1] + pr = []; pl = [] + for k in range(7): + pr.append(add('I', 'pmpy2.r {d} = {%s}, %%[f%d]' % (taps[k], k), 'pr%d_%d' % (k, g), [taps[k]], 2)) + pl.append(add('I', 'pmpy2.l {d} = {%s}, %%[f%d]' % (taps[k], k), 'pl%d_%d' % (k, g), [taps[k]], 2)) + ar = tree_build(add, pr, 'padd4', 'tr%d' % g); al = tree_build(add, pl, 'padd4', 'tl%d' % g) + ar = add('A', 'padd4 {d} = {%s}, %%[bias]' % ar, 'br%d' % g, [ar]) + al = add('A', 'padd4 {d} = {%s}, %%[bias]' % al, 'bl%d' % g, [al]) + ar = add('I', 'pshr4 {d} = {%s}, %%[sh]' % ar, 'sr%d' % g, [ar]) + al = add('I', 'pshr4 {d} = {%s}, %%[sh]' % al, 'sl%d' % g, [al]) + pk = add('I', 'pack4.sss {d} = {%s}, {%s}' % (ar, al), 'pk%d' % g, [ar, al]) + add('I', 'mux2 %%[res%d] = {%s}, 0xd8' % (g, pk), None, [pk]) + return ops + +if __name__ == '__main__': + from final import alloc + for cap in (12, 17): + sched.PRESSURE_CAP = cap + for ng in (1, 2, 3): + ops = build_wiener_h(ng); cyc = sched.schedule(ops) + nI = sum(1 for o in ops if o.unit == 'I') + try: + asg = alloc(ops, cyc); note = 'regs=%d' % len(set(asg.values())) + except RuntimeError: + note = 'SPILL' + print('cap=%-3d ngrp=%d (%2d px): I=%2d cycles=%2d floor=%4.1f %.3f cyc/px %s' + % (cap, ng, ng * 4, nI, len(cyc), nI / 2.0, len(cyc) / float(ng * 4), note)) diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/itx_dct.h dav1d-1.5.4/src/ia64/itx_dct.h --- dav1d-1.5.4.orig/src/ia64/itx_dct.h 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/itx_dct.h 2026-08-20 15:26:13.745582785 +0200 @@ -0,0 +1,472 @@ +/* + * IA-64 packed inverse DCT, four columns at a time. + * + * The column pass of inv_txfm_add_c is 86% of all 1-D transform work (measured: + * the row pass runs only last_nonzero_col+1 times, the column pass always w + * times), and four adjacent columns of the intermediate buffer are contiguous + * int32 -- tmp[i*w + x .. x+3]. So the column pass vectorises with no + * transpose at all: pack four columns into one i16x4 and run the butterfly + * elementwise. + * + * The representation follows arm64's 8bpc itx: values live in 16-bit lanes and + * only a multiply widens to 32, narrowing straight back with rounding. That + * works because for 8bpc dav1d's CLIP() bounds are exactly INT16_MIN/MAX, so + * padd2.sss/psub2.sss give the clip for free -- arm's sqadd/sqsub. + * + * The overflow-avoidance rewriting in src/itx_1d.c is dropped, as that file's + * comment explicitly permits for SIMD. The corrections are exact because 4096 + * divides 2^12: + * ((in1*1567 - in3*(3784-4096) + 2048) >> 12) - in3 + * == (in1*1567 - in3*3784 + 2048) >> 12 + * + * Lane order does not matter -- the butterfly is elementwise across columns -- + * but it must be *consistent*: pack4_sss leaves loads in {0,1,2,3} while + * pmpy2_r/l land in {0,2,1,3}, so itx_mul* go through rsh_pack_rl, which + * unswizzles. That mux2 is the one op ia64 pays here that arm does not. + */ + +#define ITXC(x) splat16((int16_t) (x)) + +/* four columns of one transform point, from contiguous int32 */ +SIMD_FN i16x4 itx_ld4(const int32_t *const p) { + return pack4_sss(ld_i32x2(p), ld_i32x2(p + 2)); +} +SIMD_FN void itx_st4(int32_t *const p, const i16x4 v) { + st_i32x2(p, sext16_lo(v)); + st_i32x2(p + 2, sext16_hi(v)); +} + +/* in[] and out[] may alias; out[i] is written only after every in[] is read. */ +SIMD_FN void dct4_col4(i16x4 *const out, const i16x4 *const in, const int tx64) +{ + i16x4 t0, t1, t2, t3; + if (tx64) { + t0 = t1 = itx_mul1(in[0], ITXC(181), 8); + t2 = itx_mul1(in[1], ITXC(1567), 12); + t3 = itx_mul1(in[1], ITXC(3784), 12); + } else { + t0 = itx_mul1(padd2_sss(in[0], in[2]), ITXC(181), 8); + t1 = itx_mul1(psub2_sss(in[0], in[2]), ITXC(181), 8); + t2 = itx_mul2(in[1], ITXC(1567), in[3], ITXC(-3784), 12); + t3 = itx_mul2(in[1], ITXC(3784), in[3], ITXC( 1567), 12); + } + out[0] = padd2_sss(t0, t3); + out[1] = padd2_sss(t1, t2); + out[2] = psub2_sss(t1, t2); + out[3] = psub2_sss(t0, t3); +} + +SIMD_FN void dct8_col4(i16x4 *const out, const i16x4 *const in, const int tx64) +{ + i16x4 e[4]; + const i16x4 ev[4] = { in[0], in[2], in[4], in[6] }; + dct4_col4(e, ev, tx64); + + i16x4 t4a, t5a, t6a, t7a; + if (tx64) { + t4a = itx_mul1(in[1], ITXC( 799), 12); + t5a = itx_mul1(in[3], ITXC(-2276), 12); + t6a = itx_mul1(in[3], ITXC( 3406), 12); + t7a = itx_mul1(in[1], ITXC( 4017), 12); + } else { + t4a = itx_mul2(in[1], ITXC( 799), in[7], ITXC(-4017), 12); + t5a = itx_mul2(in[5], ITXC(1703), in[3], ITXC(-1138), 11); + t6a = itx_mul2(in[5], ITXC(1138), in[3], ITXC( 1703), 11); + t7a = itx_mul2(in[1], ITXC(4017), in[7], ITXC( 799), 12); + } + + const i16x4 t4 = padd2_sss(t4a, t5a); + const i16x4 x5 = psub2_sss(t4a, t5a); /* t5a in the C */ + const i16x4 t7 = padd2_sss(t7a, t6a); + const i16x4 x6 = psub2_sss(t7a, t6a); /* t6a in the C */ + + const i16x4 t5 = itx_mul2(x6, ITXC(181), x5, ITXC(-181), 8); + const i16x4 t6 = itx_mul2(x6, ITXC(181), x5, ITXC( 181), 8); + + out[0] = padd2_sss(e[0], t7); + out[1] = padd2_sss(e[1], t6); + out[2] = padd2_sss(e[2], t5); + out[3] = padd2_sss(e[3], t4); + out[4] = psub2_sss(e[3], t4); + out[5] = psub2_sss(e[2], t5); + out[6] = psub2_sss(e[1], t6); + out[7] = psub2_sss(e[0], t7); +} + +/* + * dct16. The C reuses t9a/t10a/t13a/t14a across stages; here each stage gets + * its own name (suffix _b, _c) so the dataflow is explicit. + * + * Every "+ inN" / "- inN" correction in the C is folded back into the + * multiplier, which is exact since 4096 = 2^12. The one non-obvious case: + * ((-(t13*(3784-4096) + t10*1567) + 2048) >> 12) - t13 + * == (t13*-3784 + t10*-1567 + 2048) >> 12 + */ +static __attribute__((noinline)) void dct16_col4(i16x4 *const out, const i16x4 *const in, const int tx64) +{ + i16x4 e[8]; + const i16x4 ev[8] = { in[0], in[2], in[4], in[6], + in[8], in[10], in[12], in[14] }; + dct8_col4(e, ev, tx64); + + i16x4 t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a; + if (tx64) { + t8a = itx_mul1(in[1], ITXC( 401), 12); + t9a = itx_mul1(in[7], ITXC(-2598), 12); + t10a = itx_mul1(in[5], ITXC( 1931), 12); + t11a = itx_mul1(in[3], ITXC(-1189), 12); + t12a = itx_mul1(in[3], ITXC( 3920), 12); + t13a = itx_mul1(in[5], ITXC( 3612), 12); + t14a = itx_mul1(in[7], ITXC( 3166), 12); + t15a = itx_mul1(in[1], ITXC( 4076), 12); + } else { + t8a = itx_mul2(in[1], ITXC( 401), in[15], ITXC(-4076), 12); + t9a = itx_mul2(in[9], ITXC(1583), in[7], ITXC(-1299), 11); + t10a = itx_mul2(in[5], ITXC(1931), in[11], ITXC(-3612), 12); + t11a = itx_mul2(in[13], ITXC(3920), in[3], ITXC(-1189), 12); + t12a = itx_mul2(in[13], ITXC(1189), in[3], ITXC( 3920), 12); + t13a = itx_mul2(in[5], ITXC(3612), in[11], ITXC( 1931), 12); + t14a = itx_mul2(in[9], ITXC(1299), in[7], ITXC( 1583), 11); + t15a = itx_mul2(in[1], ITXC(4076), in[15], ITXC( 401), 12); + } + + const i16x4 t8 = padd2_sss(t8a, t9a); + const i16x4 t9 = psub2_sss(t8a, t9a); + const i16x4 t10 = psub2_sss(t11a, t10a); + const i16x4 t11 = padd2_sss(t11a, t10a); + const i16x4 t12 = padd2_sss(t12a, t13a); + const i16x4 t13 = psub2_sss(t12a, t13a); + const i16x4 t14 = psub2_sss(t15a, t14a); + const i16x4 t15 = padd2_sss(t15a, t14a); + + const i16x4 t9a_b = itx_mul2(t14, ITXC( 1567), t9, ITXC(-3784), 12); + const i16x4 t14a_b = itx_mul2(t14, ITXC( 3784), t9, ITXC( 1567), 12); + const i16x4 t10a_b = itx_mul2(t13, ITXC(-3784), t10, ITXC(-1567), 12); + const i16x4 t13a_b = itx_mul2(t13, ITXC( 1567), t10, ITXC(-3784), 12); + + const i16x4 t8a_c = padd2_sss(t8, t11); + const i16x4 t9_c = padd2_sss(t9a_b, t10a_b); + const i16x4 t10_c = psub2_sss(t9a_b, t10a_b); + const i16x4 t11a_c = psub2_sss(t8, t11); + const i16x4 t12a_c = psub2_sss(t15, t12); + const i16x4 t13_c = psub2_sss(t14a_b, t13a_b); + const i16x4 t14_c = padd2_sss(t14a_b, t13a_b); + const i16x4 t15a_c = padd2_sss(t15, t12); + + const i16x4 t10a_d = itx_mul2(t13_c, ITXC(181), t10_c, ITXC(-181), 8); + const i16x4 t13a_d = itx_mul2(t13_c, ITXC(181), t10_c, ITXC( 181), 8); + const i16x4 t11_d = itx_mul2(t12a_c, ITXC(181), t11a_c, ITXC(-181), 8); + const i16x4 t12_d = itx_mul2(t12a_c, ITXC(181), t11a_c, ITXC( 181), 8); + + out[ 0] = padd2_sss(e[0], t15a_c); + out[ 1] = padd2_sss(e[1], t14_c); + out[ 2] = padd2_sss(e[2], t13a_d); + out[ 3] = padd2_sss(e[3], t12_d); + out[ 4] = padd2_sss(e[4], t11_d); + out[ 5] = padd2_sss(e[5], t10a_d); + out[ 6] = padd2_sss(e[6], t9_c); + out[ 7] = padd2_sss(e[7], t8a_c); + out[ 8] = psub2_sss(e[7], t8a_c); + out[ 9] = psub2_sss(e[6], t9_c); + out[10] = psub2_sss(e[5], t10a_d); + out[11] = psub2_sss(e[4], t11_d); + out[12] = psub2_sss(e[3], t12_d); + out[13] = psub2_sss(e[2], t13a_d); + out[14] = psub2_sss(e[1], t14_c); + out[15] = psub2_sss(e[0], t15a_c); +} + +/* + * dct32. Stages are lettered A..F rather than reusing the C's names, which + * are assigned up to four times each -- that reuse is where hand transcription + * goes wrong. + * + * Every "v * (C - 4096)" term with a paired "+/- v" correction outside the + * shift collapses to "v * C": the (-4096) contributes -/+ v exactly (4096 = + * 2^12), cancelling the correction. Where the whole bracket is negated the + * cancellation still holds, e.g. + * ((-(t27a*(3784-4096) + t20a*1567) + 2048) >> 12) - t27a + * == (t27a*-3784 + t20a*-1567 + 2048) >> 12 + */ +static __attribute__((noinline)) void dct32_col4(i16x4 *const out, const i16x4 *const in, const int tx64) +{ + i16x4 e[16]; + const i16x4 ev[16] = { in[0], in[2], in[4], in[6], in[8], in[10], + in[12], in[14], in[16], in[18], in[20], in[22], + in[24], in[26], in[28], in[30] }; + dct16_col4(e, ev, tx64); + + i16x4 a16, a17, a18, a19, a20, a21, a22, a23; + i16x4 a24, a25, a26, a27, a28, a29, a30, a31; + if (tx64) { + a16 = itx_mul1(in[1], ITXC( 201), 12); + a17 = itx_mul1(in[15], ITXC(-2751), 12); + a18 = itx_mul1(in[9], ITXC( 1751), 12); + a19 = itx_mul1(in[7], ITXC(-1380), 12); + a20 = itx_mul1(in[5], ITXC( 995), 12); + a21 = itx_mul1(in[11], ITXC(-2106), 12); + a22 = itx_mul1(in[13], ITXC( 2440), 12); + a23 = itx_mul1(in[3], ITXC( -601), 12); + a24 = itx_mul1(in[3], ITXC( 4052), 12); + a25 = itx_mul1(in[13], ITXC( 3290), 12); + a26 = itx_mul1(in[11], ITXC( 3513), 12); + a27 = itx_mul1(in[5], ITXC( 3973), 12); + a28 = itx_mul1(in[7], ITXC( 3857), 12); + a29 = itx_mul1(in[9], ITXC( 3703), 12); + a30 = itx_mul1(in[15], ITXC( 3035), 12); + a31 = itx_mul1(in[1], ITXC( 4091), 12); + } else { + a16 = itx_mul2(in[1], ITXC( 201), in[31], ITXC(-4091), 12); + a17 = itx_mul2(in[17], ITXC(3035), in[15], ITXC(-2751), 12); + a18 = itx_mul2(in[9], ITXC(1751), in[23], ITXC(-3703), 12); + a19 = itx_mul2(in[25], ITXC(3857), in[7], ITXC(-1380), 12); + a20 = itx_mul2(in[5], ITXC( 995), in[27], ITXC(-3973), 12); + a21 = itx_mul2(in[21], ITXC(3513), in[11], ITXC(-2106), 12); + a22 = itx_mul2(in[13], ITXC(1220), in[19], ITXC(-1645), 11); + a23 = itx_mul2(in[29], ITXC(4052), in[3], ITXC( -601), 12); + a24 = itx_mul2(in[29], ITXC( 601), in[3], ITXC( 4052), 12); + a25 = itx_mul2(in[13], ITXC(1645), in[19], ITXC( 1220), 11); + a26 = itx_mul2(in[21], ITXC(2106), in[11], ITXC( 3513), 12); + a27 = itx_mul2(in[5], ITXC(3973), in[27], ITXC( 995), 12); + a28 = itx_mul2(in[25], ITXC(1380), in[7], ITXC( 3857), 12); + a29 = itx_mul2(in[9], ITXC(3703), in[23], ITXC( 1751), 12); + a30 = itx_mul2(in[17], ITXC(2751), in[15], ITXC( 3035), 12); + a31 = itx_mul2(in[1], ITXC(4091), in[31], ITXC( 201), 12); + } + + const i16x4 A16 = padd2_sss(a16, a17), A17 = psub2_sss(a16, a17); + const i16x4 A18 = psub2_sss(a19, a18), A19 = padd2_sss(a19, a18); + const i16x4 A20 = padd2_sss(a20, a21), A21 = psub2_sss(a20, a21); + const i16x4 A22 = psub2_sss(a23, a22), A23 = padd2_sss(a23, a22); + const i16x4 A24 = padd2_sss(a24, a25), A25 = psub2_sss(a24, a25); + const i16x4 A26 = psub2_sss(a27, a26), A27 = padd2_sss(a27, a26); + const i16x4 A28 = padd2_sss(a28, a29), A29 = psub2_sss(a28, a29); + const i16x4 A30 = psub2_sss(a31, a30), A31 = padd2_sss(a31, a30); + + const i16x4 B17 = itx_mul2(A30, ITXC( 799), A17, ITXC(-4017), 12); + const i16x4 B30 = itx_mul2(A30, ITXC( 4017), A17, ITXC( 799), 12); + const i16x4 B18 = itx_mul2(A29, ITXC(-4017), A18, ITXC( -799), 12); + const i16x4 B29 = itx_mul2(A29, ITXC( 799), A18, ITXC(-4017), 12); + const i16x4 B21 = itx_mul2(A26, ITXC( 1703), A21, ITXC(-1138), 11); + const i16x4 B26 = itx_mul2(A26, ITXC( 1138), A21, ITXC( 1703), 11); + const i16x4 B22 = itx_mul2(A25, ITXC(-1138), A22, ITXC(-1703), 11); + const i16x4 B25 = itx_mul2(A25, ITXC( 1703), A22, ITXC(-1138), 11); + + const i16x4 C16 = padd2_sss(A16, A19), C19 = psub2_sss(A16, A19); + const i16x4 C17 = padd2_sss(B17, B18), C18 = psub2_sss(B17, B18); + const i16x4 C20 = psub2_sss(A23, A20), C23 = padd2_sss(A23, A20); + const i16x4 C21 = psub2_sss(B22, B21), C22 = padd2_sss(B22, B21); + const i16x4 C24 = padd2_sss(A24, A27), C27 = psub2_sss(A24, A27); + const i16x4 C25 = padd2_sss(B25, B26), C26 = psub2_sss(B25, B26); + const i16x4 C28 = psub2_sss(A31, A28), C31 = padd2_sss(A31, A28); + const i16x4 C29 = psub2_sss(B30, B29), C30 = padd2_sss(B30, B29); + + const i16x4 D18 = itx_mul2(C29, ITXC( 1567), C18, ITXC(-3784), 12); + const i16x4 D29 = itx_mul2(C29, ITXC( 3784), C18, ITXC( 1567), 12); + const i16x4 D19 = itx_mul2(C28, ITXC( 1567), C19, ITXC(-3784), 12); + const i16x4 D28 = itx_mul2(C28, ITXC( 3784), C19, ITXC( 1567), 12); + const i16x4 D20 = itx_mul2(C27, ITXC(-3784), C20, ITXC(-1567), 12); + const i16x4 D27 = itx_mul2(C27, ITXC( 1567), C20, ITXC(-3784), 12); + const i16x4 D21 = itx_mul2(C26, ITXC(-3784), C21, ITXC(-1567), 12); + const i16x4 D26 = itx_mul2(C26, ITXC( 1567), C21, ITXC(-3784), 12); + + const i16x4 E16 = padd2_sss(C16, C23), E23 = psub2_sss(C16, C23); + const i16x4 E17 = padd2_sss(C17, C22), E22 = psub2_sss(C17, C22); + const i16x4 E18 = padd2_sss(D18, D21), E21 = psub2_sss(D18, D21); + const i16x4 E19 = padd2_sss(D19, D20), E20 = psub2_sss(D19, D20); + const i16x4 E24 = psub2_sss(C31, C24), E31 = padd2_sss(C31, C24); + const i16x4 E25 = psub2_sss(C30, C25), E30 = padd2_sss(C30, C25); + const i16x4 E26 = psub2_sss(D29, D26), E29 = padd2_sss(D29, D26); + const i16x4 E27 = psub2_sss(D28, D27), E28 = padd2_sss(D28, D27); + + const i16x4 F20 = itx_mul2(E27, ITXC(181), E20, ITXC(-181), 8); + const i16x4 F27 = itx_mul2(E27, ITXC(181), E20, ITXC( 181), 8); + const i16x4 F21 = itx_mul2(E26, ITXC(181), E21, ITXC(-181), 8); + const i16x4 F26 = itx_mul2(E26, ITXC(181), E21, ITXC( 181), 8); + const i16x4 F22 = itx_mul2(E25, ITXC(181), E22, ITXC(-181), 8); + const i16x4 F25 = itx_mul2(E25, ITXC(181), E22, ITXC( 181), 8); + const i16x4 F23 = itx_mul2(E24, ITXC(181), E23, ITXC(-181), 8); + const i16x4 F24 = itx_mul2(E24, ITXC(181), E23, ITXC( 181), 8); + + const i16x4 o[16] = { E31, E30, E29, E28, F27, F26, F25, F24, + F23, F22, F21, F20, E19, E18, E17, E16 }; + for (int i = 0; i < 16; i++) { + const i16x4 lo = e[i], hi = o[i]; + out[i] = padd2_sss(lo, hi); + out[31 - i] = psub2_sss(lo, hi); + } +} + +/* + * dct64. Stages lettered A..H; the C reassigns each t-name up to six times, + * so the letters are the only reliable way to keep the dataflow straight. + * + * Correction folding, both directions (4096 = 2^12 exactly): + * v * (C - 4096) with a paired + v -> v * C + * v * (4096 - C) with a paired - v -> v * -C + * i.e. drop the +/-4096 from the multiplier and drop the correction. + * + * The even half comes from dct32 in its tx64 form, which is why only the odd + * inputs 1,3,..,31 are read: AV1 zeroes coefficients 32..63 of a 64-point + * transform, and the whole tx64 path exists to exploit that. + */ +static __attribute__((noinline)) +void dct64_col4(i16x4 *const out, const i16x4 *const in) +{ + i16x4 e[32]; + i16x4 ev[32]; + for (int i = 0; i < 32; i++) ev[i] = in[2 * i]; + dct32_col4(e, ev, 1); + + static const int16_t k[32] = { + 101, -2824, 1660, -1474, 897, -2191, 2359, -700, + 501, -2520, 2019, -1092, 1285, -1842, 2675, -301, + 4085, 3102, 3659, 3889, 3948, 3564, 3229, 4065, + 4036, 3349, 3461, 3996, 3822, 3745, 2967, 4095 }; + static const uint8_t ki[32] = { + 1, 31, 17, 15, 9, 23, 25, 7, 5, 27, 21, 11, 13, 19, 29, 3, + 3, 29, 19, 13, 11, 21, 27, 5, 7, 25, 23, 9, 15, 17, 31, 1 }; + i16x4 a[32]; + for (int i = 0; i < 32; i++) + a[i] = itx_mul1(in[ki[i]], ITXC(k[i]), 12); + + /* A: pairwise butterfly, alternating orientation every other pair */ + i16x4 A[32]; + for (int i = 0; i < 32; i += 4) { + A[i + 0] = padd2_sss(a[i + 0], a[i + 1]); + A[i + 1] = psub2_sss(a[i + 0], a[i + 1]); + A[i + 2] = psub2_sss(a[i + 3], a[i + 2]); + A[i + 3] = padd2_sss(a[i + 3], a[i + 2]); + } +#define M(x) A[(x) - 32] + const i16x4 B33 = itx_mul2(M(33), ITXC(-4076), M(62), ITXC( 401), 12); + const i16x4 B34 = itx_mul2(M(34), ITXC( -401), M(61), ITXC(-4076), 12); + const i16x4 B37 = itx_mul2(M(37), ITXC(-1299), M(58), ITXC( 1583), 11); + const i16x4 B38 = itx_mul2(M(38), ITXC(-1583), M(57), ITXC(-1299), 11); + const i16x4 B41 = itx_mul2(M(41), ITXC(-3612), M(54), ITXC( 1931), 12); + const i16x4 B42 = itx_mul2(M(42), ITXC(-1931), M(53), ITXC(-3612), 12); + const i16x4 B45 = itx_mul2(M(45), ITXC(-1189), M(50), ITXC( 3920), 12); + const i16x4 B46 = itx_mul2(M(46), ITXC(-3920), M(49), ITXC(-1189), 12); + const i16x4 B49 = itx_mul2(M(46), ITXC(-1189), M(49), ITXC( 3920), 12); + const i16x4 B50 = itx_mul2(M(45), ITXC( 3920), M(50), ITXC( 1189), 12); + const i16x4 B53 = itx_mul2(M(42), ITXC(-3612), M(53), ITXC( 1931), 12); + const i16x4 B54 = itx_mul2(M(41), ITXC( 1931), M(54), ITXC( 3612), 12); + const i16x4 B57 = itx_mul2(M(38), ITXC(-1299), M(57), ITXC( 1583), 11); + const i16x4 B58 = itx_mul2(M(37), ITXC( 1583), M(58), ITXC( 1299), 11); + const i16x4 B61 = itx_mul2(M(34), ITXC(-4076), M(61), ITXC( 401), 12); + const i16x4 B62 = itx_mul2(M(33), ITXC( 401), M(62), ITXC( 4076), 12); + + const i16x4 C32 = padd2_sss(M(32), M(35)), C35 = psub2_sss(M(32), M(35)); + const i16x4 C33 = padd2_sss(B33, B34), C34 = psub2_sss(B33, B34); + const i16x4 C36 = psub2_sss(M(39), M(36)), C39 = padd2_sss(M(39), M(36)); + const i16x4 C37 = psub2_sss(B38, B37), C38 = padd2_sss(B38, B37); + const i16x4 C40 = padd2_sss(M(40), M(43)), C43 = psub2_sss(M(40), M(43)); + const i16x4 C41 = padd2_sss(B41, B42), C42 = psub2_sss(B41, B42); + const i16x4 C44 = psub2_sss(M(47), M(44)), C47 = padd2_sss(M(47), M(44)); + const i16x4 C45 = psub2_sss(B46, B45), C46 = padd2_sss(B46, B45); + const i16x4 C48 = padd2_sss(M(48), M(51)), C51 = psub2_sss(M(48), M(51)); + const i16x4 C49 = padd2_sss(B49, B50), C50 = psub2_sss(B49, B50); + const i16x4 C52 = psub2_sss(M(55), M(52)), C55 = padd2_sss(M(55), M(52)); + const i16x4 C53 = psub2_sss(B54, B53), C54 = padd2_sss(B54, B53); + const i16x4 C56 = padd2_sss(M(56), M(59)), C59 = psub2_sss(M(56), M(59)); + const i16x4 C57 = padd2_sss(B57, B58), C58 = psub2_sss(B57, B58); + const i16x4 C60 = psub2_sss(M(63), M(60)), C63 = padd2_sss(M(63), M(60)); + const i16x4 C61 = psub2_sss(B62, B61), C62 = padd2_sss(B62, B61); +#undef M + + const i16x4 D34 = itx_mul2(C34, ITXC(-4017), C61, ITXC( 799), 12); + const i16x4 D35 = itx_mul2(C35, ITXC(-4017), C60, ITXC( 799), 12); + const i16x4 D36 = itx_mul2(C36, ITXC( -799), C59, ITXC(-4017), 12); + const i16x4 D37 = itx_mul2(C37, ITXC( -799), C58, ITXC(-4017), 12); + const i16x4 D42 = itx_mul2(C42, ITXC(-1138), C53, ITXC( 1703), 11); + const i16x4 D43 = itx_mul2(C43, ITXC(-1138), C52, ITXC( 1703), 11); + const i16x4 D44 = itx_mul2(C44, ITXC(-1703), C51, ITXC(-1138), 11); + const i16x4 D45 = itx_mul2(C45, ITXC(-1703), C50, ITXC(-1138), 11); + const i16x4 D50 = itx_mul2(C45, ITXC(-1138), C50, ITXC( 1703), 11); + const i16x4 D51 = itx_mul2(C44, ITXC(-1138), C51, ITXC( 1703), 11); + const i16x4 D52 = itx_mul2(C43, ITXC( 1703), C52, ITXC( 1138), 11); + const i16x4 D53 = itx_mul2(C42, ITXC( 1703), C53, ITXC( 1138), 11); + const i16x4 D58 = itx_mul2(C37, ITXC(-4017), C58, ITXC( 799), 12); + const i16x4 D59 = itx_mul2(C36, ITXC(-4017), C59, ITXC( 799), 12); + const i16x4 D60 = itx_mul2(C35, ITXC( 799), C60, ITXC( 4017), 12); + const i16x4 D61 = itx_mul2(C34, ITXC( 799), C61, ITXC( 4017), 12); + + const i16x4 E32 = padd2_sss(C32, C39), E39 = psub2_sss(C32, C39); + const i16x4 E33 = padd2_sss(C33, C38), E38 = psub2_sss(C33, C38); + const i16x4 E34 = padd2_sss(D34, D37), E37 = psub2_sss(D34, D37); + const i16x4 E35 = padd2_sss(D35, D36), E36 = psub2_sss(D35, D36); + const i16x4 E40 = psub2_sss(C47, C40), E47 = padd2_sss(C47, C40); + const i16x4 E41 = psub2_sss(C46, C41), E46 = padd2_sss(C46, C41); + const i16x4 E42 = psub2_sss(D45, D42), E45 = padd2_sss(D45, D42); + const i16x4 E43 = psub2_sss(D44, D43), E44 = padd2_sss(D44, D43); + const i16x4 E48 = padd2_sss(C48, C55), E55 = psub2_sss(C48, C55); + const i16x4 E49 = padd2_sss(C49, C54), E54 = psub2_sss(C49, C54); + const i16x4 E50 = padd2_sss(D50, D53), E53 = psub2_sss(D50, D53); + const i16x4 E51 = padd2_sss(D51, D52), E52 = psub2_sss(D51, D52); + const i16x4 E56 = psub2_sss(C63, C56), E63 = padd2_sss(C63, C56); + const i16x4 E57 = psub2_sss(C62, C57), E62 = padd2_sss(C62, C57); + const i16x4 E58 = psub2_sss(D61, D58), E61 = padd2_sss(D61, D58); + const i16x4 E59 = psub2_sss(D60, D59), E60 = padd2_sss(D60, D59); + + const i16x4 F36 = itx_mul2(E36, ITXC(-3784), E59, ITXC( 1567), 12); + const i16x4 F37 = itx_mul2(E37, ITXC(-3784), E58, ITXC( 1567), 12); + const i16x4 F38 = itx_mul2(E38, ITXC(-3784), E57, ITXC( 1567), 12); + const i16x4 F39 = itx_mul2(E39, ITXC(-3784), E56, ITXC( 1567), 12); + const i16x4 F40 = itx_mul2(E40, ITXC(-1567), E55, ITXC(-3784), 12); + const i16x4 F41 = itx_mul2(E41, ITXC(-1567), E54, ITXC(-3784), 12); + const i16x4 F42 = itx_mul2(E42, ITXC(-1567), E53, ITXC(-3784), 12); + const i16x4 F43 = itx_mul2(E43, ITXC(-1567), E52, ITXC(-3784), 12); + const i16x4 F52 = itx_mul2(E43, ITXC(-3784), E52, ITXC( 1567), 12); + const i16x4 F53 = itx_mul2(E42, ITXC(-3784), E53, ITXC( 1567), 12); + const i16x4 F54 = itx_mul2(E41, ITXC(-3784), E54, ITXC( 1567), 12); + const i16x4 F55 = itx_mul2(E40, ITXC(-3784), E55, ITXC( 1567), 12); + const i16x4 F56 = itx_mul2(E39, ITXC( 1567), E56, ITXC( 3784), 12); + const i16x4 F57 = itx_mul2(E38, ITXC( 1567), E57, ITXC( 3784), 12); + const i16x4 F58 = itx_mul2(E37, ITXC( 1567), E58, ITXC( 3784), 12); + const i16x4 F59 = itx_mul2(E36, ITXC( 1567), E59, ITXC( 3784), 12); + + const i16x4 G32 = padd2_sss(E32, E47), G47 = psub2_sss(E32, E47); + const i16x4 G33 = padd2_sss(E33, E46), G46 = psub2_sss(E33, E46); + const i16x4 G34 = padd2_sss(E34, E45), G45 = psub2_sss(E34, E45); + const i16x4 G35 = padd2_sss(E35, E44), G44 = psub2_sss(E35, E44); + const i16x4 G36 = padd2_sss(F36, F43), G43 = psub2_sss(F36, F43); + const i16x4 G37 = padd2_sss(F37, F42), G42 = psub2_sss(F37, F42); + const i16x4 G38 = padd2_sss(F38, F41), G41 = psub2_sss(F38, F41); + const i16x4 G39 = padd2_sss(F39, F40), G40 = psub2_sss(F39, F40); + const i16x4 G48 = psub2_sss(E63, E48), G63 = padd2_sss(E63, E48); + const i16x4 G49 = psub2_sss(E62, E49), G62 = padd2_sss(E62, E49); + const i16x4 G50 = psub2_sss(E61, E50), G61 = padd2_sss(E61, E50); + const i16x4 G51 = psub2_sss(E60, E51), G60 = padd2_sss(E60, E51); + const i16x4 G52 = psub2_sss(F59, F52), G59 = padd2_sss(F59, F52); + const i16x4 G53 = psub2_sss(F58, F53), G58 = padd2_sss(F58, F53); + const i16x4 G54 = psub2_sss(F57, F54), G57 = padd2_sss(F57, F54); + const i16x4 G55 = psub2_sss(F56, F55), G56 = padd2_sss(F56, F55); + + const i16x4 H40 = itx_mul2(G55, ITXC(181), G40, ITXC(-181), 8); + const i16x4 H41 = itx_mul2(G54, ITXC(181), G41, ITXC(-181), 8); + const i16x4 H42 = itx_mul2(G53, ITXC(181), G42, ITXC(-181), 8); + const i16x4 H43 = itx_mul2(G52, ITXC(181), G43, ITXC(-181), 8); + const i16x4 H44 = itx_mul2(G51, ITXC(181), G44, ITXC(-181), 8); + const i16x4 H45 = itx_mul2(G50, ITXC(181), G45, ITXC(-181), 8); + const i16x4 H46 = itx_mul2(G49, ITXC(181), G46, ITXC(-181), 8); + const i16x4 H47 = itx_mul2(G48, ITXC(181), G47, ITXC(-181), 8); + const i16x4 H48 = itx_mul2(G48, ITXC(181), G47, ITXC( 181), 8); + const i16x4 H49 = itx_mul2(G49, ITXC(181), G46, ITXC( 181), 8); + const i16x4 H50 = itx_mul2(G50, ITXC(181), G45, ITXC( 181), 8); + const i16x4 H51 = itx_mul2(G51, ITXC(181), G44, ITXC( 181), 8); + const i16x4 H52 = itx_mul2(G52, ITXC(181), G43, ITXC( 181), 8); + const i16x4 H53 = itx_mul2(G53, ITXC(181), G42, ITXC( 181), 8); + const i16x4 H54 = itx_mul2(G54, ITXC(181), G41, ITXC( 181), 8); + const i16x4 H55 = itx_mul2(G55, ITXC(181), G40, ITXC( 181), 8); + + const i16x4 o[32] = { + G63, G62, G61, G60, G59, G58, G57, G56, + H55, H54, H53, H52, H51, H50, H49, H48, + H47, H46, H45, H44, H43, H42, H41, H40, + G39, G38, G37, G36, G35, G34, G33, G32 }; + for (int i = 0; i < 32; i++) { + const i16x4 lo = e[i], hi = o[i]; + out[i] = padd2_sss(lo, hi); + out[63 - i] = psub2_sss(lo, hi); + } +} diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/loopfilter.h dav1d-1.5.4/src/ia64/loopfilter.h --- dav1d-1.5.4.orig/src/ia64/loopfilter.h 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/loopfilter.h 2026-08-19 16:28:50.337835440 +0200 @@ -0,0 +1,34 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/loopfilter.h" + +bitfn_decls(void dav1d_loop_filter_dsp_init_ia64, Dav1dLoopFilterDSPContext *c); + +static ALWAYS_INLINE void loop_filter_dsp_init_ia64(Dav1dLoopFilterDSPContext *const c) { + bitfn(dav1d_loop_filter_dsp_init_ia64)(c); +} diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/loopfilter_tmpl.c dav1d-1.5.4/src/ia64/loopfilter_tmpl.c --- dav1d-1.5.4.orig/src/ia64/loopfilter_tmpl.c 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/loopfilter_tmpl.c 2026-08-20 09:11:01.664222261 +0200 @@ -0,0 +1,461 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/attributes.h" +#include "common/intops.h" + +#include "src/cpu.h" +#include "src/ia64/dav1d_simd.h" +#include "src/ia64/loopfilter.h" +#include "src/levels.h" +#include "src/loopfilter.h" + +#if BITDEPTH == 8 + +/* + * Deblocking, horizontal-edge direction only. + * + * The C filter walks four samples along the edge one at a time, and every one + * of them runs the same decision tree over the same taps. Those four are the + * natural lanes here -- but only for the horizontal-edge entry points, where + * stridea is 1 and the four samples are contiguous, so each tap row is a single + * 4-byte load. The vertical-edge entry points have stridea == stride and would + * need a 14x4 transpose to reach the same layout, so they are left on the C + * path for now. + * + * Every branch in the scalar filter becomes a mask. Nothing is skipped per + * lane: all of the candidate outputs that the block's width can produce are + * computed and then selected between, which is what makes the whole thing + * straight-line. The one branch worth keeping is the all-lanes-rejected test, + * because a 4-sample unit that no lane filters is common and skipping it also + * skips the stores. + * + * Ranges: pixels are 8-bit and the widest tap sum has 16 unit-weighted terms, + * so 255 * 16 = 4080 and everything below stays inside int16 without care. + */ + +#define LF_SIMD_FN static inline __attribute__((always_inline)) + +LF_SIMD_FN i16x4 lf_abs(const i16x4 v) { + const i16x4 z = { 0, 0, 0, 0 }; + return pmax2(v, z - v); +} + +/* Lanewise a <= b as an all-ones mask. */ +LF_SIMD_FN i16x4 lf_le(const i16x4 a, const i16x4 b) { + const i16x4 ones = { -1, -1, -1, -1 }; + return pcmp2_gt(a, b) ^ ones; +} + +LF_SIMD_FN i16x4 lf_sel(const i16x4 m, const i16x4 t, const i16x4 f) { + return (t & m) | (f & ~m); +} + +LF_SIMD_FN int lf_any(const i16x4 m) { + uint64_t bits; + memcpy(&bits, &m, 8); + return bits != 0; +} + +/* The four samples of one edge unit. dst and the row pitch are both 4-byte + * aligned on the fast path, which the caller establishes once per row. */ +LF_SIMD_FN i16x4 lf_ld(const pixel *const p, const int aligned) { + if (aligned) { + uint32_t v; + memcpy(&v, p, 4); /* provably aligned: a single ld4 */ + return zext8_lo((u8x8)(uint64_t)v); + } + return zext8_lo(ldn_u8x8(p, 4)); +} + +/* pack2.uss applies iclip_pixel() as part of the narrowing. */ +LF_SIMD_FN void lf_st(pixel *const p, const i16x4 v) { + const i16x4 z = { 0, 0, 0, 0 }; + st_u8_partial(p, pack2_uss(v, z), 4); +} + +/* ------------------------------------------------- vertical-edge transpose + * + * For a vertical edge the four samples of a unit are four *rows*, so the lanes + * the filter wants are spread one per row, while one row's fourteen taps are + * contiguous bytes. Reaching the filter's layout therefore needs a transpose. + * + * It is done in 4x4 byte blocks rather than 4x8, because dst sits on a + * 4-pixel boundary: every access below is then a provably aligned ld4/st4. + * That matters more than the slightly larger unpack count would suggest -- + * ia64 has no unaligned store at all, and st_u8_partial() would degrade to + * eight st1 per row, which costs far more than the transpose itself. + * + * A 4x4 block is 2 unpack1 + 2 unpack2 + 4 zext8 going in, and 2 pack2 + + * 4 unpack1 coming back out; sixteen I-slot ops for the round trip. + */ + +typedef uint32_t lf_u32 __attribute__((may_alias, aligned(4))); + +LF_SIMD_FN i8x8 lf_ld4(const pixel *const p, const int aligned) { + uint32_t v; + if (aligned) v = ((const lf_u32 *)p)[0]; + else memcpy(&v, p, 4); + return (i8x8)(u32x2){ v, 0 }; +} + +LF_SIMD_FN void lf_sr4(pixel *const p, const uint32_t v, const int aligned) { + if (aligned) ((lf_u32 *)p)[0] = v; + else memcpy(p, &v, 4); +} + +/* out[c] receives column c of the four rows at p, p+s, p+2s, p+3s. */ +LF_SIMD_FN void lf_t4(i16x4 *const out, const pixel *const p, + const ptrdiff_t s, const int aligned) +{ + const i8x8 r0 = lf_ld4(p, aligned); + const i8x8 r1 = lf_ld4(p + s, aligned); + const i8x8 r2 = lf_ld4(p + 2 * s, aligned); + const i8x8 r3 = lf_ld4(p + 3 * s, aligned); + const i8x8 a = unpack1_l(r1, r0); /* {r0[0],r1[0],r0[1],r1[1],..} */ + const i8x8 c = unpack1_l(r3, r2); + const u8x8 e0 = (u8x8)unpack2_l(c, a); /* cols 0,1 */ + const u8x8 e1 = (u8x8)unpack2_h(c, a); /* cols 2,3 */ + out[0] = zext8_lo(e0); out[1] = zext8_hi(e0); + out[2] = zext8_lo(e1); out[3] = zext8_hi(e1); +} + +/* Inverse of lf_t4. pack2.uss applies iclip_pixel() on the way out. */ +LF_SIMD_FN void lf_st4(pixel *const p, const ptrdiff_t s, + const i16x4 *const in, const int aligned) +{ + const u8x8 P0 = pack2_uss(in[0], in[1]); /* {c0 r0..r3, c1 r0..r3} */ + const u8x8 P1 = pack2_uss(in[2], in[3]); + const i8x8 u = unpack1_l((i8x8)P1, (i8x8)P0); + const i8x8 v = unpack1_h((i8x8)P1, (i8x8)P0); + const u32x2 w = (u32x2)unpack1_l(v, u); /* rows 0,1 */ + const u32x2 x = (u32x2)unpack1_h(v, u); /* rows 2,3 */ + lf_sr4(p, w[0], aligned); + lf_sr4(p + s, w[1], aligned); + lf_sr4(p + 2 * s, x[0], aligned); + lf_sr4(p + 3 * s, x[1], aligned); +} + +/* + * T[j] holds tap dst[j - 7], so T runs p6..q6 over j = 0..13. The 4-aligned + * byte groups of dst land on T[-1..2], T[3..6], T[7..10] and T[11..14], which + * is why the caller's array has one slot of headroom at each end. + * + * Only the groups a given width touches are moved, and the same groups are + * written back. Taps inside a loaded group that the filter does not modify + * are stored back with the value that was just read, which is a no-op: the + * edge to the left was filtered by an earlier call and the edge to the right + * has not run yet, so nothing else is in flight over those bytes. + */ +LF_SIMD_FN void lf_load_t(i16x4 *const T, const pixel *const dst, + const ptrdiff_t s, const int wd, const int aligned) +{ + if (wd == 16) { lf_t4(&T[-1], dst - 8, s, aligned); + lf_t4(&T[11], dst + 4, s, aligned); } + lf_t4(&T[3], dst - 4, s, aligned); + lf_t4(&T[7], dst, s, aligned); +} + +LF_SIMD_FN void lf_store_t(pixel *const dst, const ptrdiff_t s, + const i16x4 *const T, const int wd, const int aligned) +{ + if (wd == 16) { lf_st4(dst - 8, s, &T[-1], aligned); + lf_st4(dst + 4, s, &T[11], aligned); } + lf_st4(dst - 4, s, &T[3], aligned); + lf_st4(dst, s, &T[7], aligned); +} + +LF_SIMD_FN void lf_unit(pixel *const dst, const int E, const int I, const int H, + const ptrdiff_t sb, const int wd, const int aligned, + const int transposed) +{ + /* One slot of headroom either side: see lf_load_t(). */ + i16x4 Tbuf[16]; + i16x4 *const T = Tbuf + 1; + if (transposed) lf_load_t(T, dst, sb, wd, aligned); + + const i16x4 vE = splat16((int16_t)E); + const i16x4 vI = splat16((int16_t)I); + const i16x4 vH = splat16((int16_t)H); + const i16x4 vF = { 1, 1, 1, 1 }; /* F, with bitdepth_min_8 == 0 */ + const i16x4 k3 = { 3, 3, 3, 3 }, k4 = { 4, 4, 4, 4 }, k8 = { 8, 8, 8, 8 }; + const i16x4 k1 = { 1, 1, 1, 1 }; + const i16x4 lo = { -128, -128, -128, -128 }, hi = { 127, 127, 127, 127 }; + const i16x4 z = { 0, 0, 0, 0 }; + +#define L(k) (transposed ? T[(k) + 7] : lf_ld(dst + (k) * sb, aligned)) +#define ST(k, v) do { if (transposed) T[(k) + 7] = (v); \ + else lf_st(dst + (k) * sb, (v)); } while (0) +#define FLUSH() do { if (transposed) lf_store_t(dst, sb, T, wd, aligned); } \ + while (0) + + const i16x4 p1 = L(-2), p0 = L(-1), q0 = L(0), q1 = L(1); + + i16x4 fm = lf_le(lf_abs(p1 - p0), vI) & lf_le(lf_abs(q1 - q0), vI) & + lf_le((lf_abs(p0 - q0) << 1) + (lf_abs(p1 - q1) >> 1), vE); + + i16x4 p2 = z, q2 = z, p3 = z, q3 = z; + if (wd > 4) { + p2 = L(-3); q2 = L(2); + fm &= lf_le(lf_abs(p2 - p1), vI) & lf_le(lf_abs(q2 - q1), vI); + if (wd > 6) { + p3 = L(-4); q3 = L(3); + fm &= lf_le(lf_abs(p3 - p2), vI) & lf_le(lf_abs(q3 - q2), vI); + } + } + if (!lf_any(fm)) return; + + /* ---- narrow filter, the fallback for every lane ---- */ + const i16x4 hev = pcmp2_gt(lf_abs(p1 - p0), vH) | pcmp2_gt(lf_abs(q1 - q0), vH); + const i16x4 fh = pmin2(pmax2(p1 - q1, lo), hi); + i16x4 f = pmin2(pmax2((q0 - p0) * k3 + (fh & hev), lo), hi); + const i16x4 f1 = pmin2(f + k4, hi) >> 3; + const i16x4 f2 = pmin2(f + k3, hi) >> 3; + const i16x4 f3 = (f1 + k1) >> 1; + const i16x4 nhev = ~hev; + const i16x4 np0 = p0 + f2, nq0 = q0 - f1; + const i16x4 np1 = p1 + f3, nq1 = q1 - f3; + + /* narrow results, already merged against fm and hev */ + const i16x4 wp1 = lf_sel(fm & nhev, np1, p1); + const i16x4 wp0 = lf_sel(fm, np0, p0); + const i16x4 wq0 = lf_sel(fm, nq0, q0); + const i16x4 wq1 = lf_sel(fm & nhev, nq1, q1); + + if (wd == 4) { + ST(-2, wp1); + ST(-1, wp0); + ST(+0, wq0); + ST(+1, wq1); + FLUSH(); + return; + } + + i16x4 flat8in = lf_le(lf_abs(p2 - p0), vF) & lf_le(lf_abs(p1 - p0), vF) & + lf_le(lf_abs(q1 - q0), vF) & lf_le(lf_abs(q2 - q0), vF); + if (wd >= 8) + flat8in &= lf_le(lf_abs(p3 - p0), vF) & lf_le(lf_abs(q3 - q0), vF); + + if (wd == 6) { + const i16x4 m6 = fm & flat8in; + const i16x4 a2 = (p2 + 2 * p2 + 2 * p1 + 2 * p0 + q0 + k4) >> 3; + const i16x4 a1 = (p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + k4) >> 3; + const i16x4 b0 = (p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + k4) >> 3; + const i16x4 b1 = (p0 + 2 * q0 + 2 * q1 + 2 * q2 + q2 + k4) >> 3; + ST(-2, lf_sel(m6, a2, wp1)); + ST(-1, lf_sel(m6, a1, wp0)); + ST(+0, lf_sel(m6, b0, wq0)); + ST(+1, lf_sel(m6, b1, wq1)); + FLUSH(); + return; + } + + /* wd >= 8: the flat8in branch spans p3..q2 */ + const i16x4 c3 = (p3 + p3 + p3 + 2 * p2 + p1 + p0 + q0 + k4) >> 3; + const i16x4 c2 = (p3 + p3 + p2 + 2 * p1 + p0 + q0 + q1 + k4) >> 3; + const i16x4 c1 = (p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2 + k4) >> 3; + const i16x4 d0 = (p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3 + k4) >> 3; + const i16x4 d1 = (p1 + p0 + q0 + 2 * q1 + q2 + q3 + q3 + k4) >> 3; + const i16x4 d2 = (p0 + q0 + q1 + 2 * q2 + q3 + q3 + q3 + k4) >> 3; + + if (wd == 8) { + const i16x4 m8 = fm & flat8in; + ST(-3, lf_sel(m8, c3, p2)); + ST(-2, lf_sel(m8, c2, wp1)); + ST(-1, lf_sel(m8, c1, wp0)); + ST(+0, lf_sel(m8, d0, wq0)); + ST(+1, lf_sel(m8, d1, wq1)); + ST(+2, lf_sel(m8, d2, q2)); + FLUSH(); + return; + } + + /* wd == 16 */ + const i16x4 p6 = L(-7), p5 = L(-6), p4 = L(-5); + const i16x4 q4 = L(4), q5 = L(5), q6 = L(6); + const i16x4 flat8out = + lf_le(lf_abs(p6 - p0), vF) & lf_le(lf_abs(p5 - p0), vF) & + lf_le(lf_abs(p4 - p0), vF) & lf_le(lf_abs(q4 - q0), vF) & + lf_le(lf_abs(q5 - q0), vF) & lf_le(lf_abs(q6 - q0), vF); + + const i16x4 m16 = fm & flat8in & flat8out; + const i16x4 m8 = fm & flat8in & ~m16; + + const i16x4 e6 = (p6+p6+p6+p6+p6 + p6*2 + p5*2 + p4*2 + p3+p2+p1+p0+q0 + k8) >> 4; + const i16x4 e5 = (p6+p6+p6+p6+p6 + p5*2 + p4*2 + p3*2 + p2+p1+p0+q0+q1 + k8) >> 4; + const i16x4 e4 = (p6+p6+p6+p6 + p5 + p4*2 + p3*2 + p2*2 + p1+p0+q0+q1+q2 + k8) >> 4; + const i16x4 e3 = (p6+p6+p6 + p5+p4 + p3*2 + p2*2 + p1*2 + p0+q0+q1+q2+q3 + k8) >> 4; + const i16x4 e2 = (p6+p6 + p5+p4+p3 + p2*2 + p1*2 + p0*2 + q0+q1+q2+q3+q4 + k8) >> 4; + const i16x4 e1 = (p6 + p5+p4+p3+p2 + p1*2 + p0*2 + q0*2 + q1+q2+q3+q4+q5 + k8) >> 4; + const i16x4 g0 = (p5+p4+p3+p2+p1 + p0*2 + q0*2 + q1*2 + q2+q3+q4+q5+q6 + k8) >> 4; + const i16x4 g1 = (p4+p3+p2+p1+p0 + q0*2 + q1*2 + q2*2 + q3+q4+q5+q6+q6 + k8) >> 4; + const i16x4 g2 = (p3+p2+p1+p0+q0 + q1*2 + q2*2 + q3*2 + q4+q5+q6+q6+q6 + k8) >> 4; + const i16x4 g3 = (p2+p1+p0+q0+q1 + q2*2 + q3*2 + q4*2 + q5+q6+q6+q6+q6 + k8) >> 4; + const i16x4 g4 = (p1+p0+q0+q1+q2 + q3*2 + q4*2 + q5*2 + q6+q6+q6+q6+q6 + k8) >> 4; + const i16x4 g5 = (p0+q0+q1+q2+q3 + q4*2 + q5*2 + q6*2 + q6+q6+q6+q6+q6 + k8) >> 4; + + ST(-6, lf_sel(m16, e6, p5)); + ST(-5, lf_sel(m16, e5, p4)); + ST(-4, lf_sel(m16, e4, p3)); + ST(-3, lf_sel(m16, e3, lf_sel(m8, c3, p2))); + ST(-2, lf_sel(m16, e2, lf_sel(m8, c2, wp1))); + ST(-1, lf_sel(m16, e1, lf_sel(m8, c1, wp0))); + ST(+0, lf_sel(m16, g0, lf_sel(m8, d0, wq0))); + ST(+1, lf_sel(m16, g1, lf_sel(m8, d1, wq1))); + ST(+2, lf_sel(m16, g2, lf_sel(m8, d2, q2))); + ST(+3, lf_sel(m16, g3, q3)); + ST(+4, lf_sel(m16, g4, q4)); + ST(+5, lf_sel(m16, g5, q5)); + FLUSH(); +#undef FLUSH +#undef ST +#undef L +} + +LF_SIMD_FN void lf_dispatch(pixel *const dst, const int E, const int I, + const int H, const ptrdiff_t sb, const int wd, + const int aligned, const int transposed) +{ + switch (wd) { + case 4: lf_unit(dst, E, I, H, sb, 4, aligned, transposed); break; + case 6: lf_unit(dst, E, I, H, sb, 6, aligned, transposed); break; + case 8: lf_unit(dst, E, I, H, sb, 8, aligned, transposed); break; + default: lf_unit(dst, E, I, H, sb, 16, aligned, transposed); break; + } +} + +static void loop_filter_v_sb128y_ia64(pixel *dst, const ptrdiff_t stride, + const uint32_t *const vmask, + const uint8_t (*l)[4], ptrdiff_t b4_stride, + const Av1FilterLUT *lut, const int w + HIGHBD_DECL_SUFFIX) +{ + const unsigned vm = vmask[0] | vmask[1] | vmask[2]; + const int aligned = !((uintptr_t)dst & 3) && !(PXSTRIDE(stride) & 3); + for (unsigned x = 1; vm & ~(x - 1); x <<= 1, dst += 4, l++) { + if (vm & x) { + const int L = l[0][0] ? l[0][0] : l[-b4_stride][0]; + if (!L) continue; + const int H = L >> 4; + const int E = lut->e[L], I = lut->i[L]; + const int idx = (vmask[2] & x) ? 2 : !!(vmask[1] & x); + if (aligned) lf_dispatch(dst, E, I, H, PXSTRIDE(stride), 4 << idx, 1, 0); + else lf_dispatch(dst, E, I, H, PXSTRIDE(stride), 4 << idx, 0, 0); + } + } +} + +static void loop_filter_v_sb128uv_ia64(pixel *dst, const ptrdiff_t stride, + const uint32_t *const vmask, + const uint8_t (*l)[4], ptrdiff_t b4_stride, + const Av1FilterLUT *lut, const int w + HIGHBD_DECL_SUFFIX) +{ + const unsigned vm = vmask[0] | vmask[1]; + const int aligned = !((uintptr_t)dst & 3) && !(PXSTRIDE(stride) & 3); + for (unsigned x = 1; vm & ~(x - 1); x <<= 1, dst += 4, l++) { + if (vm & x) { + const int L = l[0][0] ? l[0][0] : l[-b4_stride][0]; + if (!L) continue; + const int H = L >> 4; + const int E = lut->e[L], I = lut->i[L]; + const int idx = !!(vmask[1] & x); + if (aligned) lf_dispatch(dst, E, I, H, PXSTRIDE(stride), 4 + 2 * idx, 1, 0); + else lf_dispatch(dst, E, I, H, PXSTRIDE(stride), 4 + 2 * idx, 0, 0); + } + } +} + +/* + * Vertical edges. The tap stride is 1 and the four samples step by |stride|, + * so these go through the transposing path; |aligned| now means the 4-byte + * groups of every row are 4-aligned, which holds whenever dst and the row + * pitch both are. + */ +static void loop_filter_h_sb128y_ia64(pixel *dst, const ptrdiff_t stride, + const uint32_t *const vmask, + const uint8_t (*l)[4], ptrdiff_t b4_stride, + const Av1FilterLUT *lut, const int h + HIGHBD_DECL_SUFFIX) +{ + const unsigned vm = vmask[0] | vmask[1] | vmask[2]; + const int aligned = !((uintptr_t)dst & 3) && !(PXSTRIDE(stride) & 3); + for (unsigned y = 1; vm & ~(y - 1); + y <<= 1, dst += 4 * PXSTRIDE(stride), l += b4_stride) + { + if (vm & y) { + const int L = l[0][0] ? l[0][0] : l[-1][0]; + if (!L) continue; + const int H = L >> 4; + const int E = lut->e[L], I = lut->i[L]; + const int idx = (vmask[2] & y) ? 2 : !!(vmask[1] & y); + if (aligned) lf_dispatch(dst, E, I, H, PXSTRIDE(stride), 4 << idx, 1, 1); + else lf_dispatch(dst, E, I, H, PXSTRIDE(stride), 4 << idx, 0, 1); + } + } +} + +static void loop_filter_h_sb128uv_ia64(pixel *dst, const ptrdiff_t stride, + const uint32_t *const vmask, + const uint8_t (*l)[4], ptrdiff_t b4_stride, + const Av1FilterLUT *lut, const int h + HIGHBD_DECL_SUFFIX) +{ + const unsigned vm = vmask[0] | vmask[1]; + const int aligned = !((uintptr_t)dst & 3) && !(PXSTRIDE(stride) & 3); + for (unsigned y = 1; vm & ~(y - 1); + y <<= 1, dst += 4 * PXSTRIDE(stride), l += b4_stride) + { + if (vm & y) { + const int L = l[0][0] ? l[0][0] : l[-1][0]; + if (!L) continue; + const int H = L >> 4; + const int E = lut->e[L], I = lut->i[L]; + const int idx = !!(vmask[1] & y); + if (aligned) lf_dispatch(dst, E, I, H, PXSTRIDE(stride), 4 + 2 * idx, 1, 1); + else lf_dispatch(dst, E, I, H, PXSTRIDE(stride), 4 + 2 * idx, 0, 1); + } + } +} + +#endif /* BITDEPTH == 8 */ + +COLD void bitfn(dav1d_loop_filter_dsp_init_ia64)(Dav1dLoopFilterDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_IA64_CPU_FLAG_SIMD)) return; + +#if BITDEPTH == 8 + c->loop_filter_sb[0][0] = loop_filter_h_sb128y_ia64; + c->loop_filter_sb[1][0] = loop_filter_h_sb128uv_ia64; + c->loop_filter_sb[0][1] = loop_filter_v_sb128y_ia64; + c->loop_filter_sb[1][1] = loop_filter_v_sb128uv_ia64; +#endif +} diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/looprestoration.h dav1d-1.5.4/src/ia64/looprestoration.h --- dav1d-1.5.4.orig/src/ia64/looprestoration.h 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/looprestoration.h 2026-08-17 21:54:29.595587184 +0200 @@ -0,0 +1,37 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/looprestoration.h" + +bitfn_decls(void dav1d_loop_restoration_dsp_init_ia64, + Dav1dLoopRestorationDSPContext *c, int bpc); + +static ALWAYS_INLINE void loop_restoration_dsp_init_ia64(Dav1dLoopRestorationDSPContext *const c, + const int bpc) +{ + bitfn(dav1d_loop_restoration_dsp_init_ia64)(c, bpc); +} diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/looprestoration_tmpl.c dav1d-1.5.4/src/ia64/looprestoration_tmpl.c --- dav1d-1.5.4.orig/src/ia64/looprestoration_tmpl.c 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/looprestoration_tmpl.c 2026-08-21 21:17:11.528731824 +0200 @@ -0,0 +1,2550 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/attributes.h" +#include "common/intops.h" + +#include "src/cpu.h" +#include "src/ia64/dav1d_simd.h" +#include "src/ia64/looprestoration.h" +#include "src/looprestoration.h" +#include "src/tables.h" + +#if BITDEPTH == 8 + +/* 390 in the generic template. 400 is used here for the same reason the ppc + * and loongarch backends use it: it keeps every intermediate row 8-byte + * aligned, which turns all of the vertical-pass loads into plain ld8. */ +#define REST_UNIT_STRIDE (400) + +/* 8bpc constants, folded from the generic code: + * round_bits_h = 3, rounding_off_h = 4, clip_limit = 1 << 13 + * round_bits_v = 11, rounding_off_v = 1024 + * round_offset = 1 << 18 + * The horizontal bias (1 << (bitdepth + 6)) and rounding_off_h are constant, so + * they seed the accumulator instead of being added at the end. */ +#define WIENER_H_BIAS ((1 << 14) + 4) +#define WIENER_H_SHIFT 3 +#define WIENER_H_CLIP 8191 +#define WIENER_V_BIAS (-(1 << 18) + 1024) +#define WIENER_V_SHIFT 11 + +/* + * Horizontal Wiener pass. + * + * The C version walks seven taps for one output pixel at a time, and on IA-64 + * each `src[idx] * fh[i]` becomes setf.sig / xma.l / getf.sig -- a serial trip + * through the FP register file inside a br.cloop that GCC cannot unroll, + * because the accumulator is a loop-carried dependency. That single loop is + * the largest item in a YouTube profile of this decoder. + * + * Here four outputs are produced at once. Two unaligned loads cover the ten + * source bytes those four outputs span; four unpack1 instructions widen them to + * 16-bit lanes; three shrp instructions slide the window to produce the + * remaining tap vectors. Each tap then costs one pmpy2.r + one pmpy2.l (four + * 16x16->32 products) and two padd4. + */ +SIMD_FN void wiener_h_4(uint16_t *const dst, const pixel *const b, + const i16x4 *const F) +{ + /* A covers bytes b[0..7], B covers b[2..9] -- together exactly the ten + * bytes taps 0..6 need for four outputs, and never one byte more than the + * scalar code would have read. */ + const u8x8 wa = ldu_u8x8(b); + const u8x8 wb = ldu_u8x8(b + 2); + const i16x4 a0 = zext8_lo(wa); /* b[0..3] */ + const i16x4 a1 = zext8_hi(wa); /* b[4..7] */ + const i16x4 b0 = zext8_lo(wb); /* b[2..5] */ + const i16x4 b1 = zext8_hi(wb); /* b[6..9] */ + const i32x2 bias = { WIENER_H_BIAS, WIENER_H_BIAS }; + + /* + * One asm block, not a chain of helpers. GCC types every asm() as + * TYPE_UNKNOWN and so issues it alone behind a stop bit; built out of three + * shrp plus fourteen pmpy2 helpers this kernel paid seventeen stop bits per + * four outputs. Bundled by hand the I-type ops (shrp/pmpy2/pshr4/pack4/ + * mux2) fill the two I slots and the A-type padd4s ride the M slots. The + * taps are also summed as a tree rather than into one accumulator, which + * would otherwise serialise the adds one per cycle. + */ + i16x4 res; + __asm__( + "{ .mii\n\t nop.m 0\n\t shrp r28 = %[a1], %[a0], 16\n\t shrp r29 = %[b1], %[b0], 16\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t shrp r30 = %[b1], %[b0], 48\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r14 = %[a0], %[f0]\n\t pmpy2.l r15 = %[a0], %[f0]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r16 = r28, %[f1]\n\t pmpy2.l r17 = r28, %[f1]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r18 = %[b0], %[f2]\n\t pmpy2.l r19 = %[b0], %[f2]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r20 = r29, %[f3]\n\t pmpy2.l r21 = r29, %[f3]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r22 = %[a1], %[f4]\n\t pmpy2.l r23 = %[a1], %[f4]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r24 = r30, %[f5]\n\t pmpy2.l r25 = r30, %[f5]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r26 = %[b1], %[f6]\n\t pmpy2.l r27 = %[b1], %[f6]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r16\n\t padd4 r15 = r15, r17\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r18 = r18, r20\n\t padd4 r19 = r19, r21\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r24\n\t padd4 r23 = r23, r25\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r18\n\t padd4 r15 = r15, r19\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r26\n\t padd4 r23 = r23, r27\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r22\n\t padd4 r15 = r15, r23\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, %[bias]\n\t padd4 r15 = r15, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pshr4 r14 = r14, %[sh]\n\t pshr4 r15 = r15, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pack4.sss r14 = r14, r15\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t mux2 %[res] = r14, 0xd8\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + : [res]"=r"(res) + : [a0]"r"(a0), [a1]"r"(a1), [b0]"r"(b0), [b1]"r"(b1), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), + [bias]"r"(bias), [sh]"i"(WIENER_H_SHIFT) + : "r14","r15","r16","r17","r18","r19","r20","r21", + "r22","r23","r24","r25","r26","r27","r28","r29","r30"); + + const i16x4 zero = { 0, 0, 0, 0 }; + const i16x4 clip = { WIENER_H_CLIP, WIENER_H_CLIP, WIENER_H_CLIP, WIENER_H_CLIP }; + st_i16x4(dst, iclip16(res, zero, clip)); +} + + +/* + * Horizontal Wiener, eight outputs per call. + * + * Identical arithmetic to wiener_h_4(), two groups deep. Alone, a group's + * fourteen pmpy2 issue in seven cycles and then spend eleven more walking the + * padd4 tree, the bias, the pshr4, the pack4 and the mux2 -- 18 cycles for + * 4.5 per pixel. Interleaved the pair costs 28 for 3.500, because the second + * group's multiplies fill the first group's tail. (Three groups would be + * better still on paper but spills: 21 of the 24 scratch GRs are already live + * here.) + * + * The window pair per group is the same as the four-wide kernel: |wa| covers + * b[0..7] and |wb| covers b[2..9], which is exactly the ten source bytes seven + * taps over four outputs touch. + */ +SIMD_FN void wiener_h_8(uint16_t *const dst, const pixel *const b, + const i16x4 *const F) +{ + const u8x8 wa0 = ldu_u8x8(b), wb0 = ldu_u8x8(b + 2); + const u8x8 wa1 = ldu_u8x8(b + 4), wb1 = ldu_u8x8(b + 6); + const i32x2 bias = { WIENER_H_BIAS, WIENER_H_BIAS }; + i16x4 r0, r1; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r3 = r0, %[wa0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r2 = r0, %[wa0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r11 = r0, %[wb0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r10 = r0, %[wb0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r9 = r0, %[wa1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r8 = r0, %[wa1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r31 = r0, %[wb1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r30 = r0, %[wb1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r29 = r2, r3, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r28 = r10, r11, 16\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r27 = r10, r11, 48\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r26 = r8, r9, 16\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r25 = r30, r31, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r24 = r30, r31, 48\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r23 = r3, %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r22 = r3, %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = r29, %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r21 = r29, %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r29 = r11, %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r20 = r11, %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r18 = r23, r3\n\t padd4 r17 = r22, r21\n\t pmpy2.r r11 = r28, %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r19 = r28, %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r21 = r2, %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r3 = r2, %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r29, r11\n\t padd4 r28 = r20, r19\n\t pmpy2.r r2 = r27, %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r22 = r27, %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r19 = r18, r23\n\t padd4 r11 = r17, r28\n\t pmpy2.r r20 = r9, %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r29 = r9, %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r28 = r21, r2\n\t padd4 r23 = r3, r22\n\t pmpy2.r r17 = r26, %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r18 = r26, %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r22 = r31, %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = r31, %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r21 = r20, r17\n\t padd4 r26 = r29, r18\n\t pmpy2.r r31 = r25, %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r3 = r25, %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r18 = r8, %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r17 = r8, %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r20 = r22, r31\n\t padd4 r25 = r2, r3\n\t pmpy2.r r8 = r24, %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r29 = r24, %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r2 = r21, r20\n\t padd4 r22 = r26, r25\n\t pmpy2.r r3 = r10, %[f6]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r31 = r10, %[f6]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r26 = r18, r8\n\t padd4 r21 = r17, r29\n\t pmpy2.r r25 = r30, %[f6]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r20 = r30, %[f6]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r28, r3\n\t padd4 r8 = r23, r31\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r26, r25\n\t padd4 r28 = r21, r20\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r31 = r19, r29\n\t padd4 r3 = r11, r8\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r21 = r2, r23\n\t padd4 r26 = r22, r28\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r8 = r31, %[bias]\n\t padd4 r29 = r3, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r3 = r21, %[bias]\n\t padd4 r31 = r26, %[bias]\n\t pshr4 r28 = r8, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r23 = r29, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r26 = r3, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r21 = r31, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r31 = r28, r23\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r3 = r26, r21\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res0] = r31, 0xd8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res1] = r3, 0xd8\n\t}\n\t" + ";;\n\t" + : [res0]"=&r"(r0), [res1]"=r"(r1) + : [wa0]"r"(wa0), [wb0]"r"(wb0), [wa1]"r"(wa1), [wb1]"r"(wb1), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), + [bias]"r"(bias), [sh]"i"(WIENER_H_SHIFT) + : "r2", "r3", "r8", "r9", "r10", "r11", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + const i16x4 zero = { 0, 0, 0, 0 }; + const i16x4 clip = { WIENER_H_CLIP, WIENER_H_CLIP, WIENER_H_CLIP, WIENER_H_CLIP }; + st_i16x4(dst, iclip16(r0, zero, clip)); + st_i16x4(dst + 4, iclip16(r1, zero, clip)); +} + +/* Scalar fallback for the edge columns and for the narrow-unit case, kept + * bit-identical to the generic template. */ +static void wiener_h_scalar_q(uint16_t *const dst, const pixel (*const left)[4], + const pixel *const src, const int16_t fh[8], + const int x, const enum LrEdgeFlags edges, + const int w) +{ + int sum = (1 << 14) + src[x] * 128; + for (int i = 0; i < 7; i++) { + const int idx = x + i - 3; + if (idx < 0) { + if (!(edges & LR_HAVE_LEFT)) + sum += src[0] * fh[i]; + else if (left) + sum += left[0][4 + idx] * fh[i]; + else + sum += src[idx] * fh[i]; + } else if (idx >= w && !(edges & LR_HAVE_RIGHT)) { + sum += src[w - 1] * fh[i]; + } else + sum += src[idx] * fh[i]; + } + dst[x] = iclip((sum + 4) >> WIENER_H_SHIFT, 0, WIENER_H_CLIP); +} + +static void wiener_filter_h_q(uint16_t *const dst, const pixel (*const left)[4], + const pixel *const src, const int16_t fh[8], + const int w, const enum LrEdgeFlags edges, + const i16x4 *const F) +{ + if (w < 6) { + for (int x = 0; x < w; x++) + wiener_h_scalar_q(dst, left, src, fh, x, edges, w); + return; + } + + int start = 3; + if (!(edges & LR_HAVE_LEFT) || left) { + for (int x = 0; x < 3; x++) + wiener_h_scalar_q(dst, left, src, fh, x, edges, w); + } else { + start = 0; + } + + int end = (edges & LR_HAVE_RIGHT) ? w : w - 3; + + int x = start; + /* Align the output pointer to a 16-bit lane group before going wide. */ + for (; x < end && (x & 3); x++) + wiener_h_scalar_q(dst, left, src, fh, x, edges, w); + for (; x + 8 <= end; x += 8) + wiener_h_8(dst + x, src + x - 3, F); + for (; x + 4 <= end; x += 4) + wiener_h_4(dst + x, src + x - 3, F); + for (; x < end; x++) + wiener_h_scalar_q(dst, left, src, fh, x, edges, w); + + for (x = end; x < w; x++) + wiener_h_scalar_q(dst, left, src, fh, x, edges, w); +} + +/* + * Vertical Wiener pass. Six or seven input rows, all 8-byte aligned thanks to + * the 400-entry stride, so these are plain ld8. The final pack2.uss performs + * iclip_pixel() as part of the narrowing. + */ +SIMD_FN i16x4 wiener_v_4(uint16_t *const *const ptrs, const uint16_t *const last, + const int i, const i16x4 *const F) +{ + /* Loads in C (plain loads bundle fine); the accumulate in one asm block for + * the same reason as wiener_h_4. */ + const i16x4 v0 = ld_i16x4(ptrs[0] + i), v1 = ld_i16x4(ptrs[1] + i); + const i16x4 v2 = ld_i16x4(ptrs[2] + i), v3 = ld_i16x4(ptrs[3] + i); + const i16x4 v4 = ld_i16x4(ptrs[4] + i), v5 = ld_i16x4(ptrs[5] + i); + const i16x4 v6 = ld_i16x4(last + i); + const i32x2 bias = { WIENER_V_BIAS, WIENER_V_BIAS }; + i16x4 res; + __asm__( + "{ .mii\n\t nop.m 0\n\t pmpy2.r r14 = %[v0], %[f0]\n\t pmpy2.l r15 = %[v0], %[f0]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r16 = %[v1], %[f1]\n\t pmpy2.l r17 = %[v1], %[f1]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r18 = %[v2], %[f2]\n\t pmpy2.l r19 = %[v2], %[f2]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r20 = %[v3], %[f3]\n\t pmpy2.l r21 = %[v3], %[f3]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r22 = %[v4], %[f4]\n\t pmpy2.l r23 = %[v4], %[f4]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r24 = %[v5], %[f5]\n\t pmpy2.l r25 = %[v5], %[f5]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r26 = %[v6], %[f6]\n\t pmpy2.l r27 = %[v6], %[f6]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r16\n\t padd4 r15 = r15, r17\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r18 = r18, r20\n\t padd4 r19 = r19, r21\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r24\n\t padd4 r23 = r23, r25\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r18\n\t padd4 r15 = r15, r19\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r26\n\t padd4 r23 = r23, r27\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r22\n\t padd4 r15 = r15, r23\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, %[bias]\n\t padd4 r15 = r15, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pshr4 r14 = r14, %[sh]\n\t pshr4 r15 = r15, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pack4.sss r14 = r14, r15\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t mux2 %[res] = r14, 0xd8\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + : [res]"=r"(res) + : [v0]"r"(v0), [v1]"r"(v1), [v2]"r"(v2), [v3]"r"(v3), + [v4]"r"(v4), [v5]"r"(v5), [v6]"r"(v6), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), + [bias]"r"(bias), [sh]"i"(WIENER_V_SHIFT) + : "r14","r15","r16","r17","r18","r19","r20","r21", + "r22","r23","r24","r25","r26","r27","r28","r29","r30"); + return res; +} + + +/* + * Both halves of one 8-pixel step in a single scheduled block. + * + * The old code called wiener_v_4() twice per iteration. Each call is about + * fourteen cycles of dependent work -- pmpy2, three levels of padd4 tree, + * bias, pshr4, pack4.sss, mux2 -- and GCC cannot overlap two asm blocks, so + * they serialised to ~28 cycles for eight outputs. + * + * Interleaved, the pair issues 37 I-type ops (28 pmpy2, 4 pshr4, 2 pack4.sss, + * 2 mux2, 1 pack2.uss), which at two I-slot ops per cycle floors the block at + * 19 cycles; it schedules in 22. The chain is shorter than that floor, so this + * is resource-bound and there is nothing further for cross-iteration + * pipelining to recover. Every padd4 is A-type and rides an M slot beside the + * multiplies -- note the bundles below are MMI pairs, not MII: two I-slot ops + * per cycle is the ceiling either way, and packing them into MII would only + * waste the M slots the tree needs. + */ +SIMD_FN u8x8 wiener_v_8(uint16_t *const *const ptrs, const uint16_t *const last, + const int i, const i16x4 *const F) +{ + const uint16_t *const r0 = ptrs[0] + i, *const r1 = ptrs[1] + i; + const uint16_t *const r2 = ptrs[2] + i, *const r3 = ptrs[3] + i; + const uint16_t *const r4 = ptrs[4] + i, *const r5 = ptrs[5] + i; + const uint16_t *const r6 = last + i; + const i16x4 v0_0 = ld_i16x4(r0), v1_0 = ld_i16x4(r0 + 4); + const i16x4 v0_1 = ld_i16x4(r1), v1_1 = ld_i16x4(r1 + 4); + const i16x4 v0_2 = ld_i16x4(r2), v1_2 = ld_i16x4(r2 + 4); + const i16x4 v0_3 = ld_i16x4(r3), v1_3 = ld_i16x4(r3 + 4); + const i16x4 v0_4 = ld_i16x4(r4), v1_4 = ld_i16x4(r4 + 4); + const i16x4 v0_5 = ld_i16x4(r5), v1_5 = ld_i16x4(r5 + 4); + const i16x4 v0_6 = ld_i16x4(r6), v1_6 = ld_i16x4(r6 + 4); + const i32x2 bias = { WIENER_V_BIAS, WIENER_V_BIAS }; + u8x8 out; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = %[v0_0], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r2 = %[v0_1], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r11 = %[v0_2], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r10 = %[v0_3], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r31 = r3, r2\n\t nop.m 0\n\t pmpy2.r r9 = %[v0_4], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r8 = %[v0_5], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r2 = r11, r10\n\t nop.m 0\n\t pmpy2.l r3 = %[v0_0], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r30 = %[v0_1], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r10 = r9, r8\n\t padd4 r11 = r31, r2\n\t pmpy2.l r29 = %[v0_2], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r28 = %[v0_3], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r8 = r3, r30\n\t nop.m 0\n\t pmpy2.l r2 = %[v0_4], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r31 = %[v0_5], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r30 = r29, r28\n\t nop.m 0\n\t pmpy2.r r3 = %[v1_0], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r9 = %[v1_1], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r28 = r2, r31\n\t padd4 r29 = r8, r30\n\t pmpy2.r r27 = %[v1_2], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r26 = %[v1_3], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r31 = r3, r9\n\t nop.m 0\n\t pmpy2.r r30 = %[v1_4], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r8 = %[v1_5], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r9 = r27, r26\n\t nop.m 0\n\t pmpy2.l r3 = %[v1_0], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[v1_1], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r26 = r30, r8\n\t padd4 r27 = r31, r9\n\t pmpy2.l r25 = %[v1_2], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r24 = %[v1_3], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r8 = r3, r2\n\t nop.m 0\n\t pmpy2.l r9 = %[v1_4], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r31 = %[v1_5], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r30 = r25, r24\n\t nop.m 0\n\t pmpy2.r r2 = %[v0_6], %[f6]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r3 = %[v0_6], %[f6]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r9, r31\n\t padd4 r22 = r8, r30\n\t pmpy2.r r24 = %[v1_6], %[f6]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r25 = %[v1_6], %[f6]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r30 = r10, r2\n\t padd4 r8 = r28, r3\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r28 = r26, r24\n\t padd4 r10 = r23, r25\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r3 = r11, r30\n\t padd4 r2 = r29, r8\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r27, r28\n\t padd4 r25 = r22, r10\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r8 = r3, %[bias]\n\t padd4 r30 = r2, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r2 = r23, %[bias]\n\t padd4 r3 = r25, %[bias]\n\t pshr4 r10 = r8, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r28 = r30, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r25 = r2, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r23 = r3, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r3 = r10, r28\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r2 = r25, r23\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r23 = r3, 0xd8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r25 = r2, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack2.uss %[out] = r23, r25\n\t}\n\t" + ";;\n\t" + : [out]"=r"(out) + : [v0_0]"r"(v0_0), [v0_1]"r"(v0_1), [v0_2]"r"(v0_2), [v0_3]"r"(v0_3), + [v0_4]"r"(v0_4), [v0_5]"r"(v0_5), [v0_6]"r"(v0_6), + [v1_0]"r"(v1_0), [v1_1]"r"(v1_1), [v1_2]"r"(v1_2), [v1_3]"r"(v1_3), + [v1_4]"r"(v1_4), [v1_5]"r"(v1_5), [v1_6]"r"(v1_6), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), + [bias]"r"(bias), [sh]"i"(WIENER_V_SHIFT) + : "r2", "r3", "r8", "r9", "r10", "r11", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + return out; +} + +static void wiener_filter_v_rows_q(pixel *const p, uint16_t *const *const ptrs, + const uint16_t *const last, const int w, + const i16x4 *const F) +{ + const int aligned = !((uintptr_t)p & 7); + int i = 0; + for (; i + 8 <= w; i += 8) + st_u8x8_sel(p + i, wiener_v_8(ptrs, last, i, F), aligned); + if (i + 4 <= w) { + st_u8_partial(p + i, pack2_uss(wiener_v_4(ptrs, last, i, F), + (i16x4){ 0 }), 4); + i += 4; + } + for (; i < w; i++) { + int sum = WIENER_V_BIAS; + for (int k = 0; k < 6; k++) + sum += ptrs[k][i] * ((const int16_t *)F)[k * 4]; + sum += last[i] * ((const int16_t *)F)[6 * 4]; + p[i] = iclip_pixel(sum >> WIENER_V_SHIFT); + } +} + +static void wiener_filter_v_q(pixel *const p, uint16_t **const ptrs, + const int w, const i16x4 *const Fv) +{ + /* The 7th row is assumed identical to the 6th; Fv[5] already carries + * fv[5] + fv[6] for this call, so only six taps are distinct. Passing + * ptrs[5] as the 7th row with a zero coefficient would cost a multiply, so + * the caller supplies a merged-coefficient vector instead. */ + wiener_filter_v_rows_q(p, ptrs, ptrs[5], w, Fv); + + for (int i = 0; i < 5; i++) + ptrs[i] = ptrs[i + 1]; +} + +static void wiener_filter_hv_q(pixel *const p, uint16_t **const ptrs, + const pixel (*const left)[4], const pixel *const src, + const int16_t (*const filter)[8], const int w, + const enum LrEdgeFlags edges, + const i16x4 *const Fh, const i16x4 *const Fv) +{ + ALIGN_STK_16(uint16_t, tmp, REST_UNIT_STRIDE,); + + wiener_filter_h_q(tmp, left, src, filter[0], w, edges, Fh); + wiener_filter_v_rows_q(p, ptrs, tmp, w, Fv); + memcpy(ptrs[6], tmp, sizeof(uint16_t) * REST_UNIT_STRIDE); + + for (int i = 0; i < 6; i++) + ptrs[i] = ptrs[i + 1]; + ptrs[6] = ptrs[0]; +} + +static void wiener_ia64(pixel *p, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, + const int w, int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(uint16_t, hor, 6 * REST_UNIT_STRIDE,); + uint16_t *ptrs[7], *rows[6]; + for (int i = 0; i < 6; i++) + rows[i] = &hor[i * REST_UNIT_STRIDE]; + + const int16_t (*const filter)[8] = params->filter; + const int16_t *const fh = filter[0]; + const int16_t *const fv = filter[1]; + + /* Broadcast the taps once per restoration unit rather than once per pixel. + * Fh[3] absorbs the separate `src[x] * 128` term of the 8bpc C code, and + * Fv[5] absorbs the duplicated 7th row of wiener_filter_v_q. */ + i16x4 Fh[7], Fv[7], Fv_last[7]; + for (int i = 0; i < 7; i++) { + Fh[i] = splat16((int16_t)(fh[i] + (i == 3 ? 128 : 0))); + Fv[i] = splat16(fv[i]); + Fv_last[i] = splat16((int16_t)(i == 5 ? fv[5] + fv[6] : (i == 6 ? 0 : fv[i]))); + } + /* wiener_filter_v_q folds tap 6 into tap 5, so its 7th coefficient is zero; + * feeding it ptrs[5] again with a zero coefficient keeps one code path. */ + + const pixel *lpf_bottom = lpf + 6 * PXSTRIDE(stride); + const pixel *src = p; + + if (edges & LR_HAVE_TOP) { + ptrs[0] = rows[0]; ptrs[1] = rows[0]; ptrs[2] = rows[1]; + ptrs[3] = rows[2]; ptrs[4] = rows[2]; ptrs[5] = rows[2]; + + wiener_filter_h_q(rows[0], NULL, lpf, fh, w, edges, Fh); + lpf += PXSTRIDE(stride); + wiener_filter_h_q(rows[1], NULL, lpf, fh, w, edges, Fh); + wiener_filter_h_q(rows[2], left, src, fh, w, edges, Fh); + left++; src += PXSTRIDE(stride); + if (--h <= 0) goto v1; + + ptrs[4] = ptrs[5] = rows[3]; + wiener_filter_h_q(rows[3], left, src, fh, w, edges, Fh); + left++; src += PXSTRIDE(stride); + if (--h <= 0) goto v2; + + ptrs[5] = rows[4]; + wiener_filter_h_q(rows[4], left, src, fh, w, edges, Fh); + left++; src += PXSTRIDE(stride); + if (--h <= 0) goto v3; + } else { + ptrs[0] = ptrs[1] = ptrs[2] = ptrs[3] = ptrs[4] = ptrs[5] = rows[0]; + + wiener_filter_h_q(rows[0], left, src, fh, w, edges, Fh); + left++; src += PXSTRIDE(stride); + if (--h <= 0) goto v1; + + ptrs[4] = ptrs[5] = rows[1]; + wiener_filter_h_q(rows[1], left, src, fh, w, edges, Fh); + left++; src += PXSTRIDE(stride); + if (--h <= 0) goto v2; + + ptrs[5] = rows[2]; + wiener_filter_h_q(rows[2], left, src, fh, w, edges, Fh); + left++; src += PXSTRIDE(stride); + if (--h <= 0) goto v3; + + ptrs[6] = rows[3]; + wiener_filter_hv_q(p, ptrs, left, src, filter, w, edges, Fh, Fv); + left++; src += PXSTRIDE(stride); p += PXSTRIDE(stride); + if (--h <= 0) goto v3; + + ptrs[6] = rows[4]; + wiener_filter_hv_q(p, ptrs, left, src, filter, w, edges, Fh, Fv); + left++; src += PXSTRIDE(stride); p += PXSTRIDE(stride); + if (--h <= 0) goto v3; + } + + ptrs[6] = ptrs[5] + REST_UNIT_STRIDE; + do { + wiener_filter_hv_q(p, ptrs, left, src, filter, w, edges, Fh, Fv); + left++; src += PXSTRIDE(stride); p += PXSTRIDE(stride); + } while (--h > 0); + + if (!(edges & LR_HAVE_BOTTOM)) goto v3; + + wiener_filter_hv_q(p, ptrs, NULL, lpf_bottom, filter, w, edges, Fh, Fv); + lpf_bottom += PXSTRIDE(stride); + p += PXSTRIDE(stride); + wiener_filter_hv_q(p, ptrs, NULL, lpf_bottom, filter, w, edges, Fh, Fv); + p += PXSTRIDE(stride); +v1: + wiener_filter_v_q(p, ptrs, w, Fv_last); + return; +v3: + wiener_filter_v_q(p, ptrs, w, Fv_last); + p += PXSTRIDE(stride); +v2: + wiener_filter_v_q(p, ptrs, w, Fv_last); + p += PXSTRIDE(stride); + goto v1; +} + +/* ---------------------------------------------- self-guided restoration */ + +/* + * The SGR chain below is ported from the generic template. The three drivers + * are pure row scheduling and are copied verbatim; the helpers they call are + * the parts that do arithmetic, and those are the ones rewritten to use packed + * ops. Keeping the drivers identical means the (fairly intricate) edge and + * row-rotation logic is shared with the reference code rather than re-derived. + */ + +static NOINLINE void rotate_q(int32_t **sumsq_ptrs, coef **sum_ptrs, int n) +{ + int32_t *tmp32 = sumsq_ptrs[0]; + coef *tmpc = sum_ptrs[0]; + for (int i = 0; i < n - 1; i++) { + sumsq_ptrs[i] = sumsq_ptrs[i + 1]; + sum_ptrs[i] = sum_ptrs[i + 1]; + } + sumsq_ptrs[n - 1] = tmp32; + sum_ptrs[n - 1] = tmpc; +} + +static NOINLINE void rotate5_x2_q(int32_t **sumsq_ptrs, coef **sum_ptrs) +{ + int32_t *tmp32[2]; + coef *tmpc[2]; + for (int i = 0; i < 2; i++) { + tmp32[i] = sumsq_ptrs[i]; + tmpc[i] = sum_ptrs[i]; + } + for (int i = 0; i < 3; i++) { + sumsq_ptrs[i] = sumsq_ptrs[i + 2]; + sum_ptrs[i] = sum_ptrs[i + 2]; + } + for (int i = 0; i < 2; i++) { + sumsq_ptrs[3 + i] = tmp32[i]; + sum_ptrs[3 + i] = tmpc[i]; + } +} + +/* + * The scalar box filters carry the window along in a/b/c(/d/e) and substitute + * the edge pixels inline. Rolling that up into a single accessor makes the + * substitution explicit, so the vector loop can simply skip the range where no + * substitution happens. Both box sizes obey the same rule. + * + * For x >= 1 the box3 window is {S(x-1), S(x), S(x+1)}; for x == 0 and x == -1 + * the carried a/b are the same S() values at negative indices, and likewise for + * box5 with a radius of two. + */ +static inline int sgr_src_q(const pixel (*const left)[4], const pixel *const src, + const int w, const enum LrEdgeFlags edges, const int i) +{ + if (i >= 0) + return (i < w || (edges & LR_HAVE_RIGHT)) ? src[i] : src[w - 1]; + if (!(edges & LR_HAVE_LEFT)) return src[0]; + return left ? left[0][4 + i] : src[i]; +} + +/* + * sum[] is coef (16-bit) and sumsq[] is 32-bit, so the two outputs are produced + * differently: the sums are a couple of padd2, while the squares come from + * pmpy2 pairs and have to be woven back into order with unpack4 because there + * is nothing to narrow them down to. + * + * Both pointers are biased by one element on entry, and the caller's buffers + * are 16-byte aligned, so &sum[x] is 8-byte aligned exactly when x == 3 mod 4. + * The vector loop is entered at that phase to keep every store aligned. + */ + + +/* + * Box filter horizontal pass, eight outputs per call, hand-bundled. + * + * The window is fetched with two aligned loads funnelled by the byte offset; + * only the two variable shifts of that sequence are I-type, everything else + * (including the *8 of the offset and the *2 of the high word) is folded into + * shladd, which is A-type and rides an M slot. The 28 I-type ops -- the + * unpack1 pair, the window shrps, the pmpy2 squares and the final unpack4 -- + * are what the schedule is built around, two per cycle. + * + * Two groups run interleaved: one group's chain (offset -> load -> funnel -> + * unpack -> shrp -> pmpy2 -> padd4 -> unpack4 -> store) is deeper than its own + * issue requirement, so alone it would leave both I units idle most cycles. + */ + +/* + * Box filter horizontal pass, eight outputs per call, hand-bundled. + * + * The window is fetched with two aligned loads funnelled by the byte offset; + * only the two variable shifts of that sequence are I-type, everything else + * (including the *8 of the offset and the *2 of the high word) is folded into + * shladd, which is A-type and rides an M slot. The 38 I-type ops -- the + * unpack1 pair, the window shrps, the pmpy2 squares and the final unpack4 -- + * are what the schedule is built around, two per cycle. + * + * Two groups run interleaved: one group's chain (offset -> load -> funnel -> + * unpack -> shrp -> pmpy2 -> padd4 -> unpack4 -> store) is deeper than its own + * issue requirement, so alone it would leave both I units idle most cycles. + */ + +/* + * Tier III box horizontal kernels; see src/ia64/looprestoration_box.S. Each + * takes the pointer to its own first tap and produces eight outputs per + * iteration, which keeps the source alignment loop-invariant. + * + * dav1d_sgr_box35_row_h_ia64 does both boxes at once: box3's three taps at + * output x (src[x-1..x+1]) are the middle three of box5's five + * (src[x-2..x+2]), so the whole front end -- extraction, unpack1s, shrps and + * pmpy2 squares -- is shared and box3 is nearly free on top of box5. + */ +void dav1d_sgr_box35_row_h_ia64(int32_t *sumsq3, coef *sum3, + int32_t *sumsq5, coef *sum5, + const pixel *src, long nblk); +void dav1d_sgr_box3_row_h_ia64(int32_t *sumsq, coef *sum, + const pixel *src, long nblk); +void dav1d_sgr_box5_row_h_ia64(int32_t *sumsq, coef *sum, + const pixel *src, long nblk); + +/* + * All three kernels take the twelve (ten for box3) source bytes they need with + * three aligned loads, so the last group reads further than the Tier II + * sgr_box*_h8 pair did. Every caller therefore holds back one block -- + * x <= w - 18 rather than w - 9 / w - 10 -- which puts the furthest read at + * src[w + 4], no further than the old code already reached, and costs eight + * outputs of scalar per row. Below w == 21 there is no vector block at all + * and the row is done entirely in scalar; restoration units that narrow occur + * only at a picture edge. + */ +/* + * How many eight-output blocks the vector kernel may run. + * + * The kernel computes unclamped box sums, so it is only valid where every tap + * is a real pixel. box5 is the binding case: its last output x reads + * src[x + 2], and outputs start at SGR_BOX_X0, so the last output + * 8*nblk + SGR_BOX_X0 - 1 needs 8*nblk + 4 <= w - 1, i.e. nblk <= (w - 5) / 8. + * That holds whatever the edge flags say, which is what lets the kernel stay + * ignorant of them; the columns that would need clamping (or the left[] array) + * are left to the scalar helpers on either side. + * + * The old bound was (w - 21) / 8 + 1 -- exactly one block short of this, and + * it gave up entirely below w = 21. Per row that moved 8 of 18 scalar output + * columns onto the vector path, and widths 13..20 vectorise at all now. + * + * On the memory side the kernel loads through three aligned, post-incrementing + * ld8 started at (src & ~7), (src & ~7) + 8 and + 16, so over nblk iterations + * the last byte it touches is (src & ~7) + 8*nblk + 15. Called as src + 1 + * that is at most src[8*nblk + 16], which for this bound reaches src[w + 11]: + * twelve bytes past the last pixel, against the 64 bytes of padding both the + * picture allocator (DAV1D_PICTURE_ALIGNMENT, and rows rounded to a multiple + * of 128 pixels) and the lpf line buffer (which over-allocates by 128 for + * precisely this reason) are documented to provide. The previous bound + * already relied on four. + */ +#define SGR_BOX_NBLK(w) ((w) >= 13 ? (long)(((w) - 5) / 8) : 0) +#define SGR_BOX_X0 3 + +/* Scalar box3 over [x0, x1). Split out so the fused box35 path below can + * reuse it for the row edges the vector kernel does not cover. */ +static NOINLINE void sgr_box3_scalar_q(int32_t *sumsq, coef *sum, + const pixel (*left)[4], + const pixel *src, const int w, + const enum LrEdgeFlags edges, + const int x0, const int x1) +{ + sumsq++; + sum++; + for (int x = x0; x < x1; x++) { + const int a_ = sgr_src_q(left, src, w, edges, x - 1); + const int b_ = sgr_src_q(left, src, w, edges, x); + const int c_ = sgr_src_q(left, src, w, edges, x + 1); + sum[x] = a_ + b_ + c_; + sumsq[x] = a_ * a_ + b_ * b_ + c_ * c_; + } +} + +static NOINLINE void sgr_box3_row_h_q(int32_t *sumsq, coef *sum, + const pixel (*left)[4], + const pixel *src, const int w, + const enum LrEdgeFlags edges) +{ + const long nblk = SGR_BOX_NBLK(w); + int x = 3 < w + 1 ? 3 : w + 1; + + sgr_box3_scalar_q(sumsq, sum, left, src, w, edges, -1, x); + if (nblk) { + /* +1 past the guard element the scalar helper skips, +3 for the first + * interior x; box3's first tap is src[x - 1]. */ + dav1d_sgr_box3_row_h_ia64(sumsq + 1 + SGR_BOX_X0, sum + 1 + SGR_BOX_X0, + src + SGR_BOX_X0 - 1, nblk); + x = SGR_BOX_X0 + 8 * (int)nblk; + } + sgr_box3_scalar_q(sumsq, sum, left, src, w, edges, x, w + 1); +} + +/* Scalar box5 over [x0, x1); the counterpart of sgr_box3_scalar_q. */ +static NOINLINE void sgr_box5_scalar_q(int32_t *sumsq, coef *sum, + const pixel (*left)[4], + const pixel *src, const int w, + const enum LrEdgeFlags edges, + const int x0, const int x1) +{ + sumsq++; + sum++; + for (int x = x0; x < x1; x++) { + const int a_ = sgr_src_q(left, src, w, edges, x - 2); + const int b_ = sgr_src_q(left, src, w, edges, x - 1); + const int c_ = sgr_src_q(left, src, w, edges, x); + const int d_ = sgr_src_q(left, src, w, edges, x + 1); + const int e_ = sgr_src_q(left, src, w, edges, x + 2); + sum[x] = a_ + b_ + c_ + d_ + e_; + sumsq[x] = a_ * a_ + b_ * b_ + c_ * c_ + d_ * d_ + e_ * e_; + } +} + +static NOINLINE void sgr_box5_row_h_q(int32_t *sumsq, coef *sum, + const pixel (*left)[4], + const pixel *src, const int w, + const enum LrEdgeFlags edges) +{ + const long nblk = SGR_BOX_NBLK(w); + int x = 3 < w + 1 ? 3 : w + 1; + + sgr_box5_scalar_q(sumsq, sum, left, src, w, edges, -1, x); + if (nblk) { + dav1d_sgr_box5_row_h_ia64(sumsq + 1 + SGR_BOX_X0, sum + 1 + SGR_BOX_X0, + src + SGR_BOX_X0 - 2, nblk); + x = SGR_BOX_X0 + 8 * (int)nblk; + } + sgr_box5_scalar_q(sumsq, sum, left, src, w, edges, x, w + 1); +} + +/* + * Tier III, and a fusion: box3's three taps at output x (src[x-1..x+1]) are + * exactly the middle three of box5's five (src[x-2..x+2]) over the same source + * row, so one kernel producing eight outputs of both shares the unaligned + * extraction, the unpack1s, all six shrps and all eighteen pmpy2 squares. + * 20 cycles per eight outputs of both, against 21 + 26 for the separate + * kernels. See src/ia64/looprestoration_box.S. + */ + + +static void sgr_box35_row_h_q(int32_t *sumsq3, coef *sum3, + int32_t *sumsq5, coef *sum5, + const pixel (*left)[4], + const pixel *src, const int w, + const enum LrEdgeFlags edges) +{ + const long nblk = SGR_BOX_NBLK(w); + if (!nblk) { + sgr_box3_row_h_q(sumsq3, sum3, left, src, w, edges); + sgr_box5_row_h_q(sumsq5, sum5, left, src, w, edges); + return; + } + const int xend = SGR_BOX_X0 + 8 * (int)nblk; + + sgr_box3_scalar_q(sumsq3, sum3, left, src, w, edges, -1, 3); + sgr_box5_scalar_q(sumsq5, sum5, left, src, w, edges, -1, 3); + + /* +1 for the leading guard element the scalar helpers skip past, +3 for + * the first interior x; src + 1 is box5's first tap, src + 3 - 2. */ + dav1d_sgr_box35_row_h_ia64(sumsq3 + 1 + 3, sum3 + 1 + 3, + sumsq5 + 1 + 3, sum5 + 1 + 3, + src + 1, nblk); + + sgr_box3_scalar_q(sumsq3, sum3, left, src, w, edges, xend, w + 1); + sgr_box5_scalar_q(sumsq5, sum5, left, src, w, edges, xend, w + 1); +} + +static NOINLINE void sgr_box3_row_v_q(int32_t **sumsq, coef **sum, + int32_t *sumsq_out, coef *sum_out, + const int w) +{ + /* Every one of these rows is a BUF_STRIDE slice of a 16-byte aligned + * buffer, so all six pointers are 8-byte aligned at x == 0 and stay so + * every four elements. No multiplies here, just padd2 / padd4. */ + const int n = w + 2; + int x = 0; + for (; x + 4 <= n; x += 4) { + st_i16x4(sum_out + x, ld_i16x4(sum[0] + x) + + ld_i16x4(sum[1] + x) + + ld_i16x4(sum[2] + x)); + st_i32x2(sumsq_out + x, ld_i32x2(sumsq[0] + x) + + ld_i32x2(sumsq[1] + x) + + ld_i32x2(sumsq[2] + x)); + st_i32x2(sumsq_out + x + 2, ld_i32x2(sumsq[0] + x + 2) + + ld_i32x2(sumsq[1] + x + 2) + + ld_i32x2(sumsq[2] + x + 2)); + } + for (; x < n; x++) { + sumsq_out[x] = sumsq[0][x] + sumsq[1][x] + sumsq[2][x]; + sum_out[x] = sum[0][x] + sum[1][x] + sum[2][x]; + } +} + +static NOINLINE void sgr_box5_row_v_q(int32_t **sumsq, coef **sum, + int32_t *sumsq_out, coef *sum_out, + const int w) +{ + const int n = w + 2; + int x = 0; + for (; x + 4 <= n; x += 4) { + st_i16x4(sum_out + x, ld_i16x4(sum[0] + x) + ld_i16x4(sum[1] + x) + + ld_i16x4(sum[2] + x) + ld_i16x4(sum[3] + x) + + ld_i16x4(sum[4] + x)); + st_i32x2(sumsq_out + x, ld_i32x2(sumsq[0] + x) + ld_i32x2(sumsq[1] + x) + + ld_i32x2(sumsq[2] + x) + ld_i32x2(sumsq[3] + x) + + ld_i32x2(sumsq[4] + x)); + st_i32x2(sumsq_out + x + 2, + ld_i32x2(sumsq[0] + x + 2) + ld_i32x2(sumsq[1] + x + 2) + + ld_i32x2(sumsq[2] + x + 2) + ld_i32x2(sumsq[3] + x + 2) + + ld_i32x2(sumsq[4] + x + 2)); + } + for (; x < n; x++) { + sumsq_out[x] = sumsq[0][x] + sumsq[1][x] + sumsq[2][x] + + sumsq[3][x] + sumsq[4][x]; + sum_out[x] = sum[0][x] + sum[1][x] + sum[2][x] + + sum[3][x] + sum[4][x]; + } +} + +/* + * p * s is the one genuine 32x32 multiply in the whole SGR chain, and pmpy2 + * only does 16x16. What rescues it is that z is clamped to 255 straight + * afterwards: clamping p first at the point where z saturates bounds p * s to + * a little over 255 << 20, so the product fits in 32 bits and can be built + * from two 16-bit halves. + * + * Splitting p at bit 15 (rather than 16) keeps both halves valid *signed* + * 16-bit pmpy2 operands. Each half sits in the low 16 bits of its 32-bit lane, + * i.e. in lanes 0 and 2, which is exactly the pair pmpy2.r consumes. + */ +SIMD_FN i32x2 sgr_p_times_s(const i32x2 p, const i16x4 sv) { + const i32x2 mask15 = { 0x7fff, 0x7fff }; + return (pmpy2_r((i16x4)(p >> 15), sv) << 15) + + pmpy2_r((i16x4)(p & mask15), sv); +} + + + + + +/* + * Eight elements per pass, hand-bundled. + * + * Every op here is one of three kinds and the mix is what sets the schedule: + * pmpy2/pshl4/pshr4/mux2/dep/extr.u are I-type and can only issue on the two + * I units, while padd4/psub4/pcmp4/and/or/xor/add are A-type ("Integer ALU -- + * I-unit or M-unit", Table 4-1) and ride the M slots. Two MMI bundles per + * cycle give four M slots and two I slots, so the floor is set by the 58 + * I-type ops -- everything else was moved off the I units to reach it: + * - b is permuted to {0,2,1,3} once with mux2, so pmpy2.r/.l land their + * results as in-order pairs and the four unpack4 that reordered the + * squares and the final products disappear; + * - z needs no clamp (p was already clamped to pmax) and no separate shift, + * so one extr.u per lane does >>20 and the byte mask together; + * - the table index arithmetic is plain add, not addp4. + * Two groups run interleaved because one group's chain is ~30 cycles deep + * against ~29 cycles of issue; alone it would idle both ports waiting. + * A third group schedules 1.13x better on paper but measured flat: it fits + * only by spending eight GCC-allocated temps, and its longer scalar tail + * (up to 11 elements instead of 7) gives back what the schedule wins. + * + * Registers are hard-allocated rather than left to GCC: 24 scratch GRs are + * needed and the operand list would otherwise blow past the 30-operand limit. + */ +SIMD_FN void sgr_calc_ab8_9(int32_t *const AA, coef *const BB, + const uint64_t pm, const uint64_t sv, + const uint64_t m15, const uint64_t r19, + const uint64_t r11, const uint8_t *const tbl) +{ + __asm__ volatile( + "{ .mmi\n\t ld8 r3 = [%[pb]]\n\t ld8 r2 = [%[pb2]]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t ld8 r11 = [%[pa]]\n\t ld8 r10 = [%[pa1]]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t ld8 r8 = [%[pa2]]\n\t ld8 r31 = [%[pa3]]\n\t mux2 r9 = r3, 0xd8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r30 = r2, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r29 = r11, 3\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r28 = r10, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r27 = r29, r11\n\t padd4 r26 = r28, r10\n\t pmpy2.r r25 = r9, r9\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r24 = r9, r9\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r28 = r8, 3\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r29 = r31, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r21 = r28, r8\n\t padd4 r20 = r29, r31\n\t pmpy2.r r19 = r30, r30\n\t}\n\t" + "{ .mmi\n\t psub4 r23 = r27, r25\n\t psub4 r22 = r26, r24\n\t pmpy2.l r18 = r30, r30\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t pcmp4.gt r29 = %[pm], r23\n\t xor r28 = r23, %[pm]\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t pcmp4.gt r24 = %[pm], r22\n\t xor r25 = r22, %[pm]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t psub4 r26 = r21, r19\n\t psub4 r27 = r20, r18\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t and r22 = r28, r29\n\t and r23 = r25, r24\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t pcmp4.gt r28 = %[pm], r26\n\t xor r29 = r26, %[pm]\n\t xor r25 = r22, %[pm]\n\t}\n\t" + "{ .mmi\n\t pcmp4.gt r18 = %[pm], r27\n\t xor r19 = r27, %[pm]\n\t xor r24 = r23, %[pm]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t and r20 = r29, r28\n\t and r21 = r19, r18\n\t pshr4 r27 = r25, 15\n\t}\n\t" + "{ .mmi\n\t and r26 = r25, %[m15]\n\t and r22 = r24, %[m15]\n\t pshr4 r23 = r24, 15\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t xor r29 = r20, %[pm]\n\t xor r28 = r21, %[pm]\n\t pmpy2.r r19 = r27, %[sv]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r18 = r23, %[sv]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t and r20 = r29, %[m15]\n\t and r27 = r28, %[m15]\n\t pshr4 r21 = r29, 15\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r23 = r28, 15\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r28 = r21, %[sv]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r29 = r23, %[sv]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r23 = r26, %[sv]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r21 = r22, %[sv]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r22 = r20, %[sv]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r26 = r27, %[sv]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r27 = r19, 15\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r20 = r18, 15\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r18 = r27, r23\n\t padd4 r19 = r20, r21\n\t pshl4 r24 = r28, 15\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r25 = r29, 15\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r21 = r24, r22\n\t padd4 r23 = r25, r26\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r20 = r18, %[r19]\n\t padd4 r27 = r19, %[r19]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r19 = r21, %[r19]\n\t padd4 r18 = r23, %[r19]\n\t extr.u r25 = r20, 52, 8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t extr.u r24 = r27, 52, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t add r23 = %[tbl], r25\n\t add r21 = %[tbl], r24\n\t extr.u r26 = r19, 52, 8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t extr.u r22 = r18, 52, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t add r17 = %[tbl], r26\n\t add r16 = %[tbl], r22\n\t extr.u r24 = r20, 20, 8\n\t}\n\t" + "{ .mmi\n\t ld1 r25 = [r23]\n\t ld1 r28 = [r21]\n\t extr.u r29 = r27, 20, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t add r22 = %[tbl], r24\n\t add r26 = %[tbl], r29\n\t extr.u r21 = r19, 20, 8\n\t}\n\t" + "{ .mmi\n\t ld1 r23 = [r17]\n\t ld1 r20 = [r16]\n\t extr.u r27 = r18, 20, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t add r24 = %[tbl], r21\n\t add r18 = %[tbl], r27\n\t dep r29 = r28, r25, 16, 8\n\t}\n\t" + "{ .mmi\n\t ld1 r16 = [r22]\n\t ld1 r17 = [r26]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t ld1 r21 = [r24]\n\t ld1 r26 = [r18]\n\t dep r22 = r20, r23, 16, 8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t dep.z r27 = r29, 32, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t dep r18 = r17, r16, 16, 8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t dep.z r24 = r22, 32, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t or r22 = r18, r27\n\t nop.m 0\n\t dep r17 = r26, r21, 16, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t or r21 = r17, r24\n\t nop.m 0\n\t pmpy2.r r18 = r22, r9\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r26 = r22, r9\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r17 = r21, r30\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r24 = r21, r30\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r30 = r18, 3\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r9 = r26, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t psub4 r27 = r30, r18\n\t psub4 r16 = r9, r26\n\t pshl4 r29 = r17, 3\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r20 = r24, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t psub4 r26 = r29, r17\n\t psub4 r18 = r20, r24\n\t pshl4 r9 = r27, 6\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r30 = r16, 6\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r20 = r9, r27\n\t padd4 r29 = r30, r16\n\t pshl4 r24 = r26, 6\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r17 = r18, 6\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r24, r26\n\t padd4 r28 = r17, r18\n\t mux2 r30 = r22, 0xd8\n\t}\n\t" + "{ .mmi\n\t padd4 r9 = r20, %[r11]\n\t padd4 r16 = r29, %[r11]\n\t mux2 r27 = r21, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r23, %[r11]\n\t padd4 r20 = r28, %[r11]\n\t pshr4 r17 = r9, 12\n\t}\n\t" + "{ .mmi\n\t st8 [%[pb]] = r30\n\t st8 [%[pb2]] = r27\n\t pshr4 r24 = r16, 12\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t st8 [%[pa]] = r17\n\t st8 [%[pa1]] = r24\n\t pshr4 r28 = r29, 12\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r23 = r20, 12\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t st8 [%[pa2]] = r28\n\t st8 [%[pa3]] = r23\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + : + : [pa]"r"(AA), [pa1]"r"(AA + 2), [pa2]"r"(AA + 4), [pa3]"r"(AA + 6), + [pa4]"r"(AA + 8), [pa5]"r"(AA + 10), + [pb]"r"(BB), [pb2]"r"(BB + 4), [pb3]"r"(BB + 8), + [pm]"r"(pm), [sv]"r"(sv), [m15]"r"(m15), [r19]"r"(r19), + [r11]"r"(r11), [tbl]"r"(tbl) + : "memory", "r2", "r3", "r8", "r9", "r10", "r11", "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); +} + +/* + * Eight elements per pass, hand-bundled. + * + * Every op here is one of three kinds and the mix is what sets the schedule: + * pmpy2/pshl4/pshr4/mux2/dep/extr.u are I-type and can only issue on the two + * I units, while padd4/psub4/pcmp4/and/or/xor/add are A-type ("Integer ALU -- + * I-unit or M-unit", Table 4-1) and ride the M slots. Two MMI bundles per + * cycle give four M slots and two I slots, so the floor is set by the 66 + * I-type ops -- everything else was moved off the I units to reach it: + * - b is permuted to {0,2,1,3} once with mux2, so pmpy2.r/.l land their + * results as in-order pairs and the four unpack4 that reordered the + * squares and the final products disappear; + * - z needs no clamp (p was already clamped to pmax) and no separate shift, + * so one extr.u per lane does >>20 and the byte mask together; + * - the table index arithmetic is plain add, not addp4. + * Two groups run interleaved because one group's chain is ~30 cycles deep + * against ~33 cycles of issue; alone it would idle both ports waiting. + * A third group schedules 1.13x better on paper but measured flat: it fits + * only by spending eight GCC-allocated temps, and its longer scalar tail + * (up to 11 elements instead of 7) gives back what the schedule wins. + * + * Registers are hard-allocated rather than left to GCC: 24 scratch GRs are + * needed and the operand list would otherwise blow past the 30-operand limit. + */ +SIMD_FN void sgr_calc_ab8_25(int32_t *const AA, coef *const BB, + const uint64_t pm, const uint64_t sv, + const uint64_t m15, const uint64_t r19, + const uint64_t r11, const uint8_t *const tbl) +{ + __asm__ volatile( + "{ .mmi\n\t ld8 r3 = [%[pb]]\n\t ld8 r2 = [%[pa]]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t ld8 r11 = [%[pa1]]\n\t ld8 r10 = [%[pb2]]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t ld8 r31 = [%[pa2]]\n\t ld8 r30 = [%[pa3]]\n\t mux2 r9 = r3, 0xd8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r8 = r2, 4\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r29 = r2, 3\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r28 = r11, 4\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r27 = r8, r29\n\t nop.m 0\n\t pshl4 r26 = r11, 3\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r25 = r10, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r8 = r28, r26\n\t padd4 r29 = r27, r2\n\t pshl4 r24 = r31, 4\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r23 = r31, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r27 = r24, r23\n\t padd4 r26 = r8, r11\n\t pshl4 r28 = r30, 4\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r22 = r30, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r21 = r28, r22\n\t padd4 r8 = r27, r31\n\t pmpy2.r r23 = r9, r9\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r24 = r9, r9\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r22 = r21, r30\n\t nop.m 0\n\t pmpy2.r r28 = r25, r25\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r27 = r25, r25\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t psub4 r21 = r29, r23\n\t psub4 r20 = r26, r24\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t psub4 r19 = r8, r28\n\t psub4 r18 = r22, r27\n\t pcmp4.gt r26 = %[pm], r20\n\t}\n\t" + "{ .mmi\n\t pcmp4.gt r24 = %[pm], r21\n\t xor r23 = r21, %[pm]\n\t xor r29 = r20, %[pm]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t pcmp4.gt r27 = %[pm], r19\n\t xor r28 = r19, %[pm]\n\t and r20 = r23, r24\n\t}\n\t" + "{ .mmi\n\t pcmp4.gt r22 = %[pm], r18\n\t xor r8 = r18, %[pm]\n\t and r21 = r29, r26\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t and r29 = r28, r27\n\t and r26 = r8, r22\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t xor r18 = r20, %[pm]\n\t xor r19 = r21, %[pm]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t xor r21 = r29, %[pm]\n\t xor r20 = r26, %[pm]\n\t pshr4 r8 = r18, 15\n\t}\n\t" + "{ .mmi\n\t and r22 = r18, %[m15]\n\t and r27 = r19, %[m15]\n\t pshr4 r28 = r19, 15\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t and r29 = r21, %[m15]\n\t and r18 = r20, %[m15]\n\t pshr4 r26 = r21, 15\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r19 = r20, 15\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r20 = r8, %[sv]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r21 = r28, %[sv]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r28 = r26, %[sv]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r8 = r19, %[sv]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r19 = r22, %[sv]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r26 = r27, %[sv]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r27 = r29, %[sv]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r22 = r18, %[sv]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r18 = r20, 15\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r29 = r21, 15\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r21 = r18, r19\n\t padd4 r20 = r29, r26\n\t pshl4 r23 = r28, 15\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r24 = r8, 15\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r26 = r23, r27\n\t padd4 r19 = r24, r22\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r29 = r21, %[r19]\n\t padd4 r18 = r20, %[r19]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r20 = r26, %[r19]\n\t padd4 r21 = r19, %[r19]\n\t extr.u r24 = r29, 52, 8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t extr.u r23 = r18, 52, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t add r19 = %[tbl], r24\n\t add r26 = %[tbl], r23\n\t extr.u r22 = r20, 52, 8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t extr.u r27 = r21, 52, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t add r17 = %[tbl], r22\n\t add r16 = %[tbl], r27\n\t extr.u r23 = r29, 20, 8\n\t}\n\t" + "{ .mmi\n\t ld1 r24 = [r19]\n\t ld1 r28 = [r26]\n\t extr.u r8 = r18, 20, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t add r27 = %[tbl], r23\n\t add r22 = %[tbl], r8\n\t extr.u r26 = r20, 20, 8\n\t}\n\t" + "{ .mmi\n\t ld1 r19 = [r17]\n\t ld1 r29 = [r16]\n\t extr.u r18 = r21, 20, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t add r23 = %[tbl], r26\n\t add r21 = %[tbl], r18\n\t dep r8 = r28, r24, 16, 8\n\t}\n\t" + "{ .mmi\n\t ld1 r16 = [r27]\n\t ld1 r17 = [r22]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t ld1 r26 = [r23]\n\t ld1 r22 = [r21]\n\t dep r27 = r29, r19, 16, 8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t dep.z r18 = r8, 32, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t dep r21 = r17, r16, 16, 8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t dep.z r23 = r27, 32, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t or r27 = r21, r18\n\t nop.m 0\n\t dep r17 = r22, r26, 16, 8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t or r26 = r17, r23\n\t nop.m 0\n\t pmpy2.r r21 = r27, r9\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r22 = r27, r9\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r17 = r26, r25\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r23 = r26, r25\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r25 = r21, 5\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r9 = r21, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r18 = r25, r9\n\t nop.m 0\n\t pshl4 r16 = r22, 5\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r8 = r22, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r25 = r16, r8\n\t padd4 r9 = r18, r21\n\t pshl4 r29 = r17, 5\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r19 = r17, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r16 = r29, r19\n\t padd4 r8 = r25, r22\n\t pshl4 r18 = r23, 5\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r21 = r23, 3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r22 = r18, r21\n\t padd4 r25 = r16, r17\n\t pshl4 r19 = r9, 2\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshl4 r29 = r8, 2\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r9 = r22, r23\n\t padd4 r18 = r19, %[r11]\n\t pshl4 r8 = r25, 2\n\t}\n\t" + "{ .mmi\n\t padd4 r16 = r29, %[r11]\n\t nop.m 0\n\t mux2 r21 = r27, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r25 = r8, %[r11]\n\t st8 [%[pb]] = r21\n\t pshl4 r29 = r9, 2\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r22 = r18, 12\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r18 = r29, %[r11]\n\t st8 [%[pa]] = r22\n\t pshr4 r9 = r16, 12\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r8 = r26, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t st8 [%[pa1]] = r9\n\t st8 [%[pb2]] = r8\n\t pshr4 r29 = r25, 12\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r22 = r18, 12\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t st8 [%[pa2]] = r29\n\t st8 [%[pa3]] = r22\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + : + : [pa]"r"(AA), [pa1]"r"(AA + 2), [pa2]"r"(AA + 4), [pa3]"r"(AA + 6), + [pa4]"r"(AA + 8), [pa5]"r"(AA + 10), + [pb]"r"(BB), [pb2]"r"(BB + 4), [pb3]"r"(BB + 8), + [pm]"r"(pm), [sv]"r"(sv), [m15]"r"(m15), [r19]"r"(r19), + [r11]"r"(r11), [tbl]"r"(tbl) + : "memory", "r2", "r3", "r8", "r9", "r10", "r11", "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); +} + +/* |n| and |one_by_x| arrive as compile-time constants so the hand-bundled + * kernel can specialise its shift/add chains on them. */ +SIMD_FN void sgr_calc_row_ab_body_q(int32_t *const AA, coef *const BB, + const int w, const int s, + const int n, const int one_by_x) +{ + /* + * p is clamped to pmax so that p * s + (1 << 19) cannot reach 256 << 20; + * that is what lets the kernel drop the umin(z, 255) and read the table + * index straight out of bits 20..27 of the product. + */ + const int32_t pmax = (int32_t)(((255 * (1 << 20)) - (1 << 19) + s - 1) / s); + const uint64_t pm = (uint64_t)(uint32_t)pmax * 0x0000000100000001ULL; + const uint64_t sv = (uint64_t)(uint16_t)s * 0x0001000100010001ULL; + const uint64_t m15 = 0x00007fff00007fffULL; + const uint64_t r19 = 0x0008000000080000ULL; + const uint64_t r11 = 0x0000080000000800ULL; + const uint8_t *const tbl = dav1d_sgr_x_by_x; + const int nw = w + 2; + + int i = 0; + if (n == 9) { + for (; i + 8 <= nw; i += 8) + sgr_calc_ab8_9(AA + i, BB + i, pm, sv, m15, r19, r11, tbl); + } else { + for (; i + 8 <= nw; i += 8) + sgr_calc_ab8_25(AA + i, BB + i, pm, sv, m15, r19, r11, tbl); + } + for (; i < nw; i++) { + const unsigned p = (unsigned)imax(AA[i] * n - BB[i] * BB[i], 0); + const unsigned z = (p * (unsigned)s + (1 << 19)) >> 20; + const unsigned x = dav1d_sgr_x_by_x[umin(z, 255)]; + AA[i] = (int32_t)((x * BB[i] * one_by_x + (1 << 11)) >> 12); + BB[i] = (coef)x; + } +} + +static NOINLINE void sgr_calc_row_ab_q(int32_t *AA, coef *BB, int w, int s, + int bitdepth_max, int n, int sgr_one_by_x) +{ + const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; + + /* The packed path needs the 8bpc shifts (both zero) and a non-zero + * strength, and specialises on the two (n, one_by_x) pairs the format + * defines. */ + if (bitdepth_min_8 == 0 && s > 0) { + if (n == 9 && sgr_one_by_x == 455) { + sgr_calc_row_ab_body_q(AA, BB, w, s, 9, 455); + return; + } + if (n == 25 && sgr_one_by_x == 164) { + sgr_calc_row_ab_body_q(AA, BB, w, s, 25, 164); + return; + } + } + + for (int i = 0; i < w + 2; i++) { + const int a = + (AA[i] + ((1 << (2 * bitdepth_min_8)) >> 1)) >> (2 * bitdepth_min_8); + const int b = + (BB[i] + ((1 << bitdepth_min_8) >> 1)) >> bitdepth_min_8; + + const unsigned p = imax(a * n - b * b, 0); + const unsigned z = (p * s + (1 << 19)) >> 20; + const unsigned x = dav1d_sgr_x_by_x[umin(z, 255)]; + + // This is where we invert A and B, so that B is of size coef. + AA[i] = (x * BB[i] * sgr_one_by_x + (1 << 11)) >> 12; + BB[i] = x; + } +} + + + +/* + * Tier III. The fused vertical-sum + calc_ab kernel is not here: it lives in + * ia64/looprestoration_sgr.S, because its schedule is limited by how many + * values can be in flight and an inline asm block gets only the 24 scratch + * GRs. With its own register frame it has ~90, which is what lets it be + * modulo-scheduled -- one 4-element group per iteration at an initiation + * interval equal to the resource floor, with the ~30-cycle dependence chain + * spread across several in-flight iterations instead of drained per block. + */ +void dav1d_sgr_vert_ab_row9_ia64(int32_t *AA, coef *BB, + int32_t *const *sumsq, coef *const *sum, + long nblk, const uint64_t *ctx); +void dav1d_sgr_vert_ab_row25_ia64(int32_t *AA, coef *BB, + int32_t *const *sumsq, coef *const *sum, + long nblk, const uint64_t *ctx); + +SIMD_FN void sgr_vert_ab_body_q(int32_t *const AA, coef *const BB, + int32_t *const *const sumsq, + coef *const *const sum, + const int w, const int s, + const int n, const int one_by_x) +{ + /* + * p is clamped to pmax so that p * s + (1 << 19) cannot reach 256 << 20; + * that is what lets the kernel drop the umin(z, 255) and read the table + * index straight out of bits 20..27 of the product. + */ + const int32_t pmax = (int32_t)(((255 * (1 << 20)) - (1 << 19) + s - 1) / s); + const uint64_t ctx[5] = { + (uint64_t)(uint32_t)pmax * 0x0000000100000001ULL, + (uint64_t)(uint16_t)s * 0x0001000100010001ULL, + 0x00007fff00007fffULL, + 0x0000080000000800ULL, /* r19 = this << 8, derived in-kernel */ + (uint64_t)(uintptr_t)dav1d_sgr_x_by_x, + }; + const int nw = w + 2; + const int nrow = n == 9 ? 3 : 5; + const long nblk = nw >> 2; + + if (n == 9) dav1d_sgr_vert_ab_row9_ia64(AA, BB, sumsq, sum, nblk, ctx); + else dav1d_sgr_vert_ab_row25_ia64(AA, BB, sumsq, sum, nblk, ctx); + + for (int i = (int)(nblk * 4); i < nw; i++) { + /* The casts reproduce what the intermediate int32_t/coef row used to + * truncate to, which is also what padd4/padd2 do lane-wise. */ + uint32_t av = 0; uint16_t bv = 0; + for (int j = 0; j < nrow; j++) { + av += (uint32_t)sumsq[j][i]; + bv = (uint16_t)(bv + (uint16_t)sum[j][i]); + } + const int32_t a = (int32_t)av; + const int b = (coef)bv; + const unsigned p = (unsigned)imax(a * n - b * b, 0); + const unsigned z = (p * (unsigned)s + (1 << 19)) >> 20; + const unsigned x = dav1d_sgr_x_by_x[umin(z, 255)]; + AA[i] = (int32_t)((x * (unsigned)b * one_by_x + (1 << 11)) >> 12); + BB[i] = (coef)x; + } +} + +/* Falls back to the separate row_v + calc_row_ab pair for the bitdepths and + * strengths the fused kernel does not specialise for. The strength bound is + * real: the kernel splats s into 16-bit lanes and pmpy2 is a signed multiply, + * so s must stay below 32768. dav1d_sgr_params tops out at 3236. */ +static NOINLINE void sgr_vert_ab_q(int32_t **sumsq, coef **sum, + int32_t *AA, coef *BB, const int w, + const int s, const int bitdepth_max, + const int n, const int one_by_x) +{ + if (bitdepth_from_max(bitdepth_max) - 8 == 0 && s > 0 && s < 32768) { + if (n == 9 && one_by_x == 455) { + sgr_vert_ab_body_q(AA, BB, sumsq, sum, w, s, 9, 455); + return; + } + if (n == 25 && one_by_x == 164) { + sgr_vert_ab_body_q(AA, BB, sumsq, sum, w, s, 25, 164); + return; + } + } + if (n == 9) sgr_box3_row_v_q(sumsq, sum, AA, BB, w); + else sgr_box5_row_v_q(sumsq, sum, AA, BB, w); + sgr_calc_row_ab_q(AA, BB, w, s, bitdepth_max, n, one_by_x); +} + +static void sgr_box3_vert_q(int32_t **sumsq, coef **sum, + int32_t *sumsq_out, coef *sum_out, + const int w, const int s, const int bitdepth_max) +{ + sgr_vert_ab_q(sumsq, sum, sumsq_out, sum_out, w, s, bitdepth_max, 9, 455); + rotate_q(sumsq, sum, 3); +} + +static void sgr_box5_vert_q(int32_t **sumsq, coef **sum, + int32_t *sumsq_out, coef *sum_out, + const int w, const int s, const int bitdepth_max) +{ + sgr_vert_ab_q(sumsq, sum, sumsq_out, sum_out, w, s, bitdepth_max, 25, 164); + rotate5_x2_q(sumsq, sum); +} + +static void sgr_box3_hv_q(int32_t **sumsq, coef **sum, + int32_t *AA, coef *BB, + const pixel (*left)[4], + const pixel *src, const int w, + const int s, + const enum LrEdgeFlags edges, + const int bitdepth_max) +{ + sgr_box3_row_h_q(sumsq[2], sum[2], left, src, w, edges); + sgr_box3_vert_q(sumsq, sum, AA, BB, w, s, bitdepth_max); +} + +/* + * The neighbourhood macros read each row at i, i+1 and i+2 (the i+1 bias is + * folded into the call). Rather than three overlapping unaligned loads per + * row, take two aligned ones and slide the window with shrp. + */ +SIMD_FN void ld3_coef(const coef *const p, i16x4 *const o0, + i16x4 *const o1, i16x4 *const o2) +{ + const i16x4 lo = ld_i16x4(p), hi = ld_i16x4(p + 4); + *o0 = lo; + *o1 = shrp_v(hi, lo, 16); + *o2 = shrp_v(hi, lo, 32); +} + +/* Same, 32 bits wide, so each offset needs two registers to cover four lanes. */ +SIMD_FN void ld3_i32(const int32_t *const p, + i32x2 *const a0, i32x2 *const a1, + i32x2 *const b0, i32x2 *const b1, + i32x2 *const c0, i32x2 *const c1) +{ + const i32x2 l0 = ld_i32x2(p), l1 = ld_i32x2(p + 2), l2 = ld_i32x2(p + 4); + *a0 = l0; *a1 = l1; + *b0 = shrp32_v(l1, l0, 32); *b1 = shrp32_v(l2, l1, 32); + *c0 = l1; *c1 = l2; +} + +SIMD_FN i16x4 mul3_16(const i16x4 v) { return v + (v + v); } +SIMD_FN i32x2 mul3_32(const i32x2 v) { return v + (v + v); } +SIMD_FN i16x4 mul5_16(const i16x4 v) { return v + (v << 2); } +SIMD_FN i32x2 mul5_32(const i32x2 v) { return v + (v << 2); } +SIMD_FN i16x4 mul6_16(const i16x4 v) { return (v << 1) + (v << 2); } +SIMD_FN i32x2 mul6_32(const i32x2 v) { return (v << 1) + (v << 2); } + +/* + * The B side (the x values from the reciprocal table) is bounded by + * 5*255*4 + 4*255*3 = 8160, so the whole eight-neighbour sum stays exact in 16 + * bits and runs four lanes at a time. The A side does not fit, so it runs two + * lanes at a time -- and because a * src[] comes out of pmpy2 deinterleaved, + * the A sums have to be woven into the same {0,2} / {1,3} order before the + * subtraction. + */ + +/* + * SGR finish filter, four outputs per pass, hand-bundled. + * + * The neighbourhood weights are applied as balanced add trees rather than + * serial accumulations, and the two heavy weights (4 and 5) use a pshl while + * 3 and 6 are built from padd alone -- the split is chosen to keep the 24 + * I-type ops (shrp, pmpy2, pack4, mux2, unpack) from outrunning the two I + * units while the A-type adds ride the M slots of the same MMI pairs. + * + * The B side is coef, so one register covers four lanes; the A side is 32-bit + * and needs a separate register per half, which is why it carries twice the + * adds for the same neighbourhood. + */ + +/* + * SGR finish filter, four outputs per pass, hand-bundled. + * + * The neighbourhood weights are applied as balanced add trees rather than + * serial accumulations, and the two heavy weights (4 and 5) use a pshl while + * 3 and 6 are built from padd alone -- the split is chosen to keep the 20 + * I-type ops (shrp, pmpy2, pack4, mux2, unpack) from outrunning the two I + * units while the A-type adds ride the M slots of the same MMI pairs. + * + * The B side is coef, so one register covers four lanes; the A side is 32-bit + * and needs a separate register per half, which is why it carries twice the + * adds for the same neighbourhood. + */ + +/* + * SGR finish filter, four outputs per pass, hand-bundled. + * + * The neighbourhood weights are applied as balanced add trees rather than + * serial accumulations, and the two heavy weights (4 and 5) use a pshl while + * 3 and 6 are built from padd alone -- the split is chosen to keep the 16 + * I-type ops (shrp, pmpy2, pack4, mux2, unpack) from outrunning the two I + * units while the A-type adds ride the M slots of the same MMI pairs. + * + * The B side is coef, so one register covers four lanes; the A side is 32-bit + * and needs a separate register per half, which is why it carries twice the + * adds for the same neighbourhood. + */ + +/* + * Tier III: the finish filter's neighbourhood combine. See + * src/ia64/looprestoration_fin.S -- one 4-output group per iteration, + * modulo-scheduled at the resource floor (13/11/8 cycles for the 3-, 2- and + * 1-row forms). As inline asm these ran ~1.4x their floor: the chain is + * longer than the floor and 24 scratch GRs cannot hide it. + */ +/* + * The rows arrive as the caller's B_ptrs/A_ptrs arrays rather than as six + * separate pointers: the IA-64 psABI passes only the first eight arguments in + * registers (out0-out7 = r32-r39), and a ninth would land on the memory stack + * at [sp+16] instead. Five arguments keeps every variant register-only and + * gives all three the same signature. + */ +void dav1d_sgr_finish_row3_ia64(coef *dst, const pixel *src, + coef *const *B_ptrs, int32_t *const *A_ptrs, + long nblk); +void dav1d_sgr_finish_row2_ia64(coef *dst, const pixel *src, + coef *const *B_ptrs, int32_t *const *A_ptrs, + long nblk); +void dav1d_sgr_finish_row1_ia64(coef *dst, const pixel *src, + coef *const *B_ptrs, int32_t *const *A_ptrs, + long nblk); + +#define FILTER_OUT_STRIDE (384) + +/* + * ld4 on the pixel row needs 4-byte alignment. Restoration units start on + * 64-pixel boundaries of an aligned picture, so this holds for every real + * call; the scalar loop below is the correctness fallback, not a fast path. + */ +static NOINLINE void sgr_finish_filter_row1_q(coef *tmp, + const pixel *src, + int32_t **A_ptrs, coef **B_ptrs, + const int w) +{ +#define EIGHT_NEIGHBORS(P, i)\ + ((P[1][i] + P[1][i - 1] + P[1][i + 1] + P[0][i] + P[2][i]) * 4 + \ + (P[0][i - 1] + P[2][i - 1] + \ + P[0][i + 1] + P[2][i + 1]) * 3) + + int i = 0; + + if (!((uintptr_t)src & 3) && w >= 4) { + const long nblk = w >> 2; + dav1d_sgr_finish_row3_ia64(tmp, src, B_ptrs, A_ptrs, nblk); + i = (int)(nblk * 4); + } + + for (; i < w; i++) { + const int a = EIGHT_NEIGHBORS(B_ptrs, i + 1); + const int b = EIGHT_NEIGHBORS(A_ptrs, i + 1); + tmp[i] = (b - a * src[i] + (1 << 8)) >> 9; + } +#undef EIGHT_NEIGHBORS +} + +static NOINLINE void sgr_finish_filter2_q(coef *tmp, + const pixel *src, + const ptrdiff_t src_stride, + int32_t **A_ptrs, coef **B_ptrs, + const int w, const int h) +{ +#define SIX_NEIGHBORS(P, i)\ + ((P[0][i] + P[1][i]) * 6 + \ + (P[0][i - 1] + P[1][i - 1] + \ + P[0][i + 1] + P[1][i + 1]) * 5) + int i = 0; + + if (!((uintptr_t)src & 3) && w >= 4) { + const long nblk = w >> 2; + dav1d_sgr_finish_row2_ia64(tmp, src, B_ptrs, A_ptrs, nblk); + i = (int)(nblk * 4); + } + + for (; i < w; i++) { + const int a = SIX_NEIGHBORS(B_ptrs, i + 1); + const int b = SIX_NEIGHBORS(A_ptrs, i + 1); + tmp[i] = (b - a * src[i] + (1 << 8)) >> 9; + } + + if (h <= 1) + return; + tmp += FILTER_OUT_STRIDE; + src += PXSTRIDE(src_stride); + const int32_t *A = &A_ptrs[1][1]; + const coef *B = &B_ptrs[1][1]; + + /* Second row: the same three offsets, but on row 1 alone. */ + i = 0; + if (!((uintptr_t)src & 3) && w >= 4) { + const long nblk = w >> 2; + dav1d_sgr_finish_row1_ia64(tmp, src, &B_ptrs[1], &A_ptrs[1], nblk); + i = (int)(nblk * 4); + } + + for (; i < w; i++) { + const int a = B[i] * 6 + (B[i - 1] + B[i + 1]) * 5; + const int b = A[i] * 6 + (A[i - 1] + A[i + 1]) * 5; + tmp[i] = (b - a * src[i] + (1 << 7)) >> 8; + } +#undef SIX_NEIGHBORS +} + +/* + * The two SGR weighting kernels, one scheduled region each. + * + * The arithmetic here is unremarkable -- a pair of 16x16->32 multiplies per + * lane, a rounding add, a narrowing shift and an add onto the destination + * pixels. What was expensive is that every one of those steps was a separate + * asm() helper (pmpy2_r, pmpy2_l, rsh_pack_rl, pack2_uss ...), and GCC types + * each asm() as TYPE_UNKNOWN, so each got its own bundle behind its own stop + * bit. The eight-pixel body measured 33 cycles against a 19-I-op floor of ten, + * with 23 nop.m and 8 nop.i of dead slots. + * + * Written as one region the same ops schedule in 12 cycles (weighted2) and 9 + * (weighted1), i.e. 1.500 and 1.125 cycles per pixel. + */ +SIMD_FN u8x8 sgr_w2_8(const u8x8 d, const i16x4 a0, const i16x4 a1, + const i16x4 b0, const i16x4 b1, + const i16x4 w0, const i16x4 w1, const i32x2 rnd) +{ + u8x8 out; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = %[a0], %[w0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r2 = %[b0], %[w1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r11 = %[a0], %[w0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[b0], %[w1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r9 = r3, r2\n\t nop.m 0\n\t pmpy2.r r8 = %[a1], %[w0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r31 = %[b1], %[w1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r3 = r11, r10\n\t padd4 r2 = r9, %[rnd]\n\t pmpy2.l r30 = %[a1], %[w0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r29 = %[b1], %[w1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r28 = r8, r31\n\t padd4 r11 = r3, %[rnd]\n\t pshr4 r10 = r2, 11\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r9 = r0, %[d]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r8 = r30, r29\n\t padd4 r31 = r28, %[rnd]\n\t pshr4 r2 = r11, 11\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r3 = r0, %[d]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r8, %[rnd]\n\t nop.m 0\n\t pshr4 r11 = r31, 11\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r28 = r10, r2\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r31 = r29, 11\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r8 = r28, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r29 = r8, r9\n\t nop.m 0\n\t pack4.sss r28 = r11, r31\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r31 = r28, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r28 = r31, r3\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack2.uss %[out] = r29, r28\n\t}\n\t" + ";;\n\t" + : [out]"=r"(out) + : [d]"r"(d), [a0]"r"(a0), [a1]"r"(a1), [b0]"r"(b0), [b1]"r"(b1), + [w0]"r"(w0), [w1]"r"(w1), [rnd]"r"(rnd) + : "r2", "r3", "r8", "r9", "r10", "r11", "r28", "r29", "r30", "r31"); + return out; +} + +SIMD_FN u8x8 sgr_w1_8(const u8x8 d, const i16x4 a0, const i16x4 a1, + const i16x4 w0, const i32x2 rnd) +{ + u8x8 out; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = %[a0], %[w0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[a0], %[w0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r11 = %[a1], %[w0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[a1], %[w0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r31 = r3, %[rnd]\n\t padd4 r30 = r2, %[rnd]\n\t unpack1.l r9 = r0, %[d]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r8 = r0, %[d]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r11, %[rnd]\n\t padd4 r28 = r10, %[rnd]\n\t pshr4 r2 = r31, 11\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r3 = r30, 11\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r30 = r29, 11\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r31 = r28, 11\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r28 = r2, r3\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r29 = r30, r31\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r31 = r28, 0xd8\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 r30 = r29, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r29 = r31, r9\n\t padd2 r28 = r30, r8\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack2.uss %[out] = r29, r28\n\t}\n\t" + ";;\n\t" + : [out]"=r"(out) + : [d]"r"(d), [a0]"r"(a0), [a1]"r"(a1), [w0]"r"(w0), [rnd]"r"(rnd) + : "r2", "r3", "r8", "r9", "r10", "r11", "r28", "r29", "r30", "r31"); + return out; +} + + +static NOINLINE void sgr_weighted_row1_q(pixel *dst, const coef *t1, + const int w, const int w1 HIGHBD_DECL_SUFFIX) +{ + /* t1 is a FILTER_OUT_STRIDE slice of a 16-byte aligned scratch buffer, so + * it is always 8-byte aligned; dst is a picture row and is not, hence the + * hoisted alignment select. pack2.uss performs the iclip_pixel() as part + * of the narrowing, so the clip is free. */ + const i16x4 wv = splat16((int16_t)w1); + const i32x2 rnd = { 1 << 10, 1 << 10 }; + const int aligned = !((uintptr_t)dst & 7); + int i = 0; + + for (; i + 8 <= w; i += 8) { + st_u8x8_sel(dst + i, + sgr_w1_8(ld_u8x8_sel(dst + i, aligned), + ld_i16x4(t1 + i), ld_i16x4(t1 + i + 4), wv, rnd), + aligned); + } + for (; i < w; i++) { + const int v = w1 * t1[i]; + dst[i] = iclip_pixel(dst[i] + ((v + (1 << 10)) >> 11)); + } +} + +static NOINLINE void sgr_weighted2_q(pixel *dst, const ptrdiff_t dst_stride, + const coef *t1, const coef *t2, + const int w, const int h, + const int w0, const int w1 HIGHBD_DECL_SUFFIX) +{ + const i16x4 w0v = splat16((int16_t)w0), w1v = splat16((int16_t)w1); + const i32x2 rnd = { 1 << 10, 1 << 10 }; + const int aligned = !(((uintptr_t)dst | (uintptr_t)dst_stride) & 7); + + for (int j = 0; j < h; j++) { + int i = 0; + for (; i + 8 <= w; i += 8) { + st_u8x8_sel(dst + i, + sgr_w2_8(ld_u8x8_sel(dst + i, aligned), + ld_i16x4(t1 + i), ld_i16x4(t1 + i + 4), + ld_i16x4(t2 + i), ld_i16x4(t2 + i + 4), + w0v, w1v, rnd), + aligned); + } + for (; i < w; i++) { + const int v = w0 * t1[i] + w1 * t2[i]; + dst[i] = iclip_pixel(dst[i] + ((v + (1 << 10)) >> 11)); + } + dst += PXSTRIDE(dst_stride); + t1 += FILTER_OUT_STRIDE; + t2 += FILTER_OUT_STRIDE; + } +} + +static NOINLINE void sgr_finish1_q(pixel **dst, const ptrdiff_t stride, + int32_t **A_ptrs, coef **B_ptrs, const int w, + const int w1 HIGHBD_DECL_SUFFIX) +{ + // Only one single row, no stride needed + ALIGN_STK_16(coef, tmp, 384,); + + sgr_finish_filter_row1_q(tmp, *dst, A_ptrs, B_ptrs, w); + sgr_weighted_row1_q(*dst, tmp, w, w1 HIGHBD_TAIL_SUFFIX); + *dst += PXSTRIDE(stride); + rotate_q(A_ptrs, B_ptrs, 3); +} + +static NOINLINE void sgr_finish2_q(pixel **dst, const ptrdiff_t stride, + int32_t **A_ptrs, coef **B_ptrs, + const int w, const int h, const int w1 + HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(coef, tmp, 2*FILTER_OUT_STRIDE,); + + sgr_finish_filter2_q(tmp, *dst, stride, A_ptrs, B_ptrs, w, h); + sgr_weighted_row1_q(*dst, tmp, w, w1 HIGHBD_TAIL_SUFFIX); + *dst += PXSTRIDE(stride); + if (h > 1) { + sgr_weighted_row1_q(*dst, tmp + FILTER_OUT_STRIDE, w, w1 HIGHBD_TAIL_SUFFIX); + *dst += PXSTRIDE(stride); + } + rotate_q(A_ptrs, B_ptrs, 2); +} + +static NOINLINE void sgr_finish_mix_q(pixel **dst, const ptrdiff_t stride, + int32_t **A5_ptrs, coef **B5_ptrs, + int32_t **A3_ptrs, coef **B3_ptrs, + const int w, const int h, + const int w0, const int w1 HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(coef, tmp5, 2*FILTER_OUT_STRIDE,); + ALIGN_STK_16(coef, tmp3, 2*FILTER_OUT_STRIDE,); + + sgr_finish_filter2_q(tmp5, *dst, stride, A5_ptrs, B5_ptrs, w, h); + sgr_finish_filter_row1_q(tmp3, *dst, A3_ptrs, B3_ptrs, w); + if (h > 1) + sgr_finish_filter_row1_q(tmp3 + FILTER_OUT_STRIDE, *dst + PXSTRIDE(stride), + &A3_ptrs[1], &B3_ptrs[1], w); + sgr_weighted2_q(*dst, stride, tmp5, tmp3, w, h, w0, w1 HIGHBD_TAIL_SUFFIX); + *dst += h*PXSTRIDE(stride); + rotate_q(A5_ptrs, B5_ptrs, 2); + rotate_q(A3_ptrs, B3_ptrs, 4); +} + + +static void sgr_3x3_ia64(pixel *dst, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, + const int w, int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ +#define BUF_STRIDE (384 + 16) + ALIGN_STK_16(int32_t, sumsq_buf, BUF_STRIDE * 3 + 16,); + ALIGN_STK_16(coef, sum_buf, BUF_STRIDE * 3 + 16,); + int32_t *sumsq_ptrs[3], *sumsq_rows[3]; + coef *sum_ptrs[3], *sum_rows[3]; + for (int i = 0; i < 3; i++) { + sumsq_rows[i] = &sumsq_buf[i * BUF_STRIDE]; + sum_rows[i] = &sum_buf[i * BUF_STRIDE]; + } + + ALIGN_STK_16(int32_t, A_buf, BUF_STRIDE * 3 + 16,); + ALIGN_STK_16(coef, B_buf, BUF_STRIDE * 3 + 16,); + int32_t *A_ptrs[3]; + coef *B_ptrs[3]; + for (int i = 0; i < 3; i++) { + A_ptrs[i] = &A_buf[i * BUF_STRIDE]; + B_ptrs[i] = &B_buf[i * BUF_STRIDE]; + } + const pixel *src = dst; + const pixel *lpf_bottom = lpf + 6*PXSTRIDE(stride); + + if (edges & LR_HAVE_TOP) { + sumsq_ptrs[0] = sumsq_rows[0]; + sumsq_ptrs[1] = sumsq_rows[1]; + sumsq_ptrs[2] = sumsq_rows[2]; + sum_ptrs[0] = sum_rows[0]; + sum_ptrs[1] = sum_rows[1]; + sum_ptrs[2] = sum_rows[2]; + + sgr_box3_row_h_q(sumsq_rows[0], sum_rows[0], NULL, lpf, w, edges); + lpf += PXSTRIDE(stride); + sgr_box3_row_h_q(sumsq_rows[1], sum_rows[1], NULL, lpf, w, edges); + + sgr_box3_hv_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + left, src, w, params->sgr.s1, edges, BITDEPTH_MAX); + left++; + src += PXSTRIDE(stride); + rotate_q(A_ptrs, B_ptrs, 3); + + if (--h <= 0) + goto vert_1; + + sgr_box3_hv_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + left, src, w, params->sgr.s1, edges, BITDEPTH_MAX); + left++; + src += PXSTRIDE(stride); + rotate_q(A_ptrs, B_ptrs, 3); + + if (--h <= 0) + goto vert_2; + } else { + sumsq_ptrs[0] = sumsq_rows[0]; + sumsq_ptrs[1] = sumsq_rows[0]; + sumsq_ptrs[2] = sumsq_rows[0]; + sum_ptrs[0] = sum_rows[0]; + sum_ptrs[1] = sum_rows[0]; + sum_ptrs[2] = sum_rows[0]; + + sgr_box3_row_h_q(sumsq_rows[0], sum_rows[0], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box3_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A_ptrs, B_ptrs, 3); + + if (--h <= 0) + goto vert_1; + + sumsq_ptrs[2] = sumsq_rows[1]; + sum_ptrs[2] = sum_rows[1]; + + sgr_box3_hv_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + left, src, w, params->sgr.s1, edges, BITDEPTH_MAX); + left++; + src += PXSTRIDE(stride); + rotate_q(A_ptrs, B_ptrs, 3); + + if (--h <= 0) + goto vert_2; + + sumsq_ptrs[2] = sumsq_rows[2]; + sum_ptrs[2] = sum_rows[2]; + } + + do { + sgr_box3_hv_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + left, src, w, params->sgr.s1, edges, BITDEPTH_MAX); + left++; + src += PXSTRIDE(stride); + + sgr_finish1_q(&dst, stride, A_ptrs, B_ptrs, + w, params->sgr.w1 HIGHBD_TAIL_SUFFIX); + } while (--h > 0); + + if (!(edges & LR_HAVE_BOTTOM)) + goto vert_2; + + sgr_box3_hv_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + NULL, lpf_bottom, w, params->sgr.s1, edges, BITDEPTH_MAX); + lpf_bottom += PXSTRIDE(stride); + + sgr_finish1_q(&dst, stride, A_ptrs, B_ptrs, + w, params->sgr.w1 HIGHBD_TAIL_SUFFIX); + + sgr_box3_hv_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + NULL, lpf_bottom, w, params->sgr.s1, edges, BITDEPTH_MAX); + + sgr_finish1_q(&dst, stride, A_ptrs, B_ptrs, + w, params->sgr.w1 HIGHBD_TAIL_SUFFIX); + return; + +vert_2: + sumsq_ptrs[2] = sumsq_ptrs[1]; + sum_ptrs[2] = sum_ptrs[1]; + sgr_box3_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + w, params->sgr.s1, BITDEPTH_MAX); + + sgr_finish1_q(&dst, stride, A_ptrs, B_ptrs, + w, params->sgr.w1 HIGHBD_TAIL_SUFFIX); + +output_1: + sumsq_ptrs[2] = sumsq_ptrs[1]; + sum_ptrs[2] = sum_ptrs[1]; + sgr_box3_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + w, params->sgr.s1, BITDEPTH_MAX); + + sgr_finish1_q(&dst, stride, A_ptrs, B_ptrs, + w, params->sgr.w1 HIGHBD_TAIL_SUFFIX); + return; + +vert_1: + sumsq_ptrs[2] = sumsq_ptrs[1]; + sum_ptrs[2] = sum_ptrs[1]; + sgr_box3_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[2], B_ptrs[2], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A_ptrs, B_ptrs, 3); + goto output_1; +} + +static void sgr_5x5_ia64(pixel *dst, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, + const int w, int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int32_t, sumsq_buf, BUF_STRIDE * 5 + 16,); + ALIGN_STK_16(coef, sum_buf, BUF_STRIDE * 5 + 16,); + int32_t *sumsq_ptrs[5], *sumsq_rows[5]; + coef *sum_ptrs[5], *sum_rows[5]; + for (int i = 0; i < 5; i++) { + sumsq_rows[i] = &sumsq_buf[i * BUF_STRIDE]; + sum_rows[i] = &sum_buf[i * BUF_STRIDE]; + } + + ALIGN_STK_16(int32_t, A_buf, BUF_STRIDE * 2 + 16,); + ALIGN_STK_16(coef, B_buf, BUF_STRIDE * 2 + 16,); + int32_t *A_ptrs[2]; + coef *B_ptrs[2]; + for (int i = 0; i < 2; i++) { + A_ptrs[i] = &A_buf[i * BUF_STRIDE]; + B_ptrs[i] = &B_buf[i * BUF_STRIDE]; + } + const pixel *src = dst; + const pixel *lpf_bottom = lpf + 6*PXSTRIDE(stride); + + if (edges & LR_HAVE_TOP) { + sumsq_ptrs[0] = sumsq_rows[0]; + sumsq_ptrs[1] = sumsq_rows[0]; + sumsq_ptrs[2] = sumsq_rows[1]; + sumsq_ptrs[3] = sumsq_rows[2]; + sumsq_ptrs[4] = sumsq_rows[3]; + sum_ptrs[0] = sum_rows[0]; + sum_ptrs[1] = sum_rows[0]; + sum_ptrs[2] = sum_rows[1]; + sum_ptrs[3] = sum_rows[2]; + sum_ptrs[4] = sum_rows[3]; + + sgr_box5_row_h_q(sumsq_rows[0], sum_rows[0], NULL, lpf, w, edges); + lpf += PXSTRIDE(stride); + sgr_box5_row_h_q(sumsq_rows[1], sum_rows[1], NULL, lpf, w, edges); + + sgr_box5_row_h_q(sumsq_rows[2], sum_rows[2], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + if (--h <= 0) + goto vert_1; + + sgr_box5_row_h_q(sumsq_rows[3], sum_rows[3], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + sgr_box5_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[1], B_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + rotate_q(A_ptrs, B_ptrs, 2); + + if (--h <= 0) + goto vert_2; + + // ptrs are rotated by 2; both [3] and [4] now point at rows[0]; set + // one of them to point at the previously unused rows[4]. + sumsq_ptrs[3] = sumsq_rows[4]; + sum_ptrs[3] = sum_rows[4]; + } else { + sumsq_ptrs[0] = sumsq_rows[0]; + sumsq_ptrs[1] = sumsq_rows[0]; + sumsq_ptrs[2] = sumsq_rows[0]; + sumsq_ptrs[3] = sumsq_rows[0]; + sumsq_ptrs[4] = sumsq_rows[0]; + sum_ptrs[0] = sum_rows[0]; + sum_ptrs[1] = sum_rows[0]; + sum_ptrs[2] = sum_rows[0]; + sum_ptrs[3] = sum_rows[0]; + sum_ptrs[4] = sum_rows[0]; + + sgr_box5_row_h_q(sumsq_rows[0], sum_rows[0], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + if (--h <= 0) + goto vert_1; + + sumsq_ptrs[4] = sumsq_rows[1]; + sum_ptrs[4] = sum_rows[1]; + + sgr_box5_row_h_q(sumsq_rows[1], sum_rows[1], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box5_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[1], B_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + rotate_q(A_ptrs, B_ptrs, 2); + + if (--h <= 0) + goto vert_2; + + sumsq_ptrs[3] = sumsq_rows[2]; + sumsq_ptrs[4] = sumsq_rows[3]; + sum_ptrs[3] = sum_rows[2]; + sum_ptrs[4] = sum_rows[3]; + + sgr_box5_row_h_q(sumsq_rows[2], sum_rows[2], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + if (--h <= 0) + goto odd; + + sgr_box5_row_h_q(sumsq_rows[3], sum_rows[3], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box5_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[1], B_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_finish2_q(&dst, stride, A_ptrs, B_ptrs, + w, 2, params->sgr.w0 HIGHBD_TAIL_SUFFIX); + + if (--h <= 0) + goto vert_2; + + // ptrs are rotated by 2; both [3] and [4] now point at rows[0]; set + // one of them to point at the previously unused rows[4]. + sumsq_ptrs[3] = sumsq_rows[4]; + sum_ptrs[3] = sum_rows[4]; + } + + do { + sgr_box5_row_h_q(sumsq_ptrs[3], sum_ptrs[3], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + if (--h <= 0) + goto odd; + + sgr_box5_row_h_q(sumsq_ptrs[4], sum_ptrs[4], left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box5_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[1], B_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_finish2_q(&dst, stride, A_ptrs, B_ptrs, + w, 2, params->sgr.w0 HIGHBD_TAIL_SUFFIX); + } while (--h > 0); + + if (!(edges & LR_HAVE_BOTTOM)) + goto vert_2; + + sgr_box5_row_h_q(sumsq_ptrs[3], sum_ptrs[3], NULL, lpf_bottom, w, edges); + lpf_bottom += PXSTRIDE(stride); + sgr_box5_row_h_q(sumsq_ptrs[4], sum_ptrs[4], NULL, lpf_bottom, w, edges); + +output_2: + sgr_box5_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[1], B_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_finish2_q(&dst, stride, A_ptrs, B_ptrs, + w, 2, params->sgr.w0 HIGHBD_TAIL_SUFFIX); + return; + +vert_2: + // Duplicate the last row twice more + sumsq_ptrs[3] = sumsq_ptrs[2]; + sumsq_ptrs[4] = sumsq_ptrs[2]; + sum_ptrs[3] = sum_ptrs[2]; + sum_ptrs[4] = sum_ptrs[2]; + goto output_2; + +odd: + // Copy the last row as padding once + sumsq_ptrs[4] = sumsq_ptrs[3]; + sum_ptrs[4] = sum_ptrs[3]; + + sgr_box5_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[1], B_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_finish2_q(&dst, stride, A_ptrs, B_ptrs, + w, 2, params->sgr.w0 HIGHBD_TAIL_SUFFIX); + +output_1: + // Duplicate the last row twice more + sumsq_ptrs[3] = sumsq_ptrs[2]; + sumsq_ptrs[4] = sumsq_ptrs[2]; + sum_ptrs[3] = sum_ptrs[2]; + sum_ptrs[4] = sum_ptrs[2]; + + sgr_box5_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[1], B_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + // Output only one row + sgr_finish2_q(&dst, stride, A_ptrs, B_ptrs, + w, 1, params->sgr.w0 HIGHBD_TAIL_SUFFIX); + return; + +vert_1: + // Copy the last row as padding once + sumsq_ptrs[4] = sumsq_ptrs[3]; + sum_ptrs[4] = sum_ptrs[3]; + + sgr_box5_vert_q(sumsq_ptrs, sum_ptrs, A_ptrs[1], B_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + rotate_q(A_ptrs, B_ptrs, 2); + + goto output_1; +} + +static void sgr_mix_ia64(pixel *dst, const ptrdiff_t stride, + const pixel (*left)[4], const pixel *lpf, + const int w, int h, + const LooprestorationParams *const params, + const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int32_t, sumsq5_buf, BUF_STRIDE * 5 + 16,); + ALIGN_STK_16(coef, sum5_buf, BUF_STRIDE * 5 + 16,); + int32_t *sumsq5_ptrs[5], *sumsq5_rows[5]; + coef *sum5_ptrs[5], *sum5_rows[5]; + for (int i = 0; i < 5; i++) { + sumsq5_rows[i] = &sumsq5_buf[i * BUF_STRIDE]; + sum5_rows[i] = &sum5_buf[i * BUF_STRIDE]; + } + ALIGN_STK_16(int32_t, sumsq3_buf, BUF_STRIDE * 3 + 16,); + ALIGN_STK_16(coef, sum3_buf, BUF_STRIDE * 3 + 16,); + int32_t *sumsq3_ptrs[3], *sumsq3_rows[3]; + coef *sum3_ptrs[3], *sum3_rows[3]; + for (int i = 0; i < 3; i++) { + sumsq3_rows[i] = &sumsq3_buf[i * BUF_STRIDE]; + sum3_rows[i] = &sum3_buf[i * BUF_STRIDE]; + } + + ALIGN_STK_16(int32_t, A5_buf, BUF_STRIDE * 2 + 16,); + ALIGN_STK_16(coef, B5_buf, BUF_STRIDE * 2 + 16,); + int32_t *A5_ptrs[2]; + coef *B5_ptrs[2]; + for (int i = 0; i < 2; i++) { + A5_ptrs[i] = &A5_buf[i * BUF_STRIDE]; + B5_ptrs[i] = &B5_buf[i * BUF_STRIDE]; + } + ALIGN_STK_16(int32_t, A3_buf, BUF_STRIDE * 4 + 16,); + ALIGN_STK_16(coef, B3_buf, BUF_STRIDE * 4 + 16,); + int32_t *A3_ptrs[4]; + coef *B3_ptrs[4]; + for (int i = 0; i < 4; i++) { + A3_ptrs[i] = &A3_buf[i * BUF_STRIDE]; + B3_ptrs[i] = &B3_buf[i * BUF_STRIDE]; + } + const pixel *src = dst; + const pixel *lpf_bottom = lpf + 6*PXSTRIDE(stride); + + if (edges & LR_HAVE_TOP) { + sumsq5_ptrs[0] = sumsq5_rows[0]; + sumsq5_ptrs[1] = sumsq5_rows[0]; + sumsq5_ptrs[2] = sumsq5_rows[1]; + sumsq5_ptrs[3] = sumsq5_rows[2]; + sumsq5_ptrs[4] = sumsq5_rows[3]; + sum5_ptrs[0] = sum5_rows[0]; + sum5_ptrs[1] = sum5_rows[0]; + sum5_ptrs[2] = sum5_rows[1]; + sum5_ptrs[3] = sum5_rows[2]; + sum5_ptrs[4] = sum5_rows[3]; + + sumsq3_ptrs[0] = sumsq3_rows[0]; + sumsq3_ptrs[1] = sumsq3_rows[1]; + sumsq3_ptrs[2] = sumsq3_rows[2]; + sum3_ptrs[0] = sum3_rows[0]; + sum3_ptrs[1] = sum3_rows[1]; + sum3_ptrs[2] = sum3_rows[2]; + + sgr_box35_row_h_q(sumsq3_rows[0], sum3_rows[0], + sumsq5_rows[0], sum5_rows[0], + NULL, lpf, w, edges); + lpf += PXSTRIDE(stride); + sgr_box35_row_h_q(sumsq3_rows[1], sum3_rows[1], + sumsq5_rows[1], sum5_rows[1], + NULL, lpf, w, edges); + + sgr_box35_row_h_q(sumsq3_rows[2], sum3_rows[2], + sumsq5_rows[2], sum5_rows[2], + left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + if (--h <= 0) + goto vert_1; + + sgr_box35_row_h_q(sumsq3_ptrs[2], sum3_ptrs[2], + sumsq5_rows[3], sum5_rows[3], + left, src, w, edges); + left++; + src += PXSTRIDE(stride); + sgr_box5_vert_q(sumsq5_ptrs, sum5_ptrs, A5_ptrs[1], B5_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + rotate_q(A5_ptrs, B5_ptrs, 2); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + if (--h <= 0) + goto vert_2; + + // ptrs are rotated by 2; both [3] and [4] now point at rows[0]; set + // one of them to point at the previously unused rows[4]. + sumsq5_ptrs[3] = sumsq5_rows[4]; + sum5_ptrs[3] = sum5_rows[4]; + } else { + sumsq5_ptrs[0] = sumsq5_rows[0]; + sumsq5_ptrs[1] = sumsq5_rows[0]; + sumsq5_ptrs[2] = sumsq5_rows[0]; + sumsq5_ptrs[3] = sumsq5_rows[0]; + sumsq5_ptrs[4] = sumsq5_rows[0]; + sum5_ptrs[0] = sum5_rows[0]; + sum5_ptrs[1] = sum5_rows[0]; + sum5_ptrs[2] = sum5_rows[0]; + sum5_ptrs[3] = sum5_rows[0]; + sum5_ptrs[4] = sum5_rows[0]; + + sumsq3_ptrs[0] = sumsq3_rows[0]; + sumsq3_ptrs[1] = sumsq3_rows[0]; + sumsq3_ptrs[2] = sumsq3_rows[0]; + sum3_ptrs[0] = sum3_rows[0]; + sum3_ptrs[1] = sum3_rows[0]; + sum3_ptrs[2] = sum3_rows[0]; + + sgr_box35_row_h_q(sumsq3_rows[0], sum3_rows[0], + sumsq5_rows[0], sum5_rows[0], + left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + if (--h <= 0) + goto vert_1; + + sumsq5_ptrs[4] = sumsq5_rows[1]; + sum5_ptrs[4] = sum5_rows[1]; + + sumsq3_ptrs[2] = sumsq3_rows[1]; + sum3_ptrs[2] = sum3_rows[1]; + + sgr_box35_row_h_q(sumsq3_rows[1], sum3_rows[1], + sumsq5_rows[1], sum5_rows[1], + left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box5_vert_q(sumsq5_ptrs, sum5_ptrs, A5_ptrs[1], B5_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + rotate_q(A5_ptrs, B5_ptrs, 2); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + if (--h <= 0) + goto vert_2; + + sumsq5_ptrs[3] = sumsq5_rows[2]; + sumsq5_ptrs[4] = sumsq5_rows[3]; + sum5_ptrs[3] = sum5_rows[2]; + sum5_ptrs[4] = sum5_rows[3]; + + sumsq3_ptrs[2] = sumsq3_rows[2]; + sum3_ptrs[2] = sum3_rows[2]; + + sgr_box35_row_h_q(sumsq3_rows[2], sum3_rows[2], + sumsq5_rows[2], sum5_rows[2], + left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + if (--h <= 0) + goto odd; + + sgr_box35_row_h_q(sumsq3_ptrs[2], sum3_ptrs[2], + sumsq5_rows[3], sum5_rows[3], + left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box5_vert_q(sumsq5_ptrs, sum5_ptrs, A5_ptrs[1], B5_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + sgr_finish_mix_q(&dst, stride, A5_ptrs, B5_ptrs, A3_ptrs, B3_ptrs, + w, 2, params->sgr.w0, params->sgr.w1 + HIGHBD_TAIL_SUFFIX); + + if (--h <= 0) + goto vert_2; + + // ptrs are rotated by 2; both [3] and [4] now point at rows[0]; set + // one of them to point at the previously unused rows[4]. + sumsq5_ptrs[3] = sumsq5_rows[4]; + sum5_ptrs[3] = sum5_rows[4]; + } + + do { + sgr_box35_row_h_q(sumsq3_ptrs[2], sum3_ptrs[2], + sumsq5_ptrs[3], sum5_ptrs[3], + left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + if (--h <= 0) + goto odd; + + sgr_box35_row_h_q(sumsq3_ptrs[2], sum3_ptrs[2], + sumsq5_ptrs[4], sum5_ptrs[4], + left, src, w, edges); + left++; + src += PXSTRIDE(stride); + + sgr_box5_vert_q(sumsq5_ptrs, sum5_ptrs, A5_ptrs[1], B5_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + sgr_finish_mix_q(&dst, stride, A5_ptrs, B5_ptrs, A3_ptrs, B3_ptrs, + w, 2, params->sgr.w0, params->sgr.w1 + HIGHBD_TAIL_SUFFIX); + } while (--h > 0); + + if (!(edges & LR_HAVE_BOTTOM)) + goto vert_2; + + sgr_box35_row_h_q(sumsq3_ptrs[2], sum3_ptrs[2], + sumsq5_ptrs[3], sum5_ptrs[3], + NULL, lpf_bottom, w, edges); + lpf_bottom += PXSTRIDE(stride); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + sgr_box35_row_h_q(sumsq3_ptrs[2], sum3_ptrs[2], + sumsq5_ptrs[4], sum5_ptrs[4], + NULL, lpf_bottom, w, edges); + +output_2: + sgr_box5_vert_q(sumsq5_ptrs, sum5_ptrs, A5_ptrs[1], B5_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + sgr_finish_mix_q(&dst, stride, A5_ptrs, B5_ptrs, A3_ptrs, B3_ptrs, + w, 2, params->sgr.w0, params->sgr.w1 + HIGHBD_TAIL_SUFFIX); + return; + +vert_2: + // Duplicate the last row twice more + sumsq5_ptrs[3] = sumsq5_ptrs[2]; + sumsq5_ptrs[4] = sumsq5_ptrs[2]; + sum5_ptrs[3] = sum5_ptrs[2]; + sum5_ptrs[4] = sum5_ptrs[2]; + + sumsq3_ptrs[2] = sumsq3_ptrs[1]; + sum3_ptrs[2] = sum3_ptrs[1]; + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + sumsq3_ptrs[2] = sumsq3_ptrs[1]; + sum3_ptrs[2] = sum3_ptrs[1]; + + goto output_2; + +odd: + // Copy the last row as padding once + sumsq5_ptrs[4] = sumsq5_ptrs[3]; + sum5_ptrs[4] = sum5_ptrs[3]; + + sumsq3_ptrs[2] = sumsq3_ptrs[1]; + sum3_ptrs[2] = sum3_ptrs[1]; + + sgr_box5_vert_q(sumsq5_ptrs, sum5_ptrs, A5_ptrs[1], B5_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + sgr_finish_mix_q(&dst, stride, A5_ptrs, B5_ptrs, A3_ptrs, B3_ptrs, + w, 2, params->sgr.w0, params->sgr.w1 + HIGHBD_TAIL_SUFFIX); + +output_1: + // Duplicate the last row twice more + sumsq5_ptrs[3] = sumsq5_ptrs[2]; + sumsq5_ptrs[4] = sumsq5_ptrs[2]; + sum5_ptrs[3] = sum5_ptrs[2]; + sum5_ptrs[4] = sum5_ptrs[2]; + + sumsq3_ptrs[2] = sumsq3_ptrs[1]; + sum3_ptrs[2] = sum3_ptrs[1]; + + sgr_box5_vert_q(sumsq5_ptrs, sum5_ptrs, A5_ptrs[1], B5_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + // Output only one row + sgr_finish_mix_q(&dst, stride, A5_ptrs, B5_ptrs, A3_ptrs, B3_ptrs, + w, 1, params->sgr.w0, params->sgr.w1 + HIGHBD_TAIL_SUFFIX); + return; + +vert_1: + // Copy the last row as padding once + sumsq5_ptrs[4] = sumsq5_ptrs[3]; + sum5_ptrs[4] = sum5_ptrs[3]; + + sumsq3_ptrs[2] = sumsq3_ptrs[1]; + sum3_ptrs[2] = sum3_ptrs[1]; + + sgr_box5_vert_q(sumsq5_ptrs, sum5_ptrs, A5_ptrs[1], B5_ptrs[1], + w, params->sgr.s0, BITDEPTH_MAX); + rotate_q(A5_ptrs, B5_ptrs, 2); + sgr_box3_vert_q(sumsq3_ptrs, sum3_ptrs, A3_ptrs[3], B3_ptrs[3], + w, params->sgr.s1, BITDEPTH_MAX); + rotate_q(A3_ptrs, B3_ptrs, 4); + + goto output_1; +} + +#endif /* BITDEPTH == 8 */ + +COLD void bitfn(dav1d_loop_restoration_dsp_init_ia64)(Dav1dLoopRestorationDSPContext *const c, + const int bpc) +{ + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_IA64_CPU_FLAG_SIMD)) return; + +#if BITDEPTH == 8 + c->wiener[0] = c->wiener[1] = wiener_ia64; + c->sgr[0] = sgr_5x5_ia64; + c->sgr[1] = sgr_3x3_ia64; + c->sgr[2] = sgr_mix_ia64; +#endif +} diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/mc.h dav1d-1.5.4/src/ia64/mc.h --- dav1d-1.5.4.orig/src/ia64/mc.h 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/mc.h 2026-08-17 21:42:00.563043910 +0200 @@ -0,0 +1,34 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/mc.h" + +bitfn_decls(void dav1d_mc_dsp_init_ia64, Dav1dMCDSPContext *c); + +static ALWAYS_INLINE void mc_dsp_init_ia64(Dav1dMCDSPContext *const c) { + bitfn(dav1d_mc_dsp_init_ia64)(c); +} diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/ia64/mc_tmpl.c dav1d-1.5.4/src/ia64/mc_tmpl.c --- dav1d-1.5.4.orig/src/ia64/mc_tmpl.c 1970-01-01 01:00:00.000000000 +0100 +++ dav1d-1.5.4/src/ia64/mc_tmpl.c 2026-08-21 10:23:59.414776465 +0200 @@ -0,0 +1,2407 @@ +/* + * Copyright © 2026, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include + +#include "common/attributes.h" +#include "common/intops.h" + +#include "src/cpu.h" +#include "src/ia64/dav1d_simd.h" +#include "src/ia64/mc.h" +#include "src/levels.h" +#include "src/mc.h" +#include "src/tables.h" + +#if BITDEPTH == 8 + +/* For 8bpc: intermediate_bits == 4 and PREP_BIAS == 0. */ +#define INTERMEDIATE_BITS 4 + +/* + * avg / w_avg / mask all reduce two int16 intermediate buffers down to pixels. + * + * The sums are accumulated in 32 bits rather than 16. dav1d's intermediates + * are bounded to fit int16 individually, but a weighted sum of two of them is + * not, and pmpy2 gives us the widening multiply for free anyway. The final + * pack2.uss performs iclip_pixel() as part of the narrowing, so the clip costs + * nothing. + */ + +/* rsh_pack_rl(), which reduces the deinterleaved pmpy2.r / pmpy2.l halves back + * to four in-order 16-bit lanes, lives in dav1d_simd.h -- the loop restoration + * kernels need it too. */ + +/* ------------------------------------------------------------------- avg */ + +/* + * avg() is (tmp1 + tmp2 + rnd) >> sh with no multiply, so unlike w_avg there is + * no xma round trip to amortise and an earlier packed attempt lost outright + * (0.81x-0.95x for w >= 16). That attempt was store-bound, not arithmetic + * bound: stu_u8x8() on an unaligned dst fell into memcpy(), which GCC expands + * as a spill of the packed register to the stack plus eight ld1/st1 -- more + * memory ops per pixel than the scalar loop it replaced. Going through + * st_u8x8_sel() with the row's alignment hoisted out keeps the store in the I + * unit and the arithmetic is then a clear win: seven parallel ops per eight + * pixels against roughly sixteen scalar ones. + * + * The sum is formed in 16 bits. That is what upstream dav1d does on x86 + * (paddw + pmulhrsw, mc_avx2.asm:5244), so the intermediate range is known to + * fit; pack2.uss then applies iclip_pixel() as part of the narrowing. + */ +SIMD_FN i16x4 avg4(const int16_t *const t1, const int16_t *const t2) { + const int16_t r = 1 << INTERMEDIATE_BITS; + const i16x4 rnd = { r, r, r, r }; + return (ld_i16x4(t1) + ld_i16x4(t2) + rnd) >> (INTERMEDIATE_BITS + 1); +} + +static void avg_ia64(pixel *dst, const ptrdiff_t dst_stride, + const int16_t *tmp1, const int16_t *tmp2, + const int w, int h HIGHBD_DECL_SUFFIX) +{ + if (w == 4) { + do { + st_u8_partial(dst, pack2_uss(avg4(tmp1, tmp2), (i16x4){0}), 4); + tmp1 += 4; tmp2 += 4; dst += PXSTRIDE(dst_stride); + } while (--h); + return; + } + + /* Both the start pointer and the row pitch have to be 8-byte aligned for + * the aligned store to stay valid for every row of the block. */ + const int aligned = !((uintptr_t)dst & 7) && !(PXSTRIDE(dst_stride) & 7); + do { + for (int x = 0; x < w; x += 8) + st_u8x8_sel(dst + x, pack2_uss(avg4(tmp1 + x, tmp2 + x), + avg4(tmp1 + x + 4, tmp2 + x + 4)), + aligned); + tmp1 += w; tmp2 += w; dst += PXSTRIDE(dst_stride); + } while (--h); +} + +/* ----------------------------------------------------------------- w_avg */ + +SIMD_FN i16x4 w_avg4(const int16_t *const t1, const int16_t *const t2, + const i16x4 wgt, const i16x4 iwgt) +{ + const i16x4 a = ld_i16x4(t1), b = ld_i16x4(t2); + const i32x2 rnd = { 8 << INTERMEDIATE_BITS, 8 << INTERMEDIATE_BITS }; + const i32x2 r = pmpy2_r(a, wgt) + pmpy2_r(b, iwgt) + rnd; + const i32x2 l = pmpy2_l(a, wgt) + pmpy2_l(b, iwgt) + rnd; + return rsh_pack_rl(r, l, INTERMEDIATE_BITS + 4); +} + +static void w_avg_ia64(pixel *dst, const ptrdiff_t dst_stride, + const int16_t *tmp1, const int16_t *tmp2, + const int w, int h, const int weight HIGHBD_DECL_SUFFIX) +{ + const i16x4 wgt = splat16((int16_t)weight); + const i16x4 iwgt = splat16((int16_t)(16 - weight)); + const int aligned = !(((uintptr_t)dst | (uintptr_t)dst_stride) & 7); + + if (w == 4) { + do { + st_u8_partial(dst, pack2_uss(w_avg4(tmp1, tmp2, wgt, iwgt), (i16x4){0}), 4); + tmp1 += 4; tmp2 += 4; dst += PXSTRIDE(dst_stride); + } while (--h); + } else { + do { + for (int x = 0; x < w; x += 8) + st_u8x8_sel(dst + x, + pack2_uss(w_avg4(tmp1 + x, tmp2 + x, wgt, iwgt), + w_avg4(tmp1 + x + 4, tmp2 + x + 4, wgt, iwgt)), + aligned); + tmp1 += w; tmp2 += w; dst += PXSTRIDE(dst_stride); + } while (--h); + } +} + +/* ------------------------------------------------------------------ mask */ + +SIMD_FN i16x4 mask4(const int16_t *const t1, const int16_t *const t2, + const i16x4 mv) +{ + const i16x4 a = ld_i16x4(t1), b = ld_i16x4(t2); + const i16x4 k64 = { 64, 64, 64, 64 }; + const i16x4 imv = k64 - mv; + const i32x2 rnd = { 32 << INTERMEDIATE_BITS, 32 << INTERMEDIATE_BITS }; + const i32x2 r = pmpy2_r(a, mv) + pmpy2_r(b, imv) + rnd; + const i32x2 l = pmpy2_l(a, mv) + pmpy2_l(b, imv) + rnd; + return rsh_pack_rl(r, l, INTERMEDIATE_BITS + 6); +} + +/* The mask is one byte per pixel, so it is fetched eight at a time and split + * with unpack1.l/.h. Fetching it four at a time through memcpy (four ld1 plus + * a shift/or chain) costs more than the arithmetic it feeds and was measured at + * 0.68x-0.93x against plain C. */ +SIMD_FN void mask_row(pixel *const dst, const int16_t *const tmp1, + const int16_t *const tmp2, const int w, + const uint8_t *const mask, const int aligned) +{ + for (int x = 0; x < w; x += 8) { + const u8x8 m = ld_u8x8_sel(mask + x, aligned); + stu_u8x8(dst + x, pack2_uss(mask4(tmp1 + x, tmp2 + x, zext8_lo(m)), + mask4(tmp1 + x + 4, tmp2 + x + 4, zext8_hi(m)))); + } +} + +static void mask_ia64(pixel *dst, const ptrdiff_t dst_stride, + const int16_t *tmp1, const int16_t *tmp2, + const int w, int h, const uint8_t *mask HIGHBD_DECL_SUFFIX) +{ + if (w == 4) { + do { + const i16x4 mv = zext8_lo(ldn_u8x8(mask, 4)); + st_u8_partial(dst, pack2_uss(mask4(tmp1, tmp2, mv), (i16x4){0}), 4); + tmp1 += 4; tmp2 += 4; mask += 4; dst += PXSTRIDE(dst_stride); + } while (--h); + return; + } + + /* w is a multiple of 8 here, so the mask pointer keeps whatever alignment + * it starts with for the whole call. */ + const int aligned = !((uintptr_t)mask & 7); + do { + if (aligned) mask_row(dst, tmp1, tmp2, w, mask, 1); + else mask_row(dst, tmp1, tmp2, w, mask, 0); + tmp1 += w; tmp2 += w; mask += w; dst += PXSTRIDE(dst_stride); + } while (--h); +} + +/* ---------------------------------------------------------------- w_mask */ + +/* + * w_mask() derives a blend weight per pixel from |tmp1 - tmp2|, applies it, and + * writes the weight out as a mask at 1x, 2x1 or 2x2 resolution. The weight and + * the blend are ordinary packed work; the only awkward part is the subsampled + * mask, which needs adjacent lanes summed. __builtin_shuffle gives that in one + * mux2 each (GCC maps it to *mux2 in vect.md), so the 2x1 reduction is three + * ops for eight pixels. + */ +SIMD_FN void wm_px4(const int16_t *const t1, const int16_t *const t2, + i16x4 *const m_out, i16x4 *const d_out) +{ + const i16x4 a = ld_i16x4(t1), b = ld_i16x4(t2); + const i16x4 zero = { 0, 0, 0, 0 }; + const i16x4 k8 = { 8, 8, 8, 8 }; + const i16x4 k38 = { 38, 38, 38, 38 }; + const i16x4 k64 = { 64, 64, 64, 64 }; + const i32x2 rnd = { 32 << INTERMEDIATE_BITS, 32 << INTERMEDIATE_BITS }; + + const i16x4 diff = a - b; + const i16x4 ad = pmax2(diff, zero - diff); + /* m = imin(38 + ((|diff| + mask_rnd) >> mask_sh), 64), mask_sh == 8 here */ + const i16x4 m = pmin2(((ad + k8) >> 8) + k38, k64); + + const i32x2 r = pmpy2_r(diff, m) + pmpy2_r(b, k64) + rnd; + const i32x2 l = pmpy2_l(diff, m) + pmpy2_l(b, k64) + rnd; + + *m_out = m; + *d_out = rsh_pack_rl(r, l, INTERMEDIATE_BITS + 6); +} + +/* {lo0+lo1, lo2+lo3, hi0+hi1, hi2+hi3} */ +SIMD_FN i16x4 wm_pairs(const i16x4 lo, const i16x4 hi) { + const i16x4 swl = __builtin_shuffle(lo, (i16x4){ 1, 0, 3, 2 }); + const i16x4 swh = __builtin_shuffle(hi, (i16x4){ 1, 0, 3, 2 }); + const i16x4 tl = lo + swl; /* lanes 0 and 2 hold the pair sums */ + const i16x4 th = hi + swh; + return __builtin_shuffle(tl, th, (i16x4){ 0, 2, 4, 6 }); +} + +#define w_mask_fns_ia64(ssn, ss_hor, ss_ver) \ +static void w_mask_##ssn##_ia64(pixel *dst, const ptrdiff_t dst_stride, \ + const int16_t *tmp1, const int16_t *tmp2, \ + const int w, int h, uint8_t *mask, \ + const int sign HIGHBD_DECL_SUFFIX) \ +{ \ + const i16x4 zero = { 0, 0, 0, 0 }; \ + const i16x4 k1 = { 1, 1, 1, 1 }, k2 = { 2, 2, 2, 2 }; \ + const i16x4 vsign = splat16((int16_t)sign); \ + if (w == 4) { \ + /* One group per row; the subsampled mask is then just two bytes. */ \ + do { \ + i16x4 m0, d0; \ + wm_px4(tmp1, tmp2, &m0, &d0); \ + st_u8_partial(dst, pack2_uss(d0, zero), 4); \ + if (!(ss_hor)) { \ + st_u8_partial(mask, pack2_uss(m0, zero), 4); \ + } else { \ + const i16x4 sw = __builtin_shuffle(m0, (i16x4){ 1, 0, 3, 2 }); \ + const i16x4 tp = m0 + sw; \ + const i16x4 ps = __builtin_shuffle(tp, (i16x4){ 0, 2, 0, 2 }); \ + if (!(ss_ver)) { \ + st_u8_partial(mask, \ + pack2_uss((ps + k1 - vsign) >> 1, zero), 2); \ + } else if (h & (ss_ver)) { \ + const i16x4 pv = zext8_lo(ldn_u8x8(mask, 2)); \ + st_u8_partial(mask, \ + pack2_uss((ps + pv + k2 - vsign) >> 2, \ + zero), 2); \ + } else { \ + st_u8_partial(mask, pack2_uss(ps, zero), 2); \ + } \ + } \ + tmp1 += 4; tmp2 += 4; dst += PXSTRIDE(dst_stride); \ + if (!(ss_ver) || (h & 1)) mask += 4 >> (ss_hor); \ + } while (--h); \ + return; \ + } \ + do { \ + for (int x = 0; x < w; x += 8) { \ + i16x4 mlo, mhi, dlo, dhi; \ + wm_px4(tmp1 + x, tmp2 + x, &mlo, &dlo); \ + wm_px4(tmp1 + x + 4, tmp2 + x + 4, &mhi, &dhi); \ + stu_u8x8(dst + x, pack2_uss(dlo, dhi)); \ + if (!(ss_hor)) { \ + stu_u8x8(mask + x, pack2_uss(mlo, mhi)); \ + } else { \ + const i16x4 ps = wm_pairs(mlo, mhi); \ + if (!(ss_ver)) { \ + st_u8_partial(mask + (x >> 1), \ + pack2_uss((ps + k1 - vsign) >> 1, zero), 4); \ + } else if (h & (ss_ver)) { \ + const i16x4 pv = zext8_lo(ldn_u8x8(mask + (x >> 1), 4)); \ + st_u8_partial(mask + (x >> 1), \ + pack2_uss((ps + pv + k2 - vsign) >> 2, \ + zero), 4); \ + } else { \ + st_u8_partial(mask + (x >> 1), pack2_uss(ps, zero), 4); \ + } \ + } \ + } \ + tmp1 += w; tmp2 += w; dst += PXSTRIDE(dst_stride); \ + if (!(ss_ver) || (h & 1)) mask += w >> (ss_hor); \ + } while (--h); \ +} + +w_mask_fns_ia64(444, 0, 0) +w_mask_fns_ia64(422, 1, 0) +w_mask_fns_ia64(420, 1, 1) + +/* ----------------------------------------------------------------- blend */ + +/* + * blend_px(a, b, m) = (a * (64 - m) + b * m + 32) >> 6 + * + * Both operands are 8-bit pixels and m <= 64, so the sum is bounded by + * 255 * 64 + 32 = 16352 and the whole thing stays exact in 16 bits. That lets + * us use pmpyshr2 (four 16x16 products, low half, one instruction) instead of + * the pmpy2 pair, and skip the 32-bit accumulate entirely. + */ +SIMD_FN i16x4 blend4(const i16x4 d, const i16x4 t, const i16x4 m) { + const i16x4 k64 = { 64, 64, 64, 64 }; + const i16x4 rnd = { 32, 32, 32, 32 }; + return (d * (k64 - m) + t * m + rnd) >> 6; +} + +SIMD_FN u8x8 blend8(const u8x8 d, const u8x8 t, const u8x8 m) { + return pack2_uss(blend4(zext8_lo(d), zext8_lo(t), zext8_lo(m)), + blend4(zext8_hi(d), zext8_hi(t), zext8_hi(m))); +} + +SIMD_FN void blend_row(pixel *const dst, const pixel *const tmp, const int w, + const uint8_t *const mask, const int aligned) +{ + for (int x = 0; x < w; x += 8) + st_u8x8_sel(dst + x, blend8(ld_u8x8_sel(dst + x, aligned), + ld_u8x8_sel(tmp + x, aligned), + ld_u8x8_sel(mask + x, aligned)), aligned); +} + +static void blend_ia64(pixel *dst, const ptrdiff_t dst_stride, const pixel *tmp, + const int w, int h, const uint8_t *mask) +{ + if (w == 4) { + do { + const u8x8 d = ldn_u8x8(dst, 4), t = ldn_u8x8(tmp, 4), + m = ldn_u8x8(mask, 4); + st_u8_partial(dst, blend8(d, t, m), 4); + dst += PXSTRIDE(dst_stride); tmp += 4; mask += 4; + } while (--h); + return; + } + + const int aligned = !(((uintptr_t)dst | (uintptr_t)dst_stride | + (uintptr_t)tmp | (uintptr_t)mask) & 7); + do { + if (aligned) blend_row(dst, tmp, w, mask, 1); + else blend_row(dst, tmp, w, mask, 0); + dst += PXSTRIDE(dst_stride); tmp += w; mask += w; + } while (--h); +} + +static void blend_v_ia64(pixel *dst, const ptrdiff_t dst_stride, + const pixel *tmp, const int w, int h) +{ + const uint8_t *const mask = &dav1d_obmc_masks[w]; + const int n = (w * 3) >> 2; + + const int aligned = !(((uintptr_t)dst | (uintptr_t)dst_stride | + (uintptr_t)tmp | (uintptr_t)mask) & 7); + do { + int x = 0; + for (; x + 8 <= n; x += 8) + st_u8x8_sel(dst + x, blend8(ld_u8x8_sel(dst + x, aligned), + ld_u8x8_sel(tmp + x, aligned), + ld_u8x8_sel(mask + x, aligned)), aligned); + /* n is 3, 6, 12, 24 or 48 -- never a multiple of 8 -- so there is + * always a short tail. Keeping it scalar avoids reading past the end + * of the obmc mask table. */ + for (; x < n; x++) + dst[x] = (uint8_t)((dst[x] * (64 - mask[x]) + tmp[x] * mask[x] + 32) >> 6); + dst += PXSTRIDE(dst_stride); tmp += w; + } while (--h); +} + +static void blend_h_ia64(pixel *dst, const ptrdiff_t dst_stride, + const pixel *tmp, const int w, int h) +{ + const uint8_t *mask = &dav1d_obmc_masks[h]; + h = (h * 3) >> 2; + + /* Unlike the other blend variants this one is also called with w == 2. */ + if (w < 8) { + do { + const int m = *mask++; + for (int x = 0; x < w; x++) + dst[x] = (uint8_t)((dst[x] * (64 - m) + tmp[x] * m + 32) >> 6); + dst += PXSTRIDE(dst_stride); tmp += w; + } while (--h); + return; + } + + const int aligned = !(((uintptr_t)dst | (uintptr_t)dst_stride | + (uintptr_t)tmp) & 7); + do { + const i16x4 m = splat16((int16_t)*mask++); + for (int x = 0; x < w; x += 8) { + const u8x8 d = ld_u8x8_sel(dst + x, aligned), + t = ld_u8x8_sel(tmp + x, aligned); + st_u8x8_sel(dst + x, pack2_uss(blend4(zext8_lo(d), zext8_lo(t), m), + blend4(zext8_hi(d), zext8_hi(t), m)), + aligned); + } + dst += PXSTRIDE(dst_stride); tmp += w; + } while (--h); +} + +/* ------------------------------------------------------- pure copy path */ + +/* + * The !fh && !fv case. These are the most frequent MC calls in the decoder and + * every one of them is short: w is 2 to 128 bytes. That is precisely the range + * where glibc's memcpy() loses -- it is a software-pipelined routine with a + * prefetch prologue, which is the right design for long copies and pure + * overhead for a 16-byte row. A packed load/store pair per eight pixels beats + * it here. (Nothing in MC is wide enough to reach the size where the tradeoff + * turns back the other way.) + */ +SIMD_FN void copy_rows(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int w, int h, const int aligned) +{ + if (w == 2) { + do { + dst[0] = src[0]; dst[1] = src[1]; + dst += dst_stride; src += src_stride; + } while (--h); + } else if (w == 4) { + do { + st_u8_partial(dst, ldn_u8x8(src, 4), 4); + dst += dst_stride; src += src_stride; + } while (--h); + } else { + do { + for (int x = 0; x < w; x += 8) + st_u8x8_sel(dst + x, ldu_u8x8(src + x), aligned); + dst += dst_stride; src += src_stride; + } while (--h); + } +} + +/* ------------------------------------------------------- 8-tap MC filters */ + +/* + * This is the hot one. Every tap of the C version is an int16 * int8 product, + * and IA-64 has no integer multiplier in the integer unit: each one becomes + * setf.sig / xma.l / getf.sig, a round trip through the FP register file. A + * 4-lane pmpy2 pair replaces eight of those with two instructions. + * + * The filter is constant for a whole block (unlike warp), so the eight + * coefficient vectors are splatted once in the driver and reused by every + * group. + * + * The horizontal pass accumulates in 16 bits and the vertical pass in 32. + * That is not an inconsistency: the largest positive tap sum in + * dav1d_mc_subpel_filters is 92, so a horizontal sum over 8-bit pixels is + * bounded by 92 * 255 = 23460 and cannot leave int16. The vertical pass runs + * over the already-filtered intermediates instead, where 92 * 9571 needs 20 + * bits, so it has to keep the pmpy2.r/.l pair. + */ + +/* Row stride of the intermediate buffer, as in the C code. */ +#define MID_STRIDE 128 + +/* 8bpc rounding constants, with intermediate_bits == 4 folded in. */ +#define H_SHIFT (6 - INTERMEDIATE_BITS) /* 2 */ +#define H_BIAS ((1 << H_SHIFT) >> 1) /* 2 */ +#define PUT_HV_SHIFT (6 + INTERMEDIATE_BITS) /* 10 */ +#define PUT_HV_BIAS ((1 << PUT_HV_SHIFT) >> 1) /* 512 */ +#define PUT_H_BIAS (32 + ((1 << H_SHIFT) >> 1)) /* 34, intermediate_rnd */ + +/* One horizontal group: four outputs, for source positions b[3] .. b[6] with + * taps reaching b[0] .. b[10]. + * + * Two 8-byte loads cover exactly those eleven bytes -- the second is taken at + * b+3 rather than b+4 so the kernel never reads a byte the C code would not. + * The eight tap vectors are then sliced back out of the two loads with shrp, + * so the source is fetched twice per group instead of eight times. */ +/* + * 32-bit accumulating horizontal kernel, kept for warp. + * + * warp cannot use the 16-bit form below: dav1d_mc_warp_filter is a good deal + * sharper than the subpel filters, with a largest positive tap sum of 175 + * against 92, so a horizontal sum reaches 175 * 255 = 44625 and leaves int16. + */ +SIMD_FN i16x4 h8_pix4_w32(const pixel *const b, const i16x4 *const F, + const i32x2 bias, const int shift) +{ + /* 32-bit accumulating horizontal kernel, for warp only: its filters have a + * largest positive tap sum of 175 against the subpel filters' 92, so a + * horizontal sum reaches 175 * 255 = 44625 and leaves int16. Bundled by + * hand for the same reason as the rest. */ + const u8x8 wa = ldu_u8x8(b); /* b[0..7] */ + const u8x8 wb = ldu_u8x8(b + 3); /* b[3..10] */ + i16x4 a0, a1, c0, c1, w1, w2, w5, w6, res; + __asm__( + "{ .mii\n\t nop.m 0\n\t unpack1.l %[a0] = r0, %[wa]\n\t unpack1.h %[a1] = r0, %[wa]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t unpack1.l %[c0] = r0, %[wb]\n\t unpack1.h %[c1] = r0, %[wb]\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t shrp %[w1] = %[a1], %[a0], 16\n\t shrp %[w2] = %[a1], %[a0], 32\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t shrp %[w5] = %[c1], %[c0], 32\n\t shrp %[w6] = %[c1], %[c0], 48\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r14 = %[a0], %[f0]\n\t pmpy2.l r15 = %[a0], %[f0]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r16 = %[w1], %[f1]\n\t pmpy2.l r17 = %[w1], %[f1]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r18 = %[w2], %[f2]\n\t pmpy2.l r19 = %[w2], %[f2]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r20 = %[c0], %[f3]\n\t pmpy2.l r21 = %[c0], %[f3]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r22 = %[a1], %[f4]\n\t pmpy2.l r23 = %[a1], %[f4]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r24 = %[w5], %[f5]\n\t pmpy2.l r25 = %[w5], %[f5]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r26 = %[w6], %[f6]\n\t pmpy2.l r27 = %[w6], %[f6]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r28 = %[c1], %[f7]\n\t pmpy2.l r29 = %[c1], %[f7]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r16\n\t padd4 r15 = r15, r17\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r18 = r18, r20\n\t padd4 r19 = r19, r21\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r24\n\t padd4 r23 = r23, r25\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r26 = r26, r28\n\t padd4 r27 = r27, r29\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r18\n\t padd4 r15 = r15, r19\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r26\n\t padd4 r23 = r23, r27\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r22\n\t padd4 r15 = r15, r23\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, %[bias]\n\t padd4 r15 = r15, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pshr4 r14 = r14, %[sh]\n\t pshr4 r15 = r15, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pack4.sss r14 = r14, r15\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t mux2 %[res] = r14, 0xd8\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + : [a0]"=&r"(a0), [a1]"=&r"(a1), [c0]"=&r"(c0), [c1]"=&r"(c1), + [w1]"=&r"(w1), [w2]"=&r"(w2), [w5]"=&r"(w5), [w6]"=&r"(w6), + [res]"=r"(res) + : [wa]"r"(wa), [wb]"r"(wb), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), [f7]"r"(F[7]), + [bias]"r"(bias), [sh]"i"(shift) + : "r14","r15","r16","r17","r18","r19","r20","r21", + "r22","r23","r24","r25","r26","r27","r28","r29"); + return res; +} + +/* + * The eight taps are summed as a balanced tree rather than into one running + * accumulator. This is not cosmetic: with `acc += tap` written eight times the + * adds form a single dependence chain, and since each padd2 has to wait for the + * previous one the loop runs at one add per cycle no matter how many issue + * slots are free -- the disassembly shows a stop bit between every pair and + * nop.m padding either side of it. Reassociating into a tree cuts the chain + * from eight deep to three and lets the independent multiplies overlap. + * Two's-complement addition is associative, so this is bit-exact even where the + * 16-bit sum is allowed to wrap. + */ +SIMD_FN i16x4 h8_core(const u8x8 wa, const u8x8 wb, const i16x4 *const F, + const i16x4 bias, const int shift) +{ + + /* + * Emitted as one asm block rather than as a chain of one-instruction + * helpers. GCC types every asm() as TYPE_UNKNOWN (recog_memoized fails on + * it), so the bundler cannot assign it to a slot and falls back to issuing + * it alone with a stop on either side. Built out of eight such helpers this + * kernel spent eight cycles on stop bits before any arithmetic retired; + * measured 12.3 -> 9.3 ticks per four-pixel group once bundled by hand. + * + * The slot assignment is the whole point. unpack1/shrp/pmpyshr2/pshr2 are + * I-type and can only issue on the two I ports, so they take the I slots of + * MII bundles; padd2 is A-type (Table 4-1: "Integer ALU -- I-unit or + * M-unit") and rides the M slots that would otherwise be nops. The taps are + * summed as a tree so the eight adds do not serialise. + */ + i16x4 a0, a1, c0, c1, s1, s2, s5, s6, res; + const i16x4 f0=F[0], f1=F[1], f2=F[2], f3=F[3]; + const i16x4 f4=F[4], f5=F[5], f6=F[6], f7=F[7]; + __asm__( + /* bytes -> 16-bit lanes: b[0..3] b[4..7] b[3..6] b[7..10] */ + "{ .mii\n\t nop.m 0\n\t unpack1.l %[a0] = r0, %[wa]\n\t unpack1.h %[a1] = r0, %[wa]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t unpack1.l %[c0] = r0, %[wb]\n\t unpack1.h %[c1] = r0, %[wb]\n\t}\n\t" + ";;\n\t" + /* the four windows that are not on a 4-byte boundary */ + "{ .mii\n\t nop.m 0\n\t shrp %[s1] = %[a1], %[a0], 16\n\t shrp %[s2] = %[a1], %[a0], 32\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t shrp %[s5] = %[c1], %[c0], 32\n\t shrp %[s6] = %[c1], %[c0], 48\n\t}\n\t" + ";;\n\t" + /* eight taps, each product overwriting its own source */ + "{ .mii\n\t nop.m 0\n\t pmpyshr2 %[a0] = %[a0], %[f0], 0\n\t pmpyshr2 %[s1] = %[s1], %[f1], 0\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpyshr2 %[s2] = %[s2], %[f2], 0\n\t pmpyshr2 %[c0] = %[c0], %[f3], 0\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpyshr2 %[a1] = %[a1], %[f4], 0\n\t pmpyshr2 %[s5] = %[s5], %[f5], 0\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpyshr2 %[s6] = %[s6], %[f6], 0\n\t pmpyshr2 %[c1] = %[c1], %[f7], 0\n\t}\n\t" + ";;\n\t" + /* reduction tree in the M slots */ + "{ .mmi\n\t padd2 %[a0] = %[a0], %[s1]\n\t padd2 %[s2] = %[s2], %[c0]\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd2 %[a1] = %[a1], %[s5]\n\t padd2 %[s6] = %[s6], %[c1]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 %[a0] = %[a0], %[s2]\n\t padd2 %[a1] = %[a1], %[s6]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 %[a0] = %[a0], %[a1]\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 %[a0] = %[a0], %[bias]\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pshr2 %[res] = %[a0], %[sh]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + : [a0]"=&r"(a0), [a1]"=&r"(a1), [c0]"=&r"(c0), [c1]"=&r"(c1), + [s1]"=&r"(s1), [s2]"=&r"(s2), [s5]"=&r"(s5), [s6]"=&r"(s6), + [res]"=r"(res) + : [wa]"r"(wa), [wb]"r"(wb), [bias]"r"(bias), [sh]"i"(shift), + [f0]"r"(f0), [f1]"r"(f1), [f2]"r"(f2), [f3]"r"(f3), + [f4]"r"(f4), [f5]"r"(f5), [f6]"r"(f6), [f7]"r"(f7)); + return res; +} + + +/* + * Horizontal 8-tap, eight pixels per call. + * + * Same reason as v8_mid8: one group's eight pmpyshr2 issue in four cycles but + * the three tree levels, the bias and the pshr2 behind them are a serial tail, + * so a lone group ran fourteen cycles for eight and a half cycles of I-type + * work. Two interleaved cost 22 instead of twenty-eight. The window + * registers arrive already extracted, so the caller's 8-way alignment switch + * (and its literal shrp counts) is untouched. + */ +SIMD_FN void h8_core8(const u8x8 wa0, const u8x8 wb0, + const u8x8 wa1, const u8x8 wb1, + const i16x4 *const F, const i16x4 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ + i16x4 r0, r1; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r3 = r0, %[wa0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r2 = r0, %[wa0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r11 = r0, %[wb0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r10 = r0, %[wb0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r9 = r0, %[wa1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r8 = r0, %[wa1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r31 = r0, %[wb1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r30 = r0, %[wb1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r29 = r2, r3, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r28 = r2, r3, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r27 = r10, r11, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r26 = r10, r11, 48\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r25 = r8, r9, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r24 = r8, r9, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r23 = r30, r31, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r22 = r30, r31, 48\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r21 = r3, %[f0], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r20 = r29, %[f1], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r29 = r28, %[f2], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r3 = r11, %[f3], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r19 = r21, r20\n\t nop.m 0\n\t pmpyshr2 r28 = r2, %[f4], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r11 = r27, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r27 = r29, r3\n\t nop.m 0\n\t pmpyshr2 r20 = r26, %[f6], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r21 = r10, %[f7], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r3 = r28, r11\n\t padd2 r29 = r19, r27\n\t pmpyshr2 r26 = r9, %[f0], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r10 = r25, %[f1], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r27 = r20, r21\n\t nop.m 0\n\t pmpyshr2 r19 = r24, %[f2], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r11 = r31, %[f3], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r31 = r26, r10\n\t padd2 r21 = r3, r27\n\t pmpyshr2 r20 = r8, %[f4], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r24 = r23, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r3 = r19, r11\n\t padd2 r27 = r29, r21\n\t pmpyshr2 r10 = r22, %[f6], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r26 = r30, %[f7], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r11 = r20, r24\n\t padd2 r19 = r31, r3\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd2 r21 = r27, %[bias]\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r3 = r10, r26\n\t nop.m 0\n\t pshr2 %[res0] = r21, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r21 = r11, r3\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r3 = r19, r21\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r21 = r3, %[bias]\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr2 %[res1] = r21, %[sh]\n\t}\n\t" + ";;\n\t" + : [res0]"=&r"(r0), [res1]"=r"(r1) + : [wa0]"r"(wa0), [wb0]"r"(wb0), [wa1]"r"(wa1), [wb1]"r"(wb1), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), [f7]"r"(F[7]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + *lo = r0; *hi = r1; +} + +SIMD_FN i16x4 h8_pix4(const pixel *const b, const i16x4 *const F, + const i16x4 bias, const int shift) +{ + return h8_core(ldu_u8x8(b), ldu_u8x8(b + 3), F, bias, shift); +} + +/* + * Horizontal pass over a whole row. + * + * Calling h8_pix4() per group pays two ldu_u8x8() per four outputs, and each of + * those is two aligned loads plus a variable shift/shift/or -- the shift + * amounts cannot be hoisted because a group advances the pointer by 4, so the + * misalignment alternates. Stepping eight outputs at a time instead advances + * the source by exactly one 64-bit word, so the row's misalignment is constant + * and every tap window is one shrp with an immediate count out of a sliding + * pair of aligned words: one load and four shrp per eight outputs, against + * eight loads and four variable-shift sequences. Measured 9.37 -> 8.32 + * cycles/px on the row. + * + * The switch specialises on the misalignment so the shrp counts are literals. + */ +/* + * --------------------------------------------------- six-tap specialisations + * + * dav1d_mc_subpel_filters is nominally eight taps, but only the SHARP set uses + * all eight. REGULAR and SMOOTH have f[0] == f[7] == 0 at every one of the + * fifteen subpel positions, and the w <= 4 filter sets are narrower still, so + * for most blocks two of the eight taps multiply by zero. Encoders pick + * REGULAR or SMOOTH for the overwhelming majority of blocks -- on the clips + * measured here, for every single one. + * + * The callers detect this (fh[0] == 0 && fh[7] == 0), hand these kernels F + * shifted down by one tap, and advance the source by one column (h) or one row + * (v). That renumbers the live taps to 0..5 and lets a quarter of the + * multiplies disappear. The horizontal extraction loses a shrp as well, though + * not two: tap 5 is shrp(c1, c0, 32) and its operand c1 is itself tap 7, so c1 + * is still unpacked even though nothing multiplies it. The vertical pass + * simply never loads the two rows it stopped using, which also shortens the + * intermediate the hv path has to build from h+7 rows to h+5. + * + * Scheduled cost against the eight-tap kernels they replace: + * h8_core8 22 -> 18 cycles / 8 px v8_mid8 24 -> 19 cycles / 8 px + * h8_core 14 -> 12 cycles / 4 px v8_mid4 16 -> 13 cycles / 4 px + */ + +SIMD_FN void h8_core8_6t(const u8x8 wa0, const u8x8 wb0, + const u8x8 wa1, const u8x8 wb1, + const i16x4 *const F, const i16x4 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ + i16x4 r0, r1; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r3 = r0, %[wa0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r2 = r0, %[wa0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r11 = r0, %[wa1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r10 = r0, %[wa1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r9 = r0, %[wb0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r8 = r0, %[wb0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r31 = r2, r3, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r30 = r2, r3, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r29 = r0, %[wb1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r28 = r0, %[wb1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r27 = r10, r11, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r26 = r10, r11, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r25 = r8, r9, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r24 = r3, %[f0], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r8 = r31, %[f1], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r3 = r30, %[f2], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r30 = r9, %[f3], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r31 = r28, r29, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r28 = r24, r8\n\t nop.m 0\n\t pmpyshr2 r9 = r11, %[f0], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r23 = r27, %[f1], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r8 = r3, r30\n\t nop.m 0\n\t pmpyshr2 r24 = r26, %[f2], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r27 = r29, %[f3], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r29 = r9, r23\n\t padd2 r26 = r28, r8\n\t pmpyshr2 r30 = r2, %[f4], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r3 = r25, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r9 = r24, r27\n\t nop.m 0\n\t pmpyshr2 r8 = r10, %[f4], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r23 = r31, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r27 = r30, r3\n\t padd2 r24 = r29, r9\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r29 = r8, r23\n\t padd2 r9 = r26, r27\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r23 = r24, r29\n\t padd2 r27 = r9, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r29 = r23, %[bias]\n\t nop.m 0\n\t pshr2 %[res0] = r27, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr2 %[res1] = r29, %[sh]\n\t}\n\t" + ";;\n\t" + : [res0]"=&r"(r0), [res1]"=r"(r1) + : [wa0]"r"(wa0), [wb0]"r"(wb0), [wa1]"r"(wa1), [wb1]"r"(wb1), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + *lo = r0; *hi = r1; +} + +SIMD_FN i16x4 h8_core_6t(const u8x8 wa0, const u8x8 wb0, + const i16x4 *const F, const i16x4 bias, const int shift) +{ + i16x4 r0; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r3 = r0, %[wa0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r2 = r0, %[wa0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r11 = r0, %[wb0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r10 = r0, %[wb0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r9 = r2, r3, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r8 = r2, r3, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r31 = r10, r11, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r30 = r3, %[f0], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r10 = r9, %[f1], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r3 = r8, %[f2], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r8 = r11, %[f3], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r9 = r2, %[f4], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r2 = r30, r10\n\t nop.m 0\n\t pmpyshr2 r11 = r31, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r10 = r3, r8\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r8 = r9, r11\n\t padd2 r3 = r2, r10\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r10 = r3, r8\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r3 = r10, %[bias]\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr2 %[res0] = r3, %[sh]\n\t}\n\t" + ";;\n\t" + : [res0]"=r"(r0) + : [wa0]"r"(wa0), [wb0]"r"(wb0), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r30", "r31"); + return r0; +} + +/* |m| points at the row supplying tap 0, i.e. two rows above the output row. */ +SIMD_FN void v8_mid8_6t(const int16_t *const m, const i16x4 *const F, + const i32x2 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ + const i16x4 s0_0 = ld_i16x4(m + 0 * MID_STRIDE + 0); + const i16x4 s1_0 = ld_i16x4(m + 1 * MID_STRIDE + 0); + const i16x4 s2_0 = ld_i16x4(m + 2 * MID_STRIDE + 0); + const i16x4 s3_0 = ld_i16x4(m + 3 * MID_STRIDE + 0); + const i16x4 s4_0 = ld_i16x4(m + 4 * MID_STRIDE + 0); + const i16x4 s5_0 = ld_i16x4(m + 5 * MID_STRIDE + 0); + const i16x4 s0_1 = ld_i16x4(m + 0 * MID_STRIDE + 4); + const i16x4 s1_1 = ld_i16x4(m + 1 * MID_STRIDE + 4); + const i16x4 s2_1 = ld_i16x4(m + 2 * MID_STRIDE + 4); + const i16x4 s3_1 = ld_i16x4(m + 3 * MID_STRIDE + 4); + const i16x4 s4_1 = ld_i16x4(m + 4 * MID_STRIDE + 4); + const i16x4 s5_1 = ld_i16x4(m + 5 * MID_STRIDE + 4); + i16x4 r0, r1; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = %[s0_0], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[s0_0], %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r11 = %[s1_0], %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[s1_0], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r9 = %[s2_0], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r8 = %[s2_0], %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r3, r11\n\t padd4 r28 = r2, r10\n\t pmpy2.r r31 = %[s3_0], %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r30 = %[s3_0], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r10 = %[s0_1], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r11 = %[s0_1], %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r2 = r9, r31\n\t padd4 r3 = r8, r30\n\t pmpy2.r r27 = %[s1_1], %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r26 = %[s1_1], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r30 = r29, r2\n\t padd4 r31 = r28, r3\n\t pmpy2.r r8 = %[s2_1], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r9 = %[s2_1], %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r28 = r10, r27\n\t padd4 r29 = r11, r26\n\t pmpy2.r r3 = %[s3_1], %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[s3_1], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r26 = %[s4_0], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r27 = %[s4_0], %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r25 = r8, r3\n\t padd4 r24 = r9, r2\n\t pmpy2.r r11 = %[s5_0], %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[s5_0], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r9 = r28, r25\n\t padd4 r8 = r29, r24\n\t pmpy2.r r2 = %[s4_1], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r3 = %[s4_1], %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r24 = r26, r11\n\t padd4 r25 = r27, r10\n\t pmpy2.r r29 = %[s5_1], %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r28 = %[s5_1], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r10 = r30, r24\n\t padd4 r11 = r31, r25\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r31 = r2, r29\n\t padd4 r30 = r3, r28\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r25 = r10, %[bias]\n\t padd4 r24 = r11, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r28 = r9, r31\n\t padd4 r29 = r8, r30\n\t pshr4 r11 = r25, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r10 = r24, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r31 = r28, %[bias]\n\t padd4 r24 = r29, %[bias]\n\t pack4.sss r30 = r11, r10\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r29 = r31, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r28 = r24, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r24 = r29, r28\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res0] = r30, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res1] = r24, 0xd8\n\t}\n\t" + ";;\n\t" + : [res0]"=&r"(r0), [res1]"=r"(r1) + : [s0_0]"r"(s0_0), [s1_0]"r"(s1_0), [s2_0]"r"(s2_0), + [s3_0]"r"(s3_0), [s4_0]"r"(s4_0), [s5_0]"r"(s5_0), + [s0_1]"r"(s0_1), [s1_1]"r"(s1_1), [s2_1]"r"(s2_1), + [s3_1]"r"(s3_1), [s4_1]"r"(s4_1), [s5_1]"r"(s5_1), + + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + *lo = r0; *hi = r1; +} + +SIMD_FN i16x4 v8_mid4_6t(const int16_t *const m, const i16x4 *const F, + const i32x2 bias, const int shift) +{ + const i16x4 s0_0 = ld_i16x4(m + 0 * MID_STRIDE + 0); + const i16x4 s1_0 = ld_i16x4(m + 1 * MID_STRIDE + 0); + const i16x4 s2_0 = ld_i16x4(m + 2 * MID_STRIDE + 0); + const i16x4 s3_0 = ld_i16x4(m + 3 * MID_STRIDE + 0); + const i16x4 s4_0 = ld_i16x4(m + 4 * MID_STRIDE + 0); + const i16x4 s5_0 = ld_i16x4(m + 5 * MID_STRIDE + 0); + i16x4 r0; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = %[s0_0], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[s0_0], %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r11 = %[s1_0], %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[s1_0], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r9 = %[s2_0], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r8 = %[s2_0], %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r3, r11\n\t padd4 r28 = r2, r10\n\t pmpy2.r r31 = %[s3_0], %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r30 = %[s3_0], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r10 = %[s4_0], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r11 = %[s4_0], %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r27 = r9, r31\n\t padd4 r26 = r8, r30\n\t pmpy2.r r2 = %[s5_0], %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r3 = %[s5_0], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r30 = r29, r27\n\t padd4 r31 = r28, r26\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r26 = r10, r2\n\t padd4 r27 = r11, r3\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r3 = r30, r26\n\t padd4 r2 = r31, r27\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r27 = r3, %[bias]\n\t padd4 r26 = r2, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r2 = r27, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r3 = r26, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r26 = r2, r3\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res0] = r26, 0xd8\n\t}\n\t" + ";;\n\t" + : [res0]"=r"(r0) + : [s0_0]"r"(s0_0), [s1_0]"r"(s1_0), [s2_0]"r"(s2_0), + [s3_0]"r"(s3_0), [s4_0]"r"(s4_0), [s5_0]"r"(s5_0), + + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r26", "r27", "r28", "r29", "r30", "r31"); + return r0; +} + + +/* + * Twelve outputs per call. Two interleaved groups still spend their last four + * cycles walking the reduction tree with the I units mostly idle; a third group + * fills that tail, taking the vertical pass from 2.375 to 2.083 cycles per pixel. + * + * Three is the ceiling, for two separate reasons that happen to bite together: + * a fourth group needs 24 source operands, and with the filters, the bias, the + * shift and four results that is 36 against GCC's limit of 30 -- and the whole + * point of this kernel is that the taps arrive in registers rather than being + * loaded behind a "memory" clobber. Twelve is also why the caller keeps the + * eight- and four-wide kernels: 12 does not divide 64 or 128, so the row ends + * with one of those. + */ +SIMD_FN void v8_mid12_6t(const int16_t *const m, const i16x4 *const F, + const i32x2 bias, const int shift, + i16x4 *const v0, i16x4 *const v1, i16x4 *const v2) +{ + const i16x4 s0_0 = ld_i16x4(m + 0 * MID_STRIDE + 0); + const i16x4 s1_0 = ld_i16x4(m + 1 * MID_STRIDE + 0); + const i16x4 s2_0 = ld_i16x4(m + 2 * MID_STRIDE + 0); + const i16x4 s3_0 = ld_i16x4(m + 3 * MID_STRIDE + 0); + const i16x4 s4_0 = ld_i16x4(m + 4 * MID_STRIDE + 0); + const i16x4 s5_0 = ld_i16x4(m + 5 * MID_STRIDE + 0); + const i16x4 s0_1 = ld_i16x4(m + 0 * MID_STRIDE + 4); + const i16x4 s1_1 = ld_i16x4(m + 1 * MID_STRIDE + 4); + const i16x4 s2_1 = ld_i16x4(m + 2 * MID_STRIDE + 4); + const i16x4 s3_1 = ld_i16x4(m + 3 * MID_STRIDE + 4); + const i16x4 s4_1 = ld_i16x4(m + 4 * MID_STRIDE + 4); + const i16x4 s5_1 = ld_i16x4(m + 5 * MID_STRIDE + 4); + const i16x4 s0_2 = ld_i16x4(m + 0 * MID_STRIDE + 8); + const i16x4 s1_2 = ld_i16x4(m + 1 * MID_STRIDE + 8); + const i16x4 s2_2 = ld_i16x4(m + 2 * MID_STRIDE + 8); + const i16x4 s3_2 = ld_i16x4(m + 3 * MID_STRIDE + 8); + const i16x4 s4_2 = ld_i16x4(m + 4 * MID_STRIDE + 8); + const i16x4 s5_2 = ld_i16x4(m + 5 * MID_STRIDE + 8); + i16x4 r0, r1, r2; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = %[s0_0], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[s0_0], %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r11 = %[s1_0], %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[s1_0], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r9 = %[s2_0], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r8 = %[s2_0], %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r3, r11\n\t padd4 r28 = r2, r10\n\t pmpy2.r r31 = %[s3_0], %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r30 = %[s3_0], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r10 = %[s0_1], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r11 = %[s0_1], %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r2 = r9, r31\n\t padd4 r3 = r8, r30\n\t pmpy2.r r27 = %[s1_1], %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r26 = %[s1_1], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r30 = r29, r2\n\t padd4 r31 = r28, r3\n\t pmpy2.r r8 = %[s2_1], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r9 = %[s2_1], %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r28 = r10, r27\n\t padd4 r29 = r11, r26\n\t pmpy2.r r3 = %[s3_1], %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[s3_1], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r26 = %[s0_2], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r27 = %[s0_2], %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r11 = r8, r3\n\t padd4 r10 = r9, r2\n\t pmpy2.r r25 = %[s1_2], %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r24 = %[s1_2], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r2 = r28, r11\n\t padd4 r3 = r29, r10\n\t pmpy2.r r9 = %[s2_2], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r8 = %[s2_2], %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r26, r25\n\t padd4 r28 = r27, r24\n\t pmpy2.r r10 = %[s3_2], %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r11 = %[s3_2], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r24 = %[s4_0], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r25 = %[s4_0], %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r9, r10\n\t padd4 r22 = r8, r11\n\t pmpy2.r r27 = %[s5_0], %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r26 = %[s5_0], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r8 = r29, r23\n\t padd4 r9 = r28, r22\n\t pmpy2.r r11 = %[s4_1], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[s4_1], %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r22 = r24, r27\n\t padd4 r23 = r25, r26\n\t pmpy2.r r28 = %[s5_1], %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r29 = %[s5_1], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r26 = r30, r22\n\t padd4 r27 = r31, r23\n\t pmpy2.r r25 = %[s4_2], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r24 = %[s4_2], %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r31 = r11, r28\n\t padd4 r30 = r10, r29\n\t pmpy2.r r21 = %[s5_2], %[f5]\n\t}\n\t" + "{ .mmi\n\t padd4 r23 = r26, %[bias]\n\t padd4 r22 = r27, %[bias]\n\t pmpy2.l r20 = %[s5_2], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r2, r31\n\t padd4 r28 = r3, r30\n\t pshr4 r27 = r23, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r26 = r22, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r25, r21\n\t padd4 r3 = r24, r20\n\t pack4.sss r30 = r27, r26\n\t}\n\t" + "{ .mmi\n\t padd4 r31 = r29, %[bias]\n\t padd4 r22 = r28, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r26 = r8, r23\n\t padd4 r27 = r9, r3\n\t pshr4 r28 = r31, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r29 = r22, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r26, %[bias]\n\t padd4 r22 = r27, %[bias]\n\t pack4.sss r3 = r28, r29\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res0] = r30, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r27 = r23, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r26 = r22, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r22 = r27, r26\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res1] = r3, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res2] = r22, 0xd8\n\t}\n\t" + ";;\n\t" + : [res0]"=&r"(r0), [res1]"=&r"(r1), [res2]"=r"(r2) + : [s0_0]"r"(s0_0), [s1_0]"r"(s1_0), [s2_0]"r"(s2_0), + [s3_0]"r"(s3_0), [s4_0]"r"(s4_0), [s5_0]"r"(s5_0), + [s0_1]"r"(s0_1), [s1_1]"r"(s1_1), [s2_1]"r"(s2_1), + [s3_1]"r"(s3_1), [s4_1]"r"(s4_1), [s5_1]"r"(s5_1), + [s0_2]"r"(s0_2), [s1_2]"r"(s1_2), [s2_2]"r"(s2_2), + [s3_2]"r"(s3_2), [s4_2]"r"(s4_2), [s5_2]"r"(s5_2), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + *v0 = r0; *v1 = r1; *v2 = r2; +} + + +/* + * Sixteen outputs per call: four interleaved groups instead of two. + * + * Two groups leave the I units idle for the last few cycles of the reduction + * tree; four keep them fed, taking the horizontal pass from 2.250 to 1.938 + * cycles per pixel. The catch is register pressure -- at the scheduler's + * default eagerness this needs more than the 24 scratch GRs exist and spills. + * Capping the in-flight set (PRESSURE_CAP=12 in the generator) makes it start + * the later groups a little further apart, which costs nothing in cycles here + * and brings it down to 21 registers. + * + * Sixteen pixels is two window words per iteration, so the caller carries four + * words (hw0..hw3) rather than three and refills two at a time. + */ +SIMD_FN void h8_core16_6t(const u8x8 wa0, const u8x8 wb0, + const u8x8 wa1, const u8x8 wb1, + const u8x8 wa2, const u8x8 wb2, + const u8x8 wa3, const u8x8 wb3, + const i16x4 *const F, const i16x4 bias, const int shift, + i16x4 *const q0, i16x4 *const q1, + i16x4 *const q2, i16x4 *const q3) +{ + i16x4 r0, r1, r2, r3; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r3 = r0, %[wa0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r2 = r0, %[wa0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r11 = r0, %[wa1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r10 = r0, %[wa1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r9 = r0, %[wa2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r8 = r0, %[wa2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r31 = r0, %[wa3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r30 = r0, %[wa3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r29 = r0, %[wb0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r28 = r0, %[wb0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r27 = r2, r3, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r26 = r2, r3, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r25 = r0, %[wb1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r24 = r0, %[wb1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r23 = r28, r29, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r22 = r3, %[f0], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r28 = r27, %[f1], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r3 = r26, %[f2], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r26 = r29, %[f3], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r27 = r24, r25, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r24 = r22, r28\n\t nop.m 0\n\t pmpyshr2 r29 = r11, %[f0], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r21 = r25, %[f3], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r28 = r3, r26\n\t nop.m 0\n\t pmpyshr2 r22 = r9, %[f0], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r25 = r31, %[f0], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r26 = r24, r28\n\t nop.m 0\n\t shrp r3 = r10, r11, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r20 = r10, r11, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r28 = r0, %[wb2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r24 = r0, %[wb2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r11 = r3, %[f1], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r19 = r20, %[f2], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r20 = r24, r28, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r3 = r28, %[f3], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r18 = r29, r11\n\t padd2 r17 = r19, r21\n\t pmpyshr2 r24 = r2, %[f4], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r28 = r23, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r21 = r18, r17\n\t nop.m 0\n\t pmpyshr2 r19 = r10, %[f4], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r11 = r27, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r17 = r24, r28\n\t nop.m 0\n\t pmpyshr2 r18 = r8, %[f4], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r27 = r20, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r24 = r19, r11\n\t padd2 r28 = r26, r17\n\t shrp r20 = r8, r9, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r10 = r8, r9, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r19 = r18, r27\n\t padd2 r11 = r21, r24\n\t unpack1.l r26 = r0, %[wb3]\n\t}\n\t" + "{ .mmi\n\t padd2 r17 = r28, %[bias]\n\t nop.m 0\n\t unpack1.h r8 = r0, %[wb3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r24 = r11, %[bias]\n\t nop.m 0\n\t shrp r28 = r30, r31, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r27 = r30, r31, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r11 = r20, %[f1], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r31 = r10, %[f2], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r10 = r8, r26, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r20 = r28, %[f1], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r28 = r22, r11\n\t padd2 r8 = r31, r3\n\t pmpyshr2 r18 = r27, %[f2], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r21 = r26, %[f3], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r26 = r25, r20\n\t padd2 r31 = r28, r8\n\t pmpyshr2 r11 = r30, %[f4], 0\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpyshr2 r27 = r10, %[f5], 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r28 = r18, r21\n\t padd2 r8 = r31, r19\n\t pshr2 %[res0] = r17, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr2 %[res1] = r24, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r21 = r11, r27\n\t padd2 r18 = r26, r28\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd2 r31 = r8, %[bias]\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r28 = r18, r21\n\t nop.m 0\n\t pshr2 %[res2] = r31, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd2 r18 = r28, %[bias]\n\t nop.m 0\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr2 %[res3] = r18, %[sh]\n\t}\n\t" + ";;\n\t" + : [res0]"=&r"(r0), [res1]"=&r"(r1), [res2]"=&r"(r2), [res3]"=r"(r3) + : + [wa0]"r"(wa0), [wb0]"r"(wb0), + [wa1]"r"(wa1), [wb1]"r"(wb1), + [wa2]"r"(wa2), [wb2]"r"(wb2), + [wa3]"r"(wa3), [wb3]"r"(wb3), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), + [f3]"r"(F[3]), [f4]"r"(F[4]), [f5]"r"(F[5]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + *q0 = r0; *q1 = r1; *q2 = r2; *q3 = r3; +} + +SIMD_FN i16x4 h8_pix4_6t(const pixel *const b, const i16x4 *const F, + const i16x4 bias, const int shift) +{ + return h8_core_6t(ldu_u8x8(b), ldu_u8x8(b + 3), F, bias, shift); +} + +#define H8_WIN(rel) ((rel) < 8 ? (u8x8)shrp64_v(hw1, hw0, ((rel) * 8)) \ + : (u8x8)shrp64_v(hw2, hw1, (((rel) - 8) * 8))) +#define H8_ROWCASE(R, CORE) \ + for (int x = 0; x < w; x += 8) { \ + i16x4 h8lo_, h8hi_; \ + CORE(H8_WIN((R) + 0), H8_WIN((R) + 3), \ + H8_WIN((R) + 4), H8_WIN((R) + 7), \ + F, bias, H_SHIFT, &h8lo_, &h8hi_); \ + st_i16x4(dst + x, h8lo_); \ + st_i16x4(dst + x + 4, h8hi_); \ + hw0 = hw1; hw1 = hw2; hw2 = *++hp; \ + } +/* Four-word form of H8_WIN, for the sixteen-wide kernel: sixteen outputs plus + * the six-tap span reach byte R+22, past what three words cover. */ +#define H8_WIN4(rel) ((rel) < 8 ? (u8x8)shrp64_v(hw1, hw0, ((rel) * 8)) \ + : (rel) < 16 ? (u8x8)shrp64_v(hw2, hw1, (((rel) - 8) * 8)) \ + : (u8x8)shrp64_v(hw3, hw2, (((rel) - 16) * 8))) +#define H8_ROW16CASE(R) \ + for (; x + 16 <= w; x += 16) { \ + i16x4 q0_, q1_, q2_, q3_; \ + h8_core16_6t(H8_WIN4((R) + 0), H8_WIN4((R) + 3), \ + H8_WIN4((R) + 4), H8_WIN4((R) + 7), \ + H8_WIN4((R) + 8), H8_WIN4((R) + 11), \ + H8_WIN4((R) + 12), H8_WIN4((R) + 15), \ + F, bias, H_SHIFT, &q0_, &q1_, &q2_, &q3_); \ + st_i16x4(dst + x, q0_); \ + st_i16x4(dst + x + 4, q1_); \ + st_i16x4(dst + x + 8, q2_); \ + st_i16x4(dst + x + 12, q3_); \ + hw0 = hw2; hw1 = hw3; hw2 = *++hp; hw3 = *++hp; \ + } +/* + * Widths are 8, 16, 32, 64 or 128, so at most one eight-wide group is ever + * left over -- an |if|, not a loop, which is also why the window needs no + * refill afterwards. Refilling it would read a further word past the row for + * a value nothing can go on to use. + */ +#define H8_ROW8TAIL(R) \ + if (x + 8 <= w) { \ + i16x4 h8lo_, h8hi_; \ + h8_core8_6t(H8_WIN4((R) + 0), H8_WIN4((R) + 3), \ + H8_WIN4((R) + 4), H8_WIN4((R) + 7), \ + F, bias, H_SHIFT, &h8lo_, &h8hi_); \ + st_i16x4(dst + x, h8lo_); \ + st_i16x4(dst + x + 4, h8hi_); \ + x += 8; \ + } +#define H8_CASE6(R) H8_ROW16CASE(R) H8_ROW8TAIL(R) +#define H8_SWITCH6 \ + switch ((int)(base & 7)) { \ + case 0: H8_CASE6(0); break; case 1: H8_CASE6(1); break; \ + case 2: H8_CASE6(2); break; case 3: H8_CASE6(3); break; \ + case 4: H8_CASE6(4); break; case 5: H8_CASE6(5); break; \ + case 6: H8_CASE6(6); break; default: H8_CASE6(7); break; \ + } + +#define H8_ROWSWITCH(CORE) \ + switch ((int)(base & 7)) { \ + case 0: H8_ROWCASE(0, CORE); break; case 1: H8_ROWCASE(1, CORE); break; \ + case 2: H8_ROWCASE(2, CORE); break; case 3: H8_ROWCASE(3, CORE); break; \ + case 4: H8_ROWCASE(4, CORE); break; case 5: H8_ROWCASE(5, CORE); break; \ + case 6: H8_ROWCASE(6, CORE); break; default: H8_ROWCASE(7, CORE); break; \ + } + +/* + * The eight-way alignment switch exists so the tap windows are shrp with + * literal counts, which costs eight copies of the loop body. A six-tap variant + * would double that again, and inlining it into both prep and put would double + * it once more -- roughly 10 KB of asm against a 16 KB L1I. Both variants are + * therefore NOINLINE with the shift hardcoded: every caller of h8_row passes + * H_SHIFT, so nothing is lost, and a per-row call is amortised over the eight + * to sixteen groups a 64- or 128-wide row contains. + */ +static NOINLINE void h8_row(int16_t *const dst, const pixel *const src, + const int w, const i16x4 *const F, const i16x4 bias) +{ + if (w == 4) { /* single group, not worth setting up a window */ + st_i16x4(dst, h8_pix4(src - 3, F, bias, H_SHIFT)); + return; + } + const uintptr_t base = (uintptr_t)(src - 3); + const uint64_t *hp = (const uint64_t *)(base & ~(uintptr_t)7); + uint64_t hw0 = hp[0], hw1 = hp[1], hw2 = hp[2]; + hp += 2; + H8_ROWSWITCH(h8_core8); +} + +/* Same row walk, six live taps. The window layout is unchanged -- taps 0..5 + * still come out of windows at base+0 and base+3 -- so only the base moves on + * by the dropped leading tap. */ +static NOINLINE void h8_row_6t(int16_t *const dst, const pixel *const src, + const int w, const i16x4 *const F, + const i16x4 bias) +{ + if (w == 4) { + st_i16x4(dst, h8_pix4_6t(src - 2, F, bias, H_SHIFT)); + return; + } + const uintptr_t base = (uintptr_t)(src - 2); + const uint64_t *hp = (const uint64_t *)(base & ~(uintptr_t)7); + uint64_t hw0 = hp[0], hw1 = hp[1], hw2 = hp[2]; + /* The fourth word is only touched by the sixteen-wide loop. Loading it for + * an eight-wide row would read further past the row than the three-word + * form already does, for nothing. */ + uint64_t hw3 = w >= 16 ? hp[3] : hw2; + hp += 3; + int x = 0; + H8_SWITCH6; +} + +/* Vertical group over the int16 intermediate buffer. This is the one vertical + * kernel that has to keep the 32-bit pmpy2.r/.l pair: its input is already + * filtered data (up to ~9571), not 8-bit pixels, so 92 * 9571 overflows int16. + * The two pixel-domain kernels below accumulate in 16 bits like h8_pix4(). |m| points at the first + * (topmost) tap row, and the rows are 8-byte aligned by construction. */ +SIMD_FN i16x4 v8_mid4(const int16_t *const m, const i16x4 *const F, + const i32x2 bias, const int shift) +{ + /* + * The loads stay in C -- plain loads bundle fine, it is only asm() that the + * bundler cannot place. Everything after them is one asm block: written as + * helpers this kernel issued sixteen separate pmpy2 asm()s, each one alone + * in a bundle behind a stop, so it spent more cycles on stop bits than on + * arithmetic. pmpy2/pack4/mux2/pshr4 are I-type and take the I slots; + * padd4 is A-type and fills the M slots beside them. + */ + const i16x4 s0 = ld_i16x4(m + 0 * MID_STRIDE), s1 = ld_i16x4(m + 1 * MID_STRIDE); + const i16x4 s2 = ld_i16x4(m + 2 * MID_STRIDE), s3 = ld_i16x4(m + 3 * MID_STRIDE); + const i16x4 s4 = ld_i16x4(m + 4 * MID_STRIDE), s5 = ld_i16x4(m + 5 * MID_STRIDE); + const i16x4 s6 = ld_i16x4(m + 6 * MID_STRIDE), s7 = ld_i16x4(m + 7 * MID_STRIDE); + i16x4 res; + __asm__( + "{ .mii\n\t nop.m 0\n\t pmpy2.r r14 = %[s0], %[f0]\n\t pmpy2.l r15 = %[s0], %[f0]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r16 = %[s1], %[f1]\n\t pmpy2.l r17 = %[s1], %[f1]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r18 = %[s2], %[f2]\n\t pmpy2.l r19 = %[s2], %[f2]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r20 = %[s3], %[f3]\n\t pmpy2.l r21 = %[s3], %[f3]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r22 = %[s4], %[f4]\n\t pmpy2.l r23 = %[s4], %[f4]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r24 = %[s5], %[f5]\n\t pmpy2.l r25 = %[s5], %[f5]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r26 = %[s6], %[f6]\n\t pmpy2.l r27 = %[s6], %[f6]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r28 = %[s7], %[f7]\n\t pmpy2.l r29 = %[s7], %[f7]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r16\n\t padd4 r15 = r15, r17\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r18 = r18, r20\n\t padd4 r19 = r19, r21\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r24\n\t padd4 r23 = r23, r25\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r26 = r26, r28\n\t padd4 r27 = r27, r29\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r18\n\t padd4 r15 = r15, r19\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r26\n\t padd4 r23 = r23, r27\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r22\n\t padd4 r15 = r15, r23\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, %[bias]\n\t padd4 r15 = r15, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pshr4 r14 = r14, %[sh]\n\t pshr4 r15 = r15, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pack4.sss r14 = r14, r15\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t mux2 %[res] = r14, 0xd8\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + : [res]"=r"(res) + : [s0]"r"(s0), [s1]"r"(s1), [s2]"r"(s2), [s3]"r"(s3), + [s4]"r"(s4), [s5]"r"(s5), [s6]"r"(s6), [s7]"r"(s7), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), [f7]"r"(F[7]), + [bias]"r"(bias), [sh]"i"(shift) + : "r14","r15","r16","r17","r18","r19","r20","r21", + "r22","r23","r24","r25","r26","r27","r28","r29"); + return res; +} + +/* Vertical group straight over pixels, eight columns at a time: one 8-byte + * load per tap row feeds both halves, and nothing is read past column x+7. */ + +/* + * Vertical 8-tap over the int16 intermediate, eight outputs per call. + * + * Two groups in one block, deliberately. A single group issues its sixteen + * pmpy2 in eight cycles (two I units) but then spends three tree levels, a + * bias, a pshr4, a pack4 and a mux2 walking the tail out -- sixteen cycles for + * ten cycles of I-type work. Interleaved, the second group's multiplies fill + * the first group's tail and the pair costs 24 cycles instead of thirty-two. + * + * padd4 is A-type and rides the M slots beside the multiplies (Table 4-1); + * only pmpy2/pshr4/pack4/mux2 compete for the two I units. + */ +SIMD_FN void v8_mid8(const int16_t *const m, const i16x4 *const F, + const i32x2 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ + const i16x4 v0_0 = ld_i16x4(m + 0 * MID_STRIDE + 0); + const i16x4 v1_0 = ld_i16x4(m + 1 * MID_STRIDE + 0); + const i16x4 v2_0 = ld_i16x4(m + 2 * MID_STRIDE + 0); + const i16x4 v3_0 = ld_i16x4(m + 3 * MID_STRIDE + 0); + const i16x4 v4_0 = ld_i16x4(m + 4 * MID_STRIDE + 0); + const i16x4 v5_0 = ld_i16x4(m + 5 * MID_STRIDE + 0); + const i16x4 v6_0 = ld_i16x4(m + 6 * MID_STRIDE + 0); + const i16x4 v7_0 = ld_i16x4(m + 7 * MID_STRIDE + 0); + const i16x4 v0_1 = ld_i16x4(m + 0 * MID_STRIDE + 4); + const i16x4 v1_1 = ld_i16x4(m + 1 * MID_STRIDE + 4); + const i16x4 v2_1 = ld_i16x4(m + 2 * MID_STRIDE + 4); + const i16x4 v3_1 = ld_i16x4(m + 3 * MID_STRIDE + 4); + const i16x4 v4_1 = ld_i16x4(m + 4 * MID_STRIDE + 4); + const i16x4 v5_1 = ld_i16x4(m + 5 * MID_STRIDE + 4); + const i16x4 v6_1 = ld_i16x4(m + 6 * MID_STRIDE + 4); + const i16x4 v7_1 = ld_i16x4(m + 7 * MID_STRIDE + 4); + i16x4 r0, r1; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = %[s0_0], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[s0_0], %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r11 = %[s1_0], %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[s1_0], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r9 = %[s2_0], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r8 = %[s2_0], %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r29 = r3, r11\n\t padd4 r28 = r2, r10\n\t pmpy2.r r31 = %[s3_0], %[f3]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r30 = %[s3_0], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r10 = %[s4_0], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r11 = %[s4_0], %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r27 = r9, r31\n\t padd4 r26 = r8, r30\n\t pmpy2.r r2 = %[s5_0], %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r3 = %[s5_0], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r8 = r29, r27\n\t padd4 r9 = r28, r26\n\t pmpy2.r r30 = %[s6_0], %[f6]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r31 = %[s6_0], %[f6]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r28 = r10, r2\n\t padd4 r29 = r11, r3\n\t pmpy2.r r26 = %[s7_0], %[f7]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r27 = %[s7_0], %[f7]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = %[s0_1], %[f0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[s0_1], %[f0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r11 = r30, r26\n\t padd4 r10 = r31, r27\n\t pmpy2.r r25 = %[s1_1], %[f1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r24 = %[s1_1], %[f1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r27 = r28, r11\n\t padd4 r26 = r29, r10\n\t pmpy2.r r31 = %[s2_1], %[f2]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r30 = %[s2_1], %[f2]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r23 = r3, r25\n\t padd4 r22 = r2, r24\n\t pmpy2.r r29 = %[s3_1], %[f3]\n\t}\n\t" + "{ .mmi\n\t padd4 r10 = r8, r27\n\t padd4 r11 = r9, r26\n\t pmpy2.l r28 = %[s3_1], %[f3]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r26 = r10, %[bias]\n\t padd4 r27 = r11, %[bias]\n\t pmpy2.r r24 = %[s4_1], %[f4]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r25 = %[s4_1], %[f4]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r2 = r31, r29\n\t padd4 r3 = r30, r28\n\t pmpy2.r r11 = %[s5_1], %[f5]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = %[s5_1], %[f5]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r30 = r23, r2\n\t padd4 r31 = r22, r3\n\t pmpy2.r r28 = %[s6_1], %[f6]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r29 = %[s6_1], %[f6]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r22 = r24, r11\n\t padd4 r23 = r25, r10\n\t pmpy2.r r3 = %[s7_1], %[f7]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = %[s7_1], %[f7]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r10 = r26, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r11 = r27, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r26 = r28, r3\n\t padd4 r25 = r29, r2\n\t pack4.sss r27 = r10, r11\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r11 = r22, r26\n\t padd4 r10 = r23, r25\n\t mux2 %[res0] = r27, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r25 = r30, r11\n\t padd4 r26 = r31, r10\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r10 = r25, %[bias]\n\t padd4 r11 = r26, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r26 = r10, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r25 = r11, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r11 = r26, r25\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res1] = r11, 0xd8\n\t}\n\t" + ";;\n\t" + : [res0]"=&r"(r0), [res1]"=r"(r1) + : [s0_0]"r"(v0_0), + [s1_0]"r"(v1_0), + [s2_0]"r"(v2_0), + [s3_0]"r"(v3_0), + [s4_0]"r"(v4_0), + [s5_0]"r"(v5_0), + [s6_0]"r"(v6_0), + [s7_0]"r"(v7_0), + [s0_1]"r"(v0_1), + [s1_1]"r"(v1_1), + [s2_1]"r"(v2_1), + [s3_1]"r"(v3_1), + [s4_1]"r"(v4_1), + [s5_1]"r"(v5_1), + [s6_1]"r"(v6_1), + [s7_1]"r"(v7_1), + [f0]"r"(F[0]), [f1]"r"(F[1]), [f2]"r"(F[2]), [f3]"r"(F[3]), + [f4]"r"(F[4]), [f5]"r"(F[5]), [f6]"r"(F[6]), [f7]"r"(F[7]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + *lo = r0; *hi = r1; +} + +SIMD_FN void v8_pix8(const pixel *const s, const ptrdiff_t stride, + const i16x4 *const F, const i16x4 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ +#define V(i) ldu_u8x8(s + (i) * stride) + const u8x8 v0 = V(0), v1 = V(1), v2 = V(2), v3 = V(3); + const u8x8 v4 = V(4), v5 = V(5), v6 = V(6), v7 = V(7); +#undef V +#define TL(i, v) pmpy2_lo(zext8_lo(v), F[(i)]) +#define TH(i, v) pmpy2_lo(zext8_hi(v), F[(i)]) + const i16x4 al = (((TL(0,v0) + TL(1,v1)) + (TL(2,v2) + TL(3,v3))) + + ((TL(4,v4) + TL(5,v5)) + (TL(6,v6) + TL(7,v7)))) + bias; + const i16x4 ah = (((TH(0,v0) + TH(1,v1)) + (TH(2,v2) + TH(3,v3))) + + ((TH(4,v4) + TH(5,v5)) + (TH(6,v6) + TH(7,v7)))) + bias; +#undef TL +#undef TH + *lo = al >> shift; + *hi = ah >> shift; +} + +/* Four-column vertical group, for w == 4 blocks. This loads eight bytes and + * uses four, i.e. it reads up to four bytes to the right of the block, the + * same over-read every other dav1d SIMD backend does here (x86 uses movq). + * Fetching only the four owned bytes means a byte-wise load, and that was + * measured at 1.06x-1.12x -- the load dominates the arithmetic completely. */ +SIMD_FN i16x4 v8_pix4(const pixel *const s, const ptrdiff_t stride, + const i16x4 *const F, const i16x4 bias, const int shift) +{ +#define T(i) pmpy2_lo(zext8_lo(ldu_u8x8(s + (i) * stride)), F[(i)]) + const i16x4 acc = (((T(0) + T(1)) + (T(2) + T(3))) + + ((T(4) + T(5)) + (T(6) + T(7)))) + bias; +#undef T + return acc >> shift; +} + +/* Six-tap forms of the two pixel-domain vertical kernels; |s| points at the + * row supplying tap 0, which is two rows above the output row, not three. */ +SIMD_FN void v8_pix8_6t(const pixel *const s, const ptrdiff_t stride, + const i16x4 *const F, const i16x4 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ +#define V(i) ldu_u8x8(s + (i) * stride) + const u8x8 v0 = V(0), v1 = V(1), v2 = V(2); + const u8x8 v3 = V(3), v4 = V(4), v5 = V(5); +#undef V +#define TL(i, v) pmpy2_lo(zext8_lo(v), F[(i)]) +#define TH(i, v) pmpy2_lo(zext8_hi(v), F[(i)]) + const i16x4 al = (((TL(0,v0) + TL(1,v1)) + (TL(2,v2) + TL(3,v3))) + + (TL(4,v4) + TL(5,v5))) + bias; + const i16x4 ah = (((TH(0,v0) + TH(1,v1)) + (TH(2,v2) + TH(3,v3))) + + (TH(4,v4) + TH(5,v5))) + bias; +#undef TL +#undef TH + *lo = al >> shift; + *hi = ah >> shift; +} + +SIMD_FN i16x4 v8_pix4_6t(const pixel *const s, const ptrdiff_t stride, + const i16x4 *const F, const i16x4 bias, const int shift) +{ +#define T(i) pmpy2_lo(zext8_lo(ldu_u8x8(s + (i) * stride)), F[(i)]) + const i16x4 acc = (((T(0) + T(1)) + (T(2) + T(3))) + (T(4) + T(5))) + bias; +#undef T + return acc >> shift; +} + +SIMD_FN void splat_filter(i16x4 *const F, const int8_t *const f) { + for (int i = 0; i < 8; i++) + F[i] = splat16(f[i]); +} + +/* Six-tap callers hand in |f| already advanced past the leading zero tap, so + * only |n| coefficients may be read: the filter rows are eight bytes and + * reading f[8] would step into the next subpel position's row. */ +SIMD_FN void splat_filter_n(i16x4 *const F, const int8_t *const f, const int n) { + for (int i = 0; i < n; i++) + F[i] = splat16(f[i]); +} + +/* Only the SHARP filter set uses all eight taps; see the six-tap kernels. */ +SIMD_FN int filter_is_6tap(const int8_t *const f) { + return !f[0] && !f[7]; +} + +/* Same selection rules as GET_H_FILTER / GET_V_FILTER in the C code. */ +#define GET_H_FILTER_IA64(mx) \ + const int8_t *const fh = !(mx) ? NULL : w > 4 ? \ + dav1d_mc_subpel_filters[filter_type & 3][(mx) - 1] : \ + dav1d_mc_subpel_filters[3 + (filter_type & 1)][(mx) - 1] + +#define GET_V_FILTER_IA64(my) \ + const int8_t *const fv = !(my) ? NULL : h > 4 ? \ + dav1d_mc_subpel_filters[filter_type >> 2][(my) - 1] : \ + dav1d_mc_subpel_filters[3 + ((filter_type >> 2) & 1)][(my) - 1] + +#define F8(p, i, F, st) \ + (F[0] * (p)[(i) + -3 * (st)] + F[1] * (p)[(i) + -2 * (st)] + \ + F[2] * (p)[(i) + -1 * (st)] + F[3] * (p)[(i) + +0 * (st)] + \ + F[4] * (p)[(i) + +1 * (st)] + F[5] * (p)[(i) + +2 * (st)] + \ + F[6] * (p)[(i) + +3 * (st)] + F[7] * (p)[(i) + +4 * (st)]) + +/* + * Chroma prediction blocks can be two pixels wide, which is narrower than a + * single packed group. Those take this scalar path, transcribed from the C + * version; there is nothing to vectorise across two columns anyway. + */ +static NOINLINE void +put_8tap_narrow(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int w, int h, + const int8_t *const fh, const int8_t *const fv) +{ + if (fh) { + if (fv) { + ALIGN_STK_16(int16_t, mid, MID_STRIDE * 135,); + int16_t *mid_ptr = mid; + int tmp_h = h + 7; + + src -= src_stride * 3; + do { + for (int x = 0; x < w; x++) + mid_ptr[x] = (int16_t)((F8(src, x, fh, 1) + H_BIAS) >> H_SHIFT); + mid_ptr += MID_STRIDE; + src += src_stride; + } while (--tmp_h); + + mid_ptr = mid + MID_STRIDE * 3; + do { + for (int x = 0; x < w; x++) + dst[x] = (pixel)iclip((F8(mid_ptr, x, fv, MID_STRIDE) + + PUT_HV_BIAS) >> PUT_HV_SHIFT, 0, 255); + mid_ptr += MID_STRIDE; + dst += dst_stride; + } while (--h); + } else { + do { + for (int x = 0; x < w; x++) + dst[x] = (pixel)iclip((F8(src, x, fh, 1) + PUT_H_BIAS) >> 6, + 0, 255); + dst += dst_stride; + src += src_stride; + } while (--h); + } + } else if (fv) { + do { + for (int x = 0; x < w; x++) + dst[x] = (pixel)iclip((F8(src, x, fv, src_stride) + 32) >> 6, + 0, 255); + dst += dst_stride; + src += src_stride; + } while (--h); + } else { + /* Hoisted so the store path is chosen once per block, not per row. */ + if (!((uintptr_t)dst & 7) && !((uintptr_t)dst_stride & 7)) + copy_rows(dst, dst_stride, src, src_stride, w, h, 1); + else + copy_rows(dst, dst_stride, src, src_stride, w, h, 0); + } +} + +static NOINLINE void +put_8tap_ia64(pixel *dst, ptrdiff_t dst_stride, + const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, const int my, + const int filter_type) +{ + GET_H_FILTER_IA64(mx); + GET_V_FILTER_IA64(my); + dst_stride = PXSTRIDE(dst_stride); + src_stride = PXSTRIDE(src_stride); + + if (w < 4) { + put_8tap_narrow(dst, dst_stride, src, src_stride, w, h, fh, fv); + return; + } + +/* st_u8x8_sel() with the row's alignment hoisted out, not stu_u8x8(): the + * latter re-tests every store and its unaligned arm is a memcpy(), which GCC + * turns into a spill plus eight st1. See the note on st_u8x8_sel(). */ + const int dal = !(((uintptr_t)dst | (uintptr_t)dst_stride) & 7); + + /* + * Six-tap dispatch. Costs two loads and two compares per block and saves a + * quarter of the multiplies in whichever pass qualifies; the vertical case + * also shortens the intermediate by two rows, so the horizontal pass runs + * h+5 times instead of h+7. + */ + const int h6 = fh && filter_is_6tap(fh); + const int v6 = fv && filter_is_6tap(fv); + + if (fh) { + i16x4 F[8]; + splat_filter_n(F, h6 ? fh + 1 : fh, h6 ? 6 : 8); + + if (fv) { + i16x4 G[8]; + splat_filter_n(G, v6 ? fv + 1 : fv, v6 ? 6 : 8); + + const i16x4 hb = { H_BIAS, H_BIAS, H_BIAS, H_BIAS }; + const i32x2 vb = { PUT_HV_BIAS, PUT_HV_BIAS }; + ALIGN_STK_16(int16_t, mid, MID_STRIDE * 135,); + const int vlead = v6 ? 2 : 3; + int16_t *mid_ptr = mid; + int tmp_h = h + 2 * vlead + 1; + + src -= src_stride * vlead; + do { + if (h6) h8_row_6t(mid_ptr, src, w, F, hb); + else h8_row (mid_ptr, src, w, F, hb); + mid_ptr += MID_STRIDE; + src += src_stride; + } while (--tmp_h); + + const int16_t *m = mid; + do { + if (w == 4) { + const i16x4 v = v6 ? v8_mid4_6t(m, G, vb, PUT_HV_SHIFT) + : v8_mid4 (m, G, vb, PUT_HV_SHIFT); + st_u8_partial(dst, pack2_uss(v, (i16x4){0}), 4); + } else if (v6) { + for (int x = 0; x < w; x += 8) { + i16x4 lo, hi; + v8_mid8_6t(m + x, G, vb, PUT_HV_SHIFT, &lo, &hi); + st_u8x8_sel(dst + x, pack2_uss(lo, hi), dal); + } + } else { + for (int x = 0; x < w; x += 8) { + i16x4 lo, hi; + v8_mid8(m + x, G, vb, PUT_HV_SHIFT, &lo, &hi); + st_u8x8_sel(dst + x, pack2_uss(lo, hi), dal); + } + } + m += MID_STRIDE; + dst += dst_stride; + } while (--h); + } else { + const i16x4 hb = { PUT_H_BIAS, PUT_H_BIAS, PUT_H_BIAS, PUT_H_BIAS }; + /* Fold the window base into |src| once; it still has to advance a + * row per iteration, so this cannot be hoisted out of the loop. */ + src += h6 - 3; + do { + if (w == 4) { + const i16x4 v = h6 ? h8_pix4_6t(src, F, hb, 6) + : h8_pix4 (src, F, hb, 6); + st_u8_partial(dst, pack2_uss(v, (i16x4){0}), 4); + } else if (h6) { + for (int x = 0; x < w; x += 8) + st_u8x8_sel(dst + x, + pack2_uss(h8_pix4_6t(src + x, F, hb, 6), + h8_pix4_6t(src + x + 4, F, hb, 6)), dal); + } else { + for (int x = 0; x < w; x += 8) + st_u8x8_sel(dst + x, + pack2_uss(h8_pix4(src + x, F, hb, 6), + h8_pix4(src + x + 4, F, hb, 6)), dal); + } + dst += dst_stride; + src += src_stride; + } while (--h); + } + } else if (fv) { + i16x4 G[8]; + splat_filter_n(G, v6 ? fv + 1 : fv, v6 ? 6 : 8); + + const i16x4 vb = { 32, 32, 32, 32 }; + src -= src_stride * (v6 ? 2 : 3); + do { + if (w == 4) { + const i16x4 v = v6 ? v8_pix4_6t(src, src_stride, G, vb, 6) + : v8_pix4 (src, src_stride, G, vb, 6); + st_u8_partial(dst, pack2_uss(v, (i16x4){0}), 4); + } else if (v6) { + for (int x = 0; x < w; x += 8) { + i16x4 lo, hi; + v8_pix8_6t(src + x, src_stride, G, vb, 6, &lo, &hi); + st_u8x8_sel(dst + x, pack2_uss(lo, hi), dal); + } + } else { + for (int x = 0; x < w; x += 8) { + i16x4 lo, hi; + v8_pix8(src + x, src_stride, G, vb, 6, &lo, &hi); + st_u8x8_sel(dst + x, pack2_uss(lo, hi), dal); + } + } + dst += dst_stride; + src += src_stride; + } while (--h); + } else { + /* Hoisted so the store path is chosen once per block, not per row. */ + if (!((uintptr_t)dst & 7) && !((uintptr_t)dst_stride & 7)) + copy_rows(dst, dst_stride, src, src_stride, w, h, 1); + else + copy_rows(dst, dst_stride, src, src_stride, w, h, 0); + } +} + +static NOINLINE void +prep_8tap_ia64(int16_t *tmp, const pixel *src, ptrdiff_t src_stride, + const int w, int h, const int mx, const int my, + const int filter_type) +{ + GET_H_FILTER_IA64(mx); + GET_V_FILTER_IA64(my); + src_stride = PXSTRIDE(src_stride); + + /* PREP_BIAS is 0 at 8bpc, so the prep paths differ from put only in the + * final shift and in storing int16 instead of clipping to a pixel. */ + /* + * Six-tap dispatch. Costs two loads and two compares per block and saves a + * quarter of the multiplies in whichever pass qualifies; the vertical case + * also shortens the intermediate by two rows, so the horizontal pass runs + * h+5 times instead of h+7. + */ + const int h6 = fh && filter_is_6tap(fh); + const int v6 = fv && filter_is_6tap(fv); + + if (fh) { + i16x4 F[8]; + splat_filter_n(F, h6 ? fh + 1 : fh, h6 ? 6 : 8); + + const i16x4 hb = { H_BIAS, H_BIAS, H_BIAS, H_BIAS }; + if (fv) { + i16x4 G[8]; + splat_filter_n(G, v6 ? fv + 1 : fv, v6 ? 6 : 8); + + const i32x2 vb = { 32, 32 }; + ALIGN_STK_16(int16_t, mid, MID_STRIDE * 135,); + const int vlead = v6 ? 2 : 3; + int16_t *mid_ptr = mid; + int tmp_h = h + 2 * vlead + 1; + + src -= src_stride * vlead; + do { + if (h6) h8_row_6t(mid_ptr, src, w, F, hb); + else h8_row (mid_ptr, src, w, F, hb); + mid_ptr += MID_STRIDE; + src += src_stride; + } while (--tmp_h); + + const int16_t *m = mid; + do { + int x = 0; + if (v6) { + for (; x + 12 <= w; x += 12) { + i16x4 a, b, c; + v8_mid12_6t(m + x, G, vb, 6, &a, &b, &c); + st_i16x4(tmp + x, a); + st_i16x4(tmp + x + 4, b); + st_i16x4(tmp + x + 8, c); + } + for (; x + 8 <= w; x += 8) { + i16x4 lo, hi; + v8_mid8_6t(m + x, G, vb, 6, &lo, &hi); + st_i16x4(tmp + x, lo); + st_i16x4(tmp + x + 4, hi); + } + for (; x < w; x += 4) + st_i16x4(tmp + x, v8_mid4_6t(m + x, G, vb, 6)); + } else { + for (; x + 8 <= w; x += 8) { + i16x4 lo, hi; + v8_mid8(m + x, G, vb, 6, &lo, &hi); + st_i16x4(tmp + x, lo); + st_i16x4(tmp + x + 4, hi); + } + for (; x < w; x += 4) + st_i16x4(tmp + x, v8_mid4(m + x, G, vb, 6)); + } + m += MID_STRIDE; + tmp += w; + } while (--h); + } else { + do { + if (h6) h8_row_6t(tmp, src, w, F, hb); + else h8_row (tmp, src, w, F, hb); + tmp += w; + src += src_stride; + } while (--h); + } + } else if (fv) { + i16x4 G[8]; + splat_filter_n(G, v6 ? fv + 1 : fv, v6 ? 6 : 8); + + const i16x4 vb = { H_BIAS, H_BIAS, H_BIAS, H_BIAS }; + src -= src_stride * (v6 ? 2 : 3); + do { + if (w == 4) { + st_i16x4(tmp, v6 ? v8_pix4_6t(src, src_stride, G, vb, H_SHIFT) + : v8_pix4 (src, src_stride, G, vb, H_SHIFT)); + } else if (v6) { + for (int x = 0; x < w; x += 8) { + i16x4 lo, hi; + v8_pix8_6t(src + x, src_stride, G, vb, H_SHIFT, &lo, &hi); + st_i16x4(tmp + x, lo); + st_i16x4(tmp + x + 4, hi); + } + } else { + for (int x = 0; x < w; x += 8) { + i16x4 lo, hi; + v8_pix8(src + x, src_stride, G, vb, H_SHIFT, &lo, &hi); + st_i16x4(tmp + x, lo); + st_i16x4(tmp + x + 4, hi); + } + } + tmp += w; + src += src_stride; + } while (--h); + } else { + /* + * No filter in either direction: widen the pixels and shift them up + * into the intermediate range. This is a fifth of all the output + * pixels prep() produces, so it is worth writing out. + * + * GCC does vectorise the obvious scalar loop into exactly the packed + * form below, but only under a runtime guard that *both* pointers are + * 8-byte aligned. |tmp| always is; |src| is a pixel pointer at an + * arbitrary column, so the guard fails for seven x positions in eight + * and the fallback is a scalar ld1/shladd/st2 chain carrying a branch + * per pixel. Going through ldu_u8x8() spends one shift pair per group + * to make the misalignment a non-event and removes the cliff. + */ + if (w == 4) { + do { + st_i16x4(tmp, zext8_lo(ldn_u8x8(src, 4)) << INTERMEDIATE_BITS); + tmp += 4; + src += src_stride; + } while (--h); + } else { + do { + for (int x = 0; x < w; x += 8) { + const u8x8 p = ldu_u8x8(src + x); + st_i16x4(tmp + x, zext8_lo(p) << INTERMEDIATE_BITS); + st_i16x4(tmp + x + 4, zext8_hi(p) << INTERMEDIATE_BITS); + } + tmp += w; + src += src_stride; + } while (--h); + } + } +} + +/* -------------------------------------------------------- warp_affine 8x8 */ + +/* + * Warp differs from the 8-tap filters in the one way that matters here: the + * filter is chosen per output *column*, so the coefficients cannot be hoisted + * out of the loop. Four filters therefore have to be transposed from + * one-filter-per-row into one-tap-per-vector before every group of four + * outputs. + * + * That transpose is cheap on this target: each filter row is exactly eight + * signed bytes, so four aligned loads plus eight unpacks produce all eight tap + * columns, and three more ops per pair sign-extend them to 16-bit lanes. With + * the coefficients in that shape both passes reduce to the same pmpy2 + * accumulate the rest of this file uses -- the horizontal one is literally + * h8_pix4() with a different filter source. + */ +SIMD_FN void warp_filters_4(const int8_t *const f0, const int8_t *const f1, + const int8_t *const f2, const int8_t *const f3, + i16x4 *const T) +{ + /* dav1d_mc_warp_filter is declared 8-byte aligned, one row per load. */ + const i8x8 r0 = *(const i8x8 *)f0, r1 = *(const i8x8 *)f1; + const i8x8 r2 = *(const i8x8 *)f2, r3 = *(const i8x8 *)f3; + + const i8x8 a0 = unpack1_l(r1, r0), a1 = unpack1_h(r1, r0); + const i8x8 b0 = unpack1_l(r3, r2), b1 = unpack1_h(r3, r2); + + /* Each c holds two tap columns: {f0[k],f1[k],f2[k],f3[k]} twice over. */ + const i8x8 c0 = unpack2_l(b0, a0), c1 = unpack2_h(b0, a0); + const i8x8 c2 = unpack2_l(b1, a1), c3 = unpack2_h(b1, a1); + + T[0] = sext8_lo(c0); T[1] = sext8_hi(c0); + T[2] = sext8_lo(c1); T[3] = sext8_hi(c1); + T[4] = sext8_lo(c2); T[5] = sext8_hi(c2); + T[6] = sext8_lo(c3); T[7] = sext8_hi(c3); +} + +/* + * Filters for the four outputs starting at parameter value t0, stepping by + * |step| per column -- the same index expression the scalar code uses. + * + * Building them is not cheap: four table rows have to be transposed from + * one-filter-per-row into one-tap-per-vector and sign-extended, which is about + * two dozen I-type ops, and warp calls this 46 times for every 8x8 block (30 in + * the horizontal pass, 16 in the vertical). That is comparable to the + * filtering itself. + * + * The saving grace is the >> 10 in the index: the warp parameter advances by a + * fraction of a filter step per row, so consecutive rows very often land on the + * same four table entries. Each call site therefore keeps its own one-entry + * cache and only rebuilds when the indices actually move. The caches are + * function-local (one per call site, alive across the block's row loop), so + * there is no shared state and nothing to reset between blocks. + */ +typedef struct { int i0, i1, i2, i3; i16x4 T[8]; } WarpFC; + +SIMD_FN void warp_fc_init(WarpFC *const c) { + c->i0 = c->i1 = c->i2 = c->i3 = -1; /* no table index is negative */ +} + +SIMD_FN const i16x4 *warp_filters_at(WarpFC *const c, const int t0, + const int step) +{ + const int i0 = 64 + ((t0 + 512) >> 10); + const int i1 = 64 + ((t0 + step + 512) >> 10); + const int i2 = 64 + ((t0 + 2 * step + 512) >> 10); + const int i3 = 64 + ((t0 + 3 * step + 512) >> 10); + if (i0 != c->i0 || i1 != c->i1 || i2 != c->i2 || i3 != c->i3) { + c->i0 = i0; c->i1 = i1; c->i2 = i2; c->i3 = i3; + warp_filters_4(dav1d_mc_warp_filter[i0], dav1d_mc_warp_filter[i1], + dav1d_mc_warp_filter[i2], dav1d_mc_warp_filter[i3], + c->T); + } + return c->T; +} + +/* Vertical group: with four consecutive columns, every tap is one aligned + * 4-lane load out of the intermediate buffer -- no window sliding needed. */ +SIMD_FN i16x4 warp_v4(const int16_t *const m, const i16x4 *const T, + const i32x2 bias, const int shift) +{ + /* Same shape as v8_mid4, and bundled for the same reason: sixteen pmpy2 + * helpers meant sixteen stop bits. Warp's taps change per column, so T is + * rebuilt by the caller, but the accumulate itself is identical. */ + const i16x4 s0 = ld_i16x4(m + 0 * 8), s1 = ld_i16x4(m + 1 * 8); + const i16x4 s2 = ld_i16x4(m + 2 * 8), s3 = ld_i16x4(m + 3 * 8); + const i16x4 s4 = ld_i16x4(m + 4 * 8), s5 = ld_i16x4(m + 5 * 8); + const i16x4 s6 = ld_i16x4(m + 6 * 8), s7 = ld_i16x4(m + 7 * 8); + i16x4 res; + __asm__( + "{ .mii\n\t nop.m 0\n\t pmpy2.r r14 = %[s0], %[f0]\n\t pmpy2.l r15 = %[s0], %[f0]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r16 = %[s1], %[f1]\n\t pmpy2.l r17 = %[s1], %[f1]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r18 = %[s2], %[f2]\n\t pmpy2.l r19 = %[s2], %[f2]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r20 = %[s3], %[f3]\n\t pmpy2.l r21 = %[s3], %[f3]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r22 = %[s4], %[f4]\n\t pmpy2.l r23 = %[s4], %[f4]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r24 = %[s5], %[f5]\n\t pmpy2.l r25 = %[s5], %[f5]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r26 = %[s6], %[f6]\n\t pmpy2.l r27 = %[s6], %[f6]\n\t}\n\t" + "{ .mii\n\t nop.m 0\n\t pmpy2.r r28 = %[s7], %[f7]\n\t pmpy2.l r29 = %[s7], %[f7]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r16\n\t padd4 r15 = r15, r17\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r18 = r18, r20\n\t padd4 r19 = r19, r21\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r24\n\t padd4 r23 = r23, r25\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r26 = r26, r28\n\t padd4 r27 = r27, r29\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r18\n\t padd4 r15 = r15, r19\n\t nop.i 0\n\t}\n\t" + "{ .mmi\n\t padd4 r22 = r22, r26\n\t padd4 r23 = r23, r27\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, r22\n\t padd4 r15 = r15, r23\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r14 = r14, %[bias]\n\t padd4 r15 = r15, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pshr4 r14 = r14, %[sh]\n\t pshr4 r15 = r15, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t pack4.sss r14 = r14, r15\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mii\n\t nop.m 0\n\t mux2 %[res] = r14, 0xd8\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + : [res]"=r"(res) + : [s0]"r"(s0), [s1]"r"(s1), [s2]"r"(s2), [s3]"r"(s3), + [s4]"r"(s4), [s5]"r"(s5), [s6]"r"(s6), [s7]"r"(s7), + [f0]"r"(T[0]), [f1]"r"(T[1]), [f2]"r"(T[2]), [f3]"r"(T[3]), + [f4]"r"(T[4]), [f5]"r"(T[5]), [f6]"r"(T[6]), [f7]"r"(T[7]), + [bias]"r"(bias), [sh]"i"(shift) + : "r14","r15","r16","r17","r18","r19","r20","r21", + "r22","r23","r24","r25","r26","r27","r28","r29"); + return res; +} + +/* Shared horizontal pass: 15 rows of 8, into a 16-byte aligned mid[]. */ + +/* + * Warp horizontal, eight pixels per call. + * + * Warp re-derives its taps between the two halves of a row, so each group + * carries its own filter set -- but the two halves are still independent, and + * that is what matters: alone, a group's sixteen pmpy2 issue in eight cycles + * and then spend a dozen more walking out the tree, the bias, the pshr4, the + * pack4 and the mux2. Interleaved the pair costs 32 cycles against forty. + */ +SIMD_FN void warp_h8(const pixel *const b0, const pixel *const b1, + const i16x4 *const F0, const i16x4 *const F1, + const i32x2 bias, const int shift, + i16x4 *const lo, i16x4 *const hi) +{ + const u8x8 wa0 = ldu_u8x8(b0), wb0 = ldu_u8x8(b0 + 3); + const u8x8 wa1 = ldu_u8x8(b1), wb1 = ldu_u8x8(b1 + 3); + i16x4 r0, r1; + __asm__( + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r3 = r0, %[wa0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r2 = r0, %[wa0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r11 = r0, %[wb0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r10 = r0, %[wb0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r9 = r0, %[wa1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r8 = r0, %[wa1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.l r31 = r0, %[wb1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t unpack1.h r30 = r0, %[wb1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r29 = r2, r3, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r28 = r2, r3, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r27 = r10, r11, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r26 = r10, r11, 48\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r25 = r8, r9, 16\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r24 = r8, r9, 32\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r23 = r30, r31, 32\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t shrp r22 = r30, r31, 48\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r21 = r3, %[f0_0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r20 = r3, %[f0_0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r3 = r29, %[f1_0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r19 = r29, %[f1_0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r29 = r28, %[f2_0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r18 = r28, %[f2_0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r16 = r21, r3\n\t padd4 r15 = r20, r19\n\t pmpy2.r r28 = r11, %[f3_0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r17 = r11, %[f3_0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r19 = r2, %[f4_0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r3 = r2, %[f4_0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r21 = r29, r28\n\t padd4 r11 = r18, r17\n\t pmpy2.r r2 = r27, %[f5_0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r20 = r27, %[f5_0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r18 = r16, r21\n\t padd4 r29 = r15, r11\n\t pmpy2.r r17 = r26, %[f6_0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r28 = r26, %[f6_0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r15 = r19, r2\n\t padd4 r16 = r3, r20\n\t pmpy2.r r11 = r10, %[f7_0]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r21 = r10, %[f7_0]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.r r20 = r9, %[f0_1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r2 = r9, %[f0_1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r9 = r17, r11\n\t padd4 r3 = r28, r21\n\t pmpy2.r r19 = r25, %[f1_1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r10 = r25, %[f1_1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r21 = r15, r9\n\t padd4 r11 = r16, r3\n\t pmpy2.r r28 = r24, %[f2_1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r17 = r24, %[f2_1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r24 = r20, r19\n\t padd4 r25 = r2, r10\n\t pmpy2.r r16 = r31, %[f3_1]\n\t}\n\t" + "{ .mmi\n\t padd4 r3 = r18, r21\n\t padd4 r9 = r29, r11\n\t pmpy2.l r15 = r31, %[f3_1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r11 = r3, %[bias]\n\t padd4 r21 = r9, %[bias]\n\t pmpy2.r r10 = r8, %[f4_1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r19 = r8, %[f4_1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r8 = r28, r16\n\t padd4 r2 = r17, r15\n\t pmpy2.r r9 = r23, %[f5_1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r3 = r23, %[f5_1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r17 = r24, r8\n\t padd4 r28 = r25, r2\n\t pmpy2.r r15 = r22, %[f6_1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r16 = r22, %[f6_1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r25 = r10, r9\n\t padd4 r24 = r19, r3\n\t pmpy2.r r2 = r30, %[f7_1]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pmpy2.l r8 = r30, %[f7_1]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r3 = r11, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r9 = r21, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r11 = r15, r2\n\t padd4 r19 = r16, r8\n\t pack4.sss r21 = r3, r9\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r9 = r25, r11\n\t padd4 r3 = r24, r19\n\t mux2 %[res0] = r21, 0xd8\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r19 = r17, r9\n\t padd4 r11 = r28, r3\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t padd4 r3 = r19, %[bias]\n\t padd4 r9 = r11, %[bias]\n\t nop.i 0\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r11 = r3, %[sh]\n\t}\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pshr4 r19 = r9, %[sh]\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t pack4.sss r9 = r11, r19\n\t}\n\t" + ";;\n\t" + "{ .mmi\n\t nop.m 0\n\t nop.m 0\n\t mux2 %[res1] = r9, 0xd8\n\t}\n\t" + ";;\n\t" + : [res0]"=&r"(r0), [res1]"=r"(r1) + : [wa0]"r"(wa0), [wb0]"r"(wb0), [wa1]"r"(wa1), [wb1]"r"(wb1), + [f0_0]"r"(F0[0]), [f0_1]"r"(F1[0]), + [f1_0]"r"(F0[1]), [f1_1]"r"(F1[1]), + [f2_0]"r"(F0[2]), [f2_1]"r"(F1[2]), + [f3_0]"r"(F0[3]), [f3_1]"r"(F1[3]), + [f4_0]"r"(F0[4]), [f4_1]"r"(F1[4]), + [f5_0]"r"(F0[5]), [f5_1]"r"(F1[5]), + [f6_0]"r"(F0[6]), [f6_1]"r"(F1[6]), + [f7_0]"r"(F0[7]), [f7_1]"r"(F1[7]), + [bias]"r"(bias), [sh]"i"(shift) + : "r2", "r3", "r8", "r9", "r10", "r11", "r15", "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"); + *lo = r0; *hi = r1; +} + +SIMD_FN void warp_h_pass(int16_t *mid_ptr, const pixel *src, + const ptrdiff_t src_stride, const int16_t *const abcd, + int mx) +{ + const i32x2 bias = { (1 << (7 - INTERMEDIATE_BITS)) >> 1, + (1 << (7 - INTERMEDIATE_BITS)) >> 1 }; + WarpFC c0, c1; + warp_fc_init(&c0); + warp_fc_init(&c1); + for (int y = 0; y < 15; y++, mx += abcd[1]) { + i16x4 lo, hi; + const i16x4 *const T0 = warp_filters_at(&c0, mx, abcd[0]); + const i16x4 *const T1 = warp_filters_at(&c1, mx + 4 * abcd[0], abcd[0]); + warp_h8(src - 3, src + 1, T0, T1, bias, 7 - INTERMEDIATE_BITS, &lo, &hi); + st_i16x4(mid_ptr, lo); + st_i16x4(mid_ptr + 4, hi); + src += src_stride; + mid_ptr += 8; + } +} + +static void warp_affine_8x8_ia64(pixel *dst, const ptrdiff_t dst_stride, + const pixel *src, const ptrdiff_t src_stride, + const int16_t *const abcd, int mx, int my + HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int16_t, mid, 15 * 8,); + + warp_h_pass(mid, src - 3 * PXSTRIDE(src_stride), PXSTRIDE(src_stride), + abcd, mx); + + const int dal = !(((uintptr_t)dst | (uintptr_t)dst_stride) & 7); + const int shift = 7 + INTERMEDIATE_BITS; + const i32x2 bias = { (1 << (7 + INTERMEDIATE_BITS)) >> 1, + (1 << (7 + INTERMEDIATE_BITS)) >> 1 }; + const int16_t *m = mid; /* &mid[3*8] - 3*8 */ + + WarpFC c0, c1; + warp_fc_init(&c0); + warp_fc_init(&c1); + for (int y = 0; y < 8; y++, my += abcd[3]) { + const i16x4 v0 = warp_v4(m, warp_filters_at(&c0, my, abcd[2]), + bias, shift); + const i16x4 v1 = warp_v4(m + 4, + warp_filters_at(&c1, my + 4 * abcd[2], abcd[2]), + bias, shift); + st_u8x8_sel(dst, pack2_uss(v0, v1), dal); + m += 8; + dst += PXSTRIDE(dst_stride); + } +} + +static void warp_affine_8x8t_ia64(int16_t *tmp, const ptrdiff_t tmp_stride, + const pixel *src, const ptrdiff_t src_stride, + const int16_t *const abcd, int mx, int my + HIGHBD_DECL_SUFFIX) +{ + ALIGN_STK_16(int16_t, mid, 15 * 8,); + + warp_h_pass(mid, src - 3 * PXSTRIDE(src_stride), PXSTRIDE(src_stride), + abcd, mx); + + /* PREP_BIAS is 0 at 8bpc, so this differs from the above only in the + * shift and in storing the intermediate rather than a clipped pixel. */ + const i32x2 bias = { 1 << 6, 1 << 6 }; + const int16_t *m = mid; + const int aligned = !(((uintptr_t)tmp | (uintptr_t)(tmp_stride * 2)) & 7); + + WarpFC c0, c1; + warp_fc_init(&c0); + warp_fc_init(&c1); + for (int y = 0; y < 8; y++, my += abcd[3]) { + const i16x4 v0 = warp_v4(m, warp_filters_at(&c0, my, abcd[2]), bias, 7); + const i16x4 v1 = warp_v4(m + 4, + warp_filters_at(&c1, my + 4 * abcd[2], abcd[2]), + bias, 7); + if (aligned) { + st_i16x4(tmp, v0); + st_i16x4(tmp + 4, v1); + } else { + stu_u8x8(tmp, (u8x8)v0); + stu_u8x8(tmp + 4, (u8x8)v1); + } + m += 8; + tmp += tmp_stride; + } +} + +#define filter_fns_ia64(type, type_h, type_v) \ +static void put_8tap_##type##_ia64(pixel *const dst, \ + const ptrdiff_t dst_stride, \ + const pixel *const src, \ + const ptrdiff_t src_stride, \ + const int w, const int h, \ + const int mx, const int my \ + HIGHBD_DECL_SUFFIX) \ +{ \ + put_8tap_ia64(dst, dst_stride, src, src_stride, w, h, mx, my, \ + type_h | (type_v << 2)); \ +} \ +static void prep_8tap_##type##_ia64(int16_t *const tmp, \ + const pixel *const src, \ + const ptrdiff_t src_stride, \ + const int w, const int h, \ + const int mx, const int my \ + HIGHBD_DECL_SUFFIX) \ +{ \ + prep_8tap_ia64(tmp, src, src_stride, w, h, mx, my, \ + type_h | (type_v << 2)); \ +} + +filter_fns_ia64(regular, DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_REGULAR) +filter_fns_ia64(regular_sharp, DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_SHARP) +filter_fns_ia64(regular_smooth, DAV1D_FILTER_8TAP_REGULAR, DAV1D_FILTER_8TAP_SMOOTH) +filter_fns_ia64(smooth, DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_SMOOTH) +filter_fns_ia64(smooth_regular, DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_REGULAR) +filter_fns_ia64(smooth_sharp, DAV1D_FILTER_8TAP_SMOOTH, DAV1D_FILTER_8TAP_SHARP) +filter_fns_ia64(sharp, DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_SHARP) +filter_fns_ia64(sharp_regular, DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_REGULAR) +filter_fns_ia64(sharp_smooth, DAV1D_FILTER_8TAP_SHARP, DAV1D_FILTER_8TAP_SMOOTH) + +#endif /* BITDEPTH == 8 */ + +COLD void bitfn(dav1d_mc_dsp_init_ia64)(Dav1dMCDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_IA64_CPU_FLAG_SIMD)) return; + +#if BITDEPTH == 8 + c->avg = avg_ia64; + c->w_avg = w_avg_ia64; + c->mask = mask_ia64; + c->w_mask[0] = w_mask_444_ia64; + c->w_mask[1] = w_mask_422_ia64; + c->w_mask[2] = w_mask_420_ia64; + c->blend = blend_ia64; + c->blend_v = blend_v_ia64; + c->blend_h = blend_h_ia64; + c->warp8x8 = warp_affine_8x8_ia64; + c->warp8x8t = warp_affine_8x8t_ia64; + +#define init_mc_fns_ia64(type, name) do { \ + c->mc [type] = put_8tap_##name##_ia64; \ + c->mct[type] = prep_8tap_##name##_ia64; \ +} while (0) + + init_mc_fns_ia64(FILTER_2D_8TAP_REGULAR, regular); + init_mc_fns_ia64(FILTER_2D_8TAP_REGULAR_SMOOTH, regular_smooth); + init_mc_fns_ia64(FILTER_2D_8TAP_REGULAR_SHARP, regular_sharp); + init_mc_fns_ia64(FILTER_2D_8TAP_SHARP_REGULAR, sharp_regular); + init_mc_fns_ia64(FILTER_2D_8TAP_SHARP_SMOOTH, sharp_smooth); + init_mc_fns_ia64(FILTER_2D_8TAP_SHARP, sharp); + init_mc_fns_ia64(FILTER_2D_8TAP_SMOOTH_REGULAR, smooth_regular); + init_mc_fns_ia64(FILTER_2D_8TAP_SMOOTH, smooth); + init_mc_fns_ia64(FILTER_2D_8TAP_SMOOTH_SHARP, smooth_sharp); +#undef init_mc_fns_ia64 +#endif +} diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/itx_tmpl.c dav1d-1.5.4/src/itx_tmpl.c --- dav1d-1.5.4.orig/src/itx_tmpl.c 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/itx_tmpl.c 2026-08-20 18:11:42.544338112 +0200 @@ -37,9 +37,128 @@ #include "src/itx.h" #include "src/itx_1d.h" +#if ARCH_IA64 && BITDEPTH == 8 +#include "src/ia64/dav1d_simd.h" +#include "src/ia64/itx_dct.h" + +/* dst rows are only guaranteed 4-byte aligned (transform blocks sit on 4-pixel + * boundaries), so the pixel loops work four at a time. That costs nothing: + * pack4.sss consumes two full i32x2 either way and it is the I-slot ops that + * set the rate, not the store width. */ +typedef uint32_t px4_t __attribute__((may_alias, aligned(4))); +#endif + #include "src/scan.h" #include "src/tables.h" +#if ARCH_IA64 && BITDEPTH == 8 +/* + * Packed inverse transform for every txtp whose *column* transform is a DCT + * (DCT_DCT, ADST_DCT, FLIPADST_DCT, H_DCT and the identity/DCT pairs). That + * is the overwhelming majority of blocks, and it is the only case the packed + * column kernels in itx_dct.h cover. + * + * The point of splitting it out of inv_txfm_add_c() is the intermediate + * buffer: once the column pass is packed, nothing downstream of the row pass + * needs more than 16 bits (for 8bpc the clip bounds *are* INT16_MIN/MAX), so + * tmp becomes int16. That halves its footprint, turns each 4-element column + * access into a single ld8/st8 instead of two ld_i32x2 plus a pack4.sss, and + * lets the shift-and-clip pass fold into the row loop -- deleting a whole + * w*sh read-modify-write sweep over tmp. + */ +static NOINLINE void +inv_txfm_add_dct_ia64(pixel *dst, const ptrdiff_t stride, coef *const coeff, + const int eob, const /*enum RectTxfmSize*/ int tx, + const int shift, const enum TxfmType txtp) +{ + const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[tx]; + const int w = 4 * t_dim->w, h = 4 * t_dim->h; + const int is_rect2 = w * 2 == h || h * 2 == w; + const int rnd = (1 << shift) >> 1; + const uint8_t *const txtps = dav1d_tx1d_types[txtp]; + const itx_1d_fn first_1d_fn = dav1d_tx1d_fns[t_dim->lw][txtps[0]]; + const int sh = imin(h, 32), sw = imin(w, 32); + + ALIGN_STK_16(int16_t, tmp, 64 * 64,); + ALIGN_STK_16(int32_t, row, 64,); + const i32x2 rndv = { rnd, rnd }; + + int last_nonzero_col; // in first 1d itx + if (txtps[0] == IDENTITY) + last_nonzero_col = eob >> (t_dim->lw + 2); + else + last_nonzero_col = dav1d_last_nonzero_col_from_eob[tx][eob]; + assert(last_nonzero_col < sh); + + int16_t *t = tmp; + for (int y = 0; y <= last_nonzero_col; y++, t += w) { + if (is_rect2) + for (int x = 0; x < sw; x++) + row[x] = (coeff[y + x * sh] * 181 + 128) >> 8; + else + for (int x = 0; x < sw; x++) + row[x] = coeff[y + x * sh]; + first_1d_fn(row, 1, INT16_MIN, INT16_MAX); + /* Round, shift and narrow straight into tmp: pack4.sss saturates to + * exactly the col_clip bounds, so this is the clip too. */ + for (int x = 0; x < w; x += 4) + st_i16x4(&t[x], pack4_sss((ld_i32x2(&row[x]) + rndv) >> shift, + (ld_i32x2(&row[x + 2]) + rndv) >> shift)); + } + if (last_nonzero_col + 1 < sh) + memset(t, 0, sizeof(*t) * (sh - last_nonzero_col - 1) * w); + + memset(coeff, 0, sizeof(*coeff) * sw * sh); + + /* + * Four adjacent columns of tmp are contiguous int16, so the column pass + * needs no transpose: one ld8 is four columns of one row. + * + * Only rows 0..sh-1 were ever written; for h == 64 the rest hold nothing, + * because AV1 zeroes coefficients 32..63 of a 64-point transform -- which + * is exactly what the tx64 path exploits. The scalar code simply never + * reads them; the packed dct64 copies its even inputs wholesale, so zero + * them here rather than read uninitialised stack. + * + * dav1d_tx1d_fns[] holds the non-internal wrappers, which pass tx64 = 0; + * only dct64 sets it, on its own nested call. + */ + for (int x = 0; x < w; x += 4) { + i16x4 v[64]; + for (int i = 0; i < sh; i++) v[i] = ld_i16x4(&tmp[i * w + x]); + for (int i = sh; i < h; i++) v[i] = (i16x4) { 0, 0, 0, 0 }; + switch (t_dim->lh) { + case 0: dct4_col4(v, v, 0); break; + case 1: dct8_col4(v, v, 0); break; + case 2: dct16_col4(v, v, 0); break; + case 3: dct32_col4(v, v, 0); break; + default: dct64_col4(v, v); break; + } + for (int i = 0; i < h; i++) st_i16x4(&tmp[i * w + x], v[i]); + } + + const int16_t *c = tmp; + if (!(((uintptr_t) dst | (uintptr_t) stride) & 3)) { + /* padd2.sss instead of padd2: v + 8 overflows int16 only for + * v >= 32760, and there the saturated 2047 and the true 2048 both + * clamp to 255 once dst (>= 0) is added. pack2.uss is then + * iclip_pixel() for free. */ + const i16x4 eight = splat16(8); + for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < w; x += 4, c += 4) { + const i16x4 v = padd2_sss(ld_i16x4(c), eight) >> 4; + const u8x8 p = (u8x8) (u32x2) { ((const px4_t *) (dst + x))[0], 0 }; + const i16x4 r = v + zext8_lo(p); + *(px4_t *) (dst + x) = ((u32x2) pack2_uss(r, r))[0]; + } + return; + } + for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < w; x++) + dst[x] = iclip_pixel(dst[x] + ((*c++ + 8) >> 4)); +} +#endif + static NOINLINE void inv_txfm_add_c(pixel *dst, const ptrdiff_t stride, coef *const coeff, const int eob, const /*enum RectTxfmSize*/ int tx, const int shift, @@ -63,6 +182,20 @@ dc = (dc * 181 + 128) >> 8; dc = (dc + rnd) >> shift; dc = (dc * 181 + 128 + 2048) >> 12; +#if ARCH_IA64 && BITDEPTH == 8 + if (!(((uintptr_t) dst | (uintptr_t) stride) & 3)) { + /* dst is 0..255, so any |dc| past 512 already saturates; clamping + * it there makes the 16-bit splat exact. */ + const i16x4 dcv = splat16((int16_t) iclip(dc, -512, 512)); + for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < w; x += 4) { + const u8x8 p = (u8x8) (u32x2) { ((const px4_t *) (dst + x))[0], 0 }; + const i16x4 v = zext8_lo(p) + dcv; + *(px4_t *) (dst + x) = ((u32x2) pack2_uss(v, v))[0]; + } + return; + } +#endif for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) for (int x = 0; x < w; x++) dst[x] = iclip_pixel(dst[x] + dc); @@ -70,6 +203,12 @@ } const uint8_t *const txtps = dav1d_tx1d_types[txtp]; +#if ARCH_IA64 && BITDEPTH == 8 + if (txtps[1] == DCT) { + inv_txfm_add_dct_ia64(dst, stride, coeff, eob, tx, shift, txtp); + return; + } +#endif const itx_1d_fn first_1d_fn = dav1d_tx1d_fns[t_dim->lw][txtps[0]]; const itx_1d_fn second_1d_fn = dav1d_tx1d_fns[t_dim->lh][txtps[1]]; const int sh = imin(h, 32), sw = imin(w, 32); @@ -83,7 +222,11 @@ const int row_clip_max = ~row_clip_min; const int col_clip_max = ~col_clip_min; - int32_t tmp[64 * 64], *c = tmp; + ALIGN_STK_16(int32_t, tmp, 64 * 64,); + int32_t *c = tmp; +#if ARCH_IA64 && BITDEPTH == 8 + const i32x2 rndv = { rnd, rnd }; +#endif int last_nonzero_col; // in first 1d itx if (txtps[1] == IDENTITY && txtps[0] != IDENTITY) { last_nonzero_col = imin(sh - 1, eob); @@ -106,13 +249,42 @@ memset(c, 0, sizeof(*c) * (sh - last_nonzero_col - 1) * w); memset(coeff, 0, sizeof(*coeff) * sw * sh); +#if ARCH_IA64 && BITDEPTH == 8 + /* For 8bpc the clip bounds are exactly INT16_MIN/MAX, which is what + * pack4.sss saturates to; sign-extending back costs one pcmp2 (A-type, + * shared between the halves) and two unpack2. */ + for (int i = 0; i < w * sh; i += 4) { + const i16x4 p = pack4_sss((ld_i32x2(&tmp[i]) + rndv) >> shift, + (ld_i32x2(&tmp[i + 2]) + rndv) >> shift); + st_i32x2(&tmp[i], sext16_lo(p)); + st_i32x2(&tmp[i + 2], sext16_hi(p)); + } +#else for (int i = 0; i < w * sh; i++) tmp[i] = iclip((tmp[i] + rnd) >> shift, col_clip_min, col_clip_max); +#endif for (int x = 0; x < w; x++) second_1d_fn(&tmp[x], w, col_clip_min, col_clip_max); c = tmp; +#if ARCH_IA64 && BITDEPTH == 8 + if (!(((uintptr_t) dst | (uintptr_t) stride) & 3)) { + /* pack2.uss is iclip_pixel() for free, and pack4.sss ahead of it is + * exact here: anything it saturates to +-32768 was already going to + * clamp to 0 or 255 once dst (0..255) was added. */ + const i32x2 eight = { 8, 8 }; + for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) + for (int x = 0; x < w; x += 4, c += 4) { + const i16x4 v = pack4_sss((ld_i32x2(c) + eight) >> 4, + (ld_i32x2(c + 2) + eight) >> 4); + const u8x8 p = (u8x8) (u32x2) { ((const px4_t *) (dst + x))[0], 0 }; + const i16x4 r = v + zext8_lo(p); + *(px4_t *) (dst + x) = ((u32x2) pack2_uss(r, r))[0]; + } + return; + } +#endif for (int y = 0; y < h; y++, dst += PXSTRIDE(stride)) for (int x = 0; x < w; x++) dst[x] = iclip_pixel(dst[x] + ((*c++ + 8) >> 4)); diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/lib.c dav1d-1.5.4/src/lib.c --- dav1d-1.5.4.orig/src/lib.c 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/lib.c 2026-08-25 17:50:00.278477250 +0200 @@ -30,6 +30,9 @@ #include #include +#if ARCH_IA64 +#include +#endif #if defined(__linux__) && HAVE_DLSYM #include @@ -106,6 +109,54 @@ #define get_stack_size_internal(attr) (0) #endif +#if ARCH_IA64 +/* Is this a Montecito with CGMT actually enabled? + * + * The two thread-policy adjustments in get_num_threads() below were measured + * on, and are only justified by, that configuration, so both halves have to + * hold before either is applied. + * + * Neither test is sufficient alone. Montecito's coarse-grained multithreading + * can be turned off in firmware, and with it off the two logical CPUs of a + * core no longer share execution resources, so there is nothing to hand back. + * Conversely, an Itanium that reports thread siblings need not be Montecito at + * all: Montvale, Tukwila, Poulson and Kittson all have siblings, and theirs is + * SMT rather than CGMT. Reserving half a core's worth of logical CPUs on a + * part with real simultaneous threads would simply cost throughput. + * + * The generation comes from CPUID register 3, which is readable at user level: + * family 0x20 covers the 9000/9100 series, and model 0x00 within it is + * Montecito (Montvale is 0x20/0x01, Tukwila 0x21, Poulson and Kittson 0x22). + * Whether CGMT is on comes from the topology, not the part number. + * + * Anything unknown -- an unreadable topology, a non-Linux host -- leaves the + * defaults alone, which is the conservative direction: a machine this was not + * measured on gets stock dav1d behaviour. */ +static COLD int ia64_cgmt_montecito(void) { + unsigned long cpuid3; + __asm__ ("mov %0 = cpuid[%1]" : "=r"(cpuid3) : "r"(3UL)); + const unsigned family = (unsigned)(cpuid3 >> 24) & 0xff; + const unsigned model = (unsigned)(cpuid3 >> 16) & 0xff; + if (family != 0x20 || model != 0x00) + return 0; + +#ifdef __linux__ + /* A cpu list with any separator in it ("0-1", or "0,4") names more than + * one logical CPU, i.e. the core's threads are shared. */ + FILE *const f = + fopen("/sys/devices/system/cpu/cpu0/topology/thread_siblings_list", "r"); + if (!f) + return 0; + char buf[64]; + const char *const line = fgets(buf, sizeof(buf), f); + fclose(f); + return line && strpbrk(line, "-,") != NULL; +#else + return 0; +#endif +} +#endif + static COLD void get_num_threads(Dav1dContext *const c, const Dav1dSettings *const s, unsigned *n_tc, unsigned *n_fc) { @@ -119,10 +170,59 @@ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, /* 26-36 */ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, /* 37-49 */ }; - *n_tc = s->n_threads ? s->n_threads : - iclip(dav1d_num_logical_processors(c), 1, DAV1D_MAX_THREADS); + const unsigned n_lp = iclip(dav1d_num_logical_processors(c), 1, + DAV1D_MAX_THREADS); +#if ARCH_IA64 + const int ia64_cgmt = ia64_cgmt_montecito(); +#endif + *n_tc = s->n_threads ? s->n_threads : n_lp; +#if ARCH_IA64 + /* ia64: leave a core's worth of logical CPUs for the caller. + * + * The Itanium 2 9000-series parts are CGMT -- the two logical CPUs of a + * core share one set of execution resources -- so a decode that fills + * every logical CPU leaves the calling application's own threads nothing + * to run on. In a player that thread is the one uploading and presenting + * frames, and delaying it does more than hurt pacing directly: on the + * r300/PCIe-x1 hardware this port targets, an upload that arrives while + * the GPU is still reading the previous frame makes Mesa rename the + * texture rather than stall, and the fresh buffer lands in GTT, where the + * GPU must DMA-read it at ~34 MB/s. That is self-sustaining -- the slow + * frame keeps the GPU busy for the next upload too -- so the whole run + * latches into a mode roughly three times too slow to present 1080p30. + * + * Measured with ffplay on a 1080p30 AV1 clip: filling all 8 logical CPUs + * latched into that mode in 3 of 5 runs and dropped 21-24 frames even in + * the runs that stayed fast; reserving two logical CPUs stayed fast in + * 10 of 10 runs and dropped 11-15. Average decode throughput is not the + * constraint here -- decode alone runs this clip at 1.56x realtime. */ + if (ia64_cgmt && !s->n_threads && n_lp > 2) + *n_tc = n_lp - 2; +#endif *n_fc = s->max_frame_delay ? umin(s->max_frame_delay, *n_tc) : *n_tc < 50 ? fc_lut[*n_tc - 1] : 8; // min(8, ceil(sqrt(n))) +#if ARCH_IA64 + /* ia64: widen the frame pipeline past ceil(sqrt(n_tc)). + * + * The default ceil(sqrt(n)) gives only 3 frame contexts at 6 threads, and + * 3 frames of pipeline is not enough slack here. This decoder is fast + * enough on average -- the clip below decodes at 1.9x realtime -- but its + * frame cost is very bimodal, and a player's own output queue is shallow + * (ffplay holds 3 pictures), so it cannot absorb the spikes on dav1d's + * behalf. Widening the frame pipeline lets the spikes overlap with the + * cheap frames around them instead of stalling output. + * + * Measured with ffplay on a 1080p30 AV1 clip, all runs display-clean: + * 3 frame contexts (the default at 6 threads) dropped 11-15 frames, 4 + * dropped 3-5, 6 dropped 1 in every run, and n_tc+4 = 10 dropped none in + * 6 of 6 runs. Going further is counterproductive -- 16 contexts costs + * enough extra start-up latency to drop the opening frame again -- so the + * pipeline is widened by a fixed 4 rather than as far as it will go. + * The cost is n_fc frame contexts of working memory instead of + * ceil(sqrt(n_tc)). */ + if (ia64_cgmt && !s->max_frame_delay) + *n_fc = umin(*n_tc + 4, DAV1D_MAX_FRAME_DELAY); +#endif } COLD int dav1d_get_frame_delay(const Dav1dSettings *const s) { diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/loopfilter_tmpl.c dav1d-1.5.4/src/loopfilter_tmpl.c --- dav1d-1.5.4.orig/src/loopfilter_tmpl.c 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/loopfilter_tmpl.c 2026-08-19 16:28:50.382804220 +0200 @@ -249,6 +249,8 @@ #include "src/arm/loopfilter.h" #elif ARCH_LOONGARCH64 #include "src/loongarch/loopfilter.h" +#elif ARCH_IA64 +#include "src/ia64/loopfilter.h" #elif ARCH_PPC64LE #include "src/ppc/loopfilter.h" #elif ARCH_X86 @@ -267,6 +269,8 @@ loop_filter_dsp_init_arm(c); #elif ARCH_LOONGARCH64 loop_filter_dsp_init_loongarch(c); +#elif ARCH_IA64 + loop_filter_dsp_init_ia64(c); #elif ARCH_PPC64LE loop_filter_dsp_init_ppc(c); #elif ARCH_X86 diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/looprestoration_tmpl.c dav1d-1.5.4/src/looprestoration_tmpl.c --- dav1d-1.5.4.orig/src/looprestoration_tmpl.c 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/looprestoration_tmpl.c 2026-08-17 21:54:30.647586542 +0200 @@ -1355,6 +1355,8 @@ #include "src/arm/looprestoration.h" #elif ARCH_LOONGARCH64 #include "src/loongarch/looprestoration.h" +#elif ARCH_IA64 +#include "src/ia64/looprestoration.h" #elif ARCH_PPC64LE #include "src/ppc/looprestoration.h" #elif ARCH_X86 @@ -1375,6 +1377,8 @@ loop_restoration_dsp_init_arm(c, bpc); #elif ARCH_LOONGARCH64 loop_restoration_dsp_init_loongarch(c, bpc); +#elif ARCH_IA64 + loop_restoration_dsp_init_ia64(c, bpc); #elif ARCH_PPC64LE loop_restoration_dsp_init_ppc(c, bpc); #elif ARCH_X86 diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/mc_tmpl.c dav1d-1.5.4/src/mc_tmpl.c --- dav1d-1.5.4.orig/src/mc_tmpl.c 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/mc_tmpl.c 2026-08-17 21:40:10.192889405 +0200 @@ -948,6 +948,8 @@ #include "src/arm/mc.h" #elif ARCH_LOONGARCH64 #include "src/loongarch/mc.h" +#elif ARCH_IA64 +#include "src/ia64/mc.h" #elif ARCH_PPC64LE #include "src/ppc/mc.h" #elif ARCH_RISCV @@ -995,6 +997,8 @@ mc_dsp_init_arm(c); #elif ARCH_LOONGARCH64 mc_dsp_init_loongarch(c); +#elif ARCH_IA64 + mc_dsp_init_ia64(c); #elif ARCH_PPC64LE mc_dsp_init_ppc(c); #elif ARCH_RISCV diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/meson.build dav1d-1.5.4/src/meson.build --- dav1d-1.5.4.orig/src/meson.build 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/meson.build 2026-08-25 18:25:36.933694825 +0200 @@ -249,6 +249,66 @@ 'loongarch/itx.S', ) libdav1d_asm_objs += libdav1d_sources_asm + elif host_machine.cpu_family() == 'ia64' + libdav1d_sources += files( + 'ia64/cpu.c', + ) + # Tier III kernels: standalone .S with their own register frame, so + # they can be modulo-scheduled. Bitdepth-independent, assembled once. + # + # These are *generated* -- the modulo scheduler and the per-kernel op + # lists live in ia64/gen, and the .S files are build artifacts that are + # not checked in. Editing generated assembly by hand would be silently + # discarded on the next build; edit the generator instead. See + # ia64/gen/README.md. + ia64_python = import('python').find_installation() + # Every generator imports these, so a change to any of them invalidates + # all three outputs. + ia64_gen_deps = files( + 'ia64/gen/modsched.py', + 'ia64/gen/sched.py', + 'ia64/gen/sgr_asm.py', + 'ia64/gen/fin_asm.py', + 'ia64/gen/boxh_asm.py', + ) + libdav1d_sources_asm = [] + foreach kernel : [ + ['looprestoration_sgr.S', 'gensgr_s.py'], + ['looprestoration_fin.S', 'genfin_s.py'], + ['looprestoration_box.S', 'genbox_s.py'], + ] + libdav1d_sources_asm += custom_target(kernel[0], + input: files('ia64/gen/' + kernel[1]), + output: kernel[0], + depend_files: ia64_gen_deps, + command: [ia64_python, '-B', '@INPUT@', '@OUTPUT@'], + ) + endforeach + # The .S files carry .explicit. Without it gas runs in auto mode, where + # it ignores their bundle templates and stops entirely and re-derives + # its own -- measured to cost a modulo-scheduled loop its whole + # initiation interval (a designed II=15 ran at 30 cycles). Check them + # with `as -x` when regenerating; the compiler driver does not. + libdav1d_asm_objs += libdav1d_sources_asm + # Tier II kernels are inline asm *inside* the C templates below: they + # were generated once and spliced in, and the C around them is + # maintained by hand. They are therefore the one place where the tree + # and its generator can drift apart, so they get an explicit check + # rather than a build step -- a block may legitimately have been + # hand-tuned after generation. Run it with: + # ninja ia64-check-generated + run_target('ia64-check-generated', + command: [ia64_python, '-B', + files('ia64/gen/verify_tier2.py'), + meson.current_source_dir() / 'ia64' / 'gen'], + ) + # The packed integer ops are base IA-64 ISA, so no extra cflags and + # no runtime dispatch tier -- everything goes in the default group. + libdav1d_arch_tmpl_sources += {'default': files( + 'ia64/loopfilter_tmpl.c', + 'ia64/looprestoration_tmpl.c', + 'ia64/mc_tmpl.c', + )} elif host_machine.cpu() == 'ppc64le' arch_flags += {'vsx': ['-maltivec', '-mvsx', '-DDAV1D_VSX']} libdav1d_sources += files( diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/src/refmvs.c dav1d-1.5.4/src/refmvs.c --- dav1d-1.5.4.orig/src/refmvs.c 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/src/refmvs.c 2026-08-21 10:22:27.791366722 +0200 @@ -172,16 +172,25 @@ } } -static inline union mv mv_projection(const union mv mv, const int num, const int den) { - static const uint16_t div_mult[32] = { - 0, 16384, 8192, 5461, 4096, 3276, 2730, 2340, - 2048, 1820, 1638, 1489, 1365, 1260, 1170, 1092, - 1024, 963, 910, 862, 819, 780, 744, 712, - 682, 655, 630, 606, 585, 564, 546, 528 - }; +static const uint16_t mv_proj_div_mult[32] = { + 0, 16384, 8192, 5461, 4096, 3276, 2730, 2340, + 2048, 1820, 1638, 1489, 1365, 1260, 1170, 1092, + 1024, 963, 910, 862, 819, 780, 744, 712, + 682, 655, 630, 606, 585, 564, 546, 528 +}; + +/* num * div_mult[den], split out so callers with a loop-invariant (num, den) + * can hoist it. That matters more than it looks on targets without an integer + * multiplier in the integer unit -- on ia64 every one of these is a + * setf.sig / xma.l / getf.sig round trip through the FP register file, and + * this one feeds the two below, so it sits on the critical path twice over. */ +static inline int mv_proj_frac(const int num, const int den) { assert(den > 0 && den < 32); assert(num > -32 && num < 32); - const int frac = num * div_mult[den]; + return num * mv_proj_div_mult[den]; +} + +static inline union mv mv_projection_frac(const union mv mv, const int frac) { const int y = mv.y * frac, x = mv.x * frac; // Round and clip according to AV1 spec section 7.9.3 return (union mv) { // 0x3fff == (1 << 14) - 1 @@ -190,6 +199,10 @@ }; } +static inline union mv mv_projection(const union mv mv, const int num, const int den) { + return mv_projection_frac(mv, mv_proj_frac(num, den)); +} + static void add_temporal_candidate(const refmvs_frame *const rf, refmvs_candidate *const mvstack, int *const cnt, const refmvs_temporal_block *const rb, @@ -714,6 +727,13 @@ const int ref = rf->mfmv_ref[n]; const int ref_sign = ref - 4; + /* frac depends only on (n, b_ref), and b_ref indexes a seven-entry + * table -- so the whole thing is known before the scan starts. */ + int frac_of_ref[8]; + for (int b = 1; b < 8; b++) { + const int r2r = rf->mfmv_ref2ref[n][b - 1]; + frac_of_ref[b] = r2r ? mv_proj_frac(ref2cur, r2r) : 0; + } const refmvs_temporal_block *r = &rf->rp_ref[ref][row_start8 * stride]; for (int y = row_start8; y < row_end8; y++) { const int y_sb_align = y & ~7; @@ -726,7 +746,7 @@ const int ref2ref = rf->mfmv_ref2ref[n][b_ref - 1]; if (!ref2ref) continue; const mv b_mv = rb->mv; - const mv offset = mv_projection(b_mv, ref2cur, ref2ref); + const mv offset = mv_projection_frac(b_mv, frac_of_ref[b_ref]); int pos_x = x + apply_sign(abs(offset.x) >> 6, offset.x ^ ref_sign); const int pos_y = y + apply_sign(abs(offset.y) >> 6, diff -urN '--exclude=build-base' '--exclude=build-ia64' '--exclude=.git' dav1d-1.5.4.orig/tests/checkasm/checkasm.c dav1d-1.5.4/tests/checkasm/checkasm.c --- dav1d-1.5.4.orig/tests/checkasm/checkasm.c 2026-07-14 08:39:09.000000000 +0200 +++ dav1d-1.5.4/tests/checkasm/checkasm.c 2026-08-17 21:40:10.193397511 +0200 @@ -79,6 +79,8 @@ #elif ARCH_LOONGARCH { "LSX", "lsx", DAV1D_LOONGARCH_CPU_FLAG_LSX }, { "LASX", "lasx", DAV1D_LOONGARCH_CPU_FLAG_LASX }, +#elif ARCH_IA64 + { "SIMD", "simd", DAV1D_IA64_CPU_FLAG_SIMD }, #elif ARCH_PPC64LE { "VSX", "vsx", DAV1D_PPC_CPU_FLAG_VSX }, { "PWR9", "pwr9", DAV1D_PPC_CPU_FLAG_PWR9 },