# --- T2-COPYRIGHT-BEGIN --- # t2/package/*/firefox/hotfix-swgl-ia64-tier3-r8-kernels.patch # Copyright (C) 2026 The T2 SDE Project # SPDX-License-Identifier: GPL-2.0 or patched project license # --- T2-COPYRIGHT-END --- # Tier III (dav1d's terminology, see package/multimedia/dav1d/ia64-simd.patch) # kernels for the R8 bilinear span with a constant row: a standalone assembly # file with its own register frame, modulo-scheduled with rotating registers # and br.ctop. One op list, four entry points that differ in the commit: # # mask mask bytes only; the generic C commit runs after it (any blend key) # bcast the mask in every channel (no color, no blend) # color muldiv255(color, mask) # over that, blended premultiplied source-over onto the destination # # The fused ones run only for a uniform color (NoColor is passed as 255, which # is exact) and no blend or exactly premultiplied source-over; clip-mask, AA # and other blend keys keep the mask kernel. ps_text_run's grayscale glyphs and # ps_quad_textured's masks are what reach this path. # # The .S is generated by a scheduler kept in the benchmark tree, not here. It # models what the Itanium 2 manual and the PMU showed matters: nops take # dispersal ports (a nop.i past the second I slot splits the issue), getf # issues only on M2, multimedia -> multimedia latency is effectively 3, and # framebuffer reads are scheduled as L2 hits with an lfetch ahead. It also # saves pr, because mov pr.rot clobbers p16-p63, which the ABI preserves. # # Itanium 2 (Montecito), cycles per pixel, byte-identical output; x86-64 for # scale is the same benchmark on a Ryzen 5900HX: # C path Tier III x86-64 # R8 mask 0.5 texel/px 38.3 6.6 31.4 # R8 mask color -- 7.9 32.4 # R8 mask color blend 51.7 10.8 40.1 --- firefox-158.0/gfx/wr/swgl/src/swgl_ext.h.vanilla +++ firefox-158.0/gfx/wr/swgl/src/swgl_ext.h @@ -631,6 +631,57 @@ #define swgl_commitTextureLinearColorR8(s, p, uv_rect, color) \ swgl_commitTextureLinear(R8, s, p, uv_rect, color, swgl_SpanLength) +#ifdef __ia64__ +// Tier III kernels for the constant-row R8 loop below, generated by +// tools/ia64-gen/genr8row_s.py. Each runs n four-pixel steps of +// textureLinearUnpackedR8 and advances x[] by n steps; they differ in the +// commit: +// mask writes the mask bytes, for the generic commit +// bcast expand_mask: the mask in every channel (no color, no blend) +// color muldiv255(color, mask) +// over that, blended premultiplied source-over onto dst +// The generator hard-codes this layout. +struct IA64R8Row { + float x[4]; + float step, min_x, max_x; + int32_t width2; + uint64_t fracy; // the y fraction in all four 16-bit lanes + uint64_t color; // one pixel of the packed color, 16-bit lanes +}; +static_assert(__builtin_offsetof(IA64R8Row, step) == 16 && + __builtin_offsetof(IA64R8Row, width2) == 28 && + __builtin_offsetof(IA64R8Row, fracy) == 32 && + __builtin_offsetof(IA64R8Row, color) == 40, + "IA64R8Row layout is hard-coded in genr8row_s.py"); +# define IA64_R8_KERNEL(variant) \ + extern "C" void swgl_r8_linear_row_##variant##_ia64( \ + uint32_t* dst, const uint8_t* row0, const uint8_t* row1, long n, \ + IA64R8Row* params) +IA64_R8_KERNEL(mask); +IA64_R8_KERNEL(bcast); +IA64_R8_KERNEL(color); +IA64_R8_KERNEL(over); +# undef IA64_R8_KERNEL + +// The fused kernels take one pixel of color. NoColor is exactly a color of +// 255: (255 * m + 255) >> 8 == m for every m in 0..255. +static ALWAYS_INLINE bool ia64_r8_color(NoColor, uint64_t& c) { + c = 0x00FF00FF00FF00FFULL; + return true; +} +static ALWAYS_INLINE bool ia64_r8_color(WideRGBA8 color, uint64_t& c) { + struct Quads { + uint64_t q[4]; + } p = bit_cast(color); + c = p.q[0]; + return p.q[1] == c && p.q[2] == c && p.q[3] == c; +} +template +static ALWAYS_INLINE bool ia64_r8_color(C, uint64_t&) { + return false; +} +#endif + // Helper function that samples from an R8 texture while expanding it to support // a differing framebuffer format. template @@ -663,6 +714,46 @@ : 0); I16 fracy = int16_t(yi & 0x7F); int width = sampler->width; + // The kernels clamp x offsets in signed 16-bit lanes. + if (width <= 32767) { + IA64R8Row params = {{x0, x1, x2, x3}, + uv_step.x, + min_uv.x, + max_uv.x, + width - 2, + uint64_t(yi & 0x7F) * 0x0001000100010001ULL, + 0}; + if constexpr (sizeof(P) == 4) { + // Commit inside the kernel when it is one the kernels implement: + // a uniform color, and no blend or exactly premultiplied source-over + // (not the clip-mask or AA variants of that key). + if (ia64_r8_color(color, params.color) && + (!BLEND || + blend_key == BLEND_KEY(GL_ONE, GL_ONE_MINUS_SRC_ALPHA))) { + long n = (span + 3) / 4; + if (BLEND) { + swgl_r8_linear_row_over_ia64(buf, row0, row1, n, ¶ms); + } else if (std::is_same_v) { + swgl_r8_linear_row_bcast_ia64(buf, row0, row1, n, ¶ms); + } else { + swgl_r8_linear_row_color_ia64(buf, row0, row1, n, ¶ms); + } + return span; + } + } + uint32_t mask[256]; + for (P* end = buf + span; buf < end;) { + int n = min(int(end - buf + 3) / 4, 256); + swgl_r8_linear_row_mask_ia64(mask, row0, row1, n, ¶ms); + for (int k = 0; k < n; k++, buf += swgl_StepSize) { + commit_blend_span( + buf, applyColor(expand_mask(buf, unpack(bit_cast( + mask[k]))), + color)); + } + } + return span; + } // GCC does not modulo-schedule here, and one iteration is a single // dependency chain from the float clamp through the byte loads to the // store, so the loop is pipelined by hand in three stages: the tap --- firefox-158.0/gfx/wr/swgl/build.rs.vanilla +++ firefox-158.0/gfx/wr/swgl/build.rs @@ -260,6 +260,14 @@ } } + // ia64: the R8 span kernels are a standalone, modulo-scheduled assembly + // file (generated outside Firefox; see its header). The compiler driver + // assembles it with the same flags, into the same archive as gl.cc. + if std::env::var("CARGO_CFG_TARGET_ARCH").as_deref() == Ok("ia64") { + println!("cargo:rerun-if-changed=src/swgl_r8row_ia64.S"); + build.file("src/swgl_r8row_ia64.S"); + } + build .file("src/gl.cc") .define("_GLIBCXX_USE_CXX11_ABI", Some("0")) --- firefox-158.0/gfx/wr/swgl/src/swgl_r8row_ia64.S.vanilla +++ firefox-158.0/gfx/wr/swgl/src/swgl_r8row_ia64.S @@ -0,0 +1,1563 @@ +/* + * Tier III: SWGL R8 bilinear row kernels, modulo-scheduled. + * + * Generated by genr8row_s.py, which lives in tools/ia64-gen/ of the T2 SWGL + * ia64 benchmark tree (not in Firefox): regenerate rather than hand-edit, the + * schedule (stages, rotating register names, bundle pairs) is derived. + * + * void swgl_r8_linear_row__ia64(uint32_t* dst, const uint8_t* row0, + * const uint8_t* row1, long n, + * IA64R8Row* params); + * + * Each runs n four-pixel steps of textureLinearUnpackedR8() for a constant + * row and leaves the four x lanes advanced by n steps in params->x: + * + * mask writes the mask bytes to dst, for the generic C commit + * bcast writes the mask to every channel of dst (no color, no blend) + * color writes muldiv255(color, mask) to dst + * over blends muldiv255(color, mask) over dst, premultiplied source-over + * + * .explicit is mandatory: in gas's default auto mode the templates and stops + * below are ignored and re-derived. + */ + + .explicit + + .text + .align 32 + .global swgl_r8_linear_row_mask_ia64 + .hidden swgl_r8_linear_row_mask_ia64 + .proc swgl_r8_linear_row_mask_ia64 +swgl_r8_linear_row_mask_ia64: + .prologue + .save ar.pfs, r124 + alloc r124 = ar.pfs, 5, 90, 0, 56 + .save ar.lc, r125 + mov r125 = ar.lc + .save pr, r126 + mov r126 = pr + .body + /* the ins are r32-r36, inside the rotating window: copy them out + before the first br.ctop */ + mov r88 = r32 + mov r89 = r33 + mov r91 = r34 + mov r97 = r35 + mov r96 = r36 + ;; + adds r90 = 1, r89 + adds r92 = 1, r91 + mov r94 = 128 + adds r98 = 28, r96 + ;; + /* IA64R8Row: float x[4], step, min_x, max_x; int32 width2; + uint64 fracy (four lanes) */ + adds r93 = 0, r96 + ;; + ldfs f33 = [r93] + adds r93 = 4, r96 + ;; + ldfs f41 = [r93] + adds r93 = 8, r96 + ;; + ldfs f49 = [r93] + adds r93 = 12, r96 + ;; + ldfs f57 = [r93] + adds r93 = 16, r96 + ;; + ldfs f6 = [r93] + adds r93 = 20, r96 + ;; + ldfs f7 = [r93] + adds r93 = 24, r96 + ;; + ldfs f8 = [r93] + ld4 r93 = [r98], 4 + ;; + ld8 r95 = [r98], 8 + mux2 r93 = r93, 0 + mux2 r94 = r94, 0 + cmp.ge p6, p0 = r0, r97 + adds r97 = -1, r97 + ;; + (p6) br.cond.dpnt.few .Ldone_mask + ;; + mov ar.lc = r97 + mov r98 = 7 + ;; + mov ar.ec = r98 + mov pr.rot = 0x10000 + ;; +.Lloop_mask: + { .mfi + (p20) psub2 r123 = r76, r72 + (p16) fadd.s.s0 f32 = f33, f6 + (p17) shr r97 = r33, 7 + } + { .mfi + nop.m 0 + (p16) fmax.s0 f9 = f33, f7 + (p18) dep r114 = r41, r37, 16, 16 + } + ;; + { .mii + nop.m 0 + (p18) mix4.r r101 = r61, r59 + (p18) pmax2 r66 = r63, r65 + } + { .mmi + nop.m 0 + nop.m 0 + (p17) and r100 = 127, r39 + } + ;; + { .mfi + (p17) and r105 = 127, r43 + (p16) fadd.s.s0 f40 = f41, f6 + (p17) shr r99 = r39, 7 + } + { .mfi + nop.m 0 + (p16) fmax.s0 f11 = f41, f7 + (p18) mix4.r r116 = r53, r51 + } + ;; + { .mii + nop.m 0 + (p17) pmin2 r98 = r97, r93 + (p17) shr r106 = r43, 7 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mfi + (p17) and r109 = 127, r45 + (p16) fadd.s.s0 f48 = f49, f6 + (p18) mix4.r r117 = r57, r55 + } + { .mfi + nop.m 0 + (p16) fmax.s0 f13 = f49, f7 + (p17) dep r115 = r100, r35, 16, 16 + } + ;; + { .mii + nop.m 0 + (p17) pmin2 r104 = r99, r93 + (p17) shr r108 = r45, 7 + } + { .mfi + nop.m 0 + (p16) fmin.s0 f10 = f9, f8 + (p18) psub2 r121 = r101, r116 + } + ;; + { .mfi + (p17) add r100 = r91, r98 + (p16) fadd.s.s0 f56 = f57, f6 + (p17) pmin2 r107 = r106, r93 + } + { .mfi + (p17) add r102 = r92, r98 + (p16) fmax.s0 f14 = f57, f7 + (p17) dep r119 = r99, r97, 16, 16 + } + ;; + { .mii + (p17) ld1 r101 = [r100] + (p17) dep r118 = r109, r105, 16, 16 + (p17) add r97 = r89, r98 + } + { .mfi + (p17) ld1 r103 = [r102] + (p16) fmin.s0 f12 = f11, f8 + (p17) add r99 = r90, r98 + } + ;; + { .mii + (p17) add r98 = r89, r104 + (p17) pmin2 r113 = r108, r93 + (p17) dep r120 = r108, r106, 16, 16 + } + { .mfi + (p17) add r100 = r90, r104 + (p16) fmin.s0 f11 = f13, f8 + (p17) add r105 = r91, r104 + } + ;; + { .mii + (p17) ld1 r102 = [r100] + (p17) add r108 = r90, r107 + (p17) add r110 = r91, r107 + } + { .mfi + (p17) add r106 = r89, r107 + (p16) fcvt.fx.trunc.s0 f9 = f10 + (p17) add r111 = r92, r107 + } + ;; + { .mii + (p17) ld1 r109 = [r108] + (p18) mix4.r r48 = r47, r114 + (p17) mix4.r r62 = r118, r115 + } + { .mfi + (p17) ld1 r112 = [r111] + (p16) fmin.s0 f13 = f14, f8 + (p17) add r100 = r92, r104 + } + ;; + { .mii + (p17) add r104 = r89, r113 + (p17) mix4.r r118 = r120, r119 + (p17) add r107 = r91, r113 + } + { .mfi + (p17) add r111 = r90, r113 + (p16) fcvt.fx.trunc.s0 f10 = f12 + (p17) add r108 = r92, r113 + } + ;; + { .mii + (p17) ld1 r114 = [r107] + (p18) pmpyshr2 r122 = r121, r95, 0 + (p19) pshr2 r120 = r70, 7 + } + { .mmf + (p17) ld1 r115 = [r108] + nop.m 0 + (p16) fcvt.fx.trunc.s0 f12 = f11 + } + ;; + { .mii + (p16) getf.sig r32 = f9 + (p20) pmpyshr2 r77 = r123, r68, 0 + (p21) pshr2 r121 = r78, 7 + } + { .mmi + (p17) ld1 r107 = [r106] + (p17) ld1 r108 = [r104] + (p18) psub2 r119 = r117, r48 + } + ;; + { .mii + (p17) ld1 r106 = [r110] + (p17) dep r60 = r115, r112, 16, 16 + (p22) pack2.uss r123 = r80, r0 + } + { .mfi + (p17) ld1 r113 = [r111] + (p16) fcvt.fx.trunc.s0 f9 = f13 + (p17) pcmp2.gt r117 = r118, r93 + } + ;; + { .mii + (p16) getf.sig r38 = f10 + (p17) dep r46 = r108, r107, 16, 16 + (p18) pshr2 r110 = r122, 7 + } + { .mmi + (p17) ld1 r40 = [r98] + (p17) ld1 r104 = [r100] + (p19) padd2 r71 = r49, r120 + } + ;; + { .mii + (p16) getf.sig r42 = f12 + (p17) dep r52 = r113, r109, 16, 16 + (p17) dep r56 = r114, r106, 16, 16 + } + { .mmi + (p17) ld1 r98 = [r99] + (p17) ld1 r100 = [r105] + (p21) padd2 r79 = r73, r121 + } + ;; + { .mii + (p17) ld1 r36 = [r97] + (p18) pmpyshr2 r69 = r119, r95, 0 + (p17) dep r58 = r104, r103, 16, 16 + } + { .mmi + (p22) st4 [r88] = r123, 4 + nop.m 0 + (p17) and r64 = r94, r117 + } + ;; + { .mii + (p16) getf.sig r44 = f9 + (p17) dep r50 = r102, r98, 16, 16 + (p17) dep r54 = r100, r101, 16, 16 + } + { .mib + (p16) and r34 = 127, r32 + (p18) padd2 r74 = r116, r110 + br.ctop.sptk.few .Lloop_mask + } + ;; + /* after n + stages - 1 rotations the last x written to f[B] is at + f[B + stages] */ + adds r98 = 0, r96 + ;; + stfs [r98] = f39 + adds r98 = 4, r96 + ;; + stfs [r98] = f47 + adds r98 = 8, r96 + ;; + stfs [r98] = f55 + adds r98 = 12, r96 + ;; + stfs [r98] = f63 +.Ldone_mask: + mov ar.lc = r125 + mov pr = r126, -1 + mov ar.pfs = r124 + br.ret.sptk.many b0 + .endp swgl_r8_linear_row_mask_ia64 + + .text + .align 32 + .global swgl_r8_linear_row_bcast_ia64 + .hidden swgl_r8_linear_row_bcast_ia64 + .proc swgl_r8_linear_row_bcast_ia64 +swgl_r8_linear_row_bcast_ia64: + .prologue + .save ar.pfs, r106 + alloc r106 = ar.pfs, 5, 72, 0, 32 + .save ar.lc, r107 + mov r107 = ar.lc + .save pr, r108 + mov r108 = pr + .body + /* the ins are r32-r36, inside the rotating window: copy them out + before the first br.ctop */ + mov r64 = r32 + mov r65 = r33 + mov r67 = r34 + mov r77 = r35 + mov r72 = r36 + ;; + adds r66 = 1, r65 + adds r68 = 1, r67 + mov r70 = 128 + adds r78 = 28, r72 + ;; + /* IA64R8Row: float x[4], step, min_x, max_x; int32 width2; + uint64 fracy (four lanes) */ + adds r69 = 0, r72 + ;; + ldfs f33 = [r69] + adds r69 = 4, r72 + ;; + ldfs f39 = [r69] + adds r69 = 8, r72 + ;; + ldfs f45 = [r69] + adds r69 = 12, r72 + ;; + ldfs f51 = [r69] + adds r69 = 16, r72 + ;; + ldfs f6 = [r69] + adds r69 = 20, r72 + ;; + ldfs f7 = [r69] + adds r69 = 24, r72 + ;; + ldfs f8 = [r69] + ld4 r69 = [r78], 4 + ;; + ld8 r71 = [r78], 8 + adds r73 = 0, r64 + adds r74 = 4, r64 + adds r75 = 8, r64 + adds r76 = 12, r64 + mux2 r69 = r69, 0 + mux2 r70 = r70, 0 + cmp.ge p6, p0 = r0, r77 + adds r77 = -1, r77 + ;; + (p6) br.cond.dpnt.few .Ldone_bcast + ;; + mov ar.lc = r77 + mov r78 = 5 + ;; + mov ar.ec = r78 + mov pr.rot = 0x10000 + ;; +.Lloop_bcast: + { .mfi + (p18) psub2 r96 = r51, r49 + (p16) fadd.s.s0 f32 = f33, f6 + (p17) pmin2 r77 = r33, r69 + } + { .mfi + nop.m 0 + (p16) fmax.s0 f9 = f33, f7 + (p17) shr r88 = r41, 7 + } + ;; + { .mii + nop.m 0 + (p17) dep r97 = r39, r35, 16, 16 + (p20) unpack1.l r105 = r61, r61 + } + { .mmi + nop.m 0 + nop.m 0 + (p17) and r84 = 127, r45 + } + ;; + { .mfi + nop.m 0 + (p16) fadd.s.s0 f38 = f39, f6 + (p17) pmin2 r83 = r37, r69 + } + { .mfi + nop.m 0 + (p16) fmax.s0 f11 = f39, f7 + (p17) shr r94 = r45, 7 + } + ;; + { .mii + (p17) add r78 = r65, r77 + (p17) pmin2 r89 = r88, r69 + (p17) dep r100 = r37, r33, 16, 16 + } + { .mii + (p17) add r79 = r66, r77 + (p17) add r81 = r67, r77 + (p17) add r82 = r68, r77 + } + ;; + { .mfi + (p17) ld1 r77 = [r78] + (p16) fadd.s.s0 f44 = f45, f6 + (p18) pmpyshr2 r102 = r55, r71, 0 + } + { .mfi + (p17) ld1 r80 = [r79] + (p16) fmax.s0 f13 = f45, f7 + (p17) dep r98 = r84, r43, 16, 16 + } + ;; + { .mii + (p17) add r84 = r65, r83 + (p17) pmin2 r95 = r94, r69 + (p17) add r86 = r67, r83 + } + { .mfi + (p17) add r85 = r66, r83 + (p16) fmin.s0 f10 = f9, f8 + (p17) add r87 = r68, r83 + } + ;; + { .mfi + (p17) add r90 = r65, r89 + (p16) fadd.s.s0 f50 = f51, f6 + (p17) add r92 = r67, r89 + } + { .mfi + (p17) add r91 = r66, r89 + (p16) fmax.s0 f14 = f51, f7 + (p17) add r93 = r68, r89 + } + ;; + { .mii + (p17) ld1 r78 = [r81] + (p18) pmpyshr2 r104 = r96, r71, 0 + (p17) dep r101 = r94, r88, 16, 16 + } + { .mmf + (p17) ld1 r79 = [r82] + nop.m 0 + (p16) fmin.s0 f12 = f11, f8 + } + ;; + { .mii + (p17) ld1 r81 = [r84] + (p17) add r89 = r66, r95 + (p17) add r94 = r67, r95 + } + { .mfi + (p17) add r88 = r65, r95 + (p16) fmin.s0 f11 = f13, f8 + (p17) add r96 = r68, r95 + } + ;; + { .mii + (p17) ld1 r82 = [r85] + (p17) mix4.r r99 = r98, r97 + (p18) pshr2 r103 = r102, 7 + } + { .mmf + (p17) ld1 r83 = [r86] + nop.m 0 + (p16) fcvt.fx.trunc.s0 f9 = f10 + } + ;; + { .mii + (p17) ld1 r84 = [r87] + (p17) dep r95 = r81, r77, 16, 16 + (p17) mix4.r r97 = r101, r100 + } + { .mmf + (p17) ld1 r85 = [r90] + nop.m 0 + (p16) fmin.s0 f13 = f14, f8 + } + ;; + { .mii + (p17) ld1 r81 = [r91] + (p17) dep r87 = r82, r80, 16, 16 + (p17) dep r90 = r83, r78, 16, 16 + } + { .mmf + (p17) ld1 r86 = [r92] + nop.m 0 + (p16) fcvt.fx.trunc.s0 f10 = f12 + } + ;; + { .mii + (p17) ld1 r80 = [r93] + (p17) dep r91 = r84, r79, 16, 16 + (p18) pshr2 r98 = r104, 7 + } + { .mfi + (p17) ld1 r78 = [r88] + (p16) fcvt.fx.trunc.s0 f12 = f11 + (p18) padd2 r56 = r47, r103 + } + ;; + { .mii + (p16) getf.sig r77 = f9 + (p19) pshr2 r93 = r59, 7 + (p20) unpack2.l r100 = r105, r105 + } + { .mmi + (p17) ld1 r79 = [r89] + (p17) ld1 r82 = [r94] + (p17) pcmp2.gt r92 = r97, r69 + } + ;; + { .mii + (p17) ld1 r83 = [r96] + (p17) dep r84 = r78, r85, 16, 16 + (p20) unpack2.h r94 = r105, r105 + } + { .mmf + nop.m 0 + nop.m 0 + (p16) fcvt.fx.trunc.s0 f9 = f13 + } + ;; + { .mii + (p16) getf.sig r78 = f10 + (p17) dep r85 = r79, r81, 16, 16 + (p17) dep r88 = r82, r86, 16, 16 + } + { .mmi + nop.m 0 + nop.m 0 + (p18) padd2 r89 = r49, r98 + } + ;; + { .mii + (p16) getf.sig r40 = f12 + (p17) dep r81 = r83, r80, 16, 16 + (p20) shr.u r96 = r100, 32 + } + { .mii + (p20) st4 [r73] = r100, 16 + (p17) and r82 = r70, r92 + (p19) padd2 r86 = r57, r93 + } + ;; + { .mii + (p20) st4 [r75] = r94, 16 + (p17) mix4.r r46 = r84, r95 + (p20) shr.u r83 = r94, 32 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p16) getf.sig r44 = f9 + (p17) mix4.r r48 = r85, r87 + (p17) mix4.r r79 = r88, r90 + } + { .mii + nop.m 0 + (p16) and r34 = 127, r77 + (p18) psub2 r80 = r89, r56 + } + ;; + { .mii + (p20) st4 [r74] = r96, 16 + (p16) shr r32 = r77, 7 + (p17) mix4.r r50 = r81, r91 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p20) st4 [r76] = r83, 16 + (p17) pmax2 r52 = r99, r82 + (p19) pack2.uss r60 = r86, r0 + } + { .mmi + nop.m 0 + nop.m 0 + (p16) and r38 = 127, r78 + } + ;; + { .mii + (p16) and r42 = 127, r40 + (p18) pmpyshr2 r58 = r80, r53, 0 + (p16) shr r36 = r78, 7 + } + { .mib + nop.m 0 + (p17) psub2 r54 = r79, r46 + br.ctop.sptk.few .Lloop_bcast + } + ;; + /* after n + stages - 1 rotations the last x written to f[B] is at + f[B + stages] */ + adds r78 = 0, r72 + ;; + stfs [r78] = f37 + adds r78 = 4, r72 + ;; + stfs [r78] = f43 + adds r78 = 8, r72 + ;; + stfs [r78] = f49 + adds r78 = 12, r72 + ;; + stfs [r78] = f55 +.Ldone_bcast: + mov ar.lc = r107 + mov pr = r108, -1 + mov ar.pfs = r106 + br.ret.sptk.many b0 + .endp swgl_r8_linear_row_bcast_ia64 + + .text + .align 32 + .global swgl_r8_linear_row_color_ia64 + .hidden swgl_r8_linear_row_color_ia64 + .proc swgl_r8_linear_row_color_ia64 +swgl_r8_linear_row_color_ia64: + .prologue + .save ar.pfs, r110 + alloc r110 = ar.pfs, 5, 76, 0, 40 + .save ar.lc, r111 + mov r111 = ar.lc + .save pr, r112 + mov r112 = pr + .body + /* the ins are r32-r36, inside the rotating window: copy them out + before the first br.ctop */ + mov r72 = r32 + mov r73 = r33 + mov r75 = r34 + mov r86 = r35 + mov r80 = r36 + ;; + adds r74 = 1, r73 + adds r76 = 1, r75 + mov r78 = 128 + adds r87 = 28, r80 + ;; + /* IA64R8Row: float x[4], step, min_x, max_x; int32 width2; + uint64 fracy (four lanes) */ + adds r77 = 0, r80 + ;; + ldfs f33 = [r77] + adds r77 = 4, r80 + ;; + ldfs f38 = [r77] + adds r77 = 8, r80 + ;; + ldfs f43 = [r77] + adds r77 = 12, r80 + ;; + ldfs f48 = [r77] + adds r77 = 16, r80 + ;; + ldfs f6 = [r77] + adds r77 = 20, r80 + ;; + ldfs f7 = [r77] + adds r77 = 24, r80 + ;; + ldfs f8 = [r77] + ld4 r77 = [r87], 4 + ;; + ld8 r79 = [r87], 8 + adds r82 = 0, r72 + adds r83 = 4, r72 + adds r84 = 8, r72 + adds r85 = 12, r72 + ;; + ld8 r81 = [r87] + mux2 r77 = r77, 0 + mux2 r78 = r78, 0 + cmp.ge p6, p0 = r0, r86 + adds r86 = -1, r86 + ;; + (p6) br.cond.dpnt.few .Ldone_color + ;; + mov ar.lc = r86 + mov r87 = 4 + ;; + mov ar.ec = r87 + mov pr.rot = 0x10000 + ;; +.Lloop_color: + { .mfi + (p17) add r86 = r73, r41 + (p16) fadd.s.s0 f32 = f33, f6 + (p17) pmin2 r96 = r47, r77 + } + { .mfi + (p17) add r88 = r74, r41 + (p16) fmax.s0 f9 = f33, f7 + (p17) dep r102 = r47, r43, 16, 16 + } + ;; + { .mii + (p17) add r87 = r75, r41 + (p17) add r89 = r73, r45 + (p17) add r90 = r74, r45 + } + { .mii + (p17) add r91 = r76, r41 + (p17) add r94 = r75, r45 + (p17) add r92 = r76, r45 + } + ;; + { .mfi + (p17) ld1 r93 = [r90] + (p16) fadd.s.s0 f37 = f38, f6 + (p17) mix4.r r104 = r51, r49 + } + { .mfi + (p17) ld1 r95 = [r92] + (p16) fmax.s0 f11 = f38, f7 + (p18) psub2 r106 = r59, r57 + } + ;; + { .mii + (p17) ld1 r90 = [r87] + (p17) add r97 = r73, r96 + (p17) add r98 = r74, r96 + } + { .mii + (p17) ld1 r92 = [r89] + (p17) add r100 = r75, r96 + (p17) add r101 = r76, r96 + } + ;; + { .mfi + (p17) ld1 r96 = [r97] + (p16) fadd.s.s0 f42 = f43, f6 + (p17) mix4.r r105 = r102, r53 + } + { .mfi + (p17) ld1 r99 = [r98] + (p16) fmax.s0 f13 = f43, f7 + (p19) pshr2.u r109 = r65, 8 + } + ;; + { .mii + (p17) ld1 r87 = [r86] + (p18) pmpyshr2 r107 = r106, r55, 0 + (p17) dep r103 = r90, r37, 16, 16 + } + { .mfi + (p17) ld1 r89 = [r88] + (p16) fmin.s0 f10 = f9, f8 + (p19) padd2 r108 = r63, r81 + } + ;; + { .mfi + (p17) ld1 r90 = [r94] + (p16) fadd.s.s0 f47 = f48, f6 + (p17) dep r98 = r96, r92, 16, 16 + } + { .mfi + (p17) ld1 r97 = [r101] + (p16) fmax.s0 f14 = f48, f7 + (p17) dep r102 = r99, r93, 16, 16 + } + ;; + { .mii + (p17) ld1 r86 = [r35] + (p19) pmpyshr2 r101 = r81, r67, 0 + (p17) dep r93 = r87, r33, 16, 16 + } + { .mfi + (p17) ld1 r92 = [r100] + (p16) fmin.s0 f12 = f11, f8 + (p17) pcmp2.gt r96 = r105, r77 + } + ;; + { .mii + (p17) ld1 r87 = [r39] + (p17) dep r99 = r97, r95, 16, 16 + (p18) pshr2 r100 = r107, 7 + } + { .mmf + (p17) ld1 r88 = [r91] + nop.m 0 + (p16) fmin.s0 f11 = f13, f8 + } + ;; + { .mii + nop.m 0 + (p17) dep r91 = r89, r86, 16, 16 + (p17) dep r94 = r92, r90, 16, 16 + } + { .mmf + nop.m 0 + nop.m 0 + (p16) fcvt.fx.trunc.s0 f9 = f10 + } + ;; + { .mii + (p17) and r90 = r78, r96 + (p17) mix4.r r95 = r98, r93 + (p17) dep r89 = r88, r87, 16, 16 + } + { .mfi + nop.m 0 + (p16) fmin.s0 f13 = f14, f8 + (p19) padd2 r105 = r101, r81 + } + ;; + { .mii + nop.m 0 + (p19) pshr2.u r98 = r61, 8 + (p19) pshr2.u r97 = r108, 8 + } + { .mfi + nop.m 0 + (p16) fcvt.fx.trunc.s0 f10 = f12 + (p18) padd2 r92 = r57, r100 + } + ;; + { .mii + nop.m 0 + (p17) mix4.r r96 = r102, r91 + (p17) mix4.r r87 = r94, r103 + } + { .mmf + nop.m 0 + nop.m 0 + (p16) fcvt.fx.trunc.s0 f12 = f11 + } + ;; + { .mii + (p16) getf.sig r86 = f9 + (p17) mix4.r r88 = r99, r89 + (p17) pmax2 r54 = r104, r90 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + nop.m 0 + (p18) mux2 r93 = r92, 0x00 + (p18) mux2 r94 = r92, 0xaa + } + { .mmf + nop.m 0 + nop.m 0 + (p16) fcvt.fx.trunc.s0 f9 = f13 + } + ;; + { .mii + (p16) getf.sig r89 = f10 + (p18) mux2 r102 = r92, 0x55 + (p18) mux2 r66 = r92, 0xff + } + { .mmi + nop.m 0 + nop.m 0 + (p17) psub2 r90 = r87, r95 + } + ;; + { .mii + (p16) getf.sig r92 = f12 + (p19) pack2.uss r100 = r109, r0 + (p19) pshr2.u r106 = r105, 8 + } + { .mmi + nop.m 0 + nop.m 0 + (p17) psub2 r91 = r88, r96 + } + ;; + { .mii + nop.m 0 + (p18) pmpyshr2 r104 = r81, r94, 0 + (p19) pack2.uss r103 = r97, r0 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p16) getf.sig r94 = f9 + (p17) pmpyshr2 r97 = r90, r79, 0 + (p19) pack2.uss r101 = r98, r0 + } + { .mmi + nop.m 0 + nop.m 0 + (p16) and r88 = 127, r86 + } + ;; + { .mii + (p19) st4 [r84] = r100, 16 + (p17) pmpyshr2 r99 = r91, r79, 0 + (p16) shr r87 = r86, 7 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p19) st4 [r83] = r103, 16 + (p18) pmpyshr2 r100 = r81, r93, 0 + (p19) pack2.uss r105 = r106, r0 + } + { .mii + nop.m 0 + (p16) and r91 = 127, r89 + (p18) padd2 r64 = r104, r81 + } + ;; + { .mii + (p19) st4 [r82] = r101, 16 + (p16) shr r90 = r89, 7 + (p17) pshr2 r98 = r97, 7 + } + { .mmi + nop.m 0 + nop.m 0 + (p16) and r93 = 127, r92 + } + ;; + { .mii + nop.m 0 + (p16) pmin2 r86 = r87, r77 + (p16) shr r42 = r92, 7 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p19) st4 [r85] = r105, 16 + (p16) dep r48 = r91, r88, 16, 16 + (p17) pshr2 r97 = r99, 7 + } + { .mii + nop.m 0 + (p16) and r92 = 127, r94 + (p18) padd2 r60 = r100, r81 + } + ;; + { .mii + nop.m 0 + (p16) pmin2 r40 = r90, r77 + (p16) shr r46 = r94, 7 + } + { .mmi + nop.m 0 + nop.m 0 + (p17) padd2 r56 = r95, r98 + } + ;; + { .mii + (p16) add r88 = r73, r86 + (p16) pmin2 r44 = r42, r77 + (p16) dep r52 = r90, r87, 16, 16 + } + { .mii + (p16) add r34 = r74, r86 + (p16) add r89 = r75, r86 + (p16) add r38 = r76, r86 + } + ;; + { .mii + (p16) ld1 r32 = [r88] + (p18) pmpyshr2 r62 = r81, r102, 0 + (p16) dep r50 = r92, r93, 16, 16 + } + { .mib + (p16) ld1 r36 = [r89] + (p17) padd2 r58 = r96, r97 + br.ctop.sptk.few .Lloop_color + } + ;; + /* after n + stages - 1 rotations the last x written to f[B] is at + f[B + stages] */ + adds r87 = 0, r80 + ;; + stfs [r87] = f36 + adds r87 = 4, r80 + ;; + stfs [r87] = f41 + adds r87 = 8, r80 + ;; + stfs [r87] = f46 + adds r87 = 12, r80 + ;; + stfs [r87] = f51 +.Ldone_color: + mov ar.lc = r111 + mov pr = r112, -1 + mov ar.pfs = r110 + br.ret.sptk.many b0 + .endp swgl_r8_linear_row_color_ia64 + + .text + .align 32 + .global swgl_r8_linear_row_over_ia64 + .hidden swgl_r8_linear_row_over_ia64 + .proc swgl_r8_linear_row_over_ia64 +swgl_r8_linear_row_over_ia64: + .prologue + .save ar.pfs, r125 + alloc r125 = ar.pfs, 5, 91, 0, 48 + .save ar.lc, r126 + mov r126 = ar.lc + .save pr, r127 + mov r127 = pr + .body + /* the ins are r32-r36, inside the rotating window: copy them out + before the first br.ctop */ + mov r80 = r32 + mov r81 = r33 + mov r83 = r34 + mov r99 = r35 + mov r88 = r36 + ;; + adds r82 = 1, r81 + adds r84 = 1, r83 + mov r86 = 128 + adds r100 = 28, r88 + ;; + /* IA64R8Row: float x[4], step, min_x, max_x; int32 width2; + uint64 fracy (four lanes) */ + adds r85 = 0, r88 + ;; + ldfs f33 = [r85] + adds r85 = 4, r88 + ;; + ldfs f38 = [r85] + adds r85 = 8, r88 + ;; + ldfs f43 = [r85] + adds r85 = 12, r88 + ;; + ldfs f48 = [r85] + adds r85 = 16, r88 + ;; + ldfs f6 = [r85] + adds r85 = 20, r88 + ;; + ldfs f7 = [r85] + adds r85 = 24, r88 + ;; + ldfs f8 = [r85] + ld4 r85 = [r100], 4 + ;; + ld8 r87 = [r100], 8 + adds r90 = 0, r80 + adds r94 = 0, r80 + adds r91 = 4, r80 + adds r95 = 4, r80 + adds r92 = 8, r80 + adds r96 = 8, r80 + adds r93 = 12, r80 + adds r97 = 12, r80 + adds r98 = 512, r80 + ;; + ld8 r89 = [r100] + mux2 r85 = r85, 0 + mux2 r86 = r86, 0 + cmp.ge p6, p0 = r0, r99 + adds r99 = -1, r99 + ;; + (p6) br.cond.dpnt.few .Ldone_over + ;; + mov ar.lc = r99 + mov r100 = 4 + ;; + mov ar.ec = r100 + mov pr.rot = 0x10000 + ;; +.Lloop_over: + { .mfi + (p16) ld4 r108 = [r94], 16 + (p16) fadd.s.s0 f32 = f33, f6 + (p17) mix4.r r100 = r37, r35 + } + { .mfi + (p16) ld4 r111 = [r95], 16 + (p16) fmax.s0 f9 = f33, f7 + (p17) mix4.r r99 = r41, r39 + } + ;; + { .mii + (p16) ld4 r115 = [r96], 16 + (p18) pmpyshr2 r106 = r89, r51, 0 + (p17) mix4.r r101 = r45, r43 + } + { .mmi + (p16) ld4 r118 = [r97], 16 + nop.m 0 + (p17) and r102 = r86, r49 + } + ;; + { .mfi + (p16) lfetch [r98], 16 + (p16) fadd.s.s0 f37 = f38, f6 + (p18) pmpyshr2 r110 = r89, r58, 0 + } + { .mfi + nop.m 0 + (p16) fmax.s0 f11 = f38, f7 + (p19) pshr2.u r117 = r77, 8 + } + ;; + { .mii + nop.m 0 + (p18) pmpyshr2 r113 = r89, r65, 0 + (p19) pack2.uss r109 = r56, r0 + } + { .mmi + nop.m 0 + nop.m 0 + (p17) psub2 r104 = r99, r33 + } + ;; + { .mfi + (p17) psub2 r105 = r101, r100 + (p16) fadd.s.s0 f42 = f43, f6 + (p18) pmpyshr2 r116 = r89, r72, 0 + } + { .mfi + (p18) padd2 r107 = r106, r89 + (p16) fmax.s0 f13 = f43, f7 + (p17) pmax2 r103 = r47, r102 + } + ;; + { .mii + (p18) padd2 r101 = r110, r89 + (p19) pack2.uss r112 = r63, r0 + (p19) pack2.uss r114 = r70, r0 + } + { .mfi + nop.m 0 + (p16) fmin.s0 f10 = f9, f8 + (p19) psub2 r119 = r79, r117 + } + ;; + { .mfi + (p19) st4 [r90] = r109, 16 + (p16) fadd.s.s0 f47 = f48, f6 + (p17) pmpyshr2 r99 = r104, r87, 0 + } + { .mfi + (p18) padd2 r110 = r113, r89 + (p16) fmax.s0 f14 = f48, f7 + nop.i 0 + } + ;; + { .mii + nop.m 0 + (p17) pmpyshr2 r102 = r105, r87, 0 + (p18) pshr2.u r106 = r107, 8 + } + { .mfi + nop.m 0 + (p16) fmin.s0 f12 = f11, f8 + (p18) padd2 r109 = r116, r89 + } + ;; + { .mii + (p19) st4 [r91] = r112, 16 + (p16) unpack1.l r52 = r0, r108 + (p18) pshr2.u r105 = r101, 8 + } + { .mmf + (p19) st4 [r92] = r114, 16 + nop.m 0 + (p16) fmin.s0 f11 = f13, f8 + } + ;; + { .mii + nop.m 0 + (p17) pshr2 r101 = r99, 7 + (p18) pshr2.u r107 = r110, 8 + } + { .mmf + nop.m 0 + nop.m 0 + (p16) fcvt.fx.trunc.s0 f9 = f10 + } + ;; + { .mii + nop.m 0 + (p17) pshr2 r104 = r102, 7 + (p18) pshr2.u r110 = r109, 8 + } + { .mfi + nop.m 0 + (p16) fmin.s0 f13 = f14, f8 + (p18) padd2 r114 = r106, r54 + } + ;; + { .mii + nop.m 0 + (p18) mux2 r102 = r106, 0xff + (p16) unpack1.l r59 = r0, r111 + } + { .mfi + nop.m 0 + (p16) fcvt.fx.trunc.s0 f10 = f12 + (p18) padd2 r117 = r105, r61 + } + ;; + { .mii + (p17) padd2 r112 = r33, r101 + (p18) mux2 r106 = r105, 0xff + (p16) unpack1.l r66 = r0, r115 + } + { .mfi + nop.m 0 + (p16) fcvt.fx.trunc.s0 f12 = f11 + (p18) padd2 r124 = r107, r68 + } + ;; + { .mii + (p16) getf.sig r99 = f9 + (p18) mux2 r108 = r107, 0xff + (p16) unpack1.l r73 = r0, r118 + } + { .mii + nop.m 0 + (p17) padd2 r101 = r100, r104 + (p18) padd2 r78 = r110, r75 + } + ;; + { .mii + nop.m 0 + (p18) pmpyshr2 r100 = r54, r102, 0 + (p18) mux2 r113 = r110, 0xff + } + { .mmf + nop.m 0 + nop.m 0 + (p16) fcvt.fx.trunc.s0 f9 = f13 + } + ;; + { .mii + (p16) getf.sig r102 = f10 + (p18) pmpyshr2 r109 = r61, r106, 0 + (p19) pack2.uss r116 = r119, r0 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p16) getf.sig r104 = f12 + (p18) pmpyshr2 r111 = r68, r108, 0 + nop.i 0 + } + { .mmi + nop.m 0 + nop.m 0 + (p17) psub2 r105 = r101, r112 + } + ;; + { .mii + nop.m 0 + (p18) pmpyshr2 r115 = r75, r113, 0 + nop.i 0 + } + { .mmi + nop.m 0 + nop.m 0 + (p18) padd2 r107 = r100, r54 + } + ;; + { .mii + (p16) getf.sig r108 = f9 + nop.i 0 + nop.i 0 + } + { .mii + (p19) st4 [r93] = r116, 16 + (p16) and r101 = 127, r99 + (p18) padd2 r110 = r109, r61 + } + ;; + { .mii + nop.m 0 + (p17) pmpyshr2 r106 = r105, r103, 0 + (p16) shr r100 = r99, 7 + } + { .mmi + nop.m 0 + nop.m 0 + (p18) padd2 r121 = r111, r68 + } + ;; + { .mii + nop.m 0 + (p18) pshr2.u r111 = r107, 8 + nop.i 0 + } + { .mii + nop.m 0 + (p16) and r103 = 127, r102 + (p18) padd2 r76 = r115, r75 + } + ;; + { .mii + nop.m 0 + (p16) shr r105 = r102, 7 + (p18) pshr2.u r115 = r110, 8 + } + { .mmi + nop.m 0 + nop.m 0 + (p16) and r107 = 127, r104 + } + ;; + { .mii + nop.m 0 + (p16) pmin2 r99 = r100, r85 + (p16) shr r110 = r104, 7 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + nop.m 0 + (p16) dep r118 = r103, r101, 16, 16 + (p17) pshr2 r113 = r106, 7 + } + { .mii + nop.m 0 + (p16) and r109 = 127, r108 + (p18) psub2 r55 = r114, r111 + } + ;; + { .mii + nop.m 0 + (p16) pmin2 r106 = r105, r85 + (p16) shr r116 = r108, 7 + } + { .mmi + nop.m 0 + nop.m 0 + (p18) psub2 r62 = r117, r115 + } + ;; + { .mii + (p16) add r101 = r81, r99 + (p16) pmin2 r111 = r110, r85 + (p16) dep r120 = r105, r100, 16, 16 + } + { .mii + (p16) add r102 = r82, r99 + (p16) add r103 = r83, r99 + (p16) add r104 = r84, r99 + } + ;; + { .mii + (p16) ld1 r99 = [r101] + (p16) dep r119 = r109, r107, 16, 16 + (p18) pshr2.u r123 = r121, 8 + } + { .mmi + (p16) ld1 r100 = [r102] + nop.m 0 + (p17) padd2 r122 = r112, r113 + } + ;; + { .mii + (p16) ld1 r101 = [r103] + (p16) pmin2 r117 = r116, r85 + (p16) add r107 = r82, r106 + } + { .mii + (p16) add r105 = r81, r106 + (p16) add r108 = r83, r106 + (p16) add r109 = r84, r106 + } + ;; + { .mii + (p16) ld1 r102 = [r104] + (p16) add r112 = r81, r111 + (p16) add r113 = r82, r111 + } + { .mii + (p16) ld1 r103 = [r105] + (p16) add r114 = r83, r111 + (p16) add r115 = r84, r111 + } + ;; + { .mii + (p16) ld1 r104 = [r107] + (p16) mix4.r r46 = r119, r118 + (p16) dep r121 = r116, r110, 16, 16 + } + { .mmi + (p16) ld1 r105 = [r108] + nop.m 0 + (p18) psub2 r69 = r124, r123 + } + ;; + { .mii + (p16) ld1 r106 = [r109] + (p16) add r110 = r81, r117 + (p16) add r111 = r82, r117 + } + { .mii + (p16) ld1 r107 = [r112] + (p16) add r116 = r83, r117 + (p16) add r118 = r84, r117 + } + ;; + { .mii + (p16) ld1 r108 = [r113] + (p16) dep r112 = r103, r99, 16, 16 + (p16) dep r34 = r104, r100, 16, 16 + } + { .mmf + (p16) ld1 r109 = [r114] + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p16) ld1 r99 = [r115] + (p16) dep r38 = r105, r101, 16, 16 + (p16) dep r42 = r106, r102, 16, 16 + } + { .mmf + (p16) ld1 r100 = [r110] + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p16) ld1 r101 = [r111] + (p16) mix4.r r105 = r121, r120 + (p17) mux2 r50 = r122, 0x00 + } + { .mmf + (p16) ld1 r102 = [r116] + nop.m 0 + nop.f 0 + } + ;; + { .mii + (p16) ld1 r103 = [r118] + (p16) dep r104 = r100, r107, 16, 16 + (p17) mux2 r57 = r122, 0x55 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + nop.m 0 + (p16) dep r36 = r101, r108, 16, 16 + (p16) dep r40 = r102, r109, 16, 16 + } + { .mmf + nop.m 0 + nop.m 0 + nop.f 0 + } + ;; + { .mii + nop.m 0 + (p16) dep r44 = r103, r99, 16, 16 + (p17) mux2 r64 = r122, 0xaa + } + { .mmi + nop.m 0 + nop.m 0 + (p16) pcmp2.gt r48 = r105, r85 + } + ;; + { .mii + nop.m 0 + (p16) mix4.r r32 = r104, r112 + (p17) mux2 r71 = r122, 0xff + } + { .mfb + nop.m 0 + nop.f 0 + br.ctop.sptk.few .Lloop_over + } + ;; + /* after n + stages - 1 rotations the last x written to f[B] is at + f[B + stages] */ + adds r100 = 0, r88 + ;; + stfs [r100] = f36 + adds r100 = 4, r88 + ;; + stfs [r100] = f41 + adds r100 = 8, r88 + ;; + stfs [r100] = f46 + adds r100 = 12, r88 + ;; + stfs [r100] = f51 +.Ldone_over: + mov ar.lc = r126 + mov pr = r127, -1 + mov ar.pfs = r125 + br.ret.sptk.many b0 + .endp swgl_r8_linear_row_over_ia64