# --- T2-COPYRIGHT-BEGIN --- # t2/package/*/firefox/up-next-jit-0030-ia64.patch.ia64 # Copyright (C) 2026 The T2 SDE Project # SPDX-License-Identifier: GPL-2.0 or patched project license # --- T2-COPYRIGHT-END --- diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/moz.configure /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/moz.configure --- firefox-153.0.1/js/moz.configure.vanilla +++ firefox-153.0.1/js/moz.configure @@ -245,6 +245,7 @@ def jit_default(target, enable_portable_ "loongarch64", "ppc64", "riscv64", + "ia64", ): return True return False @@ -323,6 +324,8 @@ def jit_codegen(jit_enabled, simulator, return namespace(ppc64=True) elif target.cpu == "riscv64": return namespace(riscv64=True) + elif target.cpu == "ia64": + return namespace(ia64=True) return namespace(**{str(target.cpu): True}) @@ -334,6 +337,7 @@ set_config("JS_CODEGEN_MIPS64", jit_code set_config("JS_CODEGEN_LOONG64", jit_codegen.loong64) set_config("JS_CODEGEN_PPC64", jit_codegen.ppc64) set_config("JS_CODEGEN_RISCV64", jit_codegen.riscv64) +set_config("JS_CODEGEN_IA64", jit_codegen.ia64) set_config("JS_CODEGEN_X86", jit_codegen.x86) set_config("JS_CODEGEN_X64", jit_codegen.x64) set_config("JS_CODEGEN_WASM32", jit_codegen.wasm32) @@ -345,6 +349,7 @@ set_define("JS_CODEGEN_MIPS64", jit_code set_define("JS_CODEGEN_LOONG64", jit_codegen.loong64) set_define("JS_CODEGEN_PPC64", jit_codegen.ppc64) set_define("JS_CODEGEN_RISCV64", jit_codegen.riscv64) +set_define("JS_CODEGEN_IA64", jit_codegen.ia64) set_define("JS_CODEGEN_X86", jit_codegen.x86) set_define("JS_CODEGEN_X64", jit_codegen.x64) set_define("JS_CODEGEN_WASM32", jit_codegen.wasm32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/Assembler.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/Assembler.h --- firefox-153.0.1/js/src/jit/Assembler.h.vanilla +++ firefox-153.0.1/js/src/jit/Assembler.h @@ -19,6 +19,8 @@ # include "jit/loong64/Assembler-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/Assembler-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/Assembler-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/Assembler-ppc64.h" #elif defined(JS_CODEGEN_WASM32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/CodeGenerator.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/CodeGenerator.h --- firefox-153.0.1/js/src/jit/CodeGenerator.h.vanilla +++ firefox-153.0.1/js/src/jit/CodeGenerator.h @@ -23,6 +23,8 @@ # include "jit/loong64/CodeGenerator-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/CodeGenerator-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/CodeGenerator-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/CodeGenerator-ppc64.h" #elif defined(JS_CODEGEN_WASM32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/EffectiveAddressAnalysis.cpp /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/EffectiveAddressAnalysis.cpp --- firefox-153.0.1/js/src/jit/EffectiveAddressAnalysis.cpp.vanilla +++ firefox-153.0.1/js/src/jit/EffectiveAddressAnalysis.cpp @@ -62,6 +62,9 @@ static bool OffsetIsSmallEnough(int32_t #elif defined(JS_CODEGEN_RISCV64) || defined(JS_CODEGEN_LOONG64) || \ defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_PPC64) return imm >= -0xFFF && imm <= 0xFFF; +#elif defined(JS_CODEGEN_IA64) + // "adds" carries a signed 14-bit immediate; anything wider needs a movl. + return imm >= -8192 && imm <= 8191; #elif defined(JS_CODEGEN_WASM32) || defined(JS_CODEGEN_NONE) return true; #else diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/FlushICache.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/FlushICache.h --- firefox-153.0.1/js/src/jit/FlushICache.h.vanilla +++ firefox-153.0.1/js/src/jit/FlushICache.h @@ -29,6 +29,40 @@ // `FlushExecutionContext` below. extern void FlushICache(void* code, size_t size); +#elif defined(JS_CODEGEN_IA64) + +// IA-64 caches are not coherent with the instruction stream, so newly written +// code must be flushed before it can be fetched. __builtin___clear_cache() is +// not usable here: GCC has no clear_cache pattern for ia64 and expands it to +// nothing at all, so this has to be the architected sequence written out by +// hand. It mirrors the kernel's flush_icache_range(): `fc` over the range at +// the 32-byte minimum line size, then sync.i to make the flushes visible to +// instruction fetch, then srlz.i to serialise this core. Plain `fc` rather than +// `fc.i` because the latter is only architected from revision 2.1 onwards. +inline void FlushICache(void* code, size_t size) { + if (size == 0) { + return; + } + uintptr_t addr = reinterpret_cast(code) & ~uintptr_t(31); + uintptr_t end = reinterpret_cast(code) + size; + asm volatile( + "1:\n\t" + "fc %0\n\t" + "add %0 = 32, %0\n\t" + ";;\n\t" + "cmp.ltu p6, p7 = %0, %1\n\t" + ";;\n\t" + "(p6) br.cond.sptk.few 1b\n\t" + ";;\n\t" + "sync.i\n\t" + ";;\n\t" + "srlz.i\n\t" + ";;\n" + : "+r"(addr) + : "r"(end) + : "p6", "p7", "memory"); +} + #elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) inline void FlushICache(void* code, size_t size) { MOZ_CRASH(); } @@ -80,6 +114,14 @@ // else this will crash. extern void FlushExecutionContextForAllThreads(); +#elif defined(JS_CODEGEN_IA64) + +// FlushICache already serialises the issuing core via srlz.i, and ia64 offers +// no syscall to do so for other threads. +inline void FlushExecutionContext() {} +inline bool CanFlushExecutionContextForAllThreads() { return false; } +inline void FlushExecutionContextForAllThreads() { MOZ_CRASH(); } + #else # error "Unknown architecture!" #endif diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/LIR.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/LIR.h --- firefox-153.0.1/js/src/jit/LIR.h.vanilla +++ firefox-153.0.1/js/src/jit/LIR.h @@ -2301,6 +2301,8 @@ AnyRegister LAllocation::toAnyRegister() # include "jit/loong64/LIR-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/LIR-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/LIR-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/LIR-ppc64.h" #elif defined(JS_CODEGEN_MIPS64) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/firefox-153.0.1/js/src/jit/LIROps.yaml b/firefox-153.0.1/js/src/jit/LIROps.yaml --- firefox-153.0.1/js/src/jit/LIROps.yaml.vanilla +++ firefox-153.0.1/js/src/jit/LIROps.yaml @@ -2247,7 +2247,7 @@ value: WordSized # Additional temp that may be used on LL/SC platforms for the flag result of the store. # Needs additional temps on LL/SC platforms to extract/insert bits of word. -#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_IA64) num_temps: 1 #elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) || defined(JS_CODEGEN_PPC64) num_temps: 3 @@ -4480,6 +4480,97 @@ - name: UDiv result_type: WordSized operands: + lhs: WordSized + rhs: WordSized + mir_op: Div + +- name: UMod + result_type: WordSized + operands: + lhs: WordSized + rhs: WordSized + mir_op: Mod + +- name: DivI64 + result_type: Int64 + operands: + lhs: WordSized + rhs: WordSized + mir_op: Div + +- name: ModI64 + result_type: Int64 + operands: + lhs: WordSized + rhs: WordSized + mir_op: Mod + +- name: UDivI64 + result_type: Int64 + operands: + lhs: WordSized + rhs: WordSized + mir_op: Div + +- name: UModI64 + result_type: Int64 + operands: + lhs: WordSized + rhs: WordSized + mir_op: Mod + +- name: ModMaskI + result_type: WordSized + operands: + input: WordSized + arguments: + shift: int32_t + num_temps: 2 + mir_op: Mod + +- name: WasmTruncateToInt64 + result_type: Int64 + operands: + input: WordSized + mir_op: true + +- name: Int64ToFloatingPoint + result_type: WordSized + operands: + input: Int64 + mir_op: true + +- name: WasmCompareExchangeI64 + result_type: Int64 + operands: + ptr: WordSized + oldValue: Int64 + newValue: Int64 + memoryBase: WordSized + mir_op: WasmCompareExchangeHeap + +- name: WasmAtomicBinopI64 + result_type: Int64 + operands: + ptr: WordSized + value: Int64 + memoryBase: WordSized + num_temps64: 1 + mir_op: WasmAtomicBinopHeap + +- name: WasmAtomicExchangeI64 + result_type: Int64 + operands: + ptr: WordSized + value: Int64 + memoryBase: WordSized + mir_op: WasmAtomicExchangeHeap +#endif + +#ifdef JS_CODEGEN_IA64 +- name: UDiv + result_type: WordSized + operands: lhs: WordSized rhs: WordSized mir_op: Div diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/Lowering.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/Lowering.h --- firefox-153.0.1/js/src/jit/Lowering.h.vanilla +++ firefox-153.0.1/js/src/jit/Lowering.h @@ -23,6 +23,8 @@ # include "jit/loong64/Lowering-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/Lowering-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/Lowering-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/Lowering-ppc64.h" #elif defined(JS_CODEGEN_WASM32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/MacroAssembler-inl.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/MacroAssembler-inl.h --- firefox-153.0.1/js/src/jit/MacroAssembler-inl.h.vanilla +++ firefox-153.0.1/js/src/jit/MacroAssembler-inl.h @@ -41,6 +41,8 @@ # include "jit/riscv64/MacroAssembler-riscv64-inl.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/MacroAssembler-ppc64-inl.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/MacroAssembler-ia64-inl.h" #elif defined(JS_CODEGEN_WASM32) # include "jit/wasm32/MacroAssembler-wasm32-inl.h" #elif !defined(JS_CODEGEN_NONE) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -23,6 +23,8 @@ # include "jit/loong64/MacroAssembler-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/MacroAssembler-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/MacroAssembler-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/MacroAssembler-ppc64.h" #elif defined(JS_CODEGEN_WASM32) @@ -145,6 +147,7 @@ #define DEFINED_ON_riscv64 #define DEFINED_ON_ppc64 #define DEFINED_ON_wasm32 +#define DEFINED_ON_ia64 #define DEFINED_ON_none // Specialize for each architecture. @@ -176,6 +179,9 @@ #elif defined(JS_CODEGEN_PPC64) # undef DEFINED_ON_ppc64 # define DEFINED_ON_ppc64 define +#elif defined(JS_CODEGEN_IA64) +# undef DEFINED_ON_ia64 +# define DEFINED_ON_ia64 define #elif defined(JS_CODEGEN_WASM32) # undef DEFINED_ON_wasm32 # define DEFINED_ON_wasm32 define @@ -411,10 +417,12 @@ // MacroAssembler high-level usage. // Flushes the assembly buffer, on platforms that need it. - void flush() PER_SHARED_ARCH; + void flush() DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); // Add a comment that is visible in the pretty printed assembly code. - void comment(const char* msg) PER_SHARED_ARCH; + void comment(const char* msg) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); // =============================================================== // Frame manipulation functions. @@ -514,9 +522,11 @@ // layout. // The size of the area used by PushRegsInMask. - static size_t PushRegsInMaskSizeInBytes(LiveRegisterSet set) PER_SHARED_ARCH; + static size_t PushRegsInMaskSizeInBytes(LiveRegisterSet set) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); - void PushRegsInMask(LiveRegisterSet set) PER_SHARED_ARCH; + void PushRegsInMask(LiveRegisterSet set) DEFINED_ON(arm, arm64, loong64, + mips64, riscv64, ppc64, x86_shared, wasm32, ia64); void PushRegsInMask(LiveGeneralRegisterSet set); // Like PushRegsInMask, but instead of pushing the registers, store them to @@ -526,27 +536,36 @@ // PushRegsInMaskSizeInBytes for this |set|. In other words, |dest.base| // must point to either the lowest address in the save area, or some address // below that. - void storeRegsInMask(LiveRegisterSet set, Address dest, - Register scratch) PER_SHARED_ARCH; + void storeRegsInMask(LiveRegisterSet set, Address dest, Register scratch) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); void PopRegsInMask(LiveRegisterSet set); void PopRegsInMask(LiveGeneralRegisterSet set); - void PopRegsInMaskIgnore(LiveRegisterSet set, - LiveRegisterSet ignore) PER_SHARED_ARCH; + void PopRegsInMaskIgnore(LiveRegisterSet set, LiveRegisterSet ignore) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // =============================================================== // Stack manipulation functions -- single registers/values. void Push(const Operand op) DEFINED_ON(x86_shared); - void Push(Register reg) PER_SHARED_ARCH; + void Push(Register reg) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); void Push(Register reg1, Register reg2, Register reg3, Register reg4) DEFINED_ON(arm64); - void Push(const Imm32 imm) PER_SHARED_ARCH; - void Push(const ImmWord imm) PER_SHARED_ARCH; - void Push(const ImmPtr imm) PER_SHARED_ARCH; - void Push(const ImmGCPtr ptr) PER_SHARED_ARCH; - void Push(FloatRegister reg) PER_SHARED_ARCH; - void PushBoxed(FloatRegister reg) PER_ARCH; + void Push(const Imm32 imm) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + void Push(const ImmWord imm) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + void Push(const ImmPtr imm) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + void Push(const ImmGCPtr ptr) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + void Push(FloatRegister reg) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + void PushBoxed(FloatRegister reg) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); void PushFlags() DEFINED_ON(x86_shared); void Push(PropertyKey key, Register scratchReg); void Push(const Address& addr); @@ -563,13 +582,17 @@ using MacroAssemblerSpecific::push; void Pop(const Operand op) DEFINED_ON(x86_shared); - void Pop(Register reg) PER_SHARED_ARCH; - void Pop(FloatRegister t) PER_SHARED_ARCH; - void Pop(const ValueOperand& val) PER_SHARED_ARCH; + void Pop(Register reg) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + void Pop(FloatRegister t) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + void Pop(const ValueOperand& val) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); void Pop(const Register64 reg); void PopFlags() DEFINED_ON(x86_shared); void PopStackPtr() - DEFINED_ON(arm, mips64, x86_shared, loong64, riscv64, ppc64, wasm32); + DEFINED_ON(arm, mips64, x86_shared, loong64, riscv64, ppc64, wasm32, + ia64); // Move the stack pointer based on the requested amount. void adjustStack(int amount); @@ -577,7 +600,8 @@ // Move the stack pointer to the specified position. It assumes the SP // register is not valid -- it uses FP to set the position. - void freeStackTo(uint32_t framePushed) PER_SHARED_ARCH; + void freeStackTo(uint32_t framePushed) DEFINED_ON(arm, arm64, loong64, + mips64, riscv64, ppc64, x86_shared, wasm32, ia64); private: // =============================================================== @@ -596,19 +620,26 @@ // The returned CodeOffset is the assembler offset for the instruction // immediately following the call; that is, for the return point. - CodeOffset call(Register reg) PER_SHARED_ARCH; - CodeOffset call(Label* label) PER_SHARED_ARCH; - CodeOffset call(const Address& addr) PER_SHARED_ARCH; + CodeOffset call(Register reg) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + CodeOffset call(Label* label) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + CodeOffset call(const Address& addr) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); - void call(ImmWord imm) PER_SHARED_ARCH; + void call(ImmWord imm) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); // Call a target native function, which is neither traceable nor movable. - void call(ImmPtr imm) PER_SHARED_ARCH; - CodeOffset call(wasm::SymbolicAddress imm) PER_SHARED_ARCH; + void call(ImmPtr imm) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + CodeOffset call(wasm::SymbolicAddress imm) DEFINED_ON(arm, arm64, loong64, + mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline CodeOffset call(const wasm::CallSiteDesc& desc, wasm::SymbolicAddress imm); // Call a target JitCode, which must be traceable, and may be movable. - void call(JitCode* c) PER_SHARED_ARCH; + void call(JitCode* c) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void call(TrampolinePtr code); @@ -616,20 +647,23 @@ inline CodeOffset call(const wasm::CallSiteDesc& desc, uint32_t funcDefIndex); inline void call(const wasm::CallSiteDesc& desc, wasm::Trap trap); - CodeOffset callWithPatch() PER_SHARED_ARCH; - void patchCall(uint32_t callerOffset, uint32_t calleeOffset) PER_SHARED_ARCH; + CodeOffset callWithPatch() DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + void patchCall(uint32_t callerOffset, uint32_t calleeOffset) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // Push the return address and make a call. On platforms where this function // is not defined, push the link register (pushReturnAddress) at the entry // point of the callee. - void callAndPushReturnAddress(Register reg) DEFINED_ON(x86_shared); - void callAndPushReturnAddress(Label* label) DEFINED_ON(x86_shared); + void callAndPushReturnAddress(Register reg) DEFINED_ON(x86_shared, ia64); + void callAndPushReturnAddress(Label* label) DEFINED_ON(x86_shared, ia64); // These do not adjust framePushed(). void pushReturnAddress() - DEFINED_ON(mips64, arm, arm64, loong64, riscv64, ppc64, wasm32); + DEFINED_ON(mips64, arm, arm64, loong64, riscv64, ppc64, wasm32, ia64); void popReturnAddress() - DEFINED_ON(mips64, arm, arm64, loong64, riscv64, ppc64, wasm32); + DEFINED_ON(mips64, arm, arm64, loong64, riscv64, ppc64, wasm32, ia64); // Useful for dealing with two-valued returns. void moveRegPair(Register src0, Register src1, Register dst0, Register dst1, @@ -645,18 +679,24 @@ // "Far jumps" provide the ability to jump to any uint32_t offset from any // other uint32_t offset without using a constant pool (thus returning a // simple CodeOffset instead of a CodeOffsetJump). - CodeOffset farJumpWithPatch() PER_SHARED_ARCH; - void patchFarJump(CodeOffset farJump, uint32_t targetOffset) PER_SHARED_ARCH; + CodeOffset farJumpWithPatch() DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + void patchFarJump(CodeOffset farJump, uint32_t targetOffset) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); static void patchFarJump(uint8_t* farJump, uint8_t* target) - DEFINED_ON(arm, arm64, x86_shared, loong64, mips64, riscv64, ppc64); + DEFINED_ON(arm, arm64, x86_shared, loong64, mips64, riscv64, ppc64, + ia64); // Emit a nop that can be patched to and from a nop and a call with int32 // relative displacement. - CodeOffset nopPatchableToCall() PER_SHARED_ARCH; + CodeOffset nopPatchableToCall() DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); void nopPatchableToCall(const wasm::CallSiteDesc& desc); - static void patchNopToCall(uint8_t* callsite, - uint8_t* target) PER_SHARED_ARCH; - static void patchCallToNop(uint8_t* callsite) PER_SHARED_ARCH; + static void patchNopToCall(uint8_t* callsite, uint8_t* target) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + static void patchCallToNop(uint8_t* callsite) DEFINED_ON(arm, arm64, loong64, + mips64, riscv64, ppc64, x86_shared, wasm32, ia64); // These methods are like movWithPatch/PatchDataWithValueCheck but allow // using pc-relative addressing on certain platforms (RIP-relative LEA on x64, @@ -664,9 +704,11 @@ // // Note: "Near" applies to ARM64 where the target must be within 1 MB (this is // release-asserted). - CodeOffset moveNearAddressWithPatch(Register dest) PER_ARCH; - static void patchNearAddressMove(CodeLocationLabel loc, - CodeLocationLabel target) PER_ARCH; + CodeOffset moveNearAddressWithPatch(Register dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + static void patchNearAddressMove(CodeLocationLabel loc, CodeLocationLabel + target) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, + wasm32, ia64); // Creates a move of a patchable 32-bit value into `dest`. On 64-bit // targets, the value (`n`) is extended to 64 bits using the target @@ -674,9 +716,9 @@ // target behaviour is only provided for `n` in the range 0 .. 2^31-1 // inclusive. CodeOffset move32WithPatch(Register dest) - DEFINED_ON(x86_shared, arm, arm64, loong64, mips64, riscv64, ppc64); + DEFINED_ON(x86_shared, arm, arm64, loong64, mips64, riscv64, ppc64, ia64); void patchMove32(CodeOffset offset, Imm32 n) - DEFINED_ON(x86_shared, arm, arm64, loong64, mips64, riscv64, ppc64); + DEFINED_ON(x86_shared, arm, arm64, loong64, mips64, riscv64, ppc64, ia64); public: // =============================================================== @@ -780,7 +822,8 @@ // Setup an ABI call for when the alignment is not known. This may need a // scratch register. - void setupUnalignedABICall(Register scratch) PER_ARCH; + void setupUnalignedABICall(Register scratch) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // Like setupUnalignedABICall, but more efficient because it doesn't push/pop // the unaligned stack pointer. The caller is responsible for restoring SP @@ -809,10 +852,10 @@ inline void callWithABI(const Address& fun, ABIType result = ABIType::General); // Like callWithABI(Register), but the target is raw JIT code implementing - // the C ABI rather than a C++ function, so on ELFv1 it must not be + // the C ABI rather than a C++ function, so on ELFv1 and IA-64 it must not be // dereferenced as a function descriptor. void callWithABIJitCode(Register fun, ABIType result = ABIType::General) - DEFINED_ON(ppc64); + DEFINED_ON(ppc64, ia64); CodeOffset callWithABI(wasm::BytecodeOffset offset, wasm::SymbolicAddress fun, mozilla::Maybe instanceOffset, @@ -830,17 +873,22 @@ void setupNativeABICall(); // Reserve the stack and resolve the arguments move. - void callWithABIPre(uint32_t* stackAdjust, - bool callFromWasm = false) PER_ARCH; + void callWithABIPre(uint32_t* stackAdjust, bool callFromWasm = false) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // Emits a call to a C/C++ function, resolving all argument moves. void callWithABINoProfiler(void* fun, ABIType result, CheckUnsafeCallWithABI check); - void callWithABINoProfiler(Register fun, ABIType result) PER_ARCH; - void callWithABINoProfiler(const Address& fun, ABIType result) PER_ARCH; + void callWithABINoProfiler(Register fun, ABIType result) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + void callWithABINoProfiler(const Address& fun, ABIType result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // Restore the stack to its state before the setup function call. - void callWithABIPost(uint32_t stackAdjust, ABIType result) PER_ARCH; + void callWithABIPost(uint32_t stackAdjust, ABIType result) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); #ifdef JS_CHECK_UNSAFE_CALL_WITH_ABI // Set the JSContext::inUnsafeCallWithABI flag using InstanceReg. @@ -948,7 +996,8 @@ // on the stack. This fake return address should never be used for resuming // any execution, and can even be an invalid pointer into the instruction // stream, as long as it does not alias any other. - uint32_t pushFakeReturnAddress(Register scratch) PER_SHARED_ARCH; + uint32_t pushFakeReturnAddress(Register scratch) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); public: // =============================================================== @@ -984,63 +1033,97 @@ // =============================================================== // Move instructions - inline void move64(Imm64 imm, Register64 dest) PER_ARCH; - inline void move64(Register64 src, Register64 dest) PER_ARCH; - - inline void moveFloat16ToGPR(FloatRegister src, - Register dest) PER_SHARED_ARCH; + inline void move64(Imm64 imm, Register64 dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void move64(Register64 src, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + + inline void moveFloat16ToGPR(FloatRegister src, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // Clears the high words of `src`. - inline void moveGPRToFloat16(Register src, - FloatRegister dest) PER_SHARED_ARCH; - - inline void moveFloat32ToGPR(FloatRegister src, - Register dest) PER_SHARED_ARCH; - inline void moveGPRToFloat32(Register src, - FloatRegister dest) PER_SHARED_ARCH; - - inline void moveDoubleToGPR64(FloatRegister src, Register64 dest) PER_ARCH; - inline void moveGPR64ToDouble(Register64 src, FloatRegister dest) PER_ARCH; + inline void moveGPRToFloat16(Register src, FloatRegister dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + inline void moveFloat32ToGPR(FloatRegister src, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void moveGPRToFloat32(Register src, FloatRegister dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + inline void moveDoubleToGPR64(FloatRegister src, Register64 dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void moveGPR64ToDouble(Register64 src, FloatRegister dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // Move the low 32-bits of a double. - inline void moveLowDoubleToGPR(FloatRegister src, - Register dest) PER_SHARED_ARCH; - - inline void move8ZeroExtend(Register src, Register dest) PER_SHARED_ARCH; - - inline void move8SignExtend(Register src, Register dest) PER_SHARED_ARCH; - inline void move16SignExtend(Register src, Register dest) PER_SHARED_ARCH; + inline void moveLowDoubleToGPR(FloatRegister src, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + inline void move8ZeroExtend(Register src, Register dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void move8SignExtend(Register src, Register dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void move16SignExtend(Register src, Register dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); // move64To32 will clear the high bits of `dest` on 64-bit systems. - inline void move64To32(Register64 src, Register dest) PER_ARCH; - - inline void move32To64ZeroExtend(Register src, Register64 dest) PER_ARCH; - - inline void move8To64SignExtend(Register src, Register64 dest) PER_ARCH; - inline void move16To64SignExtend(Register src, Register64 dest) PER_ARCH; - inline void move32To64SignExtend(Register src, Register64 dest) PER_ARCH; + inline void move64To32(Register64 src, Register dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); - inline void move8SignExtendToPtr(Register src, Register dest) PER_ARCH; - inline void move16SignExtendToPtr(Register src, Register dest) PER_ARCH; - inline void move32SignExtendToPtr(Register src, Register dest) PER_ARCH; - - inline void move32ZeroExtendToPtr(Register src, Register dest) PER_ARCH; + inline void move32To64ZeroExtend(Register src, Register64 dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void move8To64SignExtend(Register src, Register64 dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void move16To64SignExtend(Register src, Register64 dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void move32To64SignExtend(Register src, Register64 dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void move8SignExtendToPtr(Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void move16SignExtendToPtr(Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void move32SignExtendToPtr(Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void move32ZeroExtendToPtr(Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // Copy a constant, typed-register, or a ValueOperand into a ValueOperand // destination. inline void moveValue(const ConstantOrRegister& src, const ValueOperand& dest); void moveValue(const TypedOrValueRegister& src, const ValueOperand& dest); - void moveValue(const ValueOperand& src, const ValueOperand& dest) PER_ARCH; - void moveValue(const Value& src, const ValueOperand& dest) PER_ARCH; + void moveValue(const ValueOperand& src, const ValueOperand& dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + void moveValue(const Value& src, const ValueOperand& dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); void movePropertyKey(PropertyKey key, Register dest); // =============================================================== // Load instructions - inline void load32SignExtendToPtr(const Address& src, Register dest) PER_ARCH; + inline void load32SignExtendToPtr(const Address& src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); - inline void loadAbiReturnAddress(Register dest) PER_SHARED_ARCH; + inline void loadAbiReturnAddress(Register dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); // =============================================================== // Copy instructions @@ -1051,64 +1134,89 @@ // =============================================================== // Logical instructions - inline void not32(Register reg) PER_SHARED_ARCH; - inline void notPtr(Register reg) PER_ARCH; - - inline void and32(Register src, Register dest) PER_SHARED_ARCH; - inline void and32(Imm32 imm, Register dest) PER_SHARED_ARCH; - inline void and32(Imm32 imm, Register src, Register dest) PER_SHARED_ARCH; - inline void and32(Imm32 imm, const Address& dest) PER_SHARED_ARCH; - inline void and32(const Address& src, Register dest) PER_SHARED_ARCH; - - inline void andPtr(Register src, Register dest) PER_ARCH; - inline void andPtr(Imm32 imm, Register dest) PER_ARCH; - inline void andPtr(Imm32 imm, Register src, Register dest) PER_ARCH; - - inline void and64(Imm64 imm, Register64 dest) PER_ARCH; - inline void or64(Imm64 imm, Register64 dest) PER_ARCH; - inline void xor64(Imm64 imm, Register64 dest) PER_ARCH; - - inline void or32(Register src, Register dest) PER_SHARED_ARCH; - inline void or32(Imm32 imm, Register dest) PER_SHARED_ARCH; - inline void or32(Imm32 imm, Register src, Register dest) PER_SHARED_ARCH; - inline void or32(Imm32 imm, const Address& dest) PER_SHARED_ARCH; - - inline void orPtr(Register src, Register dest) PER_ARCH; - inline void orPtr(Imm32 imm, Register dest) PER_ARCH; - inline void orPtr(Imm32 imm, Register src, Register dest) PER_ARCH; - - inline void and64(Register64 src, Register64 dest) PER_ARCH; - inline void or64(Register64 src, Register64 dest) PER_ARCH; - inline void xor64(Register64 src, Register64 dest) PER_ARCH; - - inline void xor32(Register src, Register dest) PER_SHARED_ARCH; - inline void xor32(Imm32 imm, Register dest) PER_SHARED_ARCH; - inline void xor32(Imm32 imm, Register src, Register dest) PER_SHARED_ARCH; - inline void xor32(Imm32 imm, const Address& dest) PER_SHARED_ARCH; - inline void xor32(const Address& src, Register dest) PER_SHARED_ARCH; - - inline void xorPtr(Register src, Register dest) PER_ARCH; - inline void xorPtr(Imm32 imm, Register dest) PER_ARCH; - inline void xorPtr(Imm32 imm, Register src, Register dest) PER_ARCH; - - inline void and64(const Operand& src, Register64 dest) DEFINED_ON(x64); - inline void or64(const Operand& src, Register64 dest) DEFINED_ON(x64); - inline void xor64(const Operand& src, Register64 dest) DEFINED_ON(x64); + inline void not32(Register reg) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void notPtr(Register reg) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + + inline void and32(Register src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void and32(Imm32 imm, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void and32(Imm32 imm, Register src, Register dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void and32(Imm32 imm, const Address& dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void and32(const Address& src, Register dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void andPtr(Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void andPtr(Imm32 imm, Register dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void andPtr(Imm32 imm, Register src, Register dest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + + inline void and64(Imm64 imm, Register64 dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void or64(Imm64 imm, Register64 dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void xor64(Imm64 imm, Register64 dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + + inline void or32(Register src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void or32(Imm32 imm, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void or32(Imm32 imm, Register src, Register dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void or32(Imm32 imm, const Address& dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void orPtr(Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void orPtr(Imm32 imm, Register dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void orPtr(Imm32 imm, Register src, Register dest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + + inline void and64(Register64 src, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void or64(Register64 src, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void xor64(Register64 src, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + + inline void xor32(Register src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void xor32(Imm32 imm, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void xor32(Imm32 imm, Register src, Register dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void xor32(Imm32 imm, const Address& dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void xor32(const Address& src, Register dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void xorPtr(Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void xorPtr(Imm32 imm, Register dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void xorPtr(Imm32 imm, Register src, Register dest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + + inline void and64(const Operand& src, Register64 dest) DEFINED_ON(x64, ia64); + inline void or64(const Operand& src, Register64 dest) DEFINED_ON(x64, ia64); + inline void xor64(const Operand& src, Register64 dest) DEFINED_ON(x64, ia64); // =============================================================== // Swap instructions // Swap the two lower bytes and sign extend the result to 32-bit. - inline void byteSwap16SignExtend(Register reg) PER_SHARED_ARCH; + inline void byteSwap16SignExtend(Register reg) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); // Swap the two lower bytes and zero extend the result to 32-bit. - inline void byteSwap16ZeroExtend(Register reg) PER_SHARED_ARCH; + inline void byteSwap16ZeroExtend(Register reg) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); // Swap all four bytes in a 32-bit integer. - inline void byteSwap32(Register reg) PER_SHARED_ARCH; + inline void byteSwap32(Register reg) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); // Swap all eight bytes in a 64-bit integer. - inline void byteSwap64(Register64 reg) PER_ARCH; + inline void byteSwap64(Register64 reg) DEFINED_ON(mips64, arm, arm64, x86, + x64, loong64, riscv64, ppc64, wasm32, ia64); // =============================================================== // Arithmetic functions @@ -1118,75 +1226,104 @@ // explicitly requested. Instead use branch(Add|Sub|Mul|Neg) to test for // condition flags after performing arithmetic operations. - inline void add32(const Address& src, Register dest) PER_SHARED_ARCH; - inline void add32(Register src, Register dest) PER_SHARED_ARCH; - inline void add32(Imm32 imm, Register dest) PER_SHARED_ARCH; - inline void add32(Imm32 imm, Register src, Register dest) PER_SHARED_ARCH; - inline void add32(Imm32 imm, const Address& dest) PER_SHARED_ARCH; + inline void add32(const Address& src, Register dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void add32(Register src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void add32(Imm32 imm, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void add32(Imm32 imm, Register src, Register dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void add32(Imm32 imm, const Address& dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void add32(Imm32 imm, const AbsoluteAddress& dest) DEFINED_ON(x86_shared); - inline void addPtr(Register src, Register dest) PER_ARCH; + inline void addPtr(Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); inline void addPtr(Register src1, Register src2, Register dest) DEFINED_ON(arm64); - inline void addPtr(Imm32 imm, Register dest) PER_ARCH; + inline void addPtr(Imm32 imm, Register dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); inline void addPtr(Imm32 imm, Register src, Register dest) DEFINED_ON(arm64); - inline void addPtr(ImmWord imm, Register dest) PER_ARCH; + inline void addPtr(ImmWord imm, Register dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); inline void addPtr(ImmPtr imm, Register dest); - inline void addPtr(Imm32 imm, const Address& dest) PER_ARCH; + inline void addPtr(Imm32 imm, const Address& dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); inline void addPtr(Imm32 imm, const AbsoluteAddress& dest) DEFINED_ON(x86, x64); - inline void addPtr(const Address& src, Register dest) PER_ARCH; + inline void addPtr(const Address& src, Register dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); - inline void add64(Register64 src, Register64 dest) PER_ARCH; - inline void add64(Imm32 imm, Register64 dest) PER_ARCH; - inline void add64(Imm64 imm, Register64 dest) PER_ARCH; - inline void add64(const Operand& src, Register64 dest) DEFINED_ON(x64); + inline void add64(Register64 src, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void add64(Imm32 imm, Register64 dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void add64(Imm64 imm, Register64 dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void add64(const Operand& src, Register64 dest) DEFINED_ON(x64, ia64); - inline void addFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; + inline void addFloat32(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); // Compute dest=SP-imm where dest is a pointer registers and not SP. The // offset returned from sub32FromStackPtrWithPatch() must be passed to // patchSub32FromStackPtr(). - inline CodeOffset sub32FromStackPtrWithPatch(Register dest) PER_ARCH; - inline void patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) PER_ARCH; + inline CodeOffset sub32FromStackPtrWithPatch(Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); - inline void addDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; + inline void addDouble(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void addConstantDouble(double d, FloatRegister dest) DEFINED_ON(x86); - inline void sub32(const Address& src, Register dest) PER_SHARED_ARCH; - inline void sub32(Register src, Register dest) PER_SHARED_ARCH; - inline void sub32(Imm32 imm, Register dest) PER_SHARED_ARCH; - - inline void subPtr(Register src, Register dest) PER_ARCH; - inline void subPtr(Register src, const Address& dest) PER_ARCH; - inline void subPtr(Imm32 imm, Register dest) PER_ARCH; + inline void sub32(const Address& src, Register dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void sub32(Register src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void sub32(Imm32 imm, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void subPtr(Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void subPtr(Register src, const Address& dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void subPtr(Imm32 imm, Register dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); inline void subPtr(ImmWord imm, Register dest) DEFINED_ON(x86, x64); - inline void subPtr(const Address& addr, Register dest) PER_ARCH; - - inline void sub64(Register64 src, Register64 dest) PER_ARCH; - inline void sub64(Imm64 imm, Register64 dest) PER_ARCH; - inline void sub64(const Operand& src, Register64 dest) DEFINED_ON(x64); - - inline void subFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; + inline void subPtr(const Address& addr, Register dest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); - inline void subDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; - - inline void mul32(Register rhs, Register srcDest) PER_SHARED_ARCH; - inline void mul32(Imm32 imm, Register srcDest) PER_SHARED_ARCH; + inline void sub64(Register64 src, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void sub64(Imm64 imm, Register64 dest) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void sub64(const Operand& src, Register64 dest) DEFINED_ON(x64, ia64); + + inline void subFloat32(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void subDouble(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void mul32(Register rhs, Register srcDest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void mul32(Imm32 imm, Register srcDest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void mul32(Register src1, Register src2, Register dest, Label* onOver) DEFINED_ON(arm64); // Return the high word of the unsigned multiplication into |dest|. - inline void mulHighUnsigned32(Imm32 imm, Register src, - Register dest) PER_ARCH; - - inline void mulPtr(Register rhs, Register srcDest) PER_ARCH; - inline void mulPtr(ImmWord rhs, Register srcDest) PER_ARCH; + inline void mulHighUnsigned32(Imm32 imm, Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void mulPtr(Register rhs, Register srcDest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void mulPtr(ImmWord rhs, Register srcDest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); inline void mul64(const Register64& rhs, const Register64& srcDest) - DEFINED_ON(x64, arm64, mips64, loong64, riscv64, ppc64); + DEFINED_ON(x64, arm64, mips64, loong64, riscv64, ppc64, ia64); inline void mul64(const Operand& src, const Register64& dest) DEFINED_ON(x64); inline void mul64(const Operand& src, const Register64& dest, const Register temp) DEFINED_ON(x64); @@ -1194,19 +1331,25 @@ inline void mul64(Imm64 imm, const Register64& dest, const Register temp) DEFINED_ON(x86, x64, arm, mips64, loong64, riscv64, ppc64); inline void mul64(const Register64& src, const Register64& dest, - const Register temp) PER_ARCH; + const Register temp) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); inline void mul64(const Register64& src1, const Register64& src2, const Register64& dest) DEFINED_ON(arm64); inline void mul64(Imm64 src1, const Register64& src2, const Register64& dest) DEFINED_ON(arm64); - inline void mulBy3(Register src, Register dest) PER_ARCH; - - inline void mulFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; - inline void mulDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; + inline void mulBy3(Register src, Register dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); - inline void mulDoublePtr(ImmPtr imm, Register temp, - FloatRegister dest) PER_ARCH; + inline void mulFloat32(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void mulDouble(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void mulDoublePtr(ImmPtr imm, Register temp, FloatRegister dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // Perform an integer division, returning the integer part rounded toward // zero. rhs must not be zero, and the division must not overflow. @@ -1214,11 +1357,11 @@ // On ARM, the chip must have hardware division instructions. inline void quotient32(Register lhs, Register rhs, Register dest, bool isUnsigned) - DEFINED_ON(mips64, arm, arm64, loong64, riscv64, wasm32, ppc64); + DEFINED_ON(mips64, arm, arm64, loong64, riscv64, wasm32, ppc64, ia64); inline void quotient64(Register lhs, Register rhs, Register dest, bool isUnsigned) - DEFINED_ON(arm64, loong64, mips64, riscv64, ppc64); + DEFINED_ON(arm64, loong64, mips64, riscv64, ppc64, ia64); // As above, but lhs and dest must be eax and tempEdx must be edx. inline void quotient32(Register lhs, Register rhs, Register dest, @@ -1231,11 +1374,11 @@ // On ARM, the chip must have hardware division instructions. inline void remainder32(Register lhs, Register rhs, Register dest, bool isUnsigned) - DEFINED_ON(mips64, arm, arm64, loong64, riscv64, wasm32, ppc64); + DEFINED_ON(mips64, arm, arm64, loong64, riscv64, wasm32, ppc64, ia64); inline void remainder64(Register lhs, Register rhs, Register dest, bool isUnsigned) - DEFINED_ON(arm64, loong64, mips64, riscv64, ppc64); + DEFINED_ON(arm64, loong64, mips64, riscv64, ppc64, ia64); // As above, but lhs and dest must be eax and tempEdx must be edx. inline void remainder32(Register lhs, Register rhs, Register dest, @@ -1247,24 +1390,24 @@ // // This variant preserves registers, and doesn't require hardware division // instructions on ARM (will call out to a runtime routine). - void flexibleRemainder32( - Register lhs, Register rhs, Register dest, bool isUnsigned, - const LiveRegisterSet& volatileLiveRegs) PER_SHARED_ARCH; - void flexibleRemainderPtr(Register lhs, Register rhs, Register dest, - bool isUnsigned, - const LiveRegisterSet& volatileLiveRegs) PER_ARCH; + void flexibleRemainder32( Register lhs, Register rhs, Register dest, bool + isUnsigned, const LiveRegisterSet& volatileLiveRegs) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + void flexibleRemainderPtr(Register lhs, Register rhs, Register dest, bool + isUnsigned, const LiveRegisterSet& volatileLiveRegs) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // Perform an integer division, returning the integer part rounded toward // zero. rhs must not be zero, and the division must not overflow. // // This variant preserves registers, and doesn't require hardware division // instructions on ARM (will call out to a runtime routine). - void flexibleQuotient32( - Register lhs, Register rhs, Register dest, bool isUnsigned, - const LiveRegisterSet& volatileLiveRegs) PER_SHARED_ARCH; - void flexibleQuotientPtr(Register lhs, Register rhs, Register dest, - bool isUnsigned, - const LiveRegisterSet& volatileLiveRegs) PER_ARCH; + void flexibleQuotient32( Register lhs, Register rhs, Register dest, bool + isUnsigned, const LiveRegisterSet& volatileLiveRegs) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + void flexibleQuotientPtr(Register lhs, Register rhs, Register dest, bool + isUnsigned, const LiveRegisterSet& volatileLiveRegs) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // Perform an integer division, returning the integer part rounded toward // zero in the third argument register. rhs must not be zero, and the division @@ -1275,95 +1418,140 @@ // instructions on ARM (will call out to a runtime routine). // // lhs and rhs are preserved, divOutput and remOutput are clobbered. - void flexibleDivMod32( - Register lhs, Register rhs, Register divOutput, Register remOutput, - bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) PER_SHARED_ARCH; - - inline void divFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; - inline void divDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; - - inline void inc64(AbsoluteAddress dest) PER_ARCH; - - inline void neg32(Register reg) PER_SHARED_ARCH; - inline void neg64(Register64 reg) PER_ARCH; - inline void negPtr(Register reg) PER_ARCH; - - inline void negateFloat(FloatRegister reg) PER_SHARED_ARCH; - - inline void negateDouble(FloatRegister reg) PER_SHARED_ARCH; - - inline void abs32(Register src, Register dest) PER_SHARED_ARCH; - inline void absFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; - inline void absDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; - - inline void sqrtFloat32(FloatRegister src, - FloatRegister dest) PER_SHARED_ARCH; - inline void sqrtDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH; - - void floorFloat32ToInt32(FloatRegister src, Register dest, - Label* fail) PER_SHARED_ARCH; - void floorDoubleToInt32(FloatRegister src, Register dest, - Label* fail) PER_SHARED_ARCH; - - void ceilFloat32ToInt32(FloatRegister src, Register dest, - Label* fail) PER_SHARED_ARCH; - void ceilDoubleToInt32(FloatRegister src, Register dest, - Label* fail) PER_SHARED_ARCH; - - void roundFloat32ToInt32(FloatRegister src, Register dest, FloatRegister temp, - Label* fail) PER_SHARED_ARCH; + void flexibleDivMod32( Register lhs, Register rhs, Register divOutput, + Register remOutput, bool isUnsigned, const LiveRegisterSet& + volatileLiveRegs) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + + inline void divFloat32(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void divDouble(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void inc64(AbsoluteAddress dest) DEFINED_ON(mips64, arm, arm64, x86, + x64, loong64, riscv64, ppc64, wasm32, ia64); + + inline void neg32(Register reg) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void neg64(Register64 reg) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + inline void negPtr(Register reg) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + + inline void negateFloat(FloatRegister reg) DEFINED_ON(arm, arm64, loong64, + mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void negateDouble(FloatRegister reg) DEFINED_ON(arm, arm64, loong64, + mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void abs32(Register src, Register dest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void absFloat32(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void absDouble(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void sqrtFloat32(FloatRegister src, FloatRegister dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void sqrtDouble(FloatRegister src, FloatRegister dest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + + void floorFloat32ToInt32(FloatRegister src, Register dest, Label* fail) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + void floorDoubleToInt32(FloatRegister src, Register dest, Label* fail) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + void ceilFloat32ToInt32(FloatRegister src, Register dest, Label* fail) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + void ceilDoubleToInt32(FloatRegister src, Register dest, Label* fail) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + void roundFloat32ToInt32(FloatRegister src, Register dest, FloatRegister + temp, Label* fail) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); void roundDoubleToInt32(FloatRegister src, Register dest, FloatRegister temp, - Label* fail) PER_SHARED_ARCH; + Label* fail) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - void truncFloat32ToInt32(FloatRegister src, Register dest, - Label* fail) PER_SHARED_ARCH; - void truncDoubleToInt32(FloatRegister src, Register dest, - Label* fail) PER_SHARED_ARCH; - - void nearbyIntDouble(RoundingMode mode, FloatRegister src, - FloatRegister dest) PER_SHARED_ARCH; - void nearbyIntFloat32(RoundingMode mode, FloatRegister src, - FloatRegister dest) PER_SHARED_ARCH; + void truncFloat32ToInt32(FloatRegister src, Register dest, Label* fail) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + void truncDoubleToInt32(FloatRegister src, Register dest, Label* fail) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + void nearbyIntDouble(RoundingMode mode, FloatRegister src, FloatRegister + dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + void nearbyIntFloat32(RoundingMode mode, FloatRegister src, FloatRegister + dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); void signInt32(Register input, Register output); void signDouble(FloatRegister input, FloatRegister output); void signDoubleToInt32(FloatRegister input, Register output, FloatRegister temp, Label* fail); - void copySignDouble(FloatRegister lhs, FloatRegister rhs, - FloatRegister output) PER_SHARED_ARCH; - void copySignFloat32(FloatRegister lhs, FloatRegister rhs, - FloatRegister output) PER_SHARED_ARCH; + void copySignDouble(FloatRegister lhs, FloatRegister rhs, FloatRegister + output) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + void copySignFloat32(FloatRegister lhs, FloatRegister rhs, FloatRegister + output) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); // Returns a random double in range [0, 1) in |dest|. The |rng| register must // hold a pointer to a mozilla::non_crypto::XorShift128PlusRNG. void randomDouble(Register rng, FloatRegister dest, Register64 temp0, Register64 temp1); - inline void min32(Register lhs, Register rhs, Register result) PER_ARCH; - inline void min32(Register lhs, Imm32 rhs, Register result) PER_ARCH; - - inline void max32(Register lhs, Register rhs, Register result) PER_ARCH; - inline void max32(Register lhs, Imm32 rhs, Register result) PER_ARCH; - - inline void minPtr(Register lhs, Register rhs, Register result) PER_ARCH; - inline void minPtr(Register lhs, ImmWord rhs, Register result) PER_ARCH; - - inline void maxPtr(Register lhs, Register rhs, Register result) PER_ARCH; - inline void maxPtr(Register lhs, ImmWord rhs, Register result) PER_ARCH; + inline void min32(Register lhs, Register rhs, Register result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void min32(Register lhs, Imm32 rhs, Register result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void max32(Register lhs, Register rhs, Register result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void max32(Register lhs, Imm32 rhs, Register result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void minPtr(Register lhs, Register rhs, Register result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void minPtr(Register lhs, ImmWord rhs, Register result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void maxPtr(Register lhs, Register rhs, Register result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void maxPtr(Register lhs, ImmWord rhs, Register result) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // srcDest = {min,max}{Float32,Double}(srcDest, other) // For min and max, handle NaN specially if handleNaN is true. - inline void minFloat32(FloatRegister other, FloatRegister srcDest, - bool handleNaN) PER_SHARED_ARCH; - inline void minDouble(FloatRegister other, FloatRegister srcDest, - bool handleNaN) PER_SHARED_ARCH; - - inline void maxFloat32(FloatRegister other, FloatRegister srcDest, - bool handleNaN) PER_SHARED_ARCH; - inline void maxDouble(FloatRegister other, FloatRegister srcDest, - bool handleNaN) PER_SHARED_ARCH; + inline void minFloat32(FloatRegister other, FloatRegister srcDest, bool + handleNaN) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void minDouble(FloatRegister other, FloatRegister srcDest, bool + handleNaN) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + + inline void maxFloat32(FloatRegister other, FloatRegister srcDest, bool + handleNaN) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void maxDouble(FloatRegister other, FloatRegister srcDest, bool + handleNaN) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); void minMaxArrayInt32(Register array, Register result, Register temp1, Register temp2, Register temp3, bool isMax, @@ -1406,173 +1594,236 @@ // for example, the ARM assembler requires the count for 32-bit shifts to be // in the range [0,31]. - inline void lshift32(Imm32 shift, Register srcDest) PER_SHARED_ARCH; - inline void lshift32(Imm32 shift, Register src, - Register dest) PER_SHARED_ARCH; - inline void rshift32(Imm32 shift, Register srcDest) PER_SHARED_ARCH; - inline void rshift32(Imm32 shift, Register src, - Register dest) PER_SHARED_ARCH; - inline void rshift32Arithmetic(Imm32 shift, Register srcDest) PER_SHARED_ARCH; - inline void rshift32Arithmetic(Imm32 shift, Register src, - Register dest) PER_SHARED_ARCH; - - inline void lshiftPtr(Imm32 imm, Register dest) PER_ARCH; - inline void lshiftPtr(Imm32 imm, Register src, Register dest) PER_ARCH; - inline void rshiftPtr(Imm32 imm, Register dest) PER_ARCH; - inline void rshiftPtr(Imm32 imm, Register src, Register dest) PER_ARCH; - inline void rshiftPtrArithmetic(Imm32 imm, Register dest) PER_ARCH; - inline void rshiftPtrArithmetic(Imm32 imm, Register src, - Register dest) PER_ARCH; - - inline void lshift64(Imm32 imm, Register64 dest) PER_ARCH; - inline void rshift64(Imm32 imm, Register64 dest) PER_ARCH; - inline void rshift64Arithmetic(Imm32 imm, Register64 dest) PER_ARCH; + inline void lshift32(Imm32 shift, Register srcDest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void lshift32(Imm32 shift, Register src, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void rshift32(Imm32 shift, Register srcDest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void rshift32(Imm32 shift, Register src, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void rshift32Arithmetic(Imm32 shift, Register srcDest) DEFINED_ON(arm, + arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void rshift32Arithmetic(Imm32 shift, Register src, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + inline void lshiftPtr(Imm32 imm, Register dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void lshiftPtr(Imm32 imm, Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void rshiftPtr(Imm32 imm, Register dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void rshiftPtr(Imm32 imm, Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void rshiftPtrArithmetic(Imm32 imm, Register dest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void rshiftPtrArithmetic(Imm32 imm, Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void lshift64(Imm32 imm, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void rshift64(Imm32 imm, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void rshift64Arithmetic(Imm32 imm, Register64 dest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // On x86_shared these have the constraint that shift must be in CL. - inline void lshift32(Register shift, Register srcDest) PER_SHARED_ARCH; - inline void rshift32(Register shift, Register srcDest) PER_SHARED_ARCH; - inline void rshift32Arithmetic(Register shift, - Register srcDest) PER_SHARED_ARCH; - inline void lshiftPtr(Register shift, Register srcDest) PER_ARCH; - inline void rshiftPtr(Register shift, Register srcDest) PER_ARCH; - inline void rshiftPtrArithmetic(Register shift, Register srcDest) PER_ARCH; + inline void lshift32(Register shift, Register srcDest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void rshift32(Register shift, Register srcDest) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void rshift32Arithmetic(Register shift, Register srcDest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void lshiftPtr(Register shift, Register srcDest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void rshiftPtr(Register shift, Register srcDest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void rshiftPtrArithmetic(Register shift, Register srcDest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // These variants do not have the above constraint, but may emit some extra // instructions on x86_shared. They also handle shift >= 32 consistently by // masking with 0x1F (either explicitly or relying on the hardware to do // that). - inline void flexibleLshift32(Register shift, - Register srcDest) PER_SHARED_ARCH; - inline void flexibleRshift32(Register shift, - Register srcDest) PER_SHARED_ARCH; - inline void flexibleRshift32Arithmetic(Register shift, - Register srcDest) PER_SHARED_ARCH; - inline void flexibleLshiftPtr(Register shift, Register srcDest) PER_ARCH; + inline void flexibleLshift32(Register shift, Register srcDest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void flexibleRshift32(Register shift, Register srcDest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void flexibleRshift32Arithmetic(Register shift, Register srcDest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void flexibleLshiftPtr(Register shift, Register srcDest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); inline void flexibleRshiftPtr(Register shift, Register srcDest) PER_ARCH; - inline void flexibleRshiftPtrArithmetic(Register shift, - Register srcDest) PER_ARCH; - - inline void lshift64(Register shift, Register64 srcDest) PER_ARCH; - inline void rshift64(Register shift, Register64 srcDest) PER_ARCH; - inline void rshift64Arithmetic(Register shift, Register64 srcDest) PER_ARCH; + inline void flexibleRshiftPtrArithmetic(Register shift, Register srcDest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + inline void lshift64(Register shift, Register64 srcDest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void rshift64(Register shift, Register64 srcDest) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void rshift64Arithmetic(Register shift, Register64 srcDest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // =============================================================== // Rotation functions // Note: - on x86 and x64 the count register must be in CL. // - on x64 the temp register should be InvalidReg. - inline void rotateLeft(Imm32 count, Register input, - Register dest) PER_SHARED_ARCH; - inline void rotateLeft(Register count, Register input, - Register dest) PER_SHARED_ARCH; + inline void rotateLeft(Imm32 count, Register input, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void rotateLeft(Register count, Register input, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest) DEFINED_ON(x64); inline void rotateLeft64(Register count, Register64 input, Register64 dest) DEFINED_ON(x64); inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest, - Register temp) PER_ARCH; + Register temp) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, + ppc64, wasm32, ia64); inline void rotateLeft64(Register count, Register64 input, Register64 dest, - Register temp) PER_ARCH; + Register temp) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, + ppc64, wasm32, ia64); - inline void rotateRight(Imm32 count, Register input, - Register dest) PER_SHARED_ARCH; - inline void rotateRight(Register count, Register input, - Register dest) PER_SHARED_ARCH; + inline void rotateRight(Imm32 count, Register input, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void rotateRight(Register count, Register input, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); inline void rotateRight64(Imm32 count, Register64 input, Register64 dest) DEFINED_ON(x64); inline void rotateRight64(Register count, Register64 input, Register64 dest) DEFINED_ON(x64); inline void rotateRight64(Imm32 count, Register64 input, Register64 dest, - Register temp) PER_ARCH; + Register temp) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, + ppc64, wasm32, ia64); inline void rotateRight64(Register count, Register64 input, Register64 dest, - Register temp) PER_ARCH; + Register temp) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, + ppc64, wasm32, ia64); // =============================================================== // Bit counting functions // knownNotZero may be true only if the src is known not to be zero. - inline void clz32(Register src, Register dest, - bool knownNotZero) PER_SHARED_ARCH; - inline void ctz32(Register src, Register dest, - bool knownNotZero) PER_SHARED_ARCH; - - inline void clz64(Register64 src, Register64 dest) PER_ARCH; - inline void ctz64(Register64 src, Register64 dest) PER_ARCH; + inline void clz32(Register src, Register dest, bool knownNotZero) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void ctz32(Register src, Register dest, bool knownNotZero) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + inline void clz64(Register64 src, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); + inline void ctz64(Register64 src, Register64 dest) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // On x86_shared, temp may be Invalid only if the chip has the POPCNT // instruction. On ARM, temp may never be Invalid. - inline void popcnt32(Register src, Register dest, - Register temp) PER_SHARED_ARCH; + inline void popcnt32(Register src, Register dest, Register temp) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // temp may be invalid only if the chip has the POPCNT instruction. - inline void popcnt64(Register64 src, Register64 dest, Register temp) PER_ARCH; + inline void popcnt64(Register64 src, Register64 dest, Register temp) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // =============================================================== // Condition functions - inline void cmp8Set(Condition cond, Address lhs, Imm32 rhs, - Register dest) PER_SHARED_ARCH; - - inline void cmp16Set(Condition cond, Address lhs, Imm32 rhs, - Register dest) PER_SHARED_ARCH; - - template - inline void cmp32Set(Condition cond, T1 lhs, T2 rhs, - Register dest) PER_SHARED_ARCH; - - inline void cmp64Set(Condition cond, Register64 lhs, Register64 rhs, - Register dest) PER_ARCH; - - inline void cmp64Set(Condition cond, Register64 lhs, Imm64 rhs, - Register dest) PER_ARCH; - - inline void cmp64Set(Condition cond, Address lhs, Register64 rhs, - Register dest) PER_ARCH; - - inline void cmp64Set(Condition cond, Address lhs, Imm64 rhs, - Register dest) PER_ARCH; - - template - inline void cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) PER_ARCH; + inline void cmp8Set(Condition cond, Address lhs, Imm32 rhs, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + inline void cmp16Set(Condition cond, Address lhs, Imm32 rhs, Register dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + template inline void cmp32Set(Condition cond, T1 + lhs, T2 rhs, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void cmp64Set(Condition cond, Register64 lhs, Register64 rhs, Register + dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, + wasm32, ia64); + + inline void cmp64Set(Condition cond, Register64 lhs, Imm64 rhs, Register + dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, + wasm32, ia64); + + inline void cmp64Set(Condition cond, Address lhs, Register64 rhs, Register + dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, + wasm32, ia64); + + inline void cmp64Set(Condition cond, Address lhs, Imm64 rhs, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + template inline void cmpPtrSet(Condition cond, T1 + lhs, T2 rhs, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); // =============================================================== // Branch functions - inline void branch8(Condition cond, const Address& lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + inline void branch8(Condition cond, const Address& lhs, Imm32 rhs, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); // Compares the byte in |lhs| against |rhs| using a 8-bit comparison on // x86/x64 or a 32-bit comparison (all other platforms). The caller should // ensure |rhs| is a zero- resp. sign-extended byte value for cross-platform // compatible code. inline void branch8(Condition cond, const BaseIndex& lhs, Register rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branch16(Condition cond, const Address& lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + inline void branch16(Condition cond, const Address& lhs, Imm32 rhs, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branch32(Condition cond, Register lhs, Register rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branch32(Condition cond, Register lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branch32(Condition cond, Register lhs, const Address& rhs, Label* label) DEFINED_ON(arm64); - inline void branch32(Condition cond, const Address& lhs, Register rhs, - Label* label) PER_SHARED_ARCH; + inline void branch32(Condition cond, const Address& lhs, Register rhs, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branch32(Condition cond, const Address& lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); - inline void branch32(Condition cond, const AbsoluteAddress& lhs, Register rhs, - Label* label) PER_ARCH; + inline void branch32(Condition cond, const AbsoluteAddress& lhs, Register + rhs, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); inline void branch32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs, - Label* label) PER_ARCH; + Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, + ppc64, wasm32, ia64); inline void branch32(Condition cond, const BaseIndex& lhs, Register rhs, - Label* label) DEFINED_ON(arm, x86_shared); - inline void branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, x86_shared, ia64); + inline void branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branch32(Condition cond, const Operand& lhs, Register rhs, Label* label) DEFINED_ON(x86_shared); @@ -1586,84 +1837,99 @@ // GreaterThan(orEqual), Below(orEqual) and Above(orEqual). When a fail label // is not defined it will fall through to next instruction, else jump to the // fail label. - inline void branch64(Condition cond, Register64 lhs, Imm64 val, - Label* success, Label* fail = nullptr) PER_ARCH; - inline void branch64(Condition cond, Register64 lhs, Register64 rhs, - Label* success, Label* fail = nullptr) PER_ARCH; - inline void branch64(Condition cond, const Address& lhs, Imm64 val, - Label* success, Label* fail = nullptr) PER_ARCH; + inline void branch64(Condition cond, Register64 lhs, Imm64 val, Label* + success, Label* fail = nullptr) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + inline void branch64(Condition cond, Register64 lhs, Register64 rhs, Label* + success, Label* fail = nullptr) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + inline void branch64(Condition cond, const Address& lhs, Imm64 val, Label* + success, Label* fail = nullptr) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void branch64(Condition cond, const Address& lhs, Register64 rhs, - Label* success, Label* fail = nullptr) PER_ARCH; + Label* success, Label* fail = nullptr) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // Compare the value at |lhs| with the value at |rhs|. The scratch // register *must not* be the base of |lhs| or |rhs|. // Only the NotEqual and Equal conditions are allowed. inline void branch64(Condition cond, const Address& lhs, const Address& rhs, - Register scratch, Label* label) PER_ARCH; + Register scratch, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void branchPtr(Condition cond, Register lhs, Register rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, Register lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, Register lhs, ImmPtr rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, Register lhs, ImmGCPtr rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, Register lhs, ImmWord rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, const Address& lhs, Register rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, const Address& lhs, ImmPtr rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, const Address& lhs, ImmWord rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, const BaseIndex& lhs, ImmWord rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchPtr(Condition cond, const BaseIndex& lhs, Register rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, - Register rhs, Label* label) PER_ARCH; - inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, ImmWord rhs, - Label* label) PER_ARCH; + inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, Register + rhs, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); + inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, ImmWord + rhs, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); inline void branchPtr(Condition cond, wasm::SymbolicAddress lhs, Register rhs, Label* label) PER_ARCH; // Given a pointer to a GC Cell, retrieve the StoreBuffer pointer from its // chunk header, or nullptr if it is in the tenured heap. - void loadStoreBuffer(Register ptr, Register buffer) PER_ARCH; + void loadStoreBuffer(Register ptr, Register buffer) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); void branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp, - Label* label) PER_ARCH; + Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, + ppc64, wasm32, ia64); void branchPtrInNurseryChunk(Condition cond, const Address& address, Register temp, Label* label) DEFINED_ON(x86); void branchValueIsNurseryCell(Condition cond, const Address& address, - Register temp, Label* label) PER_ARCH; - void branchValueIsNurseryCell(Condition cond, ValueOperand value, - Register temp, Label* label) PER_ARCH; + Register temp, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + void branchValueIsNurseryCell(Condition cond, ValueOperand value, Register + temp, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); // This function compares a Value (lhs) which is having a private pointer // boxed inside a js::Value, with a raw pointer (rhs). - inline void branchPrivatePtr(Condition cond, const Address& lhs, Register rhs, - Label* label) PER_ARCH; + inline void branchPrivatePtr(Condition cond, const Address& lhs, Register + rhs, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); inline void branchFloat(DoubleCondition cond, FloatRegister lhs, - FloatRegister rhs, Label* label) PER_SHARED_ARCH; + FloatRegister rhs, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); // Truncate a double/float32 to int32 and when it doesn't fit an int32 it will // jump to the failure label. This particular variant is allowed to return the // value module 2**32, which isn't implemented on all architectures. - inline void branchTruncateFloat32MaybeModUint32(FloatRegister src, - Register dest, - Label* fail) PER_ARCH; - inline void branchTruncateDoubleMaybeModUint32(FloatRegister src, - Register dest, - Label* fail) PER_ARCH; + inline void branchTruncateFloat32MaybeModUint32(FloatRegister src, Register + dest, Label* fail) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); + inline void branchTruncateDoubleMaybeModUint32(FloatRegister src, Register + dest, Label* fail) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); // Truncate a double/float32 to intptr and when it doesn't fit jump to the // failure label. @@ -1677,10 +1943,12 @@ inline void branchTruncateFloat32ToInt32(FloatRegister src, Register dest, Label* fail) PER_ARCH; inline void branchTruncateDoubleToInt32(FloatRegister src, Register dest, - Label* fail) PER_ARCH; + Label* fail) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, + ppc64, wasm32, ia64); inline void branchDouble(DoubleCondition cond, FloatRegister lhs, - FloatRegister rhs, Label* label) PER_SHARED_ARCH; + FloatRegister rhs, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchDoubleNotInInt64Range(Address src, Register temp, Label* fail); @@ -1692,62 +1960,75 @@ Label* fail); // Branch if the (un)signed int64 is outside the range of a signed intptr. - inline void branchInt64NotInPtrRange(Register64 src, Label* label) PER_ARCH; - inline void branchUInt64NotInPtrRange(Register64 src, Label* label) PER_ARCH; - - template - inline void branchAdd32(Condition cond, T src, Register dest, - Label* label) PER_SHARED_ARCH; - template - inline void branchSub32(Condition cond, T src, Register dest, - Label* label) PER_SHARED_ARCH; - template - inline void branchMul32(Condition cond, T src, Register dest, - Label* label) PER_SHARED_ARCH; - template - inline void branchRshift32(Condition cond, T src, Register dest, - Label* label) PER_SHARED_ARCH; - - inline void branchNeg32(Condition cond, Register reg, - Label* label) PER_SHARED_ARCH; + inline void branchInt64NotInPtrRange(Register64 src, Label* label) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + inline void branchUInt64NotInPtrRange(Register64 src, Label* label) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + template inline void branchAdd32(Condition cond, T src, Register + dest, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + template inline void branchSub32(Condition cond, T src, Register + dest, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + template inline void branchMul32(Condition cond, T src, Register + dest, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + template inline void branchRshift32(Condition cond, T src, + Register dest, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void branchNeg32(Condition cond, Register reg, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); inline void branchAdd64(Condition cond, Imm64 imm, Register64 dest, Label* label) DEFINED_ON(x86, arm, wasm32); - template - inline void branchAddPtr(Condition cond, T src, Register dest, - Label* label) PER_SHARED_ARCH; - - template - inline void branchSubPtr(Condition cond, T src, Register dest, - Label* label) PER_SHARED_ARCH; - - inline void branchMulPtr(Condition cond, Register src, Register dest, - Label* label) PER_SHARED_ARCH; - - inline void branchNegPtr(Condition cond, Register reg, - Label* label) PER_SHARED_ARCH; - - inline void decBranchPtr(Condition cond, Register lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + template inline void branchAddPtr(Condition cond, T src, + Register dest, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + + template inline void branchSubPtr(Condition cond, T src, + Register dest, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void branchMulPtr(Condition cond, Register src, Register dest, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + + inline void branchNegPtr(Condition cond, Register reg, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + inline void decBranchPtr(Condition cond, Register lhs, Imm32 rhs, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTest32(Condition cond, Register lhs, Register rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchTest32(Condition cond, Register lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); inline void branchTest32(Condition cond, const Address& lhs, Imm32 rhh, - Label* label) PER_SHARED_ARCH; - inline void branchTest32(Condition cond, const AbsoluteAddress& lhs, - Imm32 rhs, Label* label) PER_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTest32(Condition cond, const AbsoluteAddress& lhs, Imm32 + rhs, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); inline void branchTestPtr(Condition cond, Register lhs, Register rhs, - Label* label) PER_SHARED_ARCH; - inline void branchTestPtr(Condition cond, Register lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; - inline void branchTestPtr(Condition cond, Register lhs, ImmWord rhs, - Label* label) PER_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); + inline void branchTestPtr(Condition cond, Register lhs, Imm32 rhs, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestPtr(Condition cond, Register lhs, ImmWord rhs, Label* + label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, + wasm32, ia64); inline void branchTestPtr(Condition cond, const Address& lhs, Imm32 rhs, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); // When a fail label is not defined it will fall through to next instruction, // else jump to the fail label. @@ -1755,12 +2036,13 @@ // On x86 if |lhs == rhs|, |temp| is used to generate a single branch // instruction. Otherwise |temp| is unused and can be |InvalidReg|. inline void branchTest64(Condition cond, Register64 lhs, Register64 rhs, - Register temp, Label* success, - Label* fail = nullptr) PER_ARCH; + Register temp, Label* success, Label* fail = nullptr) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); inline void branchTest64(Condition cond, Register64 lhs, Register64 rhs, Label* success, Label* fail = nullptr); - inline void branchTest64(Condition cond, Register64 lhs, Imm64 rhs, - Label* success, Label* fail = nullptr) PER_ARCH; + inline void branchTest64(Condition cond, Register64 lhs, Imm64 rhs, Label* + success, Label* fail = nullptr) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); // Branches to |label| if |reg| is false. |reg| should be a C++ bool. inline void branchIfFalseBool(Register reg, Label* label); @@ -1923,30 +2205,42 @@ // Perform a type-test on a tag of a Value (32bits boxing), or the tagged // value (64bits boxing). - inline void branchTestUndefined(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestInt32(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestDouble(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestNumber(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestBoolean(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestString(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestSymbol(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestBigInt(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestNull(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestObject(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestPrimitive(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; - inline void branchTestMagic(Condition cond, Register tag, - Label* label) PER_SHARED_ARCH; + inline void branchTestUndefined(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestInt32(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestDouble(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestNumber(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestBoolean(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestString(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestSymbol(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestBigInt(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestNull(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestObject(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestPrimitive(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline void branchTestMagic(Condition cond, Register tag, Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); void branchTestType(Condition cond, Register tag, JSValueType type, Label* label); @@ -1955,132 +2249,181 @@ // BaseIndex and ValueOperand variants clobber the ScratchReg on x64. // All Variants clobber the ScratchReg on arm64. inline void branchTestUndefined(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestUndefined(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestUndefined(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchTestInt32(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestInt32(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestInt32(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; - - inline void branchTestDouble(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestDouble(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; + inline void branchTestInt32(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestInt32(Condition cond, const BaseIndex& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestInt32(Condition cond, const ValueOperand& value, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + + inline void branchTestDouble(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestDouble(Condition cond, const BaseIndex& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestDouble(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestNumber(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchTestBoolean(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; + inline void branchTestBoolean(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestBoolean(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestBoolean(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchTestString(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestString(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; + inline void branchTestString(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestString(Condition cond, const BaseIndex& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestString(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchTestSymbol(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestSymbol(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; + inline void branchTestSymbol(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestSymbol(Condition cond, const BaseIndex& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestSymbol(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchTestBigInt(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestBigInt(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; + inline void branchTestBigInt(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestBigInt(Condition cond, const BaseIndex& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestBigInt(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchTestNull(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestNull(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestNull(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + inline void branchTestNull(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestNull(Condition cond, const BaseIndex& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestNull(Condition cond, const ValueOperand& value, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); // Clobbers the ScratchReg on x64. - inline void branchTestObject(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestObject(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; + inline void branchTestObject(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestObject(Condition cond, const BaseIndex& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestObject(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchTestGCThing(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; + inline void branchTestGCThing(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestGCThing(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestGCThing(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestPrimitive(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); - inline void branchTestMagic(Condition cond, const Address& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestMagic(Condition cond, const BaseIndex& address, - Label* label) PER_SHARED_ARCH; - inline void branchTestMagic(Condition cond, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + inline void branchTestMagic(Condition cond, const Address& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestMagic(Condition cond, const BaseIndex& address, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestMagic(Condition cond, const ValueOperand& value, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestMagic(Condition cond, const Address& valaddr, - JSWhyMagic why, Label* label) PER_ARCH; + JSWhyMagic why, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void branchTestMagic(Condition cond, const BaseIndex& valaddr, - JSWhyMagic why, Label* label) PER_ARCH; + JSWhyMagic why, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void branchTestMagicValue(Condition cond, const ValueOperand& val, JSWhyMagic why, Label* label); - void branchTestValue(Condition cond, const ValueOperand& lhs, - const Value& rhs, Label* label) PER_ARCH; - void branchTestNaNValue(Condition cond, const ValueOperand& val, - Register temp, Label* label) PER_ARCH; - - template - inline void branchTestValue(Condition cond, const T& lhs, - const ValueOperand& rhs, Label* label) PER_ARCH; + void branchTestValue(Condition cond, const ValueOperand& lhs, const Value& + rhs, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); + void branchTestNaNValue(Condition cond, const ValueOperand& val, Register + temp, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); + + template inline void branchTestValue(Condition cond, const T& + lhs, const ValueOperand& rhs, Label* label) DEFINED_ON(mips64, arm, + arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // Checks if given Value is evaluated to true or false in a condition. // The type of the value should match the type of the method. inline void branchTestInt32Truthy(bool truthy, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; - inline void branchTestDoubleTruthy(bool truthy, FloatRegister reg, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + inline void branchTestDoubleTruthy(bool truthy, FloatRegister reg, Label* + label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestBooleanTruthy(bool truthy, const ValueOperand& value, - Label* label) PER_ARCH; + Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, + ppc64, wasm32, ia64); inline void branchTestStringTruthy(bool truthy, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); inline void branchTestBigIntTruthy(bool truthy, const ValueOperand& value, - Label* label) PER_SHARED_ARCH; + Label* label) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); // Create an unconditional branch to the address given as argument. - inline void branchToComputedAddress(const BaseIndex& address) PER_ARCH; + inline void branchToComputedAddress(const BaseIndex& address) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // Subtract a constant in the range 1 .. 127 inclusive from the value stored // at `address`, write the result back to `address`, and jump to `label` if // the updated value is negative. The subtract is a 32-bit operation even // though the value to be subtracted must fit in 7 bits. - CodeOffset sub32FromMemAndBranchIfNegativeWithPatch( - Address address, Label* label) PER_SHARED_ARCH; + CodeOffset sub32FromMemAndBranchIfNegativeWithPatch(Address address, + Label* label) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // Patch in the value to be subtracted. Must be 1 .. 127 inclusive. - void patchSub32FromMemAndBranchIfNegative(CodeOffset offset, - Imm32 imm) PER_SHARED_ARCH; + void patchSub32FromMemAndBranchIfNegative(CodeOffset offset, Imm32 imm) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); private: template @@ -2089,10 +2432,9 @@ void branchPtrInNurseryChunkImpl(Condition cond, Register ptr, Label* label) DEFINED_ON(x86); - template - void branchValueIsNurseryCellImpl(Condition cond, const T& value, - Register temp, Label* label) - DEFINED_ON(arm64, x64, mips64, loong64, riscv64, ppc64); + template void branchValueIsNurseryCellImpl(Condition cond, const + T& value, Register temp, Label* label) DEFINED_ON(arm64, x64, mips64, + loong64, riscv64, ppc64, ia64); template inline void branchTestUndefinedImpl(Condition cond, const T& t, Label* label) @@ -2135,21 +2477,21 @@ DEFINED_ON(arm, arm64, x86_shared); public: - template - inline void testNumberSet(Condition cond, const T& src, - Register dest) PER_SHARED_ARCH; - template - inline void testBooleanSet(Condition cond, const T& src, - Register dest) PER_SHARED_ARCH; - template - inline void testStringSet(Condition cond, const T& src, - Register dest) PER_SHARED_ARCH; - template - inline void testSymbolSet(Condition cond, const T& src, - Register dest) PER_SHARED_ARCH; - template - inline void testBigIntSet(Condition cond, const T& src, - Register dest) PER_SHARED_ARCH; + template inline void testNumberSet(Condition cond, const T& src, + Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + template inline void testBooleanSet(Condition cond, const T& + src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + template inline void testStringSet(Condition cond, const T& src, + Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + template inline void testSymbolSet(Condition cond, const T& src, + Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); + template inline void testBigIntSet(Condition cond, const T& src, + Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); public: // The fallibleUnbox* methods below combine a Value type check with an unbox. @@ -2159,11 +2501,14 @@ // |src| and |dest| can be the same register, but |dest| may hold garbage on // failure. inline void fallibleUnboxPtr(const ValueOperand& src, Register dest, - JSValueType type, Label* fail) PER_ARCH; - inline void fallibleUnboxPtr(const Address& src, Register dest, - JSValueType type, Label* fail) PER_ARCH; - inline void fallibleUnboxPtr(const BaseIndex& src, Register dest, - JSValueType type, Label* fail) PER_ARCH; + JSValueType type, Label* fail) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + inline void fallibleUnboxPtr(const Address& src, Register dest, JSValueType + type, Label* fail) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); + inline void fallibleUnboxPtr(const BaseIndex& src, Register dest, JSValueType + type, Label* fail) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); template inline void fallibleUnboxInt32(const T& src, Register dest, Label* fail); template @@ -2177,23 +2522,29 @@ template inline void fallibleUnboxBigInt(const T& src, Register dest, Label* fail); - inline void cmp32Move32(Condition cond, Register lhs, Imm32 rhs, Register src, - Register dest) PER_SHARED_ARCH; - - inline void cmp32Move32(Condition cond, Register lhs, Register rhs, - Register src, Register dest) PER_SHARED_ARCH; + inline void cmp32Move32(Condition cond, Register lhs, Imm32 rhs, Register + src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + + inline void cmp32Move32(Condition cond, Register lhs, Register rhs, Register + src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); inline void cmp32Move32(Condition cond, Register lhs, const Address& rhs, - Register src, Register dest) PER_SHARED_ARCH; + Register src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); - inline void cmpPtrMovePtr(Condition cond, Register lhs, Imm32 rhs, - Register src, Register dest) PER_ARCH; + inline void cmpPtrMovePtr(Condition cond, Register lhs, Imm32 rhs, Register + src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); inline void cmpPtrMovePtr(Condition cond, Register lhs, Register rhs, - Register src, Register dest) PER_ARCH; + Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void cmpPtrMovePtr(Condition cond, Register lhs, const Address& rhs, - Register src, Register dest) PER_ARCH; + Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void cmp32Load32(Condition cond, Register lhs, const Address& rhs, const Address& src, Register dest) PER_SHARED_ARCH; @@ -2201,31 +2552,39 @@ inline void cmp32Load32(Condition cond, Register lhs, Register rhs, const Address& src, Register dest) PER_SHARED_ARCH; - inline void cmp32Load32(Condition cond, Register lhs, Imm32 rhs, - const Address& src, Register dest) PER_SHARED_ARCH; - - inline void cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs, - const Address& src, Register dest) PER_ARCH; - - inline void cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs, - Register src, Register dest) PER_ARCH; + inline void cmp32Load32(Condition cond, Register lhs, Imm32 rhs, const + Address& src, Register dest) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + + inline void cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs, const + Address& src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + + inline void cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs, Register + src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); inline void test32LoadPtr(Condition cond, const Address& addr, Imm32 mask, - const Address& src, Register dest) PER_ARCH; + const Address& src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, + x64, loong64, riscv64, ppc64, wasm32, ia64); inline void test32MovePtr(Condition cond, Register operand, Imm32 mask, - Register src, Register dest) PER_ARCH; + Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void test32MovePtr(Condition cond, const Address& addr, Imm32 mask, - Register src, Register dest) PER_ARCH; + Register src, Register dest) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); // Conditional move for Spectre mitigations. - inline void spectreMovePtr(Condition cond, Register src, - Register dest) PER_ARCH; + inline void spectreMovePtr(Condition cond, Register src, Register dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // Zeroes dest if the condition is true. - inline void spectreZeroRegister(Condition cond, Register scratch, - Register dest) PER_SHARED_ARCH; + inline void spectreZeroRegister(Condition cond, Register scratch, Register + dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // Performs a bounds check and zeroes the index register if out-of-bounds // (to mitigate Spectre). @@ -2235,19 +2594,19 @@ DEFINED_ON(x86); public: - inline void spectreBoundsCheck32(Register index, Register length, - Register maybeScratch, - Label* failure) PER_ARCH; + inline void spectreBoundsCheck32(Register index, Register length, Register + maybeScratch, Label* failure) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void spectreBoundsCheck32(Register index, const Address& length, - Register maybeScratch, - Label* failure) PER_ARCH; + Register maybeScratch, Label* failure) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); - inline void spectreBoundsCheckPtr(Register index, Register length, - Register maybeScratch, - Label* failure) PER_ARCH; + inline void spectreBoundsCheckPtr(Register index, Register length, Register + maybeScratch, Label* failure) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); inline void spectreBoundsCheckPtr(Register index, const Address& length, - Register maybeScratch, - Label* failure) PER_ARCH; + Register maybeScratch, Label* failure) DEFINED_ON(mips64, arm, arm64, + x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // ======================================================================== // Support for 128-bit arithmetic. @@ -2255,9 +2614,9 @@ // Produces the top 64 bits of the 128-bit value `lhsHi:lhsLo +/- // rhsHi:rhsLo`. Only used on 64-bit targets. `output` must be different // from all the other registers, on all supported targets. - inline void wasmAddSubI128HI64(Register lhsLo, Register lhsHi, Register rhsLo, - Register rhsHi, Register output, bool isAdd) - DEFINED_ON(x64, arm64, riscv64, loong64, mips64, ppc64); + inline void wasmAddSubI128HI64(Register lhsLo, Register lhsHi, Register + rhsLo, Register rhsHi, Register output, bool isAdd) DEFINED_ON(x64, + arm64, riscv64, loong64, mips64, ppc64, ia64); // Produces the top 64 bits of the 128-bit value `lhs *widen rhs`. Only used // on 64-bit targets. On x64, `lhs` must be RAX, `rhs` must be RDX, and all @@ -2269,8 +2628,7 @@ // The same, but for all other 64-bit targets. There are no restrictions on // what the registers may be. inline void wasmMulI64WideHI64(Register lhs, Register rhs, Register output, - bool isSigned) - DEFINED_ON(arm64, riscv64, loong64, mips64, ppc64); + bool isSigned) DEFINED_ON(arm64, riscv64, loong64, mips64, ppc64, ia64); // ======================================================================== // Canonicalization primitives. @@ -2292,10 +2650,12 @@ public: // ======================================================================== // Memory access primitives. - inline FaultingCodeOffset storeDouble(FloatRegister src, - const Address& dest) PER_SHARED_ARCH; - inline FaultingCodeOffset storeDouble(FloatRegister src, - const BaseIndex& dest) PER_SHARED_ARCH; + inline FaultingCodeOffset storeDouble(FloatRegister src, const Address& dest) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline FaultingCodeOffset storeDouble(FloatRegister src, const BaseIndex& + dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); inline FaultingCodeOffset storeDouble(FloatRegister src, const Operand& dest) DEFINED_ON(x86_shared); @@ -2304,24 +2664,28 @@ using MacroAssemblerSpecific::boxDouble; - inline FaultingCodeOffset storeFloat32(FloatRegister src, - const Address& dest) PER_SHARED_ARCH; - inline FaultingCodeOffset storeFloat32(FloatRegister src, - const BaseIndex& dest) PER_SHARED_ARCH; + inline FaultingCodeOffset storeFloat32(FloatRegister src, const Address& + dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + inline FaultingCodeOffset storeFloat32(FloatRegister src, const BaseIndex& + dest) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); inline FaultingCodeOffset storeFloat32(FloatRegister src, const Operand& dest) DEFINED_ON(x86_shared); - inline FaultingCodeOffset storeFloat16(FloatRegister src, const Address& dest, - Register scratch) PER_SHARED_ARCH; - inline FaultingCodeOffset storeFloat16(FloatRegister src, - const BaseIndex& dest, - Register scratch) PER_SHARED_ARCH; - - template - void storeUnboxedValue(const ConstantOrRegister& value, MIRType valueType, - const T& dest) PER_ARCH; + inline FaultingCodeOffset storeFloat16(FloatRegister src, const Address& + dest, Register scratch) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + inline FaultingCodeOffset storeFloat16(FloatRegister src, const BaseIndex& + dest, Register scratch) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); + + template void storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, const T& dest) DEFINED_ON(mips64, arm, arm64, x86, + x64, loong64, riscv64, ppc64, wasm32, ia64); - inline void memoryBarrier(MemoryBarrier barrier) PER_SHARED_ARCH; + inline void memoryBarrier(MemoryBarrier barrier) DEFINED_ON(arm, arm64, + loong64, mips64, riscv64, ppc64, x86_shared, wasm32, ia64); public: // ======================================================================== @@ -3730,27 +4094,37 @@ // Convert floating point. // temp required on x86 and x64; must be undefined on mips64 and loong64. - void convertUInt64ToFloat32(Register64 src, FloatRegister dest, Register temp) - DEFINED_ON(arm64, mips64, loong64, ppc64, riscv64, wasm32, x64, x86); + void convertUInt64ToFloat32(Register64 src, FloatRegister dest, Register + temp) DEFINED_ON(arm64, mips64, loong64, ppc64, riscv64, wasm32, x64, + x86, ia64); void convertInt64ToFloat32(Register64 src, FloatRegister dest) - DEFINED_ON(arm64, mips64, loong64, ppc64, riscv64, wasm32, x64, x86); + DEFINED_ON(arm64, mips64, loong64, ppc64, riscv64, wasm32, x64, x86, + ia64); - bool convertUInt64ToDoubleNeedsTemp() PER_ARCH; + bool convertUInt64ToDoubleNeedsTemp() + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // temp required when convertUInt64ToDoubleNeedsTemp() returns true. - void convertUInt64ToDouble(Register64 src, FloatRegister dest, - Register temp) PER_ARCH; - - void convertInt64ToDouble(Register64 src, FloatRegister dest) PER_ARCH; - - void convertIntPtrToDouble(Register src, FloatRegister dest) PER_ARCH; + void convertUInt64ToDouble(Register64 src, FloatRegister dest, Register temp) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + void convertInt64ToDouble(Register64 src, FloatRegister dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + void convertIntPtrToDouble(Register src, FloatRegister dest) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); public: // ======================================================================== // wasm support - FaultingCodeOffset wasmTrapInstruction() PER_SHARED_ARCH; + FaultingCodeOffset wasmTrapInstruction() DEFINED_ON(arm, arm64, loong64, + mips64, riscv64, ppc64, x86_shared, wasm32, ia64); void wasmTrap(wasm::Trap trap, const wasm::TrapSiteDesc& trapSiteDesc); @@ -3783,19 +4157,21 @@ // 32-bit heap lengths are limited to 4GB, and 64-bit heap lengths will be // limited to something much larger. - void wasmBoundsCheck32(Condition cond, Register index, - Register boundsCheckLimit, - Label* label) PER_SHARED_ARCH; - - void wasmBoundsCheck32(Condition cond, Register index, - Address boundsCheckLimit, - Label* label) PER_SHARED_ARCH; - - void wasmBoundsCheck64(Condition cond, Register64 index, - Register64 boundsCheckLimit, Label* label) PER_ARCH; - - void wasmBoundsCheck64(Condition cond, Register64 index, - Address boundsCheckLimit, Label* label) PER_ARCH; + void wasmBoundsCheck32(Condition cond, Register index, Register + boundsCheckLimit, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + + void wasmBoundsCheck32(Condition cond, Register index, Address + boundsCheckLimit, Label* label) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); + + void wasmBoundsCheck64(Condition cond, Register64 index, Register64 + boundsCheckLimit, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + + void wasmBoundsCheck64(Condition cond, Register64 index, Address + boundsCheckLimit, Label* label) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); // Each wasm load/store instruction appends its own wasm::Trap::OutOfBounds. void wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr, @@ -3832,14 +4208,13 @@ // register for the offset if the offset is large, and instructions to set it // up. void wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase, - Register ptr, AnyRegister output) DEFINED_ON(arm64, riscv64); + Register ptr, AnyRegister output) DEFINED_ON(arm64, riscv64, ia64); void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase, - Register ptr, Register64 output) DEFINED_ON(arm64, riscv64); + Register ptr, Register64 output) DEFINED_ON(arm64, riscv64, ia64); void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, - Register memoryBase, Register ptr) DEFINED_ON(arm64, riscv64); + Register memoryBase, Register ptr) DEFINED_ON(arm64, riscv64, ia64); void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, - Register memoryBase, Register ptr) - DEFINED_ON(arm64, riscv64); + Register memoryBase, Register ptr) DEFINED_ON(arm64, riscv64, ia64); // `ptr` will always be updated. void wasmUnalignedLoad(const wasm::MemoryAccessDesc& access, @@ -3879,53 +4254,55 @@ // The truncate-to-int32 methods do not bind the rejoin label; clients must // do so if oolWasmTruncateCheckF64ToI32() can jump to it. - void wasmTruncateDoubleToUInt32(FloatRegister input, Register output, - bool isSaturating, Label* oolEntry) PER_ARCH; - void wasmTruncateDoubleToInt32(FloatRegister input, Register output, - bool isSaturating, - Label* oolEntry) PER_SHARED_ARCH; + void wasmTruncateDoubleToUInt32(FloatRegister input, Register output, bool + isSaturating, Label* oolEntry) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + void wasmTruncateDoubleToInt32(FloatRegister input, Register output, bool + isSaturating, Label* oolEntry) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); void oolWasmTruncateCheckF64ToI32(FloatRegister input, Register output, - TruncFlags flags, - const wasm::TrapSiteDesc& trapSiteDesc, - Label* rejoin) PER_SHARED_ARCH; - - void wasmTruncateFloat32ToUInt32(FloatRegister input, Register output, - bool isSaturating, Label* oolEntry) PER_ARCH; - void wasmTruncateFloat32ToInt32(FloatRegister input, Register output, - bool isSaturating, - Label* oolEntry) PER_SHARED_ARCH; + TruncFlags flags, const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + void wasmTruncateFloat32ToUInt32(FloatRegister input, Register output, bool + isSaturating, Label* oolEntry) DEFINED_ON(mips64, arm, arm64, x86, x64, + loong64, riscv64, ppc64, wasm32, ia64); + void wasmTruncateFloat32ToInt32(FloatRegister input, Register output, bool + isSaturating, Label* oolEntry) DEFINED_ON(arm, arm64, loong64, mips64, + riscv64, ppc64, x86_shared, wasm32, ia64); void oolWasmTruncateCheckF32ToI32(FloatRegister input, Register output, - TruncFlags flags, - const wasm::TrapSiteDesc& trapSiteDesc, - Label* rejoin) PER_SHARED_ARCH; + TruncFlags flags, const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // The truncate-to-int64 methods will always bind the `oolRejoin` label // after the last emitted instruction. - void wasmTruncateDoubleToInt64(FloatRegister input, Register64 output, - bool isSaturating, Label* oolEntry, - Label* oolRejoin, FloatRegister tempDouble) - DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32, ppc64); - void wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output, - bool isSaturating, Label* oolEntry, - Label* oolRejoin, FloatRegister tempDouble) - DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32, ppc64); + void wasmTruncateDoubleToInt64(FloatRegister input, Register64 output, bool + isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister + tempDouble) DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32, + ppc64, ia64); + void wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output, bool + isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister + tempDouble) DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32, + ppc64, ia64); void oolWasmTruncateCheckF64ToI64(FloatRegister input, Register64 output, - TruncFlags flags, - const wasm::TrapSiteDesc& trapSiteDesc, - Label* rejoin) PER_SHARED_ARCH; - - void wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output, - bool isSaturating, Label* oolEntry, - Label* oolRejoin, FloatRegister tempDouble) - DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32, ppc64); - void wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output, - bool isSaturating, Label* oolEntry, - Label* oolRejoin, FloatRegister tempDouble) - DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32, ppc64); + TruncFlags flags, const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); + + void wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output, bool + isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister + tempDouble) DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32, + ppc64, ia64); + void wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output, bool + isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister + tempDouble) DEFINED_ON(arm64, x86, x64, mips64, loong64, riscv64, wasm32, + ppc64, ia64); void oolWasmTruncateCheckF32ToI64(FloatRegister input, Register64 output, - TruncFlags flags, - const wasm::TrapSiteDesc& trapSiteDesc, - Label* rejoin) PER_SHARED_ARCH; + TruncFlags flags, const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // This function takes care of loading the callee's instance and pinned regs // but it is the caller's responsibility to save/restore instance or pinned @@ -3947,14 +4324,17 @@ void wasmCollapseFrameFast(const ReturnCallAdjustmentInfo& retCallInfo); void wasmCheckSlowCallsite(Register ra, Label* notSlow, Register temp1, - Register temp2) PER_ARCH; + Register temp2) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); // Places slow class marker for tail calls. - void wasmMarkCallAsSlow() PER_ARCH; + void wasmMarkCallAsSlow() DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, + riscv64, ppc64, wasm32, ia64); // Combines slow class marker with actual assembler call. - CodeOffset wasmMarkedSlowCall(const wasm::CallSiteDesc& desc, - const Register reg) PER_SHARED_ARCH; + CodeOffset wasmMarkedSlowCall(const wasm::CallSiteDesc& desc, const Register + reg) DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); void wasmClampTable64Address(Register64 address, Register out); @@ -4230,8 +4610,9 @@ // is implementation-defined what happens if bits are lost or the value // becomes negative through the shift. On 64-bit systems, the high 32 bits of // indexTemp32 must be zero, not garbage. - void shiftIndex32AndAdd(Register indexTemp32, int shift, - Register pointer) PER_SHARED_ARCH; + void shiftIndex32AndAdd(Register indexTemp32, int shift, Register pointer) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); // The System ABI frequently states that the high bits of a 64-bit register // that holds a 32-bit return value are unpredictable, and C++ compilers will @@ -4241,13 +4622,15 @@ // convention, which requires predictable high bits. In practice, this means // that the 32-bit value will be zero-extended or sign-extended to 64 bits as // appropriate for the platform. - void widenInt32(Register r) - DEFINED_ON(arm64, x64, mips64, loong64, riscv64, ppc64); + void widenInt32(Register r) DEFINED_ON(arm64, x64, mips64, loong64, riscv64, + ppc64, ia64); // As enterFakeExitFrame(), but using register conventions appropriate for // wasm stubs. void enterFakeExitFrameForWasm(Register cxreg, Register scratch, - ExitFrameType type) PER_SHARED_ARCH; + ExitFrameType type) + DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, x86_shared, + wasm32, ia64); public: // ======================================================================== @@ -4268,7 +4651,8 @@ // ======================================================================== // Clamping functions. - inline void clampIntToUint8(Register reg) PER_SHARED_ARCH; + inline void clampIntToUint8(Register reg) DEFINED_ON(arm, arm64, loong64, + mips64, riscv64, ppc64, x86_shared, wasm32, ia64); public: // ======================================================================== @@ -4322,15 +4706,13 @@ // ARM: Registers must be distinct; `replacement` and `output` must be // (even,odd) pairs. - void compareExchange64(Synchronization sync, const Address& mem, - Register64 expected, Register64 replacement, - Register64 output) - DEFINED_ON(arm, arm64, x64, x86, mips64, loong64, riscv64, ppc64); - - void compareExchange64(Synchronization sync, const BaseIndex& mem, - Register64 expected, Register64 replacement, - Register64 output) - DEFINED_ON(arm, arm64, x64, x86, mips64, loong64, riscv64, ppc64); + void compareExchange64(Synchronization sync, const Address& mem, Register64 + expected, Register64 replacement, Register64 output) DEFINED_ON(arm, + arm64, x64, x86, mips64, loong64, riscv64, ppc64, ia64); + + void compareExchange64(Synchronization sync, const BaseIndex& mem, Register64 + expected, Register64 replacement, Register64 output) DEFINED_ON(arm, + arm64, x64, x86, mips64, loong64, riscv64, ppc64, ia64); // Exchange with memory. Return the value initially in memory. // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit @@ -4358,13 +4740,13 @@ // ARM: `value` and `output` must be distinct and (even,odd) pairs. // ARM64: `value` and `output` must be distinct. - void atomicExchange64(Synchronization sync, const Address& mem, - Register64 value, Register64 output) - DEFINED_ON(arm, arm64, x64, x86, mips64, loong64, riscv64, ppc64); - - void atomicExchange64(Synchronization sync, const BaseIndex& mem, - Register64 value, Register64 output) - DEFINED_ON(arm, arm64, x64, x86, mips64, loong64, riscv64, ppc64); + void atomicExchange64(Synchronization sync, const Address& mem, Register64 + value, Register64 output) DEFINED_ON(arm, arm64, x64, x86, mips64, + loong64, riscv64, ppc64, ia64); + + void atomicExchange64(Synchronization sync, const BaseIndex& mem, Register64 + value, Register64 output) DEFINED_ON(arm, arm64, x64, x86, mips64, + loong64, riscv64, ppc64, ia64); // Read-modify-write with memory. Return the value in memory before the // operation. @@ -4416,16 +4798,16 @@ // Registers `value`, `temp`, and `output` must all differ. void atomicFetchOp64(Synchronization sync, AtomicOp op, Register64 value, - const Address& mem, Register64 temp, Register64 output) - DEFINED_ON(arm, arm64, x64, mips64, loong64, riscv64, ppc64); + const Address& mem, Register64 temp, Register64 output) DEFINED_ON(arm, + arm64, x64, mips64, loong64, riscv64, ppc64, ia64); void atomicFetchOp64(Synchronization sync, AtomicOp op, const Address& value, const Address& mem, Register64 temp, Register64 output) DEFINED_ON(x86); void atomicFetchOp64(Synchronization sync, AtomicOp op, Register64 value, - const BaseIndex& mem, Register64 temp, Register64 output) - DEFINED_ON(arm, arm64, x64, mips64, loong64, riscv64, ppc64); + const BaseIndex& mem, Register64 temp, Register64 output) DEFINED_ON(arm, + arm64, x64, mips64, loong64, riscv64, ppc64, ia64); void atomicFetchOp64(Synchronization sync, AtomicOp op, const Address& value, const BaseIndex& mem, Register64 temp, Register64 output) @@ -4442,15 +4824,15 @@ const Address& mem) DEFINED_ON(x64); void atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value, - const Address& mem, Register64 temp) - DEFINED_ON(arm, arm64, mips64, loong64, riscv64, ppc64); + const Address& mem, Register64 temp) DEFINED_ON(arm, arm64, mips64, + loong64, riscv64, ppc64, ia64); void atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value, const BaseIndex& mem) DEFINED_ON(x64); void atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value, - const BaseIndex& mem, Register64 temp) - DEFINED_ON(arm, arm64, mips64, loong64, riscv64, ppc64); + const BaseIndex& mem, Register64 temp) DEFINED_ON(arm, arm64, mips64, + loong64, riscv64, ppc64, ia64); // 64-bit atomic load. On 64-bit systems, use regular load with // Synchronization::Load, not this method. @@ -4493,10 +4875,9 @@ Register replacement, Register output) DEFINED_ON(arm, arm64, x86_shared); - void wasmCompareExchange(const wasm::MemoryAccessDesc& access, - const BaseIndex& mem, Register expected, - Register replacement, Register output) - DEFINED_ON(arm, arm64, x86_shared); + void wasmCompareExchange(const wasm::MemoryAccessDesc& access, const + BaseIndex& mem, Register expected, Register replacement, Register output) + DEFINED_ON(arm, arm64, x86_shared, ia64); void wasmCompareExchange(const wasm::MemoryAccessDesc& access, const Address& mem, Register expected, @@ -4516,9 +4897,9 @@ const Address& mem, Register value, Register output) DEFINED_ON(arm, arm64, x86_shared); - void wasmAtomicExchange(const wasm::MemoryAccessDesc& access, - const BaseIndex& mem, Register value, Register output) - DEFINED_ON(arm, arm64, x86_shared); + void wasmAtomicExchange(const wasm::MemoryAccessDesc& access, const + BaseIndex& mem, Register value, Register output) DEFINED_ON(arm, arm64, + x86_shared, ia64); void wasmAtomicExchange(const wasm::MemoryAccessDesc& access, const Address& mem, Register value, @@ -4541,8 +4922,8 @@ Register output) DEFINED_ON(x86_shared); void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op, - Register value, const BaseIndex& mem, Register temp, - Register output) DEFINED_ON(arm, arm64, x86_shared); + Register value, const BaseIndex& mem, Register temp, Register output) + DEFINED_ON(arm, arm64, x86_shared, ia64); void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Imm32 value, const BaseIndex& mem, Register temp, @@ -4574,8 +4955,8 @@ DEFINED_ON(x86_shared); void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op, - Register value, const BaseIndex& mem, Register temp) - DEFINED_ON(arm, arm64, x86_shared); + Register value, const BaseIndex& mem, Register temp) DEFINED_ON(arm, + arm64, x86_shared, ia64); void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Imm32 value, const BaseIndex& mem, Register temp) @@ -4622,10 +5003,10 @@ Register64 replacement, Register64 output) PER_ARCH; - void wasmCompareExchange64(const wasm::MemoryAccessDesc& access, - const BaseIndex& mem, Register64 expected, - Register64 replacement, - Register64 output) PER_ARCH; + void wasmCompareExchange64(const wasm::MemoryAccessDesc& access, const + BaseIndex& mem, Register64 expected, Register64 replacement, Register64 + output) DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, + wasm32, ia64); // x86: `value` must be ecx:ebx; `output` must be edx:eax. // ARM: Registers must be distinct; `value` and `output` must be (even,odd) @@ -4634,11 +5015,13 @@ void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, const Address& mem, Register64 value, - Register64 output) PER_ARCH; - - void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, - const BaseIndex& mem, Register64 value, - Register64 output) PER_ARCH; + Register64 output) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); + + void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, const + BaseIndex& mem, Register64 value, Register64 output) DEFINED_ON(mips64, + arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, ia64); // x86: `output` must be edx:eax, `temp` must be ecx:ebx. // x64: For And, Or, and Xor `output` must be rax. @@ -4652,9 +5035,9 @@ DEFINED_ON(arm, arm64, mips64, loong64, riscv64, ppc64, x64); void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op, - Register64 value, const BaseIndex& mem, - Register64 temp, Register64 output) - DEFINED_ON(arm, arm64, mips64, loong64, riscv64, ppc64, x64); + Register64 value, const BaseIndex& mem, Register64 temp, Register64 + output) DEFINED_ON(arm, arm64, mips64, loong64, riscv64, ppc64, x64, + ia64); void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op, const Address& value, const Address& mem, @@ -4692,15 +5075,13 @@ // For additional register constraints, see the primitive 32-bit operations // and/or wasm operations above. - void compareExchangeJS(Scalar::Type arrayType, Synchronization sync, - const Address& mem, Register expected, - Register replacement, Register temp, - AnyRegister output) DEFINED_ON(arm, arm64, x86_shared); - - void compareExchangeJS(Scalar::Type arrayType, Synchronization sync, - const BaseIndex& mem, Register expected, - Register replacement, Register temp, - AnyRegister output) DEFINED_ON(arm, arm64, x86_shared); + void compareExchangeJS(Scalar::Type arrayType, Synchronization sync, const + Address& mem, Register expected, Register replacement, Register temp, + AnyRegister output) DEFINED_ON(arm, arm64, x86_shared, ia64); + + void compareExchangeJS(Scalar::Type arrayType, Synchronization sync, const + BaseIndex& mem, Register expected, Register replacement, Register temp, + AnyRegister output) DEFINED_ON(arm, arm64, x86_shared, ia64); void compareExchangeJS(Scalar::Type arrayType, Synchronization sync, const Address& mem, Register expected, @@ -4716,13 +5097,13 @@ AnyRegister output) DEFINED_ON(mips64, loong64, riscv64, ppc64); - void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync, - const Address& mem, Register value, Register temp, - AnyRegister output) DEFINED_ON(arm, arm64, x86_shared); - - void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync, - const BaseIndex& mem, Register value, Register temp, - AnyRegister output) DEFINED_ON(arm, arm64, x86_shared); + void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync, const + Address& mem, Register value, Register temp, AnyRegister output) + DEFINED_ON(arm, arm64, x86_shared, ia64); + + void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync, const + BaseIndex& mem, Register value, Register temp, AnyRegister output) + DEFINED_ON(arm, arm64, x86_shared, ia64); void atomicExchangeJS(Scalar::Type arrayType, Synchronization sync, const Address& mem, Register value, Register valueTemp, @@ -4736,15 +5117,13 @@ Register maskTemp, Register temp, AnyRegister output) DEFINED_ON(mips64, loong64, riscv64, ppc64); - void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, - AtomicOp op, Register value, const Address& mem, - Register temp1, Register temp2, AnyRegister output) - DEFINED_ON(arm, arm64, x86_shared); - - void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, - AtomicOp op, Register value, const BaseIndex& mem, - Register temp1, Register temp2, AnyRegister output) - DEFINED_ON(arm, arm64, x86_shared); + void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp + op, Register value, const Address& mem, Register temp1, Register temp2, + AnyRegister output) DEFINED_ON(arm, arm64, x86_shared, ia64); + + void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp + op, Register value, const BaseIndex& mem, Register temp1, Register temp2, + AnyRegister output) DEFINED_ON(arm, arm64, x86_shared, ia64); void atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp op, Imm32 value, const Address& mem, @@ -4768,13 +5147,13 @@ Register maskTemp, Register temp, AnyRegister output) DEFINED_ON(mips64, loong64, riscv64, ppc64); - void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync, - AtomicOp op, Register value, const Address& mem, - Register temp) DEFINED_ON(arm, arm64, x86_shared); - - void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync, - AtomicOp op, Register value, const BaseIndex& mem, - Register temp) DEFINED_ON(arm, arm64, x86_shared); + void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp + op, Register value, const Address& mem, Register temp) DEFINED_ON(arm, + arm64, x86_shared, ia64); + + void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp + op, Register value, const BaseIndex& mem, Register temp) DEFINED_ON(arm, + arm64, x86_shared, ia64); void atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp op, Imm32 value, const Address& mem, @@ -4798,7 +5177,8 @@ void atomicIsLockFreeJS(Register value, Register output); - void atomicPause() PER_SHARED_ARCH; + void atomicPause() DEFINED_ON(arm, arm64, loong64, mips64, riscv64, ppc64, + x86_shared, wasm32, ia64); // ======================================================================== // Spectre Mitigations. @@ -4828,7 +5208,8 @@ // index masking. void boundsCheck32PowerOfTwo(Register index, uint32_t length, Label* failure); - void speculationBarrier() PER_SHARED_ARCH; + void speculationBarrier() DEFINED_ON(arm, arm64, loong64, mips64, riscv64, + ppc64, x86_shared, wasm32, ia64); //}}} check_macroassembler_decl_style public: @@ -5649,7 +6030,9 @@ // Inline version of js::ClampDoubleToUint8. // This function clobbers the input register. - void clampDoubleToUint8(FloatRegister input, Register output) PER_ARCH; + void clampDoubleToUint8(FloatRegister input, Register output) + DEFINED_ON(mips64, arm, arm64, x86, x64, loong64, riscv64, ppc64, wasm32, + ia64); // If source is a double, load into dest. // If source is int32, convert to double and store in dest. @@ -5952,8 +6335,8 @@ template inline void addStackPtrTo(T t); - void subFromStackPtr(Imm32 imm32) - DEFINED_ON(mips64, loong64, riscv64, ppc64, wasm32, arm, x86, x64); + void subFromStackPtr(Imm32 imm32) DEFINED_ON(mips64, loong64, riscv64, ppc64, + wasm32, arm, x86, x64, ia64); void subFromStackPtr(Register reg); template diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/MoveEmitter.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/MoveEmitter.h --- firefox-153.0.1/js/src/jit/MoveEmitter.h.vanilla +++ firefox-153.0.1/js/src/jit/MoveEmitter.h @@ -17,6 +17,8 @@ # include "jit/loong64/MoveEmitter-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/MoveEmitter-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/MoveEmitter-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/MoveEmitter-ppc64.h" #elif defined(JS_CODEGEN_WASM32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/Registers.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/Registers.h --- firefox-153.0.1/js/src/jit/Registers.h.vanilla +++ firefox-153.0.1/js/src/jit/Registers.h @@ -20,6 +20,8 @@ # include "jit/loong64/Architecture-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/Architecture-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/Architecture-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/Architecture-ppc64.h" #elif defined(JS_CODEGEN_WASM32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/shared/Assembler-shared.h b/js/src/jit/shared/Assembler-shared.h --- firefox-153.0.1/js/src/jit/shared/Assembler-shared.h.vanilla +++ firefox-153.0.1/js/src/jit/shared/Assembler-shared.h @@ -31,16 +31,21 @@ #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \ defined(JS_CODEGEN_WASM32) || defined(JS_CODEGEN_RISCV64) || \ - defined(JS_CODEGEN_PPC64) -// Push return addresses callee-side. + defined(JS_CODEGEN_PPC64) || defined(JS_CODEGEN_IA64) +// Push return addresses callee-side. IA-64's br.call/br.ret convention +// leaves the return address in b0 exactly like a link register, so it +// belongs in this group rather than the stack-based x86/x64 style. # define JS_USE_LINK_REGISTER #endif #if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_ARM64) || \ defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) || \ - defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_PPC64) + defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_PPC64) || \ + defined(JS_CODEGEN_IA64) // JS_CODELABEL_LINKMODE gives labels additional metadata -// describing how Bind() should patch them. +// describing how Bind() should patch them. IA-64 needs this because a movl's +// 64-bit immediate is scattered across non-contiguous bundle bits, unlike a +// plain in-memory pointer -- the two patch shapes need different code. # define JS_CODELABEL_LINKMODE #endif diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/SharedICHelpers-inl.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/SharedICHelpers-inl.h --- firefox-153.0.1/js/src/jit/SharedICHelpers-inl.h.vanilla +++ firefox-153.0.1/js/src/jit/SharedICHelpers-inl.h @@ -19,6 +19,8 @@ # include "jit/loong64/SharedICHelpers-loong64-inl.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/SharedICHelpers-riscv64-inl.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/SharedICHelpers-ia64-inl.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/SharedICHelpers-ppc64-inl.h" #elif defined(JS_CODEGEN_WASM32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/SharedICHelpers.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/SharedICHelpers.h --- firefox-153.0.1/js/src/jit/SharedICHelpers.h.vanilla +++ firefox-153.0.1/js/src/jit/SharedICHelpers.h @@ -19,6 +19,8 @@ # include "jit/loong64/SharedICHelpers-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/SharedICHelpers-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/SharedICHelpers-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/SharedICHelpers-ppc64.h" #elif defined(JS_CODEGEN_WASM32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/SharedICRegisters.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/SharedICRegisters.h --- firefox-153.0.1/js/src/jit/SharedICRegisters.h.vanilla +++ firefox-153.0.1/js/src/jit/SharedICRegisters.h @@ -19,6 +19,8 @@ # include "jit/loong64/SharedICRegisters-loong64.h" #elif defined(JS_CODEGEN_RISCV64) # include "jit/riscv64/SharedICRegisters-riscv64.h" +#elif defined(JS_CODEGEN_IA64) +# include "jit/ia64/SharedICRegisters-ia64.h" #elif defined(JS_CODEGEN_PPC64) # include "jit/ppc64/SharedICRegisters-ppc64.h" #elif defined(JS_CODEGEN_WASM32) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/ia64/Architecture-ia64.cpp /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/ia64/Architecture-ia64.cpp --- firefox-153.0.1/js/src/jit/ia64/Architecture-ia64.cpp.vanilla +++ firefox-153.0.1/js/src/jit/ia64/Architecture-ia64.cpp @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#include "jit/ia64/Architecture-ia64.h" + +#include "jit/RegisterSets.h" + +namespace js { +namespace jit { + +// Singles and doubles share one architectural register on ia64, so a set that +// mentions both kinds of the same register only needs one slot pushed. +TypedRegisterSet FloatRegister::ReduceSetForPush( + const TypedRegisterSet& s) { + SetType bits = s.bits(); + return TypedRegisterSet(bits); +} + +uint32_t FloatRegister::GetPushSizeInBytes( + const TypedRegisterSet& s) { + // Spill slots are uniformly 8 bytes; ia64's 82-bit extended format is only + // used inside the register file, values in memory are IEEE doubles. + return s.size() * sizeof(double); +} + +uint32_t FloatRegister::getRegisterDumpOffsetInBytes() { + return code_ * sizeof(double); +} + +} // namespace jit +} // namespace js diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/ia64/Architecture-ia64.h b/js/src/jit/ia64/Architecture-ia64.h --- firefox-153.0.1/js/src/jit/ia64/Architecture-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/Architecture-ia64.h @@ -0,0 +1,480 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_Architecture_ia64_h +#define jit_ia64_Architecture_ia64_h + +#include +#include + +#include "jit/JitSpewer.h" +#include "jit/shared/Architecture-shared.h" +#include "js/Utility.h" + +namespace js { +namespace jit { + +// [SMDOC] IA-64 (Itanium) register model +// +// IA-64 exposes 128 general registers, but r32-r127 form the *register stack*: +// their number and identity depend on the `alloc` instruction executed by the +// current procedure, and the RSE (Register Stack Engine) spills them +// asynchronously to a separate backing store that grows upward from ar.bsp. +// Exposing rotating/stacked registers to a JIT register allocator buys little +// and costs a great deal of complexity, so this backend allocates only from the +// 32 *static* registers r0-r31, which every procedure sees identically. +// +// The register stack is still used, but only by the ABI glue: each JIT frame's +// prologue executes +// alloc rTmp = ar.pfs, 0, 0, 8, 0 +// to obtain the eight outgoing-argument registers out0-out7 (which alias r32- +// r39 in that frame) required to call C functions, and restores ar.pfs and b0 +// on return. Nothing else refers to r32+. +// +// Static register convention (IA-64 ELF psABI): +// r0 hardwired 0, writes are illegal +// r1 gp, global pointer; must be valid at every call boundary +// r2, r3 scratch, reserved here as assembler temporaries +// r4-r7 callee-saved (preserved) +// r8-r11 return values / scratch (r8 is the first return value) +// r12 sp, stack pointer; 16-byte aligned, grows down +// r13 tp, thread pointer +// r14-r31 scratch (caller-saved) +// +// Floating point registers f0 and f1 are hardwired to +0.0 and +1.0. f2-f5 and +// f16-f31 are preserved; f6-f15 and f32-f127 are scratch. As with the general +// registers only the low 32 are exposed, which keeps the register set masks a +// single 32-bit word and avoids the rotating region entirely. +// +// Predicate registers p0-p63 (p0 is hardwired true) and branch registers b0-b7 +// are not managed by the allocator; the assembler uses a fixed set of scratch +// predicates and branch registers. + +// IA-64 has no packed SIMD in the FP registers; the parallel arithmetic +// instructions operate on the general registers. No Simd128 support is exposed. +static const uint32_t SimdMemoryAlignment = 16; + +static const uint32_t WasmStackAlignment = 16; +static const uint32_t WasmTrapInstructionLength = 16; + +// See comments in wasm::GenerateFunctionPrologue. +static constexpr uint32_t WasmCheckedCallEntryOffset = 0u; + +class Registers { + public: + enum RegisterID { + r0 = 0, + r1, + r2, + r3, + r4, + r5, + r6, + r7, + r8, + r9, + r10, + r11, + r12, + r13, + r14, + r15, + r16, + r17, + r18, + r19, + r20, + r21, + r22, + r23, + r24, + r25, + r26, + r27, + r28, + r29, + r30, + r31, + // r32-r39 are the outgoing-argument window out0-out7 of a frame created by + // `alloc rN = ar.pfs, 0, 0, 8, 0` in the JIT prologue. They are named so + // that ABI calls can address them, but are never handed to the register + // allocator. + r32, + r33, + r34, + r35, + r36, + r37, + r38, + r39, + zero = r0, + gp = r1, + sp = r12, + tp = r13, + out0 = r32, + out1 = r33, + out2 = r34, + out3 = r35, + out4 = r36, + out5 = r37, + out6 = r38, + out7 = r39, + invalid_reg, + invalid_reg2 = invalid_reg + }; + + using Code = uint8_t; + using Encoding = RegisterID; + + union RegisterContent { + uintptr_t r; + }; + + using SetType = uint64_t; + + static const uint32_t Total = 40; + static const uint32_t TotalPhys = 40; + static const uint32_t Allocatable = 21; + + static const char* GetName(Code code) { + static const char* const Names[] = { + "r0", "gp", "r2", "r3", "r4", "r5", "r6", "r7", + "r8", "r9", "r10", "r11", "sp", "tp", "r14", "r15", + "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", + "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31", + "out0", "out1", "out2", "out3", "out4", "out5", "out6", "out7"}; + static_assert(Total == std::size(Names), "GetName table must be complete"); + return Names[code]; + } + static Code FromName(const char* name) { + for (size_t i = 0; i < Total; i++) { + if (strcmp(GetName(Code(i)), name) == 0) { + return Code(i); + } + } + return Code(invalid_reg); + } + + static uint32_t SetSize(SetType x) { return std::popcount(x); } + static uint32_t FirstBit(SetType x) { return std::countr_zero(x); } + static uint32_t LastBit(SetType x) { return 63 - std::countl_zero(x); } + + static const Encoding StackPointer = sp; + static const Encoding Invalid = invalid_reg; + + static const SetType AllMask = (SetType(1) << Total) - 1; + static const SetType NoneMask = 0x0; + + static const SetType ArgRegMask = + (SetType(1) << out0) | (SetType(1) << out1) | (SetType(1) << out2) | + (SetType(1) << out3) | (SetType(1) << out4) | (SetType(1) << out5) | + (SetType(1) << out6) | (SetType(1) << out7); + + // r0 (zero), r1 (gp), r12 (sp) and r13 (tp) are fixed by the ABI; r2 and r3 + // are held back as assembler scratch for materialising large immediates and + // synthesising addressing modes; r10 and r11 are the two further scratches + // the multi-instruction sequences need (see BorrowScratch); r4 is the frame + // pointer; the out0-out7 window is owned by the call sequence. + // Furthermore, the backend reserveds r7 (HeapReg) and r29 (InstanceReg). + static const SetType NonAllocatableMask = + (SetType(1) << r0) | (SetType(1) << r1) | (SetType(1) << r2) | + (SetType(1) << r3) | (SetType(1) << r4) | (SetType(1) << r10) | + (SetType(1) << r11) | (SetType(1) << r12) | (SetType(1) << r13) | + ArgRegMask | + (SetType(1) << Registers::r7) | // HeapReg + (SetType(1) << Registers::r29); // InstanceReg + + // Callee-saved static registers. + static const SetType NonVolatileMask = + (SetType(1) << r4) | (SetType(1) << r5) | (SetType(1) << r6) | + (SetType(1) << r7); + + static const SetType VolatileMask = AllMask & ~NonVolatileMask; + + static const SetType WrapperMask = VolatileMask; + + // r8 holds the first integer return value. + static const SetType JSCallMask = (SetType(1) << r8); + static const SetType CallMask = (SetType(1) << r8); + + static const SetType AllocatableMask = AllMask & ~NonAllocatableMask; + + static_assert(Allocatable == std::popcount(AllocatableMask), + "Allocatable must agree with NonAllocatableMask"); +}; + +// Smallest integer type that can hold a register bitmask. +using PackedRegisterMask = uint64_t; + +class FloatRegisters { + public: + enum FPRegisterID { + f0 = 0, // hardwired +0.0 + f1, // hardwired +1.0 + f2, + f3, + f4, + f5, + f6, + f7, + f8, + f9, + f10, + f11, + f12, + f13, + f14, + f15, + f16, + f17, + f18, + f19, + f20, + f21, + f22, + f23, + f24, + f25, + f26, + f27, + f28, + f29, + f30, + f31, + invalid_reg + }; + + using Code = uint8_t; + using Encoding = FPRegisterID; + + union RegisterContent { + float s; + double d; + }; + + // A double and a single occupy the same architectural register on IA-64 + // (registers hold an 82-bit extended value and the precision is a property of + // the instruction, not of the register), so one bit per register suffices and + // the type is tracked out of band by FloatRegister::Kind. + using SetType = uint32_t; + + static const char* GetName(Code code) { + static const char* const Names[] = { + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"}; + static_assert(Total == std::size(Names), "GetName table must be complete"); + return Names[code]; + } + static Code FromName(const char* name) { + for (size_t i = 0; i < Total; i++) { + if (strcmp(GetName(Code(i)), name) == 0) { + return Code(i); + } + } + return Code(invalid_reg); + } + + static const Code Invalid = invalid_reg; + + static const uint32_t Total = 32; + static const uint32_t TotalPhys = 32; + static const uint32_t Allocatable = 24; + + static const SetType AllMask = 0xffffffff; + static const SetType NoneMask = 0x0; + + // f0 and f1 are hardwired constants. f2-f5 are reserved as internal + // temporaries for the multi-step divide/sqrt/multiply sequences (which + // need that many simultaneously live FP registers), and f6/f7 back the + // Scratch{Double,Float32}Scope pair, mirroring the psABI's f6/f7 + // scratch-register convention. This leaves f8-f31 (24 registers) + // allocatable, matching the psABI's argument (f8-f15) and callee-saved + // (f16-f31) split. + static const SetType NonAllocatableMask = + (1 << f0) | (1 << f1) | (1 << f2) | (1 << f3) | (1 << f4) | (1 << f5) | + (1 << f6) | (1 << f7); + + // f2-f5 and f16-f31 are callee-saved; f2-f5 are already non-allocatable. + static const SetType NonVolatileMask = + (1 << f4) | (1 << f5) | (1 << f16) | (1 << f17) | (1 << f18) | + (1 << f19) | (1 << f20) | (1 << f21) | (1 << f22) | (1 << f23) | + (1 << f24) | (1 << f25) | (1 << f26) | (1 << f27) | (1 << f28) | + (1 << f29) | (1 << f30) | (1 << f31); + + static const SetType VolatileMask = AllMask & ~NonVolatileMask; + + static const SetType AllocatableMask = AllMask & ~NonAllocatableMask; + + static const SetType AllDoubleMask = AllMask; + static const SetType AllSingleMask = AllMask; + // No packed SIMD in the FP register file. + static const SetType AllSimd128Mask = 0; + + static const SetType WrapperMask = VolatileMask; +}; + +template +class TypedRegisterSet; + +struct FloatRegister { + using Codes = FloatRegisters; + using Code = Codes::Code; + using Encoding = Codes::Encoding; + using SetType = Codes::SetType; + + enum Kind : uint8_t { Single, Double }; + + Code code_; + Kind kind_; + + constexpr FloatRegister() : code_(Codes::Invalid), kind_(Double) {} + constexpr FloatRegister(Code code, Kind kind) : code_(code), kind_(kind) {} + // Not explicit: the platform register definitions in Assembler-ia64.h use + // brace initialisation from a bare register code. + constexpr FloatRegister(Code code) : code_(code), kind_(Double) {} + + static uint32_t FirstBit(SetType x) { return std::countr_zero(x); } + static uint32_t LastBit(SetType x) { return 31 - std::countl_zero(x); } + + static FloatRegister FromCode(uint32_t i) { + MOZ_ASSERT(i < Codes::Total); + return FloatRegister(Code(i), Double); + } + + bool isSingle() const { + MOZ_ASSERT(!isInvalid()); + return kind_ == Single; + } + bool isDouble() const { + MOZ_ASSERT(!isInvalid()); + return kind_ == Double; + } + bool isSimd128() const { return false; } + bool isInvalid() const { return code_ == Codes::Invalid; } + + FloatRegister asSingle() const { + MOZ_ASSERT(!isInvalid()); + return FloatRegister(code_, Single); + } + FloatRegister asDouble() const { + MOZ_ASSERT(!isInvalid()); + return FloatRegister(code_, Double); + } + FloatRegister asSimd128() const { MOZ_CRASH("no SIMD on ia64"); } + + Code code() const { + MOZ_ASSERT(!isInvalid()); + return code_; + } + Encoding encoding() const { + MOZ_ASSERT(!isInvalid()); + return Encoding(code_); + } + const char* name() const { return Codes::GetName(code_); } + + bool volatile_() const { + MOZ_ASSERT(!isInvalid()); + return !!((SetType(1) << code_) & Codes::VolatileMask); + } + + bool operator==(FloatRegister other) const { + return code_ == other.code_ && kind_ == other.kind_; + } + bool operator!=(FloatRegister other) const { return !operator==(other); } + + // Singles and doubles share one architectural register, so any two registers + // with the same code alias one another. + bool aliases(FloatRegister other) const { return code_ == other.code_; } + uint32_t numAliased() const { return 2; } + FloatRegister aliased(uint32_t aliasIdx) const { + MOZ_ASSERT(aliasIdx < 2); + if (aliasIdx == 0) { + return *this; + } + return FloatRegister(code_, kind_ == Single ? Double : Single); + } + + bool equiv(FloatRegister other) const { return kind_ == other.kind_; } + uint32_t size() const { return kind_ == Single ? sizeof(float) : sizeof(double); } + + uint32_t numAlignedAliased() const { return numAliased(); } + FloatRegister alignedAliased(uint32_t aliasIdx) const { + return aliased(aliasIdx); + } + SetType alignedOrDominatedAliasedSet() const { return SetType(1) << code_; } + + static constexpr RegTypeName DefaultType = RegTypeName::Float64; + + template + static SetType LiveAsIndexableSet(SetType s) { + return SetType(0); + } + + template + static SetType AllocatableAsIndexableSet(SetType s) { + static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable"); + return LiveAsIndexableSet(s); + } + + static TypedRegisterSet ReduceSetForPush( + const TypedRegisterSet& s); + static uint32_t GetPushSizeInBytes(const TypedRegisterSet& s); + uint32_t getRegisterDumpOffsetInBytes(); + + static uint32_t SetSize(SetType x) { return std::popcount(x); } + static Code FromName(const char* name) { return Codes::FromName(name); } +}; + +template <> +inline FloatRegister::SetType +FloatRegister::LiveAsIndexableSet(SetType set) { + return set; +} + +template <> +inline FloatRegister::SetType +FloatRegister::LiveAsIndexableSet(SetType set) { + return set; +} + +template <> +inline FloatRegister::SetType +FloatRegister::LiveAsIndexableSet(SetType set) { + return set; +} + +// Singles and doubles occupy the same register, so there are no unaliased +// doubles and no multi-register aliasing. +inline bool hasUnaliasedDouble() { return false; } +inline bool hasMultiAlias() { return false; } + +// IA-64 reserves 16 bytes of scratch at the top of the outgoing argument area +// per the psABI. +static constexpr uint32_t ShadowStackSpace = 16; + +// The IA-64 stack pointer must stay 16-byte aligned at all times. +static constexpr uint32_t ABIStackAlignment = 16; +static constexpr uint32_t CodeAlignment = 16; +static constexpr uint32_t JitStackAlignment = 16; + +static constexpr uint32_t JitStackValueAlignment = + JitStackAlignment / sizeof(Value); +static_assert(JitStackAlignment % sizeof(Value) == 0 && + JitStackValueAlignment >= 1, + "Stack alignment should be a non-zero multiple of sizeof(Value)"); + +static constexpr uint32_t SimdMemoryAlignmentBytes = 16; + +// An IP-relative branch encodes a signed 21-bit immediate scaled by the 16-byte +// bundle size, giving +/-16 MiB. +static const uint32_t JumpImmediateRange = 16 * 1024 * 1024; + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_Architecture_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/ia64/Assembler-ia64.h b/js/src/jit/ia64/Assembler-ia64.h --- firefox-153.0.1/js/src/jit/ia64/Assembler-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/Assembler-ia64.h @@ -0,0 +1,591 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_Assembler_ia64_h +#define jit_ia64_Assembler_ia64_h + +#include "mozilla/Assertions.h" + +#include + +#include "jit/FlushICache.h" +#include "jit/ia64/Architecture-ia64.h" +#include "jit/ia64/AssemblerCore-ia64.h" +#include "jit/Registers.h" +#include "jit/RegisterSets.h" +#include "jit/shared/Assembler-shared.h" +#include "jit/shared/IonAssemblerBuffer.h" + +namespace js { +namespace jit { + +class MacroAssembler; + +static constexpr Register StackPointer{Registers::sp}; +static constexpr Register FramePointer{Registers::r4}; +static constexpr Register ReturnReg{Registers::r8}; +static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::f8, + FloatRegister::Single}; +static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::f8, + FloatRegister::Double}; +static constexpr FloatRegister ReturnSimd128Reg = {FloatRegisters::invalid_reg}; +static constexpr FloatRegister ScratchSimd128Reg = { + FloatRegisters::invalid_reg}; +static constexpr FloatRegister InvalidFloatReg = {FloatRegisters::invalid_reg}; + +// f6/f7 back the two FP scratch scopes, mirroring the GPR ScratchReg/ +// SecondScratchReg split below; f2-f5 are reserved separately as internal +// temporaries for the multi-step divide/sqrt/multiply sequences in +// MacroAssembler-ia64.cpp and are never handed out via a scope. +static constexpr FloatRegister ScratchFloat32Reg = {FloatRegisters::f6, + FloatRegister::Single}; +static constexpr FloatRegister ScratchDoubleReg = {FloatRegisters::f6, + FloatRegister::Double}; +static constexpr FloatRegister SecondScratchFloat32Reg = { + FloatRegisters::f7, FloatRegister::Single}; +static constexpr FloatRegister SecondScratchDoubleReg = { + FloatRegisters::f7, FloatRegister::Double}; + +struct ScratchFloat32Scope : public AutoFloatRegisterScope { + explicit ScratchFloat32Scope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchFloat32Reg) {} +}; + +struct ScratchDoubleScope : public AutoFloatRegisterScope { + explicit ScratchDoubleScope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchDoubleReg) {} +}; + +struct SecondScratchFloat32Scope : public AutoFloatRegisterScope { + explicit SecondScratchFloat32Scope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, SecondScratchFloat32Reg) {} +}; + +struct SecondScratchDoubleScope : public AutoFloatRegisterScope { + explicit SecondScratchDoubleScope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, SecondScratchDoubleReg) {} +}; + +static constexpr Register OsrFrameReg{Registers::r15}; +static constexpr Register PreBarrierReg{Registers::r16}; +static constexpr Register InterpreterPCReg{Registers::r17}; +static constexpr Register CallTempReg0{Registers::r18}; +static constexpr Register CallTempReg1{Registers::r19}; +static constexpr Register CallTempReg2{Registers::r20}; +static constexpr Register CallTempReg3{Registers::r21}; +static constexpr Register CallTempReg4{Registers::r22}; +static constexpr Register CallTempReg5{Registers::r23}; +static constexpr Register InvalidReg{Registers::invalid_reg}; + +// r2 and r3 are reserved by Architecture-ia64.h as assembler temporaries for +// materialising large immediates and synthesising addressing modes. +static constexpr Register ScratchReg{Registers::r3}; +static constexpr Register SecondScratchReg{Registers::r2}; +// A few sequences (variable rotate, the atomic CAS retry loops) need one or two +// registers beyond those two. They are reserved rather than taken from the +// allocatable set, because the register allocator has no way to know a masm +// helper clobbered one. +static constexpr Register ThirdScratchReg{Registers::r10}; +static constexpr Register FourthScratchReg{Registers::r11}; +static constexpr Register GpReg{Registers::gp}; + +struct ScratchRegisterScope : public AutoRegisterScope { + explicit ScratchRegisterScope(MacroAssembler& masm) + : AutoRegisterScope(masm, ScratchReg) {} +}; + +struct SecondScratchRegisterScope : public AutoRegisterScope { + explicit SecondScratchRegisterScope(MacroAssembler& masm) + : AutoRegisterScope(masm, SecondScratchReg) {} +}; +static constexpr Register CallTempNonArgRegs[] = { + CallTempReg0, CallTempReg1, CallTempReg2, + CallTempReg3, CallTempReg4, CallTempReg5}; +static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs); + +static constexpr Register IntArgReg0{Registers::out0}; +static constexpr Register IntArgReg1{Registers::out1}; +static constexpr Register IntArgReg2{Registers::out2}; +static constexpr Register IntArgReg3{Registers::out3}; +static constexpr Register IntArgReg4{Registers::out4}; +static constexpr Register IntArgReg5{Registers::out5}; +static constexpr Register IntArgReg6{Registers::out6}; +static constexpr Register IntArgReg7{Registers::out7}; + +// The IA-64 psABI passes the first eight integer arguments in out0-out7 and the +// first eight FP arguments in f8-f15. +static constexpr uint32_t NumIntArgRegs = 8; +static constexpr uint32_t NumFloatArgRegs = 8; +static constexpr Register IntArgRegs[NumIntArgRegs] = { + IntArgReg0, IntArgReg1, IntArgReg2, IntArgReg3, + IntArgReg4, IntArgReg5, IntArgReg6, IntArgReg7}; +static constexpr Register HeapReg{Registers::r7}; + +static constexpr Register RegExpMatcherRegExpReg{Registers::r25}; +static constexpr Register RegExpMatcherStringReg{Registers::r27}; +static constexpr Register RegExpMatcherLastIndexReg{Registers::r28}; + +static constexpr Register RegExpExecTestRegExpReg{Registers::r25}; +static constexpr Register RegExpExecTestStringReg{Registers::r27}; + +static constexpr Register RegExpSearcherRegExpReg{Registers::r25}; +static constexpr Register RegExpSearcherStringReg{Registers::r27}; +static constexpr Register RegExpSearcherLastIndexReg{Registers::r28}; + +static constexpr Register JSReturnReg_Type{Registers::r24}; +static constexpr Register JSReturnReg_Data{Registers::r24}; +static constexpr Register JSReturnReg{Registers::r24}; + +#if defined(JS_NUNBOX32) +static constexpr ValueOperand JSReturnOperand(JSReturnReg_Type, + JSReturnReg_Data); +static constexpr Register64 ReturnReg64(ReturnReg, ReturnReg); +#elif defined(JS_PUNBOX64) +static constexpr ValueOperand JSReturnOperand(JSReturnReg); +static constexpr Register64 ReturnReg64(ReturnReg); +#else +# error "Bad architecture" +#endif + +static constexpr Register ABINonArgReg0{Registers::r14}; +static constexpr Register ABINonArgReg1{Registers::r15}; +static constexpr Register ABINonArgReg2{Registers::r16}; +static constexpr Register ABINonArgReg3{Registers::r17}; +static constexpr Register ABINonArgReturnReg0{Registers::r14}; +static constexpr Register ABINonArgReturnReg1{Registers::r15}; +static constexpr Register ABINonVolatileReg{Registers::r6}; +static constexpr Register ABINonArgReturnVolatileReg{Registers::invalid_reg}; + +static constexpr FloatRegister ABINonArgDoubleReg = { + FloatRegisters::invalid_reg}; + +static constexpr Register WasmTableCallScratchReg0{Registers::invalid_reg}; +static constexpr Register WasmTableCallScratchReg1{Registers::invalid_reg}; +static constexpr Register WasmTableCallSigReg{Registers::invalid_reg}; +static constexpr Register WasmTableCallIndexReg{Registers::invalid_reg}; +static constexpr Register InstanceReg{Registers::r29}; +static constexpr Register WasmJitEntryReturnScratch{Registers::invalid_reg}; +static constexpr Register WasmCallRefCallScratchReg0{Registers::invalid_reg}; +static constexpr Register WasmCallRefCallScratchReg1{Registers::invalid_reg}; +static constexpr Register WasmCallRefCallScratchReg2{Registers::invalid_reg}; +static constexpr Register WasmCallRefReg{Registers::invalid_reg}; +static constexpr Register WasmTailCallInstanceScratchReg{ + Registers::invalid_reg}; +static constexpr Register WasmTailCallRAScratchReg{Registers::invalid_reg}; +static constexpr Register WasmTailCallFPScratchReg{Registers::invalid_reg}; + +// ABIStackAlignment, CodeAlignment, JitStackAlignment and +// JitStackValueAlignment are defined in Architecture-ia64.h; ia64 requires +// 16-byte stack alignment throughout. + +static const Scale ScalePointer = TimesEight; + +class Assembler : public AssemblerShared { + public: + enum Condition { + Equal, + NotEqual, + Above, + AboveOrEqual, + Below, + BelowOrEqual, + GreaterThan, + GreaterThanOrEqual, + LessThan, + LessThanOrEqual, + Overflow, + CarrySet, + CarryClear, + Signed, + NotSigned, + Zero, + NonZero, + Always, + }; + + enum DoubleCondition { + DoubleOrdered, + DoubleEqual, + DoubleNotEqual, + DoubleGreaterThan, + DoubleGreaterThanOrEqual, + DoubleLessThan, + DoubleLessThanOrEqual, + DoubleUnordered, + DoubleEqualOrUnordered, + DoubleNotEqualOrUnordered, + DoubleGreaterThanOrUnordered, + DoubleGreaterThanOrEqualOrUnordered, + DoubleLessThanOrUnordered, + DoubleLessThanOrEqualOrUnordered + }; + + // ---- code emission ----------------------------------------------------- + // + // IA-64 code is a sequence of 16-byte bundles, so the buffer's instruction + // unit is a whole bundle and every offset is bundle-aligned by construction. + protected: + struct BundleUnit { + ia64::Bundle b; + }; + AssemblerBuffer m_buffer; + + public: + BufferOffset emitBundle(ia64::Bundle b) { + BufferOffset off = m_buffer.nextOffset(); + // Anything that writes raw bytes into the buffer (writeCodePointer(), and + // the jump tables built out of it) has to leave it bundle-aligned again. + MOZ_ASSERT(off.getOffset() % sizeof(ia64::Bundle) == 0); + m_buffer.putBytes(sizeof(b.lo), &b.lo); + m_buffer.putBytes(sizeof(b.hi), &b.hi); + return off; + } + + // Convenience wrappers matching the one-instruction-per-bundle strategy. + BufferOffset emitM(ia64::Insn i) { return emitBundle(ia64::BundleM(i)); } + BufferOffset emitI(ia64::Insn i) { return emitBundle(ia64::BundleI(i)); } + BufferOffset emitB(ia64::Insn i) { return emitBundle(ia64::BundleB(i)); } + BufferOffset emitF(ia64::Insn i) { return emitBundle(ia64::BundleF(i)); } + BufferOffset emitMovl(uint32_t r1, uint64_t imm, uint32_t qp = 0) { + return emitBundle(ia64::MovlBundle(r1, imm, qp)); + } + + size_t size() const { return m_buffer.size(); } + bool oom() const { return m_buffer.oom(); } + + // Bundle index of an offset, which is what IP-relative displacements count. + static int32_t BundleIndex(BufferOffset off) { + MOZ_ASSERT(off.getOffset() % sizeof(ia64::Bundle) == 0); + return int32_t(off.getOffset() / sizeof(ia64::Bundle)); + } + + // ---- label binding ----------------------------------------------------- + // + // Forward branches are emitted with their displacement field holding the + // offset of the previous branch that targets the same label, forming a chain + // whose head is Label::offset(). bind() walks that chain and rewrites each + // displacement to point at the bind site. This is the standard SpiderMonkey + // scheme; on ia64 the "displacement field" is the TGT25c immediate that + // PatchBranchDisp() rewrites, and every offset is bundle-aligned so the + // chain link is stored as a bundle index. + + public: + // Writable pointer to an already-emitted bundle. The patchable-immediate + // sequences (movl, adds) rewrite their immediate through this. + ia64::Bundle* bundleAt(BufferOffset off) { + MOZ_ASSERT(off.getOffset() % sizeof(ia64::Bundle) == 0); + return reinterpret_cast(m_buffer.data() + off.getOffset()); + } + + private: + + // Read back the raw TGT25c field, which for an unbound branch is the chain + // link rather than a real displacement. + int32_t readChainLink(BufferOffset off) { + ia64::Bundle* b = bundleAt(off); + uint64_t slot2 = (b->hi >> 23) & ((uint64_t(1) << 41) - 1); + uint32_t imm20 = uint32_t((slot2 >> 13) & 0xfffff); + uint32_t sign = uint32_t((slot2 >> 36) & 0x1); + int32_t v = int32_t(imm20 | (sign << 20)); + // sign-extend from 21 bits + if (v & (1 << 20)) { + v |= ~((1 << 21) - 1); + } + return v; + } + + public: + // Emit a branch bundle that targets |label|, returning its offset. + BufferOffset emitBranchToLabel(ia64::Insn (*make)(int32_t, uint32_t), + uint32_t qp, Label* label) { + if (label->bound()) { + BufferOffset here = m_buffer.nextOffset(); + int32_t disp = BundleIndex(BufferOffset(label->offset())) - + BundleIndex(here); + MOZ_ASSERT(ia64::BranchDispInRange(disp)); + return emitBundle(ia64::BundleB(make(disp, qp))); + } + // Unbound: link into the chain, storing the previous head as the raw field. + int32_t prev = label->used() ? BundleIndex(BufferOffset(label->offset())) + : kEndOfChain; + BufferOffset off = emitBundle(ia64::BundleB(make(prev, qp))); + label->use(off.getOffset()); + return off; + } + + void bind(Label* label) { + // Once the buffer has failed, emitBundle() still hands out advancing + // offsets but appends nothing, so a chain recorded after that point walks + // -- and PatchBranchDisp() writes -- past the end of the buffer. + if (oom()) { + return; + } + BufferOffset here = m_buffer.nextOffset(); + if (label->used()) { + BufferOffset site(label->offset()); + for (;;) { + int32_t link = readChainLink(site); + int32_t disp = BundleIndex(here) - BundleIndex(site); + MOZ_ASSERT(ia64::BranchDispInRange(disp)); + ia64::PatchBranchDisp(bundleAt(site), disp); + if (link == kEndOfChain) { + break; + } + site = BufferOffset(link * int32_t(sizeof(ia64::Bundle))); + } + } + label->bind(here.getOffset()); + } + + // Fold |label|'s pending branch chain into |target|: either patch every site + // to a bound target, or splice the chain onto target's own pending chain. + void retarget(Label* label, Label* target) { + if (oom() || !label->used()) { + return; + } + + BufferOffset site(label->offset()); + if (target->bound()) { + BufferOffset dest(target->offset()); + for (;;) { + int32_t link = readChainLink(site); + int32_t disp = BundleIndex(dest) - BundleIndex(site); + MOZ_ASSERT(ia64::BranchDispInRange(disp)); + ia64::PatchBranchDisp(bundleAt(site), disp); + if (link == kEndOfChain) { + break; + } + site = BufferOffset(link * int32_t(sizeof(ia64::Bundle))); + } + label->reset(); + return; + } + + for (;;) { + int32_t link = readChainLink(site); + if (link == kEndOfChain) { + break; + } + site = BufferOffset(link * int32_t(sizeof(ia64::Bundle))); + } + int32_t head = target->used() + ? BundleIndex(BufferOffset(target->offset())) + : kEndOfChain; + ia64::PatchBranchDisp(bundleAt(site), head); + target->use(label->offset()); + label->reset(); + } + + static const int32_t kEndOfChain = -1; + + static Condition InvertCondition(Condition cond) { + switch (cond) { + case Equal: + return NotEqual; + case NotEqual: + return Equal; + case Above: + return BelowOrEqual; + case AboveOrEqual: + return Below; + case Below: + return AboveOrEqual; + case BelowOrEqual: + return Above; + case GreaterThan: + return LessThanOrEqual; + case GreaterThanOrEqual: + return LessThan; + case LessThan: + return GreaterThanOrEqual; + case LessThanOrEqual: + return GreaterThan; + case Signed: + return NotSigned; + case NotSigned: + return Signed; + case Zero: + return NonZero; + case NonZero: + return Zero; + default: + MOZ_CRASH("unsupported condition to invert"); + } + } + + static DoubleCondition InvertCondition(DoubleCondition cond) { + switch (cond) { + case DoubleOrdered: + return DoubleUnordered; + case DoubleEqual: + return DoubleNotEqualOrUnordered; + case DoubleNotEqual: + return DoubleEqualOrUnordered; + case DoubleGreaterThan: + return DoubleLessThanOrEqualOrUnordered; + case DoubleGreaterThanOrEqual: + return DoubleLessThanOrUnordered; + case DoubleLessThan: + return DoubleGreaterThanOrEqualOrUnordered; + case DoubleLessThanOrEqual: + return DoubleGreaterThanOrUnordered; + case DoubleUnordered: + return DoubleOrdered; + case DoubleEqualOrUnordered: + return DoubleNotEqual; + case DoubleNotEqualOrUnordered: + return DoubleEqual; + case DoubleGreaterThanOrUnordered: + return DoubleLessThanOrEqual; + case DoubleGreaterThanOrEqualOrUnordered: + return DoubleLessThan; + case DoubleLessThanOrUnordered: + return DoubleGreaterThanOrEqual; + case DoubleLessThanOrEqualOrUnordered: + return DoubleGreaterThan; + default: + MOZ_CRASH("unsupported double condition to invert"); + } + } + + // The patch site is always a movl bundle (see pushWithPatch, movePtr, and + // the relocation table code), so this round-trips through + // ReadMovlImm/WriteMovlImm exactly like TraceDataRelocations does. + static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue, + ImmPtr expectedValue) { + PatchDataWithValueCheck(label, ImmWord(uintptr_t(newValue.value)), + ImmWord(uintptr_t(expectedValue.value))); + } + static void PatchDataWithValueCheck(CodeLocationLabel label, + ImmWord newValue, ImmWord expectedValue); + template + static void PatchDataWithValueCheck(CodeLocationLabel, T, S) { + MOZ_CRASH(); + } + + // Overwrites the 4 bytes immediately before |label|, matching x86/loong64: + // used only for Ion invalidation's safepoint delta, which claims those + // bytes as raw data storage in code already being invalidated, so there is + // no instruction encoding to preserve. + static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) { + *(reinterpret_cast(label.raw()) - 1) = uint32_t(imm.value); + } + + // A near call is movl(ScratchReg, target) + MovToBr(6, ScratchReg) + + // BrCall(0, 6): 3 bundles, matching call(Register) exactly. All three are + // written from scratch, because the patch site holds unrelated code. + static void PatchWrite_NearCall(CodeLocationLabel start, + CodeLocationLabel toCall); + static uint32_t PatchWrite_NearCallSize() { return 3 * sizeof(ia64::Bundle); } + + // Every patch below rewrites live instruction memory, so it has to flush the + // bundle itself: the IA-64 I-cache is not coherent with stores, and not all + // callers hold an AutoWritableJitCode (whose destructor would flush). On the + // architectures this code is shared with, either the I-cache is coherent + // (x86) or the patch target is a literal-pool data word (arm64), so the + // generic callers have never needed one. + static void ToggleToJmp(CodeLocationLabel label) { + ia64::ToggleSlot2Op(reinterpret_cast(label.raw()), 4); + FlushICache(label.raw(), sizeof(ia64::Bundle)); + } + static void ToggleToCmp(CodeLocationLabel label) { + ia64::ToggleSlot2Op(reinterpret_cast(label.raw()), 2); + FlushICache(label.raw(), sizeof(ia64::Bundle)); + } + static void ToggleCall(CodeLocationLabel label, bool enabled) { + ia64::ToggleSlot2Op(reinterpret_cast(label.raw()), + enabled ? 5 : 2); + FlushICache(label.raw(), sizeof(ia64::Bundle)); + } + + static void Bind(uint8_t*, const CodeLabel&) { MOZ_CRASH(); } + + // Reads back the immediate embedded by a movl bundle (see + // MacroAssembler-ia64.h's ReadMovlImm), e.g. JSJitFrameIter reading an + // IonScript pointer materialised by movePtr(ImmPtr). + static uintptr_t GetPointer(uint8_t* addr); + + static bool HasRoundInstruction(RoundingMode) { return false; } + + void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, + const Disassembler::HeapAccess& heapAccess) { + MOZ_CRASH(); + } + + void setUnlimitedBuffer() { MOZ_CRASH(); } +}; + +class Operand { + public: + explicit Operand(const Address&) { MOZ_CRASH(); } + explicit Operand(const Register) { MOZ_CRASH(); } + explicit Operand(const FloatRegister) { MOZ_CRASH(); } + explicit Operand(Register, Imm32) { MOZ_CRASH(); } + explicit Operand(Register, int32_t) { MOZ_CRASH(); } +}; + +// Itanium psABI: arguments are numbered positionally 1-8, and each position +// has both a GR (out0-out7) and an FR (f8-f15) slot; an argument uses +// whichever slot matches its type; but every argument, integer or FP, +// advances both slots' index (an FP argument in position N leaves out(N-1) +// unused, and vice versa). Arguments beyond the 8th spill to the stack. +class ABIArgGenerator { + unsigned argIndex_ = 0; + // The psABI reserves the low 16 bytes of the frame as the callee's scratch + // area: the callee may write there without moving SP, and GCC does, e.g. + // "mov r16 = r12; adds r12 = -N, r12; ... stf.spill [r16] = f2" to spill an + // FP register. So outgoing stack arguments start above it, and the saved + // GP/b0 pair that callWithABIPre() puts at the top of the frame stays clear + // of it even when there are no stack arguments at all. + static const uint32_t ScratchAreaSize = 16; + uint32_t stackOffset_ = ScratchAreaSize; + ABIArg current_; + + static const unsigned NumArgRegs = 8; + + public: + explicit ABIArgGenerator(ABIKind) {} + ABIArg next(MIRType type) { + switch (type) { + case MIRType::Int32: + case MIRType::Int64: + case MIRType::Pointer: + case MIRType::StackResults: + case MIRType::WasmAnyRef: + case MIRType::WasmArrayData: + if (argIndex_ < NumArgRegs) { + current_ = ABIArg(Register::FromCode(Registers::out0 + argIndex_)); + } else { + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(uintptr_t); + } + break; + case MIRType::Float32: + case MIRType::Double: + if (argIndex_ < NumArgRegs) { + current_ = ABIArg(FloatRegister( + FloatRegisters::Encoding(FloatRegisters::f8 + argIndex_), + type == MIRType::Double ? FloatRegister::Double + : FloatRegister::Single)); + } else { + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(double); + } + break; + default: + MOZ_CRASH("unexpected argument type for ia64 ABI"); + } + argIndex_++; + return current_; + } + ABIArg& current() { return current_; } + uint32_t stackBytesConsumedSoFar() const { return stackOffset_; } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_Assembler_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/ia64/AssemblerCore-ia64.h b/js/src/jit/ia64/AssemblerCore-ia64.h --- firefox-153.0.1/js/src/jit/ia64/AssemblerCore-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/AssemblerCore-ia64.h @@ -0,0 +1,1069 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_AssemblerCore_ia64_h +#define jit_ia64_AssemblerCore_ia64_h + +#include "mozilla/Assertions.h" + +#include + +// [SMDOC] IA-64 instruction and bundle encoding +// +// IA-64 does not have a linear instruction stream: instructions are packed +// three at a time into 128-bit (16-byte) *bundles*: +// +// bits 4:0 template +// bits 45:5 slot 0 (41 bits) +// bits 86:46 slot 1 (41 bits) +// bits 127:87 slot 2 (41 bits) +// +// The 5-bit template selects, for each slot, which execution unit the +// instruction in it must belong to (M = memory, I = integer/shift, F = float, +// B = branch, L+X = the two-slot long-immediate form), and where the *stop +// bits* fall. A stop bit terminates an instruction group: instructions within a +// group must have no register dependencies, because the hardware issues them in +// parallel with no interlocks. +// +// A JIT could schedule aggressively and pack three instructions per bundle, but +// correctness comes first: this backend emits one instruction per bundle, +// padding the unused slots with unit-appropriate nops and always setting the +// stop bit at the end of the bundle. That makes every instruction its own +// instruction group, so no dependency analysis is required and the emitted code +// is correct by construction. It costs 16 bytes per instruction; packing is a +// later optimisation that can be layered on without changing any caller. +// +// All bit positions below are taken from the binutils IA-64 opcode tables +// (opcodes/ia64-opc-*.c), which are the authoritative encoding description, and +// the encoder is verified byte-for-byte against GNU as by +// jit/ia64/test-encoding.cpp. + +namespace js { +namespace jit { +namespace ia64 { + +using Insn = uint64_t; // a 41-bit instruction, right-aligned + +struct Bundle { + uint64_t lo; + uint64_t hi; +}; + +// Template encodings. The trailing underscore marks a stop bit at the end of +// the bundle; "_" inside the name marks an internal stop. +enum Template : uint8_t { + tMII = 0x00, + tMII_ = 0x01, + tMI_I = 0x02, + tMI_I_ = 0x03, + tMLX = 0x04, + tMLX_ = 0x05, + tMMI = 0x08, + tMMI_ = 0x09, + tM_MI = 0x0a, + tM_MI_ = 0x0b, + tMFI = 0x0c, + tMFI_ = 0x0d, + tMMF = 0x0e, + tMMF_ = 0x0f, + tMIB = 0x10, + tMIB_ = 0x11, + tMBB = 0x12, + tMBB_ = 0x13, + tBBB = 0x16, + tBBB_ = 0x17, + tMMB = 0x18, + tMMB_ = 0x19, + tMFB = 0x1c, + tMFB_ = 0x1d, +}; + +// Pack three 41-bit slots and a template into a 16-byte bundle. +inline Bundle MakeBundle(Template tmpl, Insn s0, Insn s1, Insn s2) { + const uint64_t kSlotMask = (uint64_t(1) << 41) - 1; + s0 &= kSlotMask; + s1 &= kSlotMask; + s2 &= kSlotMask; + + Bundle b; + // lo = template | slot0 << 5 | low 18 bits of slot1 << 46 + b.lo = uint64_t(tmpl & 0x1f) | (s0 << 5) | (s1 << 46); + // hi = high 23 bits of slot1 | slot2 << 23 + b.hi = (s1 >> 18) | (s2 << 23); + return b; +} + +// --------------------------------------------------------------------------- +// Instruction field helpers. Positions per binutils opcodes/ia64-opc-*.c. +// --------------------------------------------------------------------------- + +inline Insn fOp(uint32_t x) { return Insn(x & 0xf) << 37; } +inline Insn fQp(uint32_t x) { return Insn(x & 0x3f); } +// Only r0-r39 exist in this backend's register model: r0-r31 static, r32-r39 +// the out0-out7 window sized by the prologue's `alloc`. Encoding anything else +// -- in particular Registers::invalid_reg, which is 40 -- names a stacked +// register outside the current frame, and IA-64 raises an Illegal Operation +// fault (SIGILL/ILL_ILLOPC) rather than faulting at the point of the mistake. +inline void AssertGpr(uint32_t x) { MOZ_ASSERT(x < 40, "invalid GR encoding"); } + +inline Insn fR1(uint32_t x) { + AssertGpr(x); + return Insn(x & 0x7f) << 6; +} +inline Insn fR2(uint32_t x) { + AssertGpr(x); + return Insn(x & 0x7f) << 13; +} +inline Insn fR3(uint32_t x) { + AssertGpr(x); + return Insn(x & 0x7f) << 20; +} +inline Insn fX2a(uint32_t x) { return Insn(x & 0x3) << 34; } +inline Insn fX2b(uint32_t x) { return Insn(x & 0x3) << 27; } +inline Insn fX4(uint32_t x) { return Insn(x & 0xf) << 29; } +inline Insn fVe(uint32_t x) { return Insn(x & 0x1) << 33; } + +// imm14, as used by A4 (adds): imm[6:0] -> 19:13, imm[12:7] -> 32:27, +// imm[13] (sign) -> 36. +inline Insn fImm14(int64_t v) { + uint64_t u = uint64_t(v); + return (Insn((u >> 0) & 0x7f) << 13) | (Insn((u >> 7) & 0x3f) << 27) | + (Insn((u >> 13) & 0x1) << 36); +} + +// --------------------------------------------------------------------------- +// A-type (ALU) instructions, major opcode 8. +// --------------------------------------------------------------------------- + +// A1: r1 = r2 r3 +inline Insn A1(uint32_t x4, uint32_t x2b, uint32_t r1, uint32_t r2, + uint32_t r3, uint32_t qp = 0) { + return fOp(8) | fX2a(0) | fVe(0) | fX4(x4) | fX2b(x2b) | fR3(r3) | fR2(r2) | + fR1(r1) | fQp(qp); +} + +inline Insn Add(uint32_t r1, uint32_t r2, uint32_t r3, uint32_t qp = 0) { + return A1(0, 0, r1, r2, r3, qp); +} +inline Insn Sub(uint32_t r1, uint32_t r2, uint32_t r3, uint32_t qp = 0) { + return A1(1, 1, r1, r2, r3, qp); +} +inline Insn And(uint32_t r1, uint32_t r2, uint32_t r3, uint32_t qp = 0) { + return A1(3, 0, r1, r2, r3, qp); +} +inline Insn Andcm(uint32_t r1, uint32_t r2, uint32_t r3, uint32_t qp = 0) { + return A1(3, 1, r1, r2, r3, qp); +} + +// A3: and r1 = imm8, r3. sign -> bit 36, imm7b -> bits 19:13 (the same split +// as fImm14 minus its extra 6 immediate bits). +inline Insn fImm8(int64_t v) { + return (Insn(v & 0x7f) << 13) | (Insn((v >> 7) & 1) << 36); +} +inline Insn AndImm(uint32_t r1, int64_t imm8, uint32_t r3, uint32_t qp = 0) { + return fOp(8) | fX2a(0) | fVe(0) | fX4(0xb) | fX2b(0) | fImm8(imm8) | + fR3(r3) | fR1(r1) | fQp(qp); +} +inline Insn Or(uint32_t r1, uint32_t r2, uint32_t r3, uint32_t qp = 0) { + return A1(3, 2, r1, r2, r3, qp); +} +inline Insn Xor(uint32_t r1, uint32_t r2, uint32_t r3, uint32_t qp = 0) { + return A1(3, 3, r1, r2, r3, qp); +} + +// A4: r1 = imm14 + r3 ("adds"). Also the canonical register move when +// imm == 0, which is how "mov r1 = r3" is encoded. +inline Insn Adds(uint32_t r1, int64_t imm14, uint32_t r3, uint32_t qp = 0) { + return fOp(8) | fX2a(2) | fVe(0) | fImm14(imm14) | fR3(r3) | fR1(r1) | + fQp(qp); +} + +inline Insn MovReg(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return Adds(r1, 0, r3, qp); +} + +// --------------------------------------------------------------------------- +// A6: compare two registers and write a pair of predicates. +// cmp.rel p1, p2 = r2, r3 +// tb -> bit 36 x2 -> bits 35:34 ta -> bit 33 +// p2 -> bits 32:27 r3 -> 26:20 r2 -> 19:13 c -> bit 12 p1 -> 11:6 +// Predicates are 6 bits (p0-p63), so p1 does not reach bit 12. +// --------------------------------------------------------------------------- + +inline Insn fP1(uint32_t x) { return Insn(x & 0x3f) << 6; } +inline Insn fP2(uint32_t x) { return Insn(x & 0x3f) << 27; } +inline Insn fTa(uint32_t x) { return Insn(x & 0x1) << 33; } +inline Insn fTb(uint32_t x) { return Insn(x & 0x1) << 36; } +inline Insn fC(uint32_t x) { return Insn(x & 0x1) << 12; } +inline Insn fX2(uint32_t x) { return Insn(x & 0x3) << 34; } + +inline Insn CmpA6(uint32_t op, uint32_t p1, uint32_t p2, uint32_t r2, + uint32_t r3, uint32_t qp = 0) { + return fOp(op) | fTb(0) | fX2(0) | fTa(0) | fP2(p2) | fR3(r3) | fR2(r2) | + fC(0) | fP1(p1) | fQp(qp); +} + +inline Insn CmpEq(uint32_t p1, uint32_t p2, uint32_t r2, uint32_t r3, + uint32_t qp = 0) { + return CmpA6(0xe, p1, p2, r2, r3, qp); +} +// cmp.ne is cmp.eq with the two predicate destinations exchanged. +inline Insn CmpNe(uint32_t p1, uint32_t p2, uint32_t r2, uint32_t r3, + uint32_t qp = 0) { + return CmpA6(0xe, p2, p1, r2, r3, qp); +} +inline Insn CmpLt(uint32_t p1, uint32_t p2, uint32_t r2, uint32_t r3, + uint32_t qp = 0) { + return CmpA6(0xc, p1, p2, r2, r3, qp); +} +inline Insn CmpLtu(uint32_t p1, uint32_t p2, uint32_t r2, uint32_t r3, + uint32_t qp = 0) { + return CmpA6(0xd, p1, p2, r2, r3, qp); +} + +// --------------------------------------------------------------------------- +// M-type (memory) instructions, major opcode 4. +// m -> bit 36 x -> bit 27 +// x6a -> bits 35:30 hint -> bits 29:28 +// --------------------------------------------------------------------------- + +inline Insn fM(uint32_t x) { return Insn(x & 0x1) << 36; } +inline Insn fXm(uint32_t x) { return Insn(x & 0x1) << 27; } +inline Insn fX6a(uint32_t x) { return Insn(x & 0x3f) << 30; } +inline Insn fHint(uint32_t x) { return Insn(x & 0x3) << 28; } + +// M1: r1 = [r3] +inline Insn LoadM1(uint32_t x6a, uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return fOp(4) | fM(0) | fXm(0) | fX6a(x6a) | fHint(0) | fR3(r3) | fR1(r1) | + fQp(qp); +} + +// M4: [r3] = r2 +inline Insn StoreM4(uint32_t x6a, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return fOp(4) | fM(0) | fXm(0) | fX6a(x6a) | fHint(0) | fR3(r3) | fR2(r2) | + fQp(qp); +} + +inline Insn Ld1(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return LoadM1(0x00, r1, r3, qp); +} +inline Insn Ld2(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return LoadM1(0x01, r1, r3, qp); +} +inline Insn Ld4(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return LoadM1(0x02, r1, r3, qp); +} +inline Insn Ld8(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return LoadM1(0x03, r1, r3, qp); +} + +inline Insn St1(uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return StoreM4(0x30, r3, r2, qp); +} +inline Insn St2(uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return StoreM4(0x31, r3, r2, qp); +} +inline Insn St4(uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return StoreM4(0x32, r3, r2, qp); +} +inline Insn St8(uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return StoreM4(0x33, r3, r2, qp); +} + +// M6/M9: floating-point loads and stores. Same M1/M4 shape but with major +// opcode 6 instead of 4, and f1/f2 in place of the integer register fields +// (they share the same 7-bit slot). +// f1/f2 share the r1/r2 bit position (6:12 and 13:19 respectively), so the +// generic fR1/fR2 helpers double as the FP register fields here. +inline Insn LoadF6(uint32_t x6a, uint32_t f1, uint32_t r3, uint32_t qp = 0) { + return fOp(6) | fM(0) | fXm(0) | fX6a(x6a) | fHint(0) | fR3(r3) | fR1(f1) | + fQp(qp); +} +inline Insn StoreF9(uint32_t x6a, uint32_t r3, uint32_t f2, uint32_t qp = 0) { + return fOp(6) | fM(0) | fXm(0) | fX6a(x6a) | fHint(0) | fR3(r3) | fR2(f2) | + fQp(qp); +} +inline Insn Ldfs(uint32_t f1, uint32_t r3, uint32_t qp = 0) { + return LoadF6(0x02, f1, r3, qp); +} +inline Insn Ldfd(uint32_t f1, uint32_t r3, uint32_t qp = 0) { + return LoadF6(0x03, f1, r3, qp); +} +inline Insn Stfs(uint32_t r3, uint32_t f2, uint32_t qp = 0) { + return StoreF9(0x32, r3, f2, qp); +} +inline Insn Stfd(uint32_t r3, uint32_t f2, uint32_t qp = 0) { + return StoreF9(0x33, r3, f2, qp); +} + +// M-type atomics and semaphores. The acquire/release loads and stores are +// ordinary M1/M4 with a different x6a; the read-modify-write forms set x +// (bit 27). +inline Insn Ld8Acq(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return LoadM1(0x17, r1, r3, qp); +} +inline Insn Ld4Acq(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return LoadM1(0x16, r1, r3, qp); +} +inline Insn St8Rel(uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return StoreM4(0x37, r3, r2, qp); +} +inline Insn St4Rel(uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return StoreM4(0x36, r3, r2, qp); +} + +// M29: mov ar = r2. Used to load ar.ccv (AR 32) with the compare value before +// a cmpxchg -- the hardware compares against ar.ccv's *exact* bit pattern +// (see [[ia64-llvm-cmpxchg-ccv-missing-zext]]: a real LLVM backend bug came +// from feeding it a non-zero-extended comparand for a narrow cmpxchg), so +// callers must zero-extend to the access width first. +inline Insn MovToAr(uint32_t x6b, uint32_t ar, uint32_t r2, uint32_t qp = 0) { + return fOp(1) | fM(0) | (Insn(x6b & 0x3f) << 27) | fR3(ar) | fR2(r2) | + fQp(qp); +} +inline Insn MovToArCcv(uint32_t r2, uint32_t qp = 0) { + return MovToAr(0x2a, 32, r2, qp); +} + +// M16: r1 = cmpxchg[r3], r2, ar.ccv -- the comparand comes from ar.ccv, which +// the caller must have loaded beforehand. +inline Insn CmpxchgM16(uint32_t x6a, uint32_t r1, uint32_t r3, uint32_t r2, + uint32_t qp = 0) { + return fOp(4) | fM(0) | fXm(1) | fX6a(x6a) | fHint(0) | fR3(r3) | fR2(r2) | + fR1(r1) | fQp(qp); +} + +inline Insn Cmpxchg8Acq(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x03, r1, r3, r2, qp); +} +inline Insn Cmpxchg8Rel(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x07, r1, r3, r2, qp); +} +inline Insn Cmpxchg4Acq(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x02, r1, r3, r2, qp); +} +inline Insn Cmpxchg4Rel(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x06, r1, r3, r2, qp); +} +inline Insn Xchg8(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x0b, r1, r3, r2, qp); +} +inline Insn Xchg4(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x0a, r1, r3, r2, qp); +} +inline Insn Xchg2(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x09, r1, r3, r2, qp); +} +inline Insn Xchg1(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x08, r1, r3, r2, qp); +} +inline Insn Cmpxchg2Acq(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x01, r1, r3, r2, qp); +} +inline Insn Cmpxchg2Rel(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x05, r1, r3, r2, qp); +} +inline Insn Cmpxchg1Acq(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x00, r1, r3, r2, qp); +} +inline Insn Cmpxchg1Rel(uint32_t r1, uint32_t r3, uint32_t r2, uint32_t qp = 0) { + return CmpxchgM16(0x04, r1, r3, r2, qp); +} + +// M17: fetchadd. The increment is not a plain immediate: bits 14:13 select a +// magnitude from {16, 8, 4, 1} and bit 15 is the sign. Derived from GNU as by +// assembling every legal increment. +inline Insn fInc3(int32_t inc) { + uint32_t mag = uint32_t(inc < 0 ? -inc : inc); + uint32_t code = mag == 16 ? 0 : mag == 8 ? 1 : mag == 4 ? 2 : 3; + return (Insn(code) << 13) | (Insn(inc < 0 ? 1 : 0) << 15); +} + +inline Insn FetchaddM17(uint32_t x6a, uint32_t r1, uint32_t r3, int32_t inc, + uint32_t qp = 0) { + return fOp(4) | fM(0) | fXm(1) | fX6a(x6a) | fHint(0) | fR3(r3) | fInc3(inc) | + fR1(r1) | fQp(qp); +} + +inline Insn Fetchadd8Acq(uint32_t r1, uint32_t r3, int32_t inc, + uint32_t qp = 0) { + return FetchaddM17(0x13, r1, r3, inc, qp); +} +inline Insn Fetchadd8Rel(uint32_t r1, uint32_t r3, int32_t inc, + uint32_t qp = 0) { + return FetchaddM17(0x17, r1, r3, inc, qp); +} +inline Insn Fetchadd4Acq(uint32_t r1, uint32_t r3, int32_t inc, + uint32_t qp = 0) { + return FetchaddM17(0x12, r1, r3, inc, qp); +} +inline Insn Fetchadd4Rel(uint32_t r1, uint32_t r3, int32_t inc, + uint32_t qp = 0) { + return FetchaddM17(0x16, r1, r3, inc, qp); +} + +// M18/M19: move between a general register and a floating-point register. +// getf is major op 4, setf major op 6; both set x (bit 27). +inline Insn GetfM19(uint32_t x6a, uint32_t r1, uint32_t f2, uint32_t qp = 0) { + return fOp(4) | fM(0) | fXm(1) | fX6a(x6a) | fR1(r1) | + (Insn(f2 & 0x7f) << 13) | fQp(qp); +} +inline Insn SetfM18(uint32_t x6a, uint32_t f1, uint32_t r2, uint32_t qp = 0) { + return fOp(6) | fM(0) | fXm(1) | fX6a(x6a) | (Insn(f1 & 0x7f) << 6) | + fR2(r2) | fQp(qp); +} + +inline Insn GetfSig(uint32_t r1, uint32_t f2, uint32_t qp = 0) { + return GetfM19(0x1c, r1, f2, qp); +} +inline Insn GetfExp(uint32_t r1, uint32_t f2, uint32_t qp = 0) { + return GetfM19(0x1d, r1, f2, qp); +} +inline Insn GetfS(uint32_t r1, uint32_t f2, uint32_t qp = 0) { + return GetfM19(0x1e, r1, f2, qp); +} +inline Insn GetfD(uint32_t r1, uint32_t f2, uint32_t qp = 0) { + return GetfM19(0x1f, r1, f2, qp); +} +inline Insn SetfSig(uint32_t f1, uint32_t r2, uint32_t qp = 0) { + return SetfM18(0x1c, f1, r2, qp); +} +inline Insn SetfExp(uint32_t f1, uint32_t r2, uint32_t qp = 0) { + return SetfM18(0x1d, f1, r2, qp); +} +inline Insn SetfS(uint32_t f1, uint32_t r2, uint32_t qp = 0) { + return SetfM18(0x1e, f1, r2, qp); +} +inline Insn SetfD(uint32_t f1, uint32_t r2, uint32_t qp = 0) { + return SetfM18(0x1f, f1, r2, qp); +} + +// --------------------------------------------------------------------------- +// M34: alloc r1 = ar.pfs, ins, locals, outs, rot +// +// Establishes this procedure's register-stack frame and saves the previous +// frame marker into r1. The JIT prologue uses +// alloc rN = ar.pfs, 0, 0, 8, 0 +// to obtain out0-out7 (r32-r39) for calling C functions. +// +// Field layout derived from GNU as by varying each operand: +// sof = ins + locals + outs -> bits 19:13 +// sol = ins + locals -> bits 26:20 +// sor = rot / 8 -> bits 30:27 +// op = 1, x3 = 6 (bits 35:33), r1 -> bits 12:6 +// alloc cannot be predicated, so qp is always 0. +// --------------------------------------------------------------------------- + +inline Insn fX3(uint32_t x) { return Insn(x & 0x7) << 33; } + +inline Insn Alloc(uint32_t r1, uint32_t ins, uint32_t locals, uint32_t outs, + uint32_t rot = 0) { + uint32_t sof = ins + locals + outs; + uint32_t sol = ins + locals; + uint32_t sor = rot / 8; + return fOp(1) | fX3(6) | (Insn(sor & 0xf) << 27) | (Insn(sol & 0x7f) << 20) | + (Insn(sof & 0x7f) << 13) | fR1(r1); +} + +// --------------------------------------------------------------------------- +// B-type (branch) instructions. +// btype -> bits 8:6 pa -> bit 12 b1 -> bits 8:6 +// b2 -> bits 15:13 wha -> bits 34:33 whc -> bits 34:32 +// x6 -> bits 32:27 d -> bit 35 +// --------------------------------------------------------------------------- + +inline Insn fBtype(uint32_t x) { return Insn(x & 0x7) << 6; } +inline Insn fB1(uint32_t x) { return Insn(x & 0x7) << 6; } +inline Insn fB2(uint32_t x) { return Insn(x & 0x7) << 13; } +inline Insn fPa(uint32_t x) { return Insn(x & 0x1) << 12; } +inline Insn fWha(uint32_t x) { return Insn(x & 0x3) << 33; } +inline Insn fWhc(uint32_t x) { return Insn(x & 0x7) << 32; } +inline Insn fD(uint32_t x) { return Insn(x & 0x1) << 35; } +inline Insn fX6b(uint32_t x) { return Insn(x & 0x3f) << 27; } + +// B4: br.cond.sptk b2 -- indirect branch through a branch register. +inline Insn BrCond(uint32_t b2, uint32_t qp = 0) { + return fOp(0) | fX6b(0x20) | fBtype(0) | fPa(0) | fWha(0) | fD(0) | fB2(b2) | + fQp(qp); +} + +// B4: br.ret.sptk.many b2 +inline Insn BrRet(uint32_t b2, uint32_t qp = 0) { + return fOp(0) | fX6b(0x21) | fBtype(4) | fPa(1) | fWha(0) | fD(0) | fB2(b2) | + fQp(qp); +} + +// B5: br.call.sptk.many b1 = b2 -- indirect call, return address into b1. +inline Insn BrCall(uint32_t b1, uint32_t b2, uint32_t qp = 0) { + return fOp(1) | fPa(1) | fWhc(1) | fD(0) | fB1(b1) | fB2(b2) | fQp(qp); +} + +// IP-relative displacements (TGT25c) are measured in whole 16-byte bundles and +// encoded as a signed 21-bit value: imm20 in bits 32:13 and the sign in bit 36. +// That gives +/-2^20 bundles, i.e. +/-16 MiB. Layout confirmed against GNU as by +// assembling branches over varying distances in both directions. +static const int32_t MaxBranchBundleDisp = (1 << 20) - 1; +static const int32_t MinBranchBundleDisp = -(1 << 20); + +inline bool BranchDispInRange(int32_t bundleDisp) { + return bundleDisp >= MinBranchBundleDisp && bundleDisp <= MaxBranchBundleDisp; +} + +inline Insn fTgt25c(int32_t bundleDisp) { + uint32_t u = uint32_t(bundleDisp); + return (Insn(u & 0xfffff) << 13) | (Insn((u >> 20) & 0x1) << 36); +} + +// B1: br.cond.sptk +inline Insn BrCondRel(int32_t bundleDisp, uint32_t qp = 0) { + return fOp(4) | fBtype(0) | fPa(0) | fWha(0) | fD(0) | fTgt25c(bundleDisp) | + fQp(qp); +} + +// B3: br.call.sptk.many b1 = +inline Insn BrCallRel(uint32_t b1, int32_t bundleDisp, uint32_t qp = 0) { + return fOp(5) | fPa(1) | fWha(0) | fD(0) | fB1(b1) | fTgt25c(bundleDisp) | + fQp(qp); +} + +// B9: break.b. Major opcode 0; the 21-bit immediate is spread the same way +// as the M/I/F/B nops below (low 20 bits at 25:6, top bit at 36). +inline Insn BreakB(uint32_t imm21, uint32_t qp = 0) { + return fOp(0) | (Insn(imm21 & 0xfffff) << 6) | + (Insn((imm21 >> 20) & 1) << 36) | fQp(qp); +} + +// Rewrite the IP-relative displacement of a branch already emitted into slot 2 +// of |b|, leaving every other field alone. This is what label binding uses to +// resolve forward branches. +inline void PatchBranchDisp(Bundle* b, int32_t bundleDisp) { + const uint64_t kSlotMask = (uint64_t(1) << 41) - 1; + // slot 2 occupies bundle bits 127:87, i.e. hi bits 63:23. + Insn slot2 = (b->hi >> 23) & kSlotMask; + Insn clearMask = (Insn(0xfffff) << 13) | (Insn(1) << 36); + slot2 = (slot2 & ~clearMask) | fTgt25c(bundleDisp); + b->hi = (b->hi & ((uint64_t(1) << 23) - 1)) | (slot2 << 23); +} + +// Toggle a lone B-slot bundle (BundleB) between a real branch/call (op 4/5) +// and a nop (op 2), in place, without disturbing anything else in the slot +// -- op sits in the same 4 high bits (37:40) for all three, well clear of +// the tgt25c/imm21 payload and qp, so flipping just those bits round-trips +// cleanly and preserves whatever displacement a br.cond/br.call form had. +inline void ToggleSlot2Op(Bundle* b, uint32_t op) { + const uint64_t kSlotMask = (uint64_t(1) << 41) - 1; + Insn slot2 = (b->hi >> 23) & kSlotMask; + slot2 = (slot2 & ~(Insn(0xf) << 37)) | fOp(op); + b->hi = (b->hi & ((uint64_t(1) << 23) - 1)) | (slot2 << 23); +} + +// --------------------------------------------------------------------------- +// Moves to and from branch registers and ar.pfs. An indirect call needs its +// target in a branch register, and a JIT frame must save/restore b0 (the return +// pointer) and ar.pfs (the previous frame marker written by alloc). +// +// The binutils macros for these overlap ambiguously (tag13 and x3 both claim +// bit 33), so the base patterns below were taken from GNU as directly and the +// operand field positions confirmed by varying each operand: +// mov bN = rM b -> bits 8:6 r2 -> bits 19:13 +// mov rN = bM r1 -> bits 12:6 b2 -> bits 15:13 +// mov rN = ar.pfs r1 -> bits 12:6 +// mov ar.pfs = rN r2 -> bits 19:13 +// --------------------------------------------------------------------------- + +inline Insn fBr(uint32_t x) { return Insn(x & 0x7) << 6; } + +// mov b1 = r2 +inline Insn MovToBr(uint32_t b1, uint32_t r2, uint32_t qp = 0) { + return Insn(0x0e00100000ull) | fBr(b1) | fR2(r2) | fQp(qp); +} + +// mov r1 = b2 +inline Insn MovFromBr(uint32_t r1, uint32_t b2, uint32_t qp = 0) { + return Insn(0x0188000000ull) | fR1(r1) | (Insn(b2 & 0x7) << 13) | fQp(qp); +} + +// mov r1 = ar.pfs +inline Insn MovFromPfs(uint32_t r1, uint32_t qp = 0) { + return Insn(0x0194000000ull) | fR1(r1) | fQp(qp); +} + +// mov ar.pfs = r2 +inline Insn MovToPfs(uint32_t r2, uint32_t qp = 0) { + return Insn(0x0154000000ull) | fR2(r2) | fQp(qp); +} + +// --------------------------------------------------------------------------- +// I-type shifts, extracts and deposits. +// +// The register-variable shifts (I5/I7, major opcode 7) are the multimedia +// shift group restricted to a full 64-bit element by za/zb: +// za -> bit 36 zb -> bit 33 ve -> bit 32 +// x2a -> 35:34 x2c -> 31:30 x2b -> 29:28 +// Note the I-unit ve and x2b sit at different positions than the A-unit ones. +// +// The immediate forms (I11/I12, major opcode 5) are really extr/dep.z: +// shr r1 = r3, n == extr r1 = r3, n, 64-n +// shl r1 = r2, n == dep.z r1 = r2, n, 64-n +// x2 -> 35:34 x -> bit 33 ya -> bit 13 yb -> bit 26 +// pos6 -> 19:14 len6 -> 32:27 (len - 1) cpos6a -> 25:20 (63 - pos) +// --------------------------------------------------------------------------- + +inline Insn fZa(uint32_t x) { return Insn(x & 0x1) << 36; } +inline Insn fZb(uint32_t x) { return Insn(x & 0x1) << 33; } +inline Insn fVeI(uint32_t x) { return Insn(x & 0x1) << 32; } +inline Insn fX2bI(uint32_t x) { return Insn(x & 0x3) << 28; } +inline Insn fX2c(uint32_t x) { return Insn(x & 0x3) << 30; } +inline Insn fXi(uint32_t x) { return Insn(x & 0x1) << 33; } +inline Insn fYa(uint32_t x) { return Insn(x & 0x1) << 13; } +inline Insn fYb(uint32_t x) { return Insn(x & 0x1) << 26; } +inline Insn fPos6(uint32_t x) { return Insn(x & 0x3f) << 14; } +inline Insn fLen6(uint32_t len) { return Insn((len - 1) & 0x3f) << 27; } +inline Insn fCpos6a(uint32_t x) { return Insn((63 - x) & 0x3f) << 20; } + +inline Insn ShiftVar(uint32_t x2b, uint32_t x2c, uint32_t r1, uint32_t r2, + uint32_t r3, uint32_t qp = 0) { + return fOp(7) | fZa(1) | fZb(1) | fVeI(0) | fX2a(0) | fX2c(x2c) | fX2bI(x2b) | + fR3(r3) | fR2(r2) | fR1(r1) | fQp(qp); +} + +// shl r1 = value, count -- the count is r3 here, but r2 for the shift-rights. +inline Insn Shl(uint32_t r1, uint32_t value, uint32_t count, uint32_t qp = 0) { + return ShiftVar(0, 1, r1, value, count, qp); +} +inline Insn Shr(uint32_t r1, uint32_t value, uint32_t count, uint32_t qp = 0) { + return ShiftVar(2, 0, r1, count, value, qp); +} +inline Insn ShrU(uint32_t r1, uint32_t value, uint32_t count, uint32_t qp = 0) { + return ShiftVar(0, 0, r1, count, value, qp); +} + +// I11: extr / extr.u r1 = r3, pos, len +inline Insn Extr(uint32_t r1, uint32_t r3, uint32_t pos, uint32_t len, + uint32_t qp = 0) { + return fOp(5) | fX2(1) | fXi(0) | fYa(1) | fLen6(len) | fR3(r3) | fPos6(pos) | + fR1(r1) | fQp(qp); +} +inline Insn ExtrU(uint32_t r1, uint32_t r3, uint32_t pos, uint32_t len, + uint32_t qp = 0) { + return fOp(5) | fX2(1) | fXi(0) | fYa(0) | fLen6(len) | fR3(r3) | fPos6(pos) | + fR1(r1) | fQp(qp); +} + +// I12: dep.z r1 = r2, pos, len +inline Insn DepZ(uint32_t r1, uint32_t r2, uint32_t pos, uint32_t len, + uint32_t qp = 0) { + return fOp(5) | fX2(1) | fXi(1) | fYb(0) | fLen6(len) | fCpos6a(pos) | + fR2(r2) | fR1(r1) | fQp(qp); +} + +inline Insn ShlImm(uint32_t r1, uint32_t r2, uint32_t count, uint32_t qp = 0) { + return DepZ(r1, r2, count, 64 - count, qp); +} +inline Insn ShrImm(uint32_t r1, uint32_t r3, uint32_t count, uint32_t qp = 0) { + return Extr(r1, r3, count, 64 - count, qp); +} +inline Insn ShrUImm(uint32_t r1, uint32_t r3, uint32_t count, uint32_t qp = 0) { + return ExtrU(r1, r3, count, 64 - count, qp); +} + +// I29: sign/zero extend. Major opcode 0, x3 = 0, x6 selects the width. +inline Insn ExtendI29(uint32_t x6, uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return fOp(0) | fX3(0) | fX6b(x6) | fR3(r3) | fR1(r1) | fQp(qp); +} + +inline Insn Zxt1(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x10, r1, r3, qp); +} +inline Insn Zxt2(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x11, r1, r3, qp); +} +inline Insn Zxt4(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x12, r1, r3, qp); +} +inline Insn Sxt1(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x14, r1, r3, qp); +} +inline Insn Sxt2(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x15, r1, r3, qp); +} +inline Insn Sxt4(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x16, r1, r3, qp); +} + +// I29: czx (count trailing/leading zero bytes), same shape as sxt/zxt. +inline Insn Czx1L(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x18, r1, r3, qp); +} +inline Insn Czx2L(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x19, r1, r3, qp); +} +inline Insn Czx1R(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x1c, r1, r3, qp); +} +inline Insn Czx2R(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return ExtendI29(0x1d, r1, r3, qp); +} + +// I3: mux1. Shares the za/zb/ve/x2a/x2b/x2c field layout with ShiftVar/pmin2 +// et al (major opcode 7), but with a different fixed field combination and a +// third register operand carrying the permute-type immediate (MBTYPE4) +// instead of a register. +inline Insn Mux1(uint32_t r1, uint32_t r2, uint32_t mbtype4, uint32_t qp = 0) { + return fOp(7) | fZa(0) | fZb(0) | fVeI(0) | fX2a(3) | fX2bI(2) | fX2c(2) | + fR3(mbtype4) | fR2(r2) | fR1(r1) | fQp(qp); +} +static const uint32_t kMux1Rev = 0xb; + +// I9: popcnt. +inline Insn Popcnt(uint32_t r1, uint32_t r3, uint32_t qp = 0) { + return fOp(7) | fZa(0) | fZb(1) | fVeI(0) | fX2a(1) | fX2bI(1) | fX2c(2) | + fR3(r3) | fR1(r1) | fQp(qp); +} + +// I10: shrp. Major opcode 5, x2 (bits 35:34) = 3, x (bit 33) = 0; the shift +// count reuses the 6-bit field at 32:27 that fX6b already models. +inline Insn Shrp(uint32_t r1, uint32_t r2, uint32_t r3, uint32_t count, + uint32_t qp = 0) { + return fOp(5) | fX2(3) | fX6b(count) | fR3(r3) | fR2(r2) | fR1(r1) | + fQp(qp); +} + +// --------------------------------------------------------------------------- +// F-type (floating point). +// +// IA-64 has no plain fadd/fmul: everything is the fused multiply-add +// fma.pc.sf f1 = f3, f4, f2 -> f1 = f3 * f4 + f2 +// with f0 (0.0) and f1 (1.0) used as identity operands. fadd is fma with +// f4 = 1, fmpy is fma with f2 = 0, and fnorm is fma with f4 = 1, f2 = 0. +// +// f1 -> 12:6 f2 -> 19:13 f3 -> 26:20 f4 -> 33:27 +// sf -> 35:34 x (opcode extension) -> bit 36 +// +// The .d suffix rounds to IEEE double, which is what the JS engine wants; +// the unsuffixed form keeps the 82-bit register format. +// --------------------------------------------------------------------------- + +inline Insn fF1(uint32_t x) { return Insn(x & 0x7f) << 6; } +inline Insn fF2(uint32_t x) { return Insn(x & 0x7f) << 13; } +inline Insn fF3(uint32_t x) { return Insn(x & 0x7f) << 20; } +inline Insn fF4(uint32_t x) { return Insn(x & 0x7f) << 27; } +inline Insn fSf(uint32_t x) { return Insn(x & 0x3) << 34; } +inline Insn fXa(uint32_t x) { return Insn(x & 0x1) << 36; } + +// Status field: sf0 is the architected default the JIT uses throughout. +enum FpSf : uint32_t { sf0 = 0, sf1 = 1, sf2 = 2, sf3 = 3 }; + +// F1: f1 = f3 * f4 (+/-) f2 +inline Insn FmaF1(uint32_t op, uint32_t xa, uint32_t f1, uint32_t f3, + uint32_t f4, uint32_t f2, uint32_t sf, uint32_t qp) { + return fOp(op) | fXa(xa) | fSf(sf) | fF4(f4) | fF3(f3) | fF2(f2) | fF1(f1) | + fQp(qp); +} + +inline Insn Fma(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0x8, 0, f1, f3, f4, f2, sf, qp); +} +inline Insn FmaS(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0x8, 1, f1, f3, f4, f2, sf, qp); +} +inline Insn FmaD(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0x9, 0, f1, f3, f4, f2, sf, qp); +} +inline Insn Fms(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0xa, 0, f1, f3, f4, f2, sf, qp); +} +inline Insn FmsS(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0xa, 1, f1, f3, f4, f2, sf, qp); +} +inline Insn FmsD(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0xb, 0, f1, f3, f4, f2, sf, qp); +} +inline Insn Fnma(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0xc, 0, f1, f3, f4, f2, sf, qp); +} +inline Insn FnmaS(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0xc, 1, f1, f3, f4, f2, sf, qp); +} +inline Insn FnmaD(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t sf = sf0, uint32_t qp = 0) { + return FmaF1(0xd, 0, f1, f3, f4, f2, sf, qp); +} + +// Register f0 reads as +0.0 and f1 as +1.0, which is what makes the pseudo-ops +// below assemble to a bare fma. +static const uint32_t fpZero = 0; +static const uint32_t fpOne = 1; + +inline Insn FaddD(uint32_t f1, uint32_t f3, uint32_t f2, uint32_t sf = sf0, + uint32_t qp = 0) { + return FmaD(f1, f3, fpOne, f2, sf, qp); +} +inline Insn FsubD(uint32_t f1, uint32_t f3, uint32_t f2, uint32_t sf = sf0, + uint32_t qp = 0) { + return FmsD(f1, f3, fpOne, f2, sf, qp); +} +inline Insn FmpyD(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t sf = sf0, + uint32_t qp = 0) { + return FmaD(f1, f3, f4, fpZero, sf, qp); +} +inline Insn FnormD(uint32_t f1, uint32_t f3, uint32_t sf = sf0, + uint32_t qp = 0) { + return FmaD(f1, f3, fpOne, fpZero, sf, qp); +} +inline Insn Fnorm(uint32_t f1, uint32_t f3, uint32_t sf = sf0, + uint32_t qp = 0) { + return Fma(f1, f3, fpOne, fpZero, sf, qp); +} +inline Insn FmovD(uint32_t f1, uint32_t f3, uint32_t qp = 0) { + return FnormD(f1, f3, sf0, qp); +} + +// F6/F11: conversions. Major opcode 0, xb (bit 33) = 0, x6 -> 32:27. +inline Insn FcvtF11(uint32_t x6, uint32_t f1, uint32_t f2, uint32_t sf, + uint32_t qp) { + return fOp(0) | fZb(0) | fX6b(x6) | fSf(sf) | fF2(f2) | fF1(f1) | fQp(qp); +} + +inline Insn FcvtFx(uint32_t f1, uint32_t f2, uint32_t sf = sf0, + uint32_t qp = 0) { + return FcvtF11(0x18, f1, f2, sf, qp); +} +inline Insn FcvtFxu(uint32_t f1, uint32_t f2, uint32_t sf = sf0, + uint32_t qp = 0) { + return FcvtF11(0x19, f1, f2, sf, qp); +} +inline Insn FcvtFxTrunc(uint32_t f1, uint32_t f2, uint32_t sf = sf0, + uint32_t qp = 0) { + return FcvtF11(0x1a, f1, f2, sf, qp); +} +inline Insn FcvtFxuTrunc(uint32_t f1, uint32_t f2, uint32_t sf = sf0, + uint32_t qp = 0) { + return FcvtF11(0x1b, f1, f2, sf, qp); +} +// fcvt.xf has no status field. +inline Insn FcvtXf(uint32_t f1, uint32_t f2, uint32_t qp = 0) { + return FcvtF11(0x1c, f1, f2, 0, qp); +} + +// F4: fcmp.rel.sf p1, p2 = f2, f3 +// ra -> bit 33 rb -> bit 36 ta -> bit 12 +inline Insn fRa(uint32_t x) { return Insn(x & 0x1) << 33; } +inline Insn fRb(uint32_t x) { return Insn(x & 0x1) << 36; } + +// fccTa (bit 12) selects the .unc completer: when qp is false, p1/p2 are +// simply left at 0 rather than holding the relation, which lets a second +// fcmp be chained off the first predicate to build "ordered AND rel" in one +// extra instruction (see emitCompareDouble in MacroAssembler-ia64.cpp). This +// is a different field from the A-type compare's fTa (also bit 33, unlike +// this one) despite the similar name in the manual. +inline Insn fFccTa(uint32_t x) { return Insn(x & 0x1) << 12; } + +inline Insn FcmpF4(uint32_t ra, uint32_t rb, uint32_t p1, uint32_t p2, + uint32_t f2, uint32_t f3, uint32_t sf, uint32_t qp, + uint32_t ta = 0) { + return fOp(4) | fRb(rb) | fSf(sf) | fRa(ra) | fP2(p2) | fF3(f3) | fF2(f2) | + fFccTa(ta) | fP1(p1) | fQp(qp); +} + +inline Insn FcmpEq(uint32_t p1, uint32_t p2, uint32_t f2, uint32_t f3, + uint32_t sf = sf0, uint32_t qp = 0) { + return FcmpF4(0, 0, p1, p2, f2, f3, sf, qp); +} +inline Insn FcmpLt(uint32_t p1, uint32_t p2, uint32_t f2, uint32_t f3, + uint32_t sf = sf0, uint32_t qp = 0) { + return FcmpF4(0, 1, p1, p2, f2, f3, sf, qp); +} +inline Insn FcmpLe(uint32_t p1, uint32_t p2, uint32_t f2, uint32_t f3, + uint32_t sf = sf0, uint32_t qp = 0) { + return FcmpF4(1, 0, p1, p2, f2, f3, sf, qp); +} +inline Insn FcmpUnord(uint32_t p1, uint32_t p2, uint32_t f2, uint32_t f3, + uint32_t sf = sf0, uint32_t qp = 0) { + return FcmpF4(1, 1, p1, p2, f2, f3, sf, qp); +} +inline Insn FcmpEqUnc(uint32_t p1, uint32_t p2, uint32_t f2, uint32_t f3, + uint32_t sf, uint32_t qp) { + return FcmpF4(0, 0, p1, p2, f2, f3, sf, qp, 1); +} +inline Insn FcmpLtUnc(uint32_t p1, uint32_t p2, uint32_t f2, uint32_t f3, + uint32_t sf, uint32_t qp) { + return FcmpF4(0, 1, p1, p2, f2, f3, sf, qp, 1); +} +inline Insn FcmpLeUnc(uint32_t p1, uint32_t p2, uint32_t f2, uint32_t f3, + uint32_t sf, uint32_t qp) { + return FcmpF4(1, 0, p1, p2, f2, f3, sf, qp, 1); +} + +// F6: frcpa/frsqrta. Major opcode 0, xb (bit 33) selects the pair, q (bit 36) +// selects frcpa (0) vs frsqrta (1). Both write a status predicate p2 in +// addition to f1. +inline Insn fXb(uint32_t x) { return Insn(x & 0x1) << 33; } +inline Insn fQ(uint32_t x) { return Insn(x & 0x1) << 36; } + +inline Insn Frcpa(uint32_t f1, uint32_t p2, uint32_t f2, uint32_t f3, + uint32_t sf = sf0, uint32_t qp = 0) { + return fOp(0) | fQ(0) | fXb(1) | fSf(sf) | fF3(f3) | fF2(f2) | fP2(p2) | + fF1(f1) | fQp(qp); +} +inline Insn Frsqrta(uint32_t f1, uint32_t p2, uint32_t f3, uint32_t sf = sf0, + uint32_t qp = 0) { + return fOp(0) | fQ(1) | fXb(1) | fSf(sf) | fF3(f3) | fP2(p2) | fF1(f1) | + fQp(qp); +} + +// F2: xma. Fused 64x64 integer multiply-add via the significand path. xa is +// bit 36, x2 is bits 35:34. +inline Insn XmaF2(uint32_t x2, uint32_t f1, uint32_t f3, uint32_t f4, + uint32_t f2, uint32_t qp) { + return fOp(0xe) | fXa(1) | fX2(x2) | fF4(f4) | fF3(f3) | fF2(f2) | fF1(f1) | + fQp(qp); +} +inline Insn XmaL(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t qp = 0) { + return XmaF2(0, f1, f3, f4, f2, qp); +} +inline Insn XmaH(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t qp = 0) { + return XmaF2(3, f1, f3, f4, f2, qp); +} +inline Insn XmaHu(uint32_t f1, uint32_t f3, uint32_t f4, uint32_t f2, + uint32_t qp = 0) { + return XmaF2(2, f1, f3, f4, f2, qp); +} + +// F9: fmerge. Major opcode 0, xb=0, x6 -> bits 32:27. +inline Insn FmergeF9(uint32_t x6, uint32_t f1, uint32_t f2, uint32_t f3, + uint32_t qp) { + return fOp(0) | fZb(0) | fX6b(x6) | fF3(f3) | fF2(f2) | fF1(f1) | fQp(qp); +} +inline Insn FmergeS(uint32_t f1, uint32_t f2, uint32_t f3, uint32_t qp = 0) { + return FmergeF9(0x10, f1, f2, f3, qp); +} +inline Insn FmergeNs(uint32_t f1, uint32_t f2, uint32_t f3, uint32_t qp = 0) { + return FmergeF9(0x11, f1, f2, f3, qp); +} +inline Insn FmergeSe(uint32_t f1, uint32_t f2, uint32_t f3, uint32_t qp = 0) { + return FmergeF9(0x12, f1, f2, f3, qp); +} + +// F5: fclass. Major opcode 5, ta (bit 12) = 0 for the non-.unc form. The +// class-test immediate is split across two fields: a 7-bit "magnitude" mask +// at bits 26:20 (zero/unorm/norm/inf/snan/qnan/nat, one bit each, matching +// the low-to-high order of the FclassMask constants below) and a 2-bit sign +// selector at bits 34:33 (bit 33 = test positive, bit 34 = test negative). +// GNU as sets both sign bits automatically for the sign-independent magnitude +// classes (zero/unorm/norm/inf); the NaN/NaT classes carry no sign bits. +enum FclassMask : uint32_t { + FclassZero = 1u << 0, + FclassUnorm = 1u << 1, + FclassNorm = 1u << 2, + FclassInf = 1u << 3, + FclassSNaN = 1u << 4, + FclassQNaN = 1u << 5, + FclassNat = 1u << 6, + // The two hardware sign-select bits (bits 33:34), settable independently. + FclassSignPos = 1u << 7, + FclassSignNeg = 1u << 8, + FclassSignBoth = FclassSignPos | FclassSignNeg, +}; +static const uint32_t FclassNaN = FclassSNaN | FclassQNaN; +static const uint32_t FclassAnySignZero = FclassZero | FclassSignBoth; +static const uint32_t FclassAnySignInf = FclassInf | FclassSignBoth; +static const uint32_t FclassPosInf = FclassInf | FclassSignPos; + +inline Insn Fclass(uint32_t p1, uint32_t p2, uint32_t f2, uint32_t mask, + uint32_t qp = 0) { + return fOp(5) | fTa(0) | (Insn((mask >> 7) & 0x3) << 33) | + (Insn(mask & 0x7f) << 20) | fF2(f2) | fP2(p2) | fP1(p1) | fQp(qp); +} + +// --------------------------------------------------------------------------- +// Nops. The nop encoding is the same 41-bit pattern in every unit; only the +// slot it occupies determines which unit executes it. +// M-unit nop: major op 0, x3=0, x6=1 +// I-unit nop: major op 0, x3=0, x6=1 +// B-unit nop: major op 2, x6=0 +// F-unit nop: major op 0, x=0, x6=1 +// --------------------------------------------------------------------------- + +inline Insn NopM(uint32_t imm21 = 0) { + return fOp(0) | (Insn(1) << 27) | (Insn(imm21 & 0xfffff) << 6) | + (Insn((imm21 >> 20) & 1) << 36); +} +inline Insn NopI(uint32_t imm21 = 0) { return NopM(imm21); } +inline Insn NopF(uint32_t imm21 = 0) { return NopM(imm21); } +inline Insn NopB(uint32_t imm21 = 0) { + return fOp(2) | (Insn(imm21 & 0xfffff) << 6) | + (Insn((imm21 >> 20) & 1) << 36); +} + +// --------------------------------------------------------------------------- +// movl: load a full 64-bit immediate. This is the only two-slot instruction: +// it occupies the L and X slots of an MLX bundle, with the immediate scattered +// across both. The layout below was derived from GNU as by assembling movl +// with one immediate bit set at a time and observing which bundle bit moved: +// +// imm[6:0] -> X slot 19:13 (imm7b) +// imm[15:7] -> X slot 35:27 (imm9d) +// imm[20:16] -> X slot 26:22 (imm5c) +// imm[21] -> X slot 21 (ic) +// imm[62:22] -> the whole L slot +// imm[63] -> X slot 36 (i, the sign bit) +// +// vc (X slot bit 20) is 0 for movl. +// --------------------------------------------------------------------------- + +inline Bundle MovlBundle(uint32_t r1, uint64_t imm, uint32_t qp = 0) { + Insn l = Insn((imm >> 22) & ((uint64_t(1) << 41) - 1)); + + Insn x = fOp(6) | fR1(r1) | fQp(qp); + x |= Insn((imm >> 0) & 0x7f) << 13; // imm7b + x |= Insn((imm >> 7) & 0x1ff) << 27; // imm9d + x |= Insn((imm >> 16) & 0x1f) << 22; // imm5c + x |= Insn((imm >> 21) & 0x1) << 21; // ic + x |= Insn((imm >> 63) & 0x1) << 36; // i + + // Slot 0 of an MLX bundle is an M slot; pad it with an M nop. + return MakeBundle(tMLX_, NopM(), l, x); +} + +// --------------------------------------------------------------------------- +// One-instruction-per-bundle emission helpers. +// --------------------------------------------------------------------------- + +// An M-unit instruction alone in a bundle: MII with nops and a trailing stop. +inline Bundle BundleM(Insn m) { return MakeBundle(tMII_, m, NopI(), NopI()); } + +// An I-unit instruction alone in a bundle: the first slot of MII must be an +// M-unit slot, so pad it with an M nop. +inline Bundle BundleI(Insn i) { + return MakeBundle(tMII_, NopM(), i, NopI()); +} + +// A B-unit instruction alone in a bundle. +inline Bundle BundleB(Insn b) { + return MakeBundle(tMIB_, NopM(), NopI(), b); +} + +// An F-unit instruction alone in a bundle. +inline Bundle BundleF(Insn f) { + return MakeBundle(tMFI_, NopM(), f, NopI()); +} + +} // namespace ia64 +} // namespace jit +} // namespace js + +#endif /* jit_ia64_AssemblerCore_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/firefox-153.0.1/js/src/jit/ia64/CodeGenerator-ia64.cpp b/firefox-153.0.1/js/src/jit/ia64/CodeGenerator-ia64.cpp --- firefox-153.0.1/js/src/jit/ia64/CodeGenerator-ia64.cpp.vanilla +++ firefox-153.0.1/js/src/jit/ia64/CodeGenerator-ia64.cpp @@ -0,0 +1,2373 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#include "jit/ia64/CodeGenerator-ia64.h" + +#include "mozilla/MathAlgorithms.h" + +#include "jit/CodeGenerator.h" +#include "jit/InlineScriptTree.h" +#include "jit/JitRuntime.h" +#include "jit/MIR-wasm.h" +#include "jit/MIR.h" +#include "jit/MIRGraph.h" + +#include "jit/shared/CodeGenerator-shared-inl.h" + +using namespace js; +using namespace js::jit; + +using JS::GenericNaN; +using mozilla::NegativeInfinity; + +// r0 is hardwired to zero and f0 to +0.0, so neither needs materialising. +static constexpr Register Zero{Registers::zero}; +static constexpr FloatRegister FPZero{FloatRegisters::f0, FloatRegister::Double}; +static constexpr FloatRegister FPZeroSingle{FloatRegisters::f0, + FloatRegister::Single}; + +// IA-64 is a three-address machine, so the arithmetic below is emitted through +// the instruction encoders rather than the two-address MacroAssembler forms: +// that avoids having to shuffle registers when the output aliases an input. +// +// Emit3() is for A-type encoders (add, sub, and, or, xor) only, which issue on +// either an M or an I slot. The variable shifts are I-unit only and must go +// through Emit3I(): in slot 0 of an MII bundle the hardware decodes one as an +// M-unit memory op instead (shl r14=r14,r2 reads as ldf8.a f14=[r2]). +static void Emit3(MacroAssembler& masm, + ia64::Insn (*make)(uint32_t, uint32_t, uint32_t, uint32_t), + Register d, Register a, Register b) { + masm.emitM(make(d.encoding(), a.encoding(), b.encoding(), 0)); +} + +static void Emit3I(MacroAssembler& masm, + ia64::Insn (*make)(uint32_t, uint32_t, uint32_t, uint32_t), + Register d, Register a, Register b) { + masm.emitI(make(d.encoding(), a.encoding(), b.encoding(), 0)); +} + +static void SignExtend32(MacroAssembler& masm, Register dest, Register src) { + masm.emitI(ia64::Sxt4(dest.encoding(), src.encoding())); +} + +static void ZeroExtend32(MacroAssembler& masm, Register dest, Register src) { + masm.emitI(ia64::Zxt4(dest.encoding(), src.encoding())); +} + +CodeGeneratorIa64::CodeGeneratorIa64(MIRGenerator* gen, LIRGraph* graph, + MacroAssembler* masm, + const wasm::CodeMetadata* wasmCodeMeta) + : CodeGeneratorShared(gen, graph, masm, wasmCodeMeta) {} + +void CodeGeneratorIa64::branchToBlock(FloatFormat fmt, FloatRegister lhs, + FloatRegister rhs, MBasicBlock* mir, + Assembler::DoubleCondition cond) { + Label* label = skipTrivialBlocks(mir)->lir()->label(); + if (fmt == DoubleFloat) { + masm.branchDouble(cond, lhs, rhs, label); + } else { + masm.branchFloat(cond, lhs, rhs, label); + } +} + +MoveOperand CodeGeneratorIa64::toMoveOperand(LAllocation a) const { + if (a.isGeneralReg()) { + return MoveOperand(ToRegister(a)); + } + if (a.isFloatReg()) { + return MoveOperand(ToFloatRegister(a)); + } + MoveOperand::Kind kind = a.isStackArea() ? MoveOperand::Kind::EffectiveAddress + : MoveOperand::Kind::Memory; + return MoveOperand(ToAddress(a), kind); +} + +void CodeGeneratorIa64::emitBailoutOOL(LSnapshot* snapshot) { + masm.push(Imm32(snapshot->snapshotOffset())); + masm.jump(&deoptLabel_); +} + +void CodeGeneratorIa64::bailoutFrom(Label* label, LSnapshot* snapshot) { + MOZ_ASSERT_IF(!masm.oom(), label->used()); + MOZ_ASSERT_IF(!masm.oom(), !label->bound()); + + encode(snapshot); + + InlineScriptTree* tree = snapshot->mir()->block()->trackedTree(); + auto* ool = new (alloc()) LambdaOutOfLineCode( + [=, this](OutOfLineCode& ool) { emitBailoutOOL(snapshot); }); + addOutOfLineCode(ool, + new (alloc()) BytecodeSite(tree, tree->script()->code())); + + masm.retarget(label, ool->entry()); +} + +void CodeGeneratorIa64::bailout(LSnapshot* snapshot) { + Label label; + masm.jump(&label); + bailoutFrom(&label, snapshot); +} + +bool CodeGeneratorIa64::generateOutOfLineCode() { + if (!CodeGeneratorShared::generateOutOfLineCode()) { + return false; + } + + if (deoptLabel_.used()) { + // All non-table-based bailouts land here. + masm.bind(&deoptLabel_); + + // Push the frame size so the handler can recover the IonScript. + masm.push(Imm32(frameSize())); + + TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler(); + masm.jump(handler); + } + + return !masm.oom(); +} + +class js::jit::OutOfLineTableSwitch + : public OutOfLineCodeBase { + MTableSwitch* mir_; + CodeLabel jumpLabel_; + + void accept(CodeGeneratorIa64* codegen) { + codegen->visitOutOfLineTableSwitch(this); + } + + public: + explicit OutOfLineTableSwitch(MTableSwitch* mir) : mir_(mir) {} + + MTableSwitch* mir() const { return mir_; } + + CodeLabel* jumpLabel() { return &jumpLabel_; } +}; + +void CodeGeneratorIa64::emitTableSwitchDispatch(MTableSwitch* mir, + Register index, + Register base) { + Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label(); + + if (mir->low() != 0) { + masm.sub32(Imm32(mir->low()), index); + } + + int32_t cases = mir->numCases(); + masm.branch32(Assembler::AboveOrEqual, index, Imm32(cases), defaultcase); + + // The case entries do not exist yet, so the table is emitted out of line and + // its address patched in afterwards. + OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(mir); + addOutOfLineCode(ool, mir); + + masm.mov(ool->jumpLabel(), base); + BaseIndex pointer(base, index, ScalePointer); + masm.branchToComputedAddress(pointer); +} + +void CodeGeneratorIa64::generateInvalidateEpilogue() { + // Leave room for the OsiPoint patching so it cannot overwrite the epilogue. + // One IA-64 bundle is 16 bytes, comfortably more than a pointer. + masm.nop(); + + masm.bind(&invalidate_); + + // InvalidationBailoutStack ends in {ionScript_, osiPointReturnAddress_}, so + // the return address of the patched-in call has to be spilled here: it is in + // b0 rather than already on the stack the way a x86 call leaves it. + masm.pushReturnAddress(); + + invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1))); + + TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk(); + masm.jump(thunk); +} + +void CodeGeneratorIa64::visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool) { + MTableSwitch* mir = ool->mir(); + + masm.haltingAlign(sizeof(void*)); + masm.bind(ool->jumpLabel()); + masm.addCodeLabel(*ool->jumpLabel()); + + for (size_t i = 0; i < mir->numCases(); i++) { + LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir(); + Label* caseheader = caseblock->label(); + uint32_t caseoffset = caseheader->offset(); + + // Table entries are absolute addresses and must be patched once codegen + // has finished. + CodeLabel cl; + masm.writeCodePointer(&cl); + cl.target()->bind(caseoffset); + masm.addCodeLabel(cl); + } + + masm.alignToBundle(); +} + +void CodeGeneratorIa64::visitOutOfLineWasmTruncateCheck( + OutOfLineWasmTruncateCheck* ool) { + FloatRegister input = ool->input(); + Register output = ool->output(); + Register64 output64 = ool->output64(); + MIRType fromType = ool->fromType(); + MIRType toType = ool->toType(); + Label* oolRejoin = ool->rejoin(); + TruncFlags flags = ool->flags(); + wasm::TrapSiteDesc off = ool->trapSiteDesc(); + + if (fromType == MIRType::Float32) { + if (toType == MIRType::Int32) { + masm.oolWasmTruncateCheckF32ToI32(input, output, flags, off, oolRejoin); + } else if (toType == MIRType::Int64) { + masm.oolWasmTruncateCheckF32ToI64(input, output64, flags, off, oolRejoin); + } else { + MOZ_CRASH("unexpected type"); + } + } else if (fromType == MIRType::Double) { + if (toType == MIRType::Int32) { + masm.oolWasmTruncateCheckF64ToI32(input, output, flags, off, oolRejoin); + } else if (toType == MIRType::Int64) { + masm.oolWasmTruncateCheckF64ToI64(input, output64, flags, off, oolRejoin); + } else { + MOZ_CRASH("unexpected type"); + } + } else { + MOZ_CRASH("unexpected type"); + } +} + +// The flexible*Ptr/32 helpers divide inline (frcpa + Newton) and never call +// out, so they ignore the volatile set entirely. Passing liveVolatileRegs(ins) +// instead would dereference ins->safepoint(), which is null on the LIR nodes +// that reach here. +void CodeGeneratorIa64::emitBigIntPtrDiv(LBigIntPtrDiv* ins, Register dividend, + Register divisor, Register output) { + masm.flexibleQuotientPtr(dividend, divisor, output, /* isUnsigned = */ false, + LiveRegisterSet()); +} + +void CodeGeneratorIa64::emitBigIntPtrMod(LBigIntPtrMod* ins, Register dividend, + Register divisor, Register output) { + masm.flexibleRemainderPtr(dividend, divisor, output, /* isUnsigned = */ false, + LiveRegisterSet()); +} + +// =========================================================================== +// Boxing. + +void CodeGenerator::visitBox(LBox* ins) { + const LAllocation* in = ins->payload(); + ValueOperand result = ToOutValue(ins); + + masm.moveValue(TypedOrValueRegister(ins->type(), ToAnyRegister(in)), result); +} + +void CodeGenerator::visitUnbox(LUnbox* ins) { + MUnbox* mir = ins->mir(); + + Register result = ToRegister(ins->output()); + + if (mir->fallible()) { + ValueOperand value = ToValue(ins->input()); + Label bail; + switch (mir->type()) { + case MIRType::Int32: + masm.fallibleUnboxInt32(value, result, &bail); + break; + case MIRType::Boolean: + masm.fallibleUnboxBoolean(value, result, &bail); + break; + case MIRType::Object: + masm.fallibleUnboxObject(value, result, &bail); + break; + case MIRType::String: + masm.fallibleUnboxString(value, result, &bail); + break; + case MIRType::Symbol: + masm.fallibleUnboxSymbol(value, result, &bail); + break; + case MIRType::BigInt: + masm.fallibleUnboxBigInt(value, result, &bail); + break; + default: + MOZ_CRASH("Given MIRType cannot be unboxed."); + } + bailoutFrom(&bail, ins->snapshot()); + return; + } + + LAllocation* input = ins->getOperand(LUnbox::Input); + if (input->isGeneralReg()) { + // ValueOperand's Register constructor is explicit; passing the bare + // Register instead binds to the catch-all MOZ_CRASH template overload. + ValueOperand inputReg = ValueOperand(ToRegister(input)); + switch (mir->type()) { + case MIRType::Int32: + masm.unboxInt32(inputReg, result); + break; + case MIRType::Boolean: + masm.unboxBoolean(inputReg, result); + break; + case MIRType::Object: + masm.unboxObject(inputReg, result); + break; + case MIRType::String: + masm.unboxString(inputReg, result); + break; + case MIRType::Symbol: + masm.unboxSymbol(inputReg, result); + break; + case MIRType::BigInt: + masm.unboxBigInt(inputReg, result); + break; + default: + MOZ_CRASH("Given MIRType cannot be unboxed."); + } + return; + } + + Address inputAddr = ToAddress(input); + switch (mir->type()) { + case MIRType::Int32: + masm.unboxInt32(inputAddr, result); + break; + case MIRType::Boolean: + masm.unboxBoolean(inputAddr, result); + break; + case MIRType::Object: + masm.unboxObject(inputAddr, result); + break; + case MIRType::String: + masm.unboxString(inputAddr, result); + break; + case MIRType::Symbol: + masm.unboxSymbol(inputAddr, result); + break; + case MIRType::BigInt: + masm.unboxBigInt(inputAddr, result); + break; + default: + MOZ_CRASH("Given MIRType cannot be unboxed."); + } +} + +// =========================================================================== +// Integer arithmetic. + +template +static void TrapIfDivideByZero(MacroAssembler& masm, LIR* lir, Register rhs) { + auto* mir = lir->mir(); + MOZ_ASSERT(mir->trapOnError()); + + if (mir->canBeDivideByZero()) { + Label nonZero; + masm.branchPtr(Assembler::NotEqual, rhs, Zero, &nonZero); + masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->trapSiteDesc()); + masm.bind(&nonZero); + } +} + +void CodeGenerator::visitAddI(LAddI* ins) { + const LAllocation* rhs = ins->rhs(); + Register lhs = ToRegister(ins->lhs()); + Register dest = ToRegister(ins->output()); + + if (!ins->snapshot()) { + if (rhs->isConstant()) { + masm.move32(lhs, dest); + masm.add32(Imm32(ToInt32(rhs)), dest); + } else { + Emit3(masm, ia64::Add, dest, lhs, ToRegister(rhs)); + } + SignExtend32(masm, dest, dest); + return; + } + + // The 64-bit sum of two canonical (sign-extended) int32 values is exact, so + // the add overflowed exactly when truncating it back to 32 bits changes it. + SecondScratchRegisterScope scratch(masm); + if (rhs->isConstant()) { + masm.move32(Imm32(ToInt32(rhs)), scratch); + Emit3(masm, ia64::Add, scratch, lhs, scratch); + } else { + Emit3(masm, ia64::Add, scratch, lhs, ToRegister(rhs)); + } + SignExtend32(masm, dest, scratch); + bailoutCmpPtr(Assembler::NotEqual, dest, Register(scratch), ins->snapshot()); +} + +void CodeGenerator::visitAddIntPtr(LAddIntPtr* ins) { + Register lhs = ToRegister(ins->lhs()); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + + if (rhs->isConstant()) { + masm.movePtr(lhs, dest); + masm.addPtr(ImmWord(ToIntPtr(rhs)), dest); + } else { + Emit3(masm, ia64::Add, dest, lhs, ToRegister(rhs)); + } +} + +void CodeGenerator::visitAddI64(LAddI64* ins) { + Register lhs = ToRegister64(ins->lhs()).reg; + LInt64Allocation rhs = ins->rhs(); + Register dest = ToOutRegister64(ins).reg; + + if (IsConstant(rhs)) { + masm.movePtr(lhs, dest); + masm.add64(Imm64(ToInt64(rhs)), Register64(dest)); + } else { + Emit3(masm, ia64::Add, dest, lhs, ToRegister64(rhs).reg); + } +} + +void CodeGenerator::visitSubI(LSubI* ins) { + const LAllocation* rhs = ins->rhs(); + Register lhs = ToRegister(ins->lhs()); + Register dest = ToRegister(ins->output()); + + if (!ins->snapshot()) { + if (rhs->isConstant()) { + masm.move32(lhs, dest); + masm.sub32(Imm32(ToInt32(rhs)), dest); + } else { + Emit3(masm, ia64::Sub, dest, lhs, ToRegister(rhs)); + } + SignExtend32(masm, dest, dest); + return; + } + + SecondScratchRegisterScope scratch(masm); + if (rhs->isConstant()) { + masm.move32(Imm32(ToInt32(rhs)), scratch); + Emit3(masm, ia64::Sub, scratch, lhs, scratch); + } else { + Emit3(masm, ia64::Sub, scratch, lhs, ToRegister(rhs)); + } + SignExtend32(masm, dest, scratch); + bailoutCmpPtr(Assembler::NotEqual, dest, Register(scratch), ins->snapshot()); +} + +void CodeGenerator::visitSubIntPtr(LSubIntPtr* ins) { + Register lhs = ToRegister(ins->lhs()); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + + if (rhs->isConstant()) { + masm.movePtr(lhs, dest); + masm.subPtr(Imm32(ToIntPtr(rhs)), dest); + } else { + Emit3(masm, ia64::Sub, dest, lhs, ToRegister(rhs)); + } +} + +void CodeGenerator::visitSubI64(LSubI64* ins) { + Register lhs = ToRegister64(ins->lhs()).reg; + LInt64Allocation rhs = ins->rhs(); + Register dest = ToOutRegister64(ins).reg; + + if (IsConstant(rhs)) { + masm.movePtr(lhs, dest); + masm.sub64(Imm64(ToInt64(rhs)), Register64(dest)); + } else { + Emit3(masm, ia64::Sub, dest, lhs, ToRegister64(rhs).reg); + } +} + +void CodeGenerator::visitMulI(LMulI* ins) { + Register lhs = ToRegister(ins->lhs()); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + MMul* mul = ins->mir(); + + MOZ_ASSERT_IF(mul->mode() == MMul::Integer, + !mul->canBeNegativeZero() && !mul->canOverflow()); + + if (rhs->isConstant()) { + int32_t constant = ToInt32(rhs); + + // Bailout on -0.0. + if (mul->canBeNegativeZero() && constant <= 0) { + Assembler::Condition cond = + (constant == 0) ? Assembler::LessThan : Assembler::Equal; + bailoutCmp32(cond, lhs, Imm32(0), ins->snapshot()); + } + + if (!mul->canOverflow()) { + masm.move32(lhs, dest); + masm.mul32(Imm32(constant), dest); + SignExtend32(masm, dest, dest); + return; + } + + switch (constant) { + case -1: + bailoutCmp32(Assembler::Equal, lhs, Imm32(INT32_MIN), ins->snapshot()); + Emit3(masm, ia64::Sub, dest, Zero, lhs); + SignExtend32(masm, dest, dest); + return; + case 0: + masm.move32(Imm32(0), dest); + return; + case 1: + masm.move32(lhs, dest); + return; + default: + break; + } + + Label overflow; + masm.move32(lhs, dest); + masm.branchMul32(Assembler::Overflow, Imm32(constant), dest, &overflow); + bailoutFrom(&overflow, ins->snapshot()); + SignExtend32(masm, dest, dest); + return; + } + + Register rhsReg = ToRegister(rhs); + if (mul->canOverflow()) { + Label overflow; + // Multiplication is commutative, so pick the operand ordering that avoids + // clobbering an input with the (two-address) accumulator. + if (dest == rhsReg) { + masm.branchMul32(Assembler::Overflow, lhs, dest, &overflow); + } else { + masm.move32(lhs, dest); + masm.branchMul32(Assembler::Overflow, rhsReg, dest, &overflow); + } + bailoutFrom(&overflow, ins->snapshot()); + } else { + if (dest == rhsReg) { + masm.mul32(lhs, dest); + } else { + masm.move32(lhs, dest); + masm.mul32(rhsReg, dest); + } + } + SignExtend32(masm, dest, dest); + + if (mul->canBeNegativeZero()) { + Label done; + masm.branch32(Assembler::NotEqual, dest, Imm32(0), &done); + + // A zero result is -0 if either operand was negative, which must be a + // double, so bail out. + SecondScratchRegisterScope scratch(masm); + Emit3(masm, ia64::Or, scratch, lhs, rhsReg); + bailoutCmp32(Assembler::LessThan, Register(scratch), Imm32(0), + ins->snapshot()); + + masm.bind(&done); + } +} + +void CodeGenerator::visitMulIntPtr(LMulIntPtr* ins) { + Register lhs = ToRegister(ins->lhs()); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + + if (rhs->isConstant()) { + masm.movePtr(lhs, dest); + masm.mulPtr(ImmWord(ToIntPtr(rhs)), dest); + return; + } + + Register rhsReg = ToRegister(rhs); + if (dest == rhsReg) { + masm.mulPtr(lhs, dest); + } else { + masm.movePtr(lhs, dest); + masm.mulPtr(rhsReg, dest); + } +} + +void CodeGenerator::visitMulI64(LMulI64* ins) { + Register lhs = ToRegister64(ins->lhs()).reg; + LInt64Allocation rhs = ins->rhs(); + Register dest = ToOutRegister64(ins).reg; + + if (IsConstant(rhs)) { + masm.movePtr(lhs, dest); + masm.mulPtr(ImmWord(ToInt64(rhs)), dest); + return; + } + + Register rhsReg = ToRegister64(rhs).reg; + if (dest == rhsReg) { + masm.mulPtr(lhs, dest); + } else { + masm.movePtr(lhs, dest); + masm.mulPtr(rhsReg, dest); + } +} + +void CodeGenerator::visitDivI(LDivI* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register dest = ToRegister(ins->output()); + Register temp = ToRegister(ins->temp0()); + MDiv* mir = ins->mir(); + + Label done; + + if (mir->canBeDivideByZero()) { + if (mir->trapOnError()) { + TrapIfDivideByZero(masm, ins, rhs); + } else if (mir->canTruncateInfinities()) { + // Truncated division by zero is zero (Infinity|0 == 0). + Label notzero; + masm.branch32(Assembler::NotEqual, rhs, Imm32(0), ¬zero); + masm.move32(Imm32(0), dest); + masm.jump(&done); + masm.bind(¬zero); + } else { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot()); + } + } + + // (INT32_MIN / -1) yields INT32_MIN but should be -(double)INT32_MIN. + if (mir->canBeNegativeOverflow() && + (mir->trapOnError() || !mir->canTruncateOverflow())) { + Label notMinInt; + masm.branch32(Assembler::NotEqual, lhs, Imm32(INT32_MIN), ¬MinInt); + + if (mir->trapOnError()) { + Label ok; + masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), &ok); + masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->trapSiteDesc()); + masm.bind(&ok); + } else { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, rhs, Imm32(-1), ins->snapshot()); + } + masm.bind(¬MinInt); + } + + // Negative zero: lhs == 0 && rhs < 0. + if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) { + Label nonzero; + masm.branch32(Assembler::NotEqual, lhs, Imm32(0), &nonzero); + bailoutCmp32(Assembler::LessThan, rhs, Imm32(0), ins->snapshot()); + masm.bind(&nonzero); + } + + if (mir->canTruncateRemainder()) { + masm.flexibleQuotient32(lhs, rhs, dest, /* isUnsigned = */ false, + LiveRegisterSet()); + } else { + MOZ_ASSERT(mir->fallible()); + masm.flexibleDivMod32(lhs, rhs, dest, temp, /* isUnsigned = */ false, + LiveRegisterSet()); + // A non-zero remainder means the result is not an integer. + bailoutCmp32(Assembler::NotEqual, temp, Imm32(0), ins->snapshot()); + } + + masm.bind(&done); +} + +void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) { + Register lhs = ToRegister(ins->numerator()); + Register dest = ToRegister(ins->output()); + int32_t shift = ins->shift(); + MOZ_ASSERT(0 <= shift && shift <= 31); + + if (shift == 0) { + masm.move32(lhs, dest); + return; + } + + MDiv* mir = ins->mir(); + SecondScratchRegisterScope scratch(masm); + + if (!mir->isTruncated()) { + // A non-zero remainder means the result is a double. + masm.emitI(ia64::ShlImm(scratch.encoding(), lhs.encoding(), 32 - shift)); + bailoutCmpPtr(Assembler::NotEqual, Register(scratch), Zero, + ins->snapshot()); + } + + if (!mir->canBeNegativeDividend()) { + masm.emitI(ia64::ShrImm(dest.encoding(), lhs.encoding(), shift)); + return; + } + + // Bias a negative numerator so the arithmetic shift rounds toward zero. See + // "Signed Division by a Known Power of 2" in Hacker's Delight. + masm.emitI(ia64::ShrImm(scratch.encoding(), lhs.encoding(), 31)); + masm.emitI(ia64::ShrUImm(scratch.encoding(), scratch.encoding(), 32 - shift)); + Emit3(masm, ia64::Add, scratch, scratch, lhs); + masm.emitI(ia64::ShrImm(dest.encoding(), scratch.encoding(), shift)); + SignExtend32(masm, dest, dest); +} + +void CodeGenerator::visitModI(LModI* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register dest = ToRegister(ins->output()); + MMod* mir = ins->mir(); + Label done; + + if (mir->canBeDivideByZero()) { + if (mir->trapOnError()) { + TrapIfDivideByZero(masm, ins, rhs); + } else if (mir->isTruncated()) { + // Truncated modulo by zero yields integer zero. + Label yNonZero; + masm.branch32(Assembler::NotEqual, rhs, Imm32(0), &yNonZero); + masm.move32(Imm32(0), dest); + masm.jump(&done); + masm.bind(&yNonZero); + } else { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot()); + } + } + + masm.flexibleRemainder32(lhs, rhs, dest, /* isUnsigned = */ false, + LiveRegisterSet()); + + if (mir->canBeNegativeDividend() && !mir->isTruncated()) { + MOZ_ASSERT(mir->fallible()); + // A zero result with a negative dividend is the double -0.0. This also + // covers lhs == INT32_MIN with rhs == -1. + masm.branch32(Assembler::NotEqual, dest, Imm32(0), &done); + bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot()); + } + masm.bind(&done); +} + +void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) { + Register in = ToRegister(ins->input()); + Register out = ToRegister(ins->output()); + MMod* mir = ins->mir(); + Label negative, done; + + masm.branch32(Assembler::LessThan, in, Imm32(0), &negative); + { + masm.move32(in, out); + masm.and32(Imm32((1 << ins->shift()) - 1), out); + masm.jump(&done); + } + { + masm.bind(&negative); + Emit3(masm, ia64::Sub, out, Zero, in); + masm.and32(Imm32((1 << ins->shift()) - 1), out); + Emit3(masm, ia64::Sub, out, Zero, out); + SignExtend32(masm, out, out); + } + if (mir->canBeNegativeDividend() && !mir->isTruncated()) { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, out, Imm32(0), ins->snapshot()); + } + masm.bind(&done); +} + +void CodeGenerator::visitModMaskI(LModMaskI* ins) { + Register src = ToRegister(ins->input()); + Register dest = ToRegister(ins->output()); + Register tmp0 = ToRegister(ins->temp0()); + Register tmp1 = ToRegister(ins->temp1()); + MMod* mir = ins->mir(); + + // x % (2^n - 1) via repeated folding of the digit sum in base 2^n. + int32_t shift = ins->shift(); + MOZ_ASSERT(shift > 0 && shift < 32); + int32_t mask = (1 << shift) - 1; + + Label loop, done, negative; + + masm.move32(src, tmp0); + masm.move32(Imm32(0), dest); + + masm.branch32(Assembler::GreaterThanOrEqual, tmp0, Imm32(0), &loop); + Emit3(masm, ia64::Sub, tmp0, Zero, tmp0); + masm.bind(&loop); + { + Label exit; + masm.branch32(Assembler::Equal, tmp0, Imm32(0), &exit); + masm.move32(tmp0, tmp1); + masm.and32(Imm32(mask), tmp1); + masm.add32(tmp1, dest); + // Reduce on every digit: the running sum of all the base-2^n digits of a + // 32-bit value reaches 11 * mask, so folding it once at the end leaves a + // result that is still several multiples of the modulus too large. + Label noFold; + masm.branch32(Assembler::LessThan, dest, Imm32(mask), &noFold); + masm.sub32(Imm32(mask), dest); + masm.bind(&noFold); + masm.rshift32(Imm32(shift), tmp0); + masm.jump(&loop); + masm.bind(&exit); + } + + masm.branch32(Assembler::GreaterThanOrEqual, src, Imm32(0), &done); + masm.bind(&negative); + Emit3(masm, ia64::Sub, dest, Zero, dest); + SignExtend32(masm, dest, dest); + if (!mir->isTruncated() && mir->canBeNegativeDividend()) { + MOZ_ASSERT(mir->fallible()); + bailoutCmp32(Assembler::Equal, dest, Imm32(0), ins->snapshot()); + } + masm.bind(&done); +} + +void CodeGenerator::visitUDiv(LUDiv* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register output = ToRegister(ins->output()); + Label done; + + MDiv* mir = ins->mir(); + + if (mir->canBeDivideByZero()) { + if (mir->trapOnError()) { + TrapIfDivideByZero(masm, ins, rhs); + } else if (mir->isTruncated()) { + // Infinity|0 == 0 + Label nonZero; + masm.branch32(Assembler::NotEqual, rhs, Imm32(0), &nonZero); + masm.move32(Imm32(0), output); + masm.jump(&done); + masm.bind(&nonZero); + } else { + bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot()); + } + } + + masm.flexibleQuotient32(lhs, rhs, output, /* isUnsigned = */ true, + LiveRegisterSet()); + + // An unsigned division can produce a value that is not a signed int32. + if (!mir->isTruncated()) { + bailoutCmp32(Assembler::LessThan, output, Imm32(0), ins->snapshot()); + } + + masm.bind(&done); +} + +void CodeGenerator::visitUMod(LUMod* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register output = ToRegister(ins->output()); + Label done; + + MMod* mir = ins->mir(); + + if (mir->canBeDivideByZero()) { + if (mir->trapOnError()) { + TrapIfDivideByZero(masm, ins, rhs); + } else if (mir->isTruncated()) { + // NaN|0 == 0 + Label nonZero; + masm.branch32(Assembler::NotEqual, rhs, Imm32(0), &nonZero); + masm.move32(Imm32(0), output); + masm.jump(&done); + masm.bind(&nonZero); + } else { + bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot()); + } + } + + masm.flexibleRemainder32(lhs, rhs, output, /* isUnsigned = */ true, + LiveRegisterSet()); + + if (!mir->isTruncated()) { + bailoutCmp32(Assembler::LessThan, output, Imm32(0), ins->snapshot()); + } + + masm.bind(&done); +} + +void CodeGenerator::visitDivI64(LDivI64* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register output = ToRegister(ins->output()); + + MDiv* div = ins->mir(); + + TrapIfDivideByZero(masm, ins, rhs); + + if (div->canBeNegativeOverflow()) { + Label notOverflow; + masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), ¬Overflow); + masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), ¬Overflow); + masm.wasmTrap(wasm::Trap::IntegerOverflow, div->trapSiteDesc()); + masm.bind(¬Overflow); + } + + masm.flexibleQuotientPtr(lhs, rhs, output, /* isUnsigned = */ false, + LiveRegisterSet()); +} + +void CodeGenerator::visitModI64(LModI64* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register output = ToRegister(ins->output()); + + TrapIfDivideByZero(masm, ins, rhs); + + masm.flexibleRemainderPtr(lhs, rhs, output, /* isUnsigned = */ false, + LiveRegisterSet()); +} + +void CodeGenerator::visitUDivI64(LUDivI64* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register output = ToRegister(ins->output()); + + TrapIfDivideByZero(masm, ins, rhs); + + masm.flexibleQuotientPtr(lhs, rhs, output, /* isUnsigned = */ true, + LiveRegisterSet()); +} + +void CodeGenerator::visitUModI64(LUModI64* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register output = ToRegister(ins->output()); + + TrapIfDivideByZero(masm, ins, rhs); + + masm.flexibleRemainderPtr(lhs, rhs, output, /* isUnsigned = */ true, + LiveRegisterSet()); +} + +void CodeGenerator::visitNegI(LNegI* ins) { + Register input = ToRegister(ins->input()); + Register output = ToRegister(ins->output()); + + Emit3(masm, ia64::Sub, output, Zero, input); + SignExtend32(masm, output, output); +} + +void CodeGenerator::visitNegI64(LNegI64* ins) { + Register input = ToRegister64(ins->input()).reg; + Register output = ToOutRegister64(ins).reg; + + Emit3(masm, ia64::Sub, output, Zero, input); +} + +// =========================================================================== +// Bitwise operations. + +void CodeGenerator::visitBitNotI(LBitNotI* ins) { + Register input = ToRegister(ins->input()); + Register dest = ToRegister(ins->output()); + + // ~x == -x - 1 + Emit3(masm, ia64::Sub, dest, Zero, input); + masm.emitM(ia64::Adds(dest.encoding(), -1, dest.encoding())); +} + +void CodeGenerator::visitBitNotI64(LBitNotI64* ins) { + Register input = ToRegister64(ins->input()).reg; + Register dest = ToOutRegister64(ins).reg; + + Emit3(masm, ia64::Sub, dest, Zero, input); + masm.emitM(ia64::Adds(dest.encoding(), -1, dest.encoding())); +} + +void CodeGenerator::visitBitOpI(LBitOpI* ins) { + Register lhs = ToRegister(ins->lhs()); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + + ia64::Insn (*op)(uint32_t, uint32_t, uint32_t, uint32_t) = nullptr; + switch (ins->bitop()) { + case JSOp::BitOr: + op = ia64::Or; + break; + case JSOp::BitXor: + op = ia64::Xor; + break; + case JSOp::BitAnd: + op = ia64::And; + break; + default: + MOZ_CRASH("unexpected binary opcode"); + } + + if (rhs->isConstant()) { + SecondScratchRegisterScope scratch(masm); + masm.move32(Imm32(ToInt32(rhs)), scratch); + Emit3(masm, op, dest, lhs, scratch); + } else { + Emit3(masm, op, dest, lhs, ToRegister(rhs)); + } +} + +void CodeGenerator::visitBitOpI64(LBitOpI64* ins) { + Register lhs = ToRegister64(ins->lhs()).reg; + LInt64Allocation rhs = ins->rhs(); + Register dest = ToOutRegister64(ins).reg; + + ia64::Insn (*op)(uint32_t, uint32_t, uint32_t, uint32_t) = nullptr; + switch (ins->bitop()) { + case JSOp::BitOr: + op = ia64::Or; + break; + case JSOp::BitXor: + op = ia64::Xor; + break; + case JSOp::BitAnd: + op = ia64::And; + break; + default: + MOZ_CRASH("unexpected binary opcode"); + } + + if (IsConstant(rhs)) { + SecondScratchRegisterScope scratch(masm); + masm.movePtr(ImmWord(ToInt64(rhs)), scratch); + Emit3(masm, op, dest, lhs, scratch); + } else { + Emit3(masm, op, dest, lhs, ToRegister64(rhs).reg); + } +} + +void CodeGenerator::visitShiftI(LShiftI* ins) { + Register lhs = ToRegister(ins->lhs()); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + + if (rhs->isConstant()) { + int32_t shift = ToInt32(rhs) & 0x1F; + switch (ins->bitop()) { + case JSOp::Lsh: + if (shift) { + masm.emitI(ia64::ShlImm(dest.encoding(), lhs.encoding(), shift)); + SignExtend32(masm, dest, dest); + } else { + masm.move32(lhs, dest); + } + break; + case JSOp::Rsh: + if (shift) { + masm.emitI(ia64::ShrImm(dest.encoding(), lhs.encoding(), shift)); + } else { + masm.move32(lhs, dest); + } + break; + case JSOp::Ursh: + if (shift) { + ZeroExtend32(masm, dest, lhs); + masm.emitI(ia64::ShrUImm(dest.encoding(), dest.encoding(), shift)); + SignExtend32(masm, dest, dest); + } else { + // x >>> 0 can produce a value outside the int32 range. + if (ins->mir()->toUrsh()->fallible()) { + bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot()); + } + masm.move32(lhs, dest); + } + break; + default: + MOZ_CRASH("Unexpected shift op"); + } + return; + } + + SecondScratchRegisterScope count(masm); + masm.move32(Imm32(0x1F), count); + Emit3(masm, ia64::And, count, ToRegister(rhs), count); + + switch (ins->bitop()) { + case JSOp::Lsh: + Emit3I(masm, ia64::Shl, dest, lhs, count); + SignExtend32(masm, dest, dest); + break; + case JSOp::Rsh: + Emit3I(masm, ia64::Shr, dest, lhs, count); + break; + case JSOp::Ursh: + ZeroExtend32(masm, dest, lhs); + Emit3I(masm, ia64::ShrU, dest, dest, count); + SignExtend32(masm, dest, dest); + if (ins->mir()->toUrsh()->fallible()) { + bailoutCmp32(Assembler::LessThan, dest, Imm32(0), ins->snapshot()); + } + break; + default: + MOZ_CRASH("Unexpected shift op"); + } +} + +void CodeGenerator::visitShiftIntPtr(LShiftIntPtr* ins) { + Register lhs = ToRegister(ins->lhs()); + const LAllocation* rhs = ins->rhs(); + Register dest = ToRegister(ins->output()); + + if (rhs->isConstant()) { + auto shamt = ToIntPtr(rhs) & 0x3F; + if (!shamt) { + masm.movePtr(lhs, dest); + return; + } + switch (ins->bitop()) { + case JSOp::Lsh: + masm.emitI(ia64::ShlImm(dest.encoding(), lhs.encoding(), shamt)); + break; + case JSOp::Rsh: + masm.emitI(ia64::ShrImm(dest.encoding(), lhs.encoding(), shamt)); + break; + case JSOp::Ursh: + masm.emitI(ia64::ShrUImm(dest.encoding(), lhs.encoding(), shamt)); + break; + default: + MOZ_CRASH("Unexpected shift op"); + } + return; + } + + SecondScratchRegisterScope count(masm); + masm.move32(Imm32(0x3F), count); + Emit3(masm, ia64::And, count, ToRegister(rhs), count); + + switch (ins->bitop()) { + case JSOp::Lsh: + Emit3I(masm, ia64::Shl, dest, lhs, count); + break; + case JSOp::Rsh: + Emit3I(masm, ia64::Shr, dest, lhs, count); + break; + case JSOp::Ursh: + Emit3I(masm, ia64::ShrU, dest, lhs, count); + break; + default: + MOZ_CRASH("Unexpected shift op"); + } +} + +void CodeGenerator::visitShiftI64(LShiftI64* ins) { + Register lhs = ToRegister64(ins->lhs()).reg; + const LAllocation* rhs = ins->rhs(); + Register dest = ToOutRegister64(ins).reg; + + if (rhs->isConstant()) { + int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F); + if (!shift) { + masm.movePtr(lhs, dest); + return; + } + switch (ins->bitop()) { + case JSOp::Lsh: + masm.emitI(ia64::ShlImm(dest.encoding(), lhs.encoding(), shift)); + break; + case JSOp::Rsh: + masm.emitI(ia64::ShrImm(dest.encoding(), lhs.encoding(), shift)); + break; + case JSOp::Ursh: + masm.emitI(ia64::ShrUImm(dest.encoding(), lhs.encoding(), shift)); + break; + default: + MOZ_CRASH("Unexpected shift op"); + } + return; + } + + SecondScratchRegisterScope count(masm); + masm.move32(Imm32(0x3F), count); + Emit3(masm, ia64::And, count, ToRegister(rhs), count); + + switch (ins->bitop()) { + case JSOp::Lsh: + Emit3I(masm, ia64::Shl, dest, lhs, count); + break; + case JSOp::Rsh: + Emit3I(masm, ia64::Shr, dest, lhs, count); + break; + case JSOp::Ursh: + Emit3I(masm, ia64::ShrU, dest, lhs, count); + break; + default: + MOZ_CRASH("Unexpected shift op"); + } +} + +void CodeGenerator::visitUrshD(LUrshD* ins) { + Register lhs = ToRegister(ins->lhs()); + Register temp = ToRegister(ins->temp0()); + const LAllocation* rhs = ins->rhs(); + FloatRegister out = ToFloatRegister(ins->output()); + + ZeroExtend32(masm, temp, lhs); + if (rhs->isConstant()) { + int32_t shift = ToInt32(rhs) & 0x1F; + if (shift) { + masm.emitI(ia64::ShrUImm(temp.encoding(), temp.encoding(), shift)); + } + } else { + SecondScratchRegisterScope count(masm); + masm.move32(Imm32(0x1F), count); + Emit3(masm, ia64::And, count, ToRegister(rhs), count); + Emit3I(masm, ia64::ShrU, temp, temp, count); + } + + masm.convertUInt32ToDouble(temp, out); +} + +// =========================================================================== +// Integer/pointer width conversions. + +void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* ins) { + const LAllocation* input = ins->input(); + Register output = ToRegister(ins->output()); + + if (ins->mir()->isUnsigned()) { + ZeroExtend32(masm, output, ToRegister(input)); + } else { + SignExtend32(masm, output, ToRegister(input)); + } +} + +void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* ins) { + LInt64Allocation input = ins->input(); + Register output = ToRegister(ins->output()); + + if (!ins->mir()->bottomHalf()) { + // Arithmetic-shifting the full 64-bit value right by 32 both selects the + // high word and sign-extends it as an int32 in the one instruction. + if (input.value().isMemory()) { + masm.loadPtr(ToAddress(input), output); + } else { + masm.movePtr(ToRegister64(input).reg, output); + } + masm.emitI(ia64::ShrImm(output.encoding(), output.encoding(), 32)); + return; + } + + if (input.value().isMemory()) { + masm.load32(ToAddress(input), output); + } else { + SignExtend32(masm, output, ToRegister64(input).reg); + } +} + +void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* ins) { + Register64 input = ToRegister64(ins->input()); + Register64 output = ToOutRegister64(ins); + switch (ins->mir()->mode()) { + case MSignExtendInt64::Byte: + masm.emitI(ia64::Sxt1(output.reg.encoding(), input.reg.encoding())); + break; + case MSignExtendInt64::Half: + masm.emitI(ia64::Sxt2(output.reg.encoding(), input.reg.encoding())); + break; + case MSignExtendInt64::Word: + SignExtend32(masm, output.reg, input.reg); + break; + } +} + +void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* ins) { + Register input = ToRegister(ins->input()); + Register output = ToRegister(ins->output()); + MOZ_ASSERT(input == output); + ZeroExtend32(masm, output, input); +} + +void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* ins) { + Register input = ToRegister(ins->input()); + Register output = ToRegister(ins->output()); + MOZ_ASSERT(input == output); + ZeroExtend32(masm, output, input); +} + +// =========================================================================== +// Floating point. + +void CodeGenerator::visitMinMaxD(LMinMaxD* ins) { + FloatRegister first = ToFloatRegister(ins->first()); + FloatRegister second = ToFloatRegister(ins->second()); + + MOZ_ASSERT(first == ToFloatRegister(ins->output())); + + if (ins->mir()->isMax()) { + masm.maxDouble(second, first, true); + } else { + masm.minDouble(second, first, true); + } +} + +void CodeGenerator::visitMinMaxF(LMinMaxF* ins) { + FloatRegister first = ToFloatRegister(ins->first()); + FloatRegister second = ToFloatRegister(ins->second()); + + MOZ_ASSERT(first == ToFloatRegister(ins->output())); + + if (ins->mir()->isMax()) { + masm.maxFloat32(second, first, true); + } else { + masm.minFloat32(second, first, true); + } +} + +void CodeGenerator::visitNegD(LNegD* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + FloatRegister output = ToFloatRegister(ins->output()); + + if (input != output) { + masm.moveDouble(input, output); + } + masm.negateDouble(output); +} + +void CodeGenerator::visitNegF(LNegF* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + FloatRegister output = ToFloatRegister(ins->output()); + + if (input != output) { + masm.moveFloat32(input, output); + } + masm.negateFloat(output); +} + +void CodeGenerator::visitMathD(LMathD* ins) { + FloatRegister src1 = ToFloatRegister(ins->lhs()); + FloatRegister src2 = ToFloatRegister(ins->rhs()); + FloatRegister output = ToFloatRegister(ins->output()); + + // lowerForFPU reuses the first input as the output. + MOZ_ASSERT(src1 == output); + + switch (ins->jsop()) { + case JSOp::Add: + masm.addDouble(src2, output); + break; + case JSOp::Sub: + masm.subDouble(src2, output); + break; + case JSOp::Mul: + masm.mulDouble(src2, output); + break; + case JSOp::Div: + masm.divDouble(src2, output); + break; + default: + MOZ_CRASH("unexpected opcode"); + } +} + +void CodeGenerator::visitMathF(LMathF* ins) { + FloatRegister src1 = ToFloatRegister(ins->lhs()); + FloatRegister src2 = ToFloatRegister(ins->rhs()); + FloatRegister output = ToFloatRegister(ins->output()); + + MOZ_ASSERT(src1 == output); + + switch (ins->jsop()) { + case JSOp::Add: + masm.addFloat32(src2, output); + break; + case JSOp::Sub: + masm.subFloat32(src2, output); + break; + case JSOp::Mul: + masm.mulFloat32(src2, output); + break; + case JSOp::Div: + masm.divFloat32(src2, output); + break; + default: + MOZ_CRASH("unexpected opcode"); + } +} + +void CodeGenerator::visitPowHalfD(LPowHalfD* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + FloatRegister output = ToFloatRegister(ins->output()); + + Label done, skip; + + // Math.pow(-Infinity, 0.5) == Infinity. LPowHalfD uses useRegisterAtStart, + // so |output| may be the very register holding |input|: the probe constant + // has to go somewhere else or it destroys the value being tested. + { + ScratchDoubleScope scratch(masm); + masm.loadConstantDouble(NegativeInfinity(), scratch); + masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, input, scratch, + &skip); + masm.loadConstantDouble(NegativeInfinity(), output); + masm.negateDouble(output); + masm.jump(&done); + } + masm.bind(&skip); + + // Adding +0.0 turns -0 into 0 without disturbing any other value. + if (input != output) { + masm.moveDouble(input, output); + } + masm.addDouble(FPZero, output); + masm.sqrtDouble(output, output); + + masm.bind(&done); +} + +void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) { + emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()), + ins->mir()); +} + +void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) { + emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()), + ins->mir()); +} + +void CodeGenerator::visitWasmBuiltinTruncateDToInt32( + LWasmBuiltinTruncateDToInt32* ins) { + emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()), + ins->mir()); +} + +void CodeGenerator::visitWasmBuiltinTruncateFToInt32( + LWasmBuiltinTruncateFToInt32* ins) { + emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()), + ins->mir()); +} + +void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* ins) { + auto input = ToFloatRegister(ins->input()); + auto output = ToRegister(ins->output()); + + MWasmTruncateToInt32* mir = ins->mir(); + MIRType fromType = mir->input()->type(); + + MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32); + + bool isSaturating = mir->isSaturating(); + + OutOfLineWasmTruncateCheck* ool = nullptr; + Label* oolEntry = nullptr; + if (!isSaturating) { + ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output); + addOutOfLineCode(ool, mir); + oolEntry = ool->entry(); + } + + if (fromType == MIRType::Double) { + if (mir->isUnsigned()) { + masm.wasmTruncateDoubleToUInt32(input, output, isSaturating, oolEntry); + } else { + masm.wasmTruncateDoubleToInt32(input, output, isSaturating, oolEntry); + } + } else { + if (mir->isUnsigned()) { + masm.wasmTruncateFloat32ToUInt32(input, output, isSaturating, oolEntry); + } else { + masm.wasmTruncateFloat32ToInt32(input, output, isSaturating, oolEntry); + } + } + + if (ool) { + masm.bind(ool->rejoin()); + } +} + +void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + Register64 output = ToOutRegister64(ins); + + MWasmTruncateToInt64* mir = ins->mir(); + MIRType fromType = mir->input()->type(); + + MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32); + + bool isSaturating = mir->isSaturating(); + + OutOfLineWasmTruncateCheck* ool = nullptr; + Label* oolEntry = nullptr; + Label* oolRejoin = nullptr; + if (!isSaturating) { + ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output); + addOutOfLineCode(ool, mir); + + oolEntry = ool->entry(); + oolRejoin = ool->rejoin(); + } + + if (fromType == MIRType::Double) { + if (mir->isUnsigned()) { + masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry, + oolRejoin, InvalidFloatReg); + } else { + masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry, + oolRejoin, InvalidFloatReg); + } + } else { + if (mir->isUnsigned()) { + masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry, + oolRejoin, InvalidFloatReg); + } else { + masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry, + oolRejoin, InvalidFloatReg); + } + } +} + +void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* ins) { + Register64 input = ToRegister64(ins->input()); + FloatRegister output = ToFloatRegister(ins->output()); + + MIRType outputType = ins->mir()->type(); + MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32); + + if (outputType == MIRType::Double) { + if (ins->mir()->isUnsigned()) { + masm.convertUInt64ToDouble(input, output, Register::Invalid()); + } else { + masm.convertInt64ToDouble(input, output); + } + } else { + if (ins->mir()->isUnsigned()) { + masm.convertUInt64ToFloat32(input, output, Register::Invalid()); + } else { + masm.convertInt64ToFloat32(input, output); + } + } +} + +void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* ins) { + masm.convertUInt32ToDouble(ToRegister(ins->input()), + ToFloatRegister(ins->output())); +} + +void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* ins) { + masm.convertUInt32ToFloat32(ToRegister(ins->input()), + ToFloatRegister(ins->output())); +} + +// =========================================================================== +// Comparisons and branches. + +// IA-64 has no condition codes, so a boolean result is produced by branching +// over the two constants. +static void EmitSetFromBranch(MacroAssembler& masm, Register dest, + Label* isTrue) { + Label done; + masm.move32(Imm32(0), dest); + masm.jump(&done); + masm.bind(isTrue); + masm.move32(Imm32(1), dest); + masm.bind(&done); +} + +void CodeGenerator::visitCompareD(LCompareD* ins) { + FloatRegister lhs = ToFloatRegister(ins->left()); + FloatRegister rhs = ToFloatRegister(ins->right()); + Register dest = ToRegister(ins->output()); + + Assembler::DoubleCondition cond = JSOpToDoubleCondition(ins->mir()->jsop()); + Label isTrue; + masm.branchDouble(cond, lhs, rhs, &isTrue); + EmitSetFromBranch(masm, dest, &isTrue); +} + +void CodeGenerator::visitCompareF(LCompareF* ins) { + FloatRegister lhs = ToFloatRegister(ins->left()); + FloatRegister rhs = ToFloatRegister(ins->right()); + Register dest = ToRegister(ins->output()); + + Assembler::DoubleCondition cond = JSOpToDoubleCondition(ins->mir()->jsop()); + Label isTrue; + masm.branchFloat(cond, lhs, rhs, &isTrue); + EmitSetFromBranch(masm, dest, &isTrue); +} + +void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* ins) { + FloatRegister lhs = ToFloatRegister(ins->left()); + FloatRegister rhs = ToFloatRegister(ins->right()); + + Assembler::DoubleCondition cond = + JSOpToDoubleCondition(ins->cmpMir()->jsop()); + MBasicBlock* ifTrue = ins->ifTrue(); + MBasicBlock* ifFalse = ins->ifFalse(); + + if (isNextBlock(ifFalse->lir())) { + branchToBlock(DoubleFloat, lhs, rhs, ifTrue, cond); + } else { + branchToBlock(DoubleFloat, lhs, rhs, ifFalse, + Assembler::InvertCondition(cond)); + jumpToBlock(ifTrue); + } +} + +void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* ins) { + FloatRegister lhs = ToFloatRegister(ins->left()); + FloatRegister rhs = ToFloatRegister(ins->right()); + + Assembler::DoubleCondition cond = + JSOpToDoubleCondition(ins->cmpMir()->jsop()); + MBasicBlock* ifTrue = ins->ifTrue(); + MBasicBlock* ifFalse = ins->ifFalse(); + + if (isNextBlock(ifFalse->lir())) { + branchToBlock(SingleFloat, lhs, rhs, ifTrue, cond); + } else { + branchToBlock(SingleFloat, lhs, rhs, ifFalse, + Assembler::InvertCondition(cond)); + jumpToBlock(ifTrue); + } +} + +void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + + MBasicBlock* ifTrue = ins->ifTrue(); + MBasicBlock* ifFalse = ins->ifFalse(); + + // 0, -0 and NaN are all falsey. f0 is hardwired to +0.0. + if (isNextBlock(ifFalse->lir())) { + branchToBlock(DoubleFloat, input, FPZero, ifTrue, + Assembler::DoubleNotEqual); + } else { + branchToBlock(DoubleFloat, input, FPZero, ifFalse, + Assembler::DoubleEqualOrUnordered); + jumpToBlock(ifTrue); + } +} + +void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* ins) { + FloatRegister input = ToFloatRegister(ins->input()); + + MBasicBlock* ifTrue = ins->ifTrue(); + MBasicBlock* ifFalse = ins->ifFalse(); + + if (isNextBlock(ifFalse->lir())) { + branchToBlock(SingleFloat, input, FPZeroSingle, ifTrue, + Assembler::DoubleNotEqual); + } else { + branchToBlock(SingleFloat, input, FPZeroSingle, ifFalse, + Assembler::DoubleEqualOrUnordered); + jumpToBlock(ifTrue); + } +} + +void CodeGenerator::visitNotD(LNotD* ins) { + FloatRegister in = ToFloatRegister(ins->input()); + Register dest = ToRegister(ins->output()); + + Label isTrue; + masm.branchDouble(Assembler::DoubleEqualOrUnordered, in, FPZero, &isTrue); + EmitSetFromBranch(masm, dest, &isTrue); +} + +void CodeGenerator::visitNotF(LNotF* ins) { + FloatRegister in = ToFloatRegister(ins->input()); + Register dest = ToRegister(ins->output()); + + Label isTrue; + masm.branchFloat(Assembler::DoubleEqualOrUnordered, in, FPZeroSingle, + &isTrue); + EmitSetFromBranch(masm, dest, &isTrue); +} + +// =========================================================================== +// Addressing. + +void CodeGenerator::visitEffectiveAddress3(LEffectiveAddress3* ins) { + const MEffectiveAddress3* mir = ins->mir(); + Register base = ToRegister(ins->base()); + Register index = ToRegister(ins->index()); + Register output = ToRegister(ins->output()); + uint32_t shift = uint32_t(mir->scale()); + + if (shift) { + // The scaled index goes via the scratch so |base| survives an output that + // aliases it. + SecondScratchRegisterScope scratch(masm); + masm.emitI(ia64::ShlImm(scratch.encoding(), index.encoding(), shift)); + Emit3(masm, ia64::Add, output, base, scratch); + } else { + Emit3(masm, ia64::Add, output, base, index); + } + if (mir->displacement()) { + masm.add32(Imm32(mir->displacement()), output); + } + SignExtend32(masm, output, output); +} + +void CodeGenerator::visitEffectiveAddress2(LEffectiveAddress2* ins) { + const MEffectiveAddress2* mir = ins->mir(); + Register index = ToRegister(ins->index()); + Register output = ToRegister(ins->output()); + uint32_t shift = uint32_t(mir->scale()); + + if (shift) { + masm.emitI(ia64::ShlImm(output.encoding(), index.encoding(), shift)); + } else { + masm.move32(index, output); + } + if (mir->displacement()) { + masm.add32(Imm32(mir->displacement()), output); + } + SignExtend32(masm, output, output); +} + +// =========================================================================== +// Wasm memory. + +void CodeGenerator::visitWasmLoad(LWasmLoad* ins) { + const MWasmLoad* mir = ins->mir(); + + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptr = ToRegister(ins->ptr()); + + if (mir->base()->type() == MIRType::Int32) { + ZeroExtend32(masm, ptr, ptr); + } + masm.wasmLoad(mir->access(), memoryBase, ptr, ToAnyRegister(ins->output())); +} + +void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* ins) { + const MWasmLoad* mir = ins->mir(); + + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptr = ToRegister(ins->ptr()); + + if (mir->base()->type() == MIRType::Int32) { + ZeroExtend32(masm, ptr, ptr); + } + masm.wasmLoadI64(mir->access(), memoryBase, ptr, ToOutRegister64(ins)); +} + +void CodeGenerator::visitWasmStore(LWasmStore* ins) { + const MWasmStore* mir = ins->mir(); + + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptr = ToRegister(ins->ptr()); + + if (mir->base()->type() == MIRType::Int32) { + ZeroExtend32(masm, ptr, ptr); + } + masm.wasmStore(mir->access(), ToAnyRegister(ins->value()), memoryBase, ptr); +} + +void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* ins) { + const MWasmStore* mir = ins->mir(); + + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptr = ToRegister(ins->ptr()); + + if (mir->base()->type() == MIRType::Int32) { + ZeroExtend32(masm, ptr, ptr); + } + masm.wasmStoreI64(mir->access(), ToRegister64(ins->value()), memoryBase, ptr); +} + +void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) { + const MAsmJSLoadHeap* mir = ins->mir(); + const LAllocation* ptr = ins->ptr(); + const LDefinition* out = ins->output(); + const LAllocation* boundsCheckLimit = ins->boundsCheckLimit(); + + Scalar::Type accessType = mir->access().type(); + bool isFloat = Scalar::isFloatingType(accessType); + int size = Scalar::byteSize(accessType) * 8; + + auto load = [&](const auto& addr) { + if (isFloat) { + if (size == 32) { + masm.loadFloat32(addr, ToFloatRegister(out)); + } else { + masm.loadDouble(addr, ToFloatRegister(out)); + } + } else { + switch (size) { + case 8: + if (Scalar::isSignedIntType(accessType)) { + masm.load8SignExtend(addr, ToRegister(out)); + } else { + masm.load8ZeroExtend(addr, ToRegister(out)); + } + break; + case 16: + if (Scalar::isSignedIntType(accessType)) { + masm.load16SignExtend(addr, ToRegister(out)); + } else { + masm.load16ZeroExtend(addr, ToRegister(out)); + } + break; + case 32: + masm.load32(addr, ToRegister(out)); + break; + default: + MOZ_CRASH("unexpected access size"); + } + } + }; + + if (ptr->isConstant()) { + MOZ_ASSERT(!mir->needsBoundsCheck()); + int32_t ptrImm = ptr->toConstant()->toInt32(); + MOZ_ASSERT(ptrImm >= 0); + load(Address(HeapReg, ptrImm)); + return; + } + + Register ptrReg = ToRegister(ptr); + + if (!mir->needsBoundsCheck()) { + load(BaseIndex(HeapReg, ptrReg, TimesOne)); + return; + } + + Label done, outOfRange; + masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg, + ToRegister(boundsCheckLimit), &outOfRange); + load(BaseIndex(HeapReg, ptrReg, TimesOne)); + masm.jump(&done); + masm.bind(&outOfRange); + if (isFloat) { + if (size == 32) { + masm.loadConstantFloat32(float(GenericNaN()), ToFloatRegister(out)); + } else { + masm.loadConstantDouble(GenericNaN(), ToFloatRegister(out)); + } + } else { + masm.move32(Imm32(0), ToRegister(out)); + } + masm.bind(&done); +} + +void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) { + const MAsmJSStoreHeap* mir = ins->mir(); + const LAllocation* value = ins->value(); + const LAllocation* ptr = ins->ptr(); + const LAllocation* boundsCheckLimit = ins->boundsCheckLimit(); + + Scalar::Type accessType = mir->access().type(); + bool isFloat = Scalar::isFloatingType(accessType); + int size = Scalar::byteSize(accessType) * 8; + + auto store = [&](const auto& addr) { + if (isFloat) { + if (size == 32) { + masm.storeFloat32(ToFloatRegister(value), addr); + } else { + masm.storeDouble(ToFloatRegister(value), addr); + } + } else { + switch (size) { + case 8: + masm.store8(ToRegister(value), addr); + break; + case 16: + masm.store16(ToRegister(value), addr); + break; + case 32: + masm.store32(ToRegister(value), addr); + break; + default: + MOZ_CRASH("unexpected access size"); + } + } + }; + + if (ptr->isConstant()) { + MOZ_ASSERT(!mir->needsBoundsCheck()); + int32_t ptrImm = ptr->toConstant()->toInt32(); + MOZ_ASSERT(ptrImm >= 0); + store(Address(HeapReg, ptrImm)); + return; + } + + Register ptrReg = ToRegister(ptr); + + if (!mir->needsBoundsCheck()) { + store(BaseIndex(HeapReg, ptrReg, TimesOne)); + return; + } + + Label outOfRange; + masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg, + ToRegister(boundsCheckLimit), &outOfRange); + store(BaseIndex(HeapReg, ptrReg, TimesOne)); + masm.bind(&outOfRange); +} + +void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* ins) { + MWasmAddOffset* mir = ins->mir(); + Register base = ToRegister(ins->base()); + Register out = ToRegister(ins->output()); + + Label ok; + masm.move32(base, out); + masm.branchAdd32(Assembler::CarryClear, Imm32(mir->offset()), out, &ok); + masm.wasmTrap(wasm::Trap::OutOfBounds, mir->trapSiteDesc()); + masm.bind(&ok); +} + +void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* ins) { + MWasmAddOffset* mir = ins->mir(); + Register64 base = ToRegister64(ins->base()); + Register64 out = ToOutRegister64(ins); + + Label ok; + masm.movePtr(base.reg, out.reg); + masm.branchAddPtr(Assembler::CarryClear, ImmWord(mir->offset()), out.reg, + &ok); + masm.wasmTrap(wasm::Trap::OutOfBounds, mir->trapSiteDesc()); + masm.bind(&ok); +} + +void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) { + const MWasmStackArg* mir = ins->mir(); + if (ins->arg()->isConstant()) { + masm.storePtr(ImmWord(ToInt32(ins->arg())), + Address(StackPointer, mir->spOffset())); + } else if (ins->arg()->isGeneralReg()) { + masm.storePtr(ToRegister(ins->arg()), + Address(StackPointer, mir->spOffset())); + } else if (mir->input()->type() == MIRType::Double) { + masm.storeDouble(ToFloatRegister(ins->arg()), + Address(StackPointer, mir->spOffset())); + } else { + masm.storeFloat32(ToFloatRegister(ins->arg()), + Address(StackPointer, mir->spOffset())); + } +} + +void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) { + const MWasmStackArg* mir = ins->mir(); + Address dst(StackPointer, mir->spOffset()); + if (IsConstant(ins->arg())) { + masm.store64(Imm64(ToInt64(ins->arg())), dst); + } else { + masm.store64(ToRegister64(ins->arg()), dst); + } +} + +void CodeGenerator::visitWasmSelect(LWasmSelect* ins) { + MIRType mirType = ins->mir()->type(); + + Register cond = ToRegister(ins->condExpr()); + const LAllocation* falseExpr = ins->falseExpr(); + + Label done; + masm.branchTest32(Assembler::NonZero, cond, cond, &done); + + if (mirType == MIRType::Int32 || mirType == MIRType::WasmAnyRef) { + Register out = ToRegister(ins->output()); + MOZ_ASSERT(ToRegister(ins->trueExpr()) == out, + "true expr input is reused for output"); + if (falseExpr->isGeneralReg()) { + masm.movePtr(ToRegister(falseExpr), out); + } else { + masm.loadPtr(ToAddress(falseExpr), out); + } + masm.bind(&done); + return; + } + + FloatRegister out = ToFloatRegister(ins->output()); + MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out, + "true expr input is reused for output"); + + if (falseExpr->isFloatReg()) { + if (mirType == MIRType::Float32) { + masm.moveFloat32(ToFloatRegister(falseExpr), out); + } else if (mirType == MIRType::Double) { + masm.moveDouble(ToFloatRegister(falseExpr), out); + } else { + MOZ_CRASH("unhandled type in visitWasmSelect!"); + } + } else { + if (mirType == MIRType::Float32) { + masm.loadFloat32(ToAddress(falseExpr), out); + } else if (mirType == MIRType::Double) { + masm.loadDouble(ToAddress(falseExpr), out); + } else { + MOZ_CRASH("unhandled type in visitWasmSelect!"); + } + } + masm.bind(&done); +} + +void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* ins) { + MOZ_ASSERT(ins->mir()->type() == MIRType::Int64); + + Register cond = ToRegister(ins->condExpr()); + LInt64Allocation falseExpr = ins->falseExpr(); + + Register64 out = ToOutRegister64(ins); + MOZ_ASSERT(ToRegister64(ins->trueExpr()) == out, + "true expr is reused for input"); + + Label done; + masm.branchTest32(Assembler::NonZero, cond, cond, &done); + if (falseExpr.value().isGeneralReg()) { + masm.movePtr(ToRegister(falseExpr.value()), out.reg); + } else { + masm.loadPtr(ToAddress(falseExpr.value()), out.reg); + } + masm.bind(&done); +} + +void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) { + MOZ_CRASH("ia64 does not specialize compare-and-select"); +} + +// =========================================================================== +// Atomics. + +void CodeGenerator::visitCompareExchangeTypedArrayElement( + LCompareExchangeTypedArrayElement* lir) { + Register elements = ToRegister(lir->elements()); + AnyRegister output = ToAnyRegister(lir->output()); + Register temp = ToTempRegisterOrInvalid(lir->temp0()); + + Register oldval = ToRegister(lir->oldval()); + Register newval = ToRegister(lir->newval()); + + Scalar::Type arrayType = lir->mir()->arrayType(); + + auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType); + + dest.match([&](const auto& dest) { + masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval, + newval, temp, output); + }); +} + +void CodeGenerator::visitAtomicExchangeTypedArrayElement( + LAtomicExchangeTypedArrayElement* lir) { + Register elements = ToRegister(lir->elements()); + AnyRegister output = ToAnyRegister(lir->output()); + Register temp = ToTempRegisterOrInvalid(lir->temp0()); + + Register value = ToRegister(lir->value()); + + Scalar::Type arrayType = lir->mir()->arrayType(); + + auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType); + + dest.match([&](const auto& dest) { + masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp, + output); + }); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinop( + LAtomicTypedArrayElementBinop* lir) { + MOZ_ASSERT(!lir->mir()->isForEffect()); + + AnyRegister output = ToAnyRegister(lir->output()); + Register elements = ToRegister(lir->elements()); + Register temp1 = ToTempRegisterOrInvalid(lir->temp0()); + Register temp2 = ToTempRegisterOrInvalid(lir->temp1()); + Register value = ToRegister(lir->value()); + + Scalar::Type arrayType = lir->mir()->arrayType(); + AtomicOp atomicOp = lir->mir()->operation(); + + auto mem = ToAddressOrBaseIndex(elements, lir->index(), arrayType); + + mem.match([&](const auto& mem) { + masm.atomicFetchOpJS(arrayType, Synchronization::Full(), atomicOp, value, + mem, temp1, temp2, output); + }); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect( + LAtomicTypedArrayElementBinopForEffect* lir) { + MOZ_ASSERT(lir->mir()->isForEffect()); + + Register elements = ToRegister(lir->elements()); + Register value = ToRegister(lir->value()); + + Scalar::Type arrayType = lir->mir()->arrayType(); + AtomicOp atomicOp = lir->mir()->operation(); + + auto mem = ToAddressOrBaseIndex(elements, lir->index(), arrayType); + + mem.match([&](const auto& mem) { + masm.atomicEffectOpJS(arrayType, Synchronization::Full(), atomicOp, value, + mem, ToRegister(lir->temp0())); + }); +} + +void CodeGenerator::visitCompareExchangeTypedArrayElement64( + LCompareExchangeTypedArrayElement64* lir) { + Register elements = ToRegister(lir->elements()); + Register64 oldval = ToRegister64(lir->oldval()); + Register64 newval = ToRegister64(lir->newval()); + Register64 out = ToOutRegister64(lir); + Scalar::Type arrayType = lir->mir()->arrayType(); + + auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType); + + dest.match([&](const auto& dest) { + masm.compareExchange64(Synchronization::Full(), dest, oldval, newval, out); + }); +} + +void CodeGenerator::visitAtomicExchangeTypedArrayElement64( + LAtomicExchangeTypedArrayElement64* lir) { + Register elements = ToRegister(lir->elements()); + Register64 value = ToRegister64(lir->value()); + Register64 out = ToOutRegister64(lir); + Scalar::Type arrayType = lir->mir()->arrayType(); + + auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType); + + dest.match([&](const auto& dest) { + masm.atomicExchange64(Synchronization::Full(), dest, value, out); + }); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinop64( + LAtomicTypedArrayElementBinop64* lir) { + MOZ_ASSERT(!lir->mir()->isForEffect()); + + Register elements = ToRegister(lir->elements()); + Register64 value = ToRegister64(lir->value()); + Register64 temp = ToRegister64(lir->temp0()); + Register64 out = ToOutRegister64(lir); + + Scalar::Type arrayType = lir->mir()->arrayType(); + AtomicOp atomicOp = lir->mir()->operation(); + + auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType); + + dest.match([&](const auto& dest) { + masm.atomicFetchOp64(Synchronization::Full(), atomicOp, value, dest, temp, + out); + }); +} + +void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64( + LAtomicTypedArrayElementBinopForEffect64* lir) { + MOZ_ASSERT(lir->mir()->isForEffect()); + + Register elements = ToRegister(lir->elements()); + Register64 value = ToRegister64(lir->value()); + Register64 temp = ToRegister64(lir->temp0()); + + Scalar::Type arrayType = lir->mir()->arrayType(); + AtomicOp atomicOp = lir->mir()->operation(); + + auto dest = ToAddressOrBaseIndex(elements, lir->index(), arrayType); + + dest.match([&](const auto& dest) { + masm.atomicEffectOp64(Synchronization::Full(), atomicOp, value, dest, temp); + }); +} + +void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) { + Register elements = ToRegister(lir->elements()); + Register64 out = ToOutRegister64(lir); + + Scalar::Type storageType = lir->mir()->storageType(); + + auto source = ToAddressOrBaseIndex(elements, lir->index(), storageType); + + auto sync = Synchronization::Load(); + masm.memoryBarrierBefore(sync); + source.match([&](const auto& source) { masm.load64(source, out); }); + masm.memoryBarrierAfter(sync); +} + +void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) { + Register elements = ToRegister(lir->elements()); + Register64 value = ToRegister64(lir->value()); + + Scalar::Type writeType = lir->mir()->writeType(); + + auto dest = ToAddressOrBaseIndex(elements, lir->index(), writeType); + + auto sync = Synchronization::Store(); + masm.memoryBarrierBefore(sync); + dest.match([&](const auto& dest) { masm.store64(value, dest); }); + masm.memoryBarrierAfter(sync); +} + +void CodeGenerator::visitWasmCompareExchangeHeap( + LWasmCompareExchangeHeap* ins) { + MWasmCompareExchangeHeap* mir = ins->mir(); + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptrReg = ToRegister(ins->ptr()); + BaseIndex srcAddr(memoryBase, ptrReg, TimesOne, mir->access().offset32()); + + Register oldval = ToRegister(ins->oldValue()); + Register newval = ToRegister(ins->newValue()); + + masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval, + ToRegister(ins->output())); +} + +void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) { + MWasmAtomicExchangeHeap* mir = ins->mir(); + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptrReg = ToRegister(ins->ptr()); + Register value = ToRegister(ins->value()); + BaseIndex srcAddr(memoryBase, ptrReg, TimesOne, mir->access().offset32()); + + masm.wasmAtomicExchange(mir->access(), srcAddr, value, + ToRegister(ins->output())); +} + +void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) { + MOZ_ASSERT(ins->mir()->hasUses()); + + MWasmAtomicBinopHeap* mir = ins->mir(); + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptrReg = ToRegister(ins->ptr()); + Register temp = ToTempRegisterOrInvalid(ins->temp0()); + + BaseIndex srcAddr(memoryBase, ptrReg, TimesOne, mir->access().offset32()); + + masm.wasmAtomicFetchOp(mir->access(), mir->operation(), + ToRegister(ins->value()), srcAddr, temp, + ToRegister(ins->output())); +} + +void CodeGenerator::visitWasmAtomicBinopHeapForEffect( + LWasmAtomicBinopHeapForEffect* ins) { + MOZ_ASSERT(!ins->mir()->hasUses()); + + MWasmAtomicBinopHeap* mir = ins->mir(); + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptrReg = ToRegister(ins->ptr()); + + BaseIndex srcAddr(memoryBase, ptrReg, TimesOne, mir->access().offset32()); + masm.wasmAtomicEffectOp(mir->access(), mir->operation(), + ToRegister(ins->value()), srcAddr, InvalidReg); +} + +void CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* ins) { + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptr = ToRegister(ins->ptr()); + Register64 oldValue = ToRegister64(ins->oldValue()); + Register64 newValue = ToRegister64(ins->newValue()); + Register64 output = ToOutRegister64(ins); + uint32_t offset = ins->mir()->access().offset32(); + + BaseIndex addr(memoryBase, ptr, TimesOne, offset); + masm.wasmCompareExchange64(ins->mir()->access(), addr, oldValue, newValue, + output); +} + +void CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* ins) { + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptr = ToRegister(ins->ptr()); + Register64 value = ToRegister64(ins->value()); + Register64 output = ToOutRegister64(ins); + uint32_t offset = ins->mir()->access().offset32(); + + BaseIndex addr(memoryBase, ptr, TimesOne, offset); + masm.wasmAtomicExchange64(ins->mir()->access(), addr, value, output); +} + +void CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* ins) { + Register memoryBase = ToRegister(ins->memoryBase()); + Register ptr = ToRegister(ins->ptr()); + Register64 value = ToRegister64(ins->value()); + Register64 output = ToOutRegister64(ins); + Register64 temp = ToRegister64(ins->temp0()); + uint32_t offset = ins->mir()->access().offset32(); + + BaseIndex addr(memoryBase, ptr, TimesOne, offset); + + masm.wasmAtomicFetchOp64(ins->mir()->access(), ins->mir()->operation(), value, + addr, temp, output); +} + +void CodeGenerator::visitWasmMulI64WideHI64(LWasmMulI64WideHI64* ins) { + Register lhs = ToRegister(ins->lhs()); + Register rhs = ToRegister(ins->rhs()); + Register output = ToRegister(ins->output()); + MOZ_ASSERT(output != lhs && output != rhs); + masm.wasmMulI64WideHI64(lhs, rhs, output, ins->isSigned()); +} + +// =========================================================================== +// SIMD: IA-64 has no packed SIMD in the FP register file. + +void CodeGenerator::visitSimd128(LSimd128* ins) { MOZ_CRASH("No SIMD"); } + +void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmBinarySimd128WithConstant( + LWasmBinarySimd128WithConstant* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmVariableShiftSimd128( + LWasmVariableShiftSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmConstantShiftSimd128( + LWasmConstantShiftSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmSignReplicationSimd128( + LWasmSignReplicationSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReplaceInt64LaneSimd128( + LWasmReplaceInt64LaneSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReduceAndBranchSimd128( + LWasmReduceAndBranchSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmReduceSimd128ToInt64( + LWasmReduceSimd128ToInt64* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) { + MOZ_CRASH("No SIMD"); +} + +void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) { + MOZ_CRASH("No SIMD"); +} diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/ia64/CodeGenerator-ia64.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/ia64/CodeGenerator-ia64.h --- firefox-153.0.1/js/src/jit/ia64/CodeGenerator-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/CodeGenerator-ia64.h @@ -0,0 +1,109 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_CodeGenerator_ia64_h +#define jit_ia64_CodeGenerator_ia64_h + +#include "jit/ia64/Assembler-ia64.h" +#include "jit/shared/CodeGenerator-shared.h" + +namespace js { +namespace jit { + +class CodeGeneratorIa64; +class OutOfLineTableSwitch; + +using OutOfLineWasmTruncateCheck = + OutOfLineWasmTruncateCheckBase; + +class CodeGeneratorIa64 : public CodeGeneratorShared { + protected: + CodeGeneratorIa64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm, + const wasm::CodeMetadata* wasmCodeMeta); + + NonAssertingLabel deoptLabel_; + + MoveOperand toMoveOperand(LAllocation a) const; + + template + void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branch32(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } + template + void bailoutTest32(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branchTest32(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } + template + void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs, + LSnapshot* snapshot) { + Label bail; + masm.branchPtr(c, lhs, rhs, &bail); + bailoutFrom(&bail, snapshot); + } + void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) { + Label bail; + masm.branchTest32(Assembler::Zero, reg, Imm32(0xFF), &bail); + bailoutFrom(&bail, snapshot); + } + + void emitBailoutOOL(LSnapshot* snapshot); + void bailoutFrom(Label* label, LSnapshot* snapshot); + void bailout(LSnapshot* snapshot); + + bool generateOutOfLineCode(); + + template + void branchToBlock(Register lhs, T rhs, MBasicBlock* mir, + Assembler::Condition cond) { + masm.branch32(cond, lhs, rhs, skipTrivialBlocks(mir)->lir()->label()); + } + + enum FloatFormat { SingleFloat, DoubleFloat }; + void branchToBlock(FloatFormat fmt, FloatRegister lhs, FloatRegister rhs, + MBasicBlock* mir, Assembler::DoubleCondition cond); + + // Directs control flow to the true block if |cond| holds, else to the false + // block, falling through where possible. + template + void emitBranch(Register lhs, T rhs, Assembler::Condition cond, + MBasicBlock* mirTrue, MBasicBlock* mirFalse) { + if (isNextBlock(mirFalse->lir())) { + branchToBlock(lhs, rhs, mirTrue, cond); + } else { + branchToBlock(lhs, rhs, mirFalse, Assembler::InvertCondition(cond)); + jumpToBlock(mirTrue); + } + } + + void emitTableSwitchDispatch(MTableSwitch* mir, Register index, + Register base); + + void generateInvalidateEpilogue(); + + public: + // Out of line visitors. + void visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool); + void visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool); + + protected: + void emitBigIntPtrDiv(LBigIntPtrDiv* ins, Register dividend, Register divisor, + Register output); + void emitBigIntPtrMod(LBigIntPtrMod* ins, Register dividend, Register divisor, + Register output); +}; + +using CodeGeneratorSpecific = CodeGeneratorIa64; + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_CodeGenerator_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/ia64/LIR-ia64.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/ia64/LIR-ia64.h --- firefox-153.0.1/js/src/jit/ia64/LIR-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/LIR-ia64.h @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_LIR_ia64_h +#define jit_ia64_LIR_ia64_h + +namespace js { +namespace jit { + +// PUNBOX64: a JS::Value lives in a single 64-bit register, so LUnbox takes the +// box as one operand and needs no separate type/payload accessors. +class LUnbox : public LInstructionHelper<1, BOX_PIECES, 0> { + public: + LIR_HEADER(Unbox); + + explicit LUnbox(const LAllocation& input) : LInstructionHelper(classOpcode) { + setOperand(0, input); + } + + static const size_t Input = 0; + + LBoxAllocation input() const { return getBoxOperand(Input); } + + MUnbox* mir() const { return mir_->toUnbox(); } + const char* extraName() const { return StringFromMIRType(mir()->type()); } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_LIR_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/firefox-153.0.1/js/src/jit/ia64/Lowering-ia64.cpp b/firefox-153.0.1/js/src/jit/ia64/Lowering-ia64.cpp --- firefox-153.0.1/js/src/jit/ia64/Lowering-ia64.cpp.vanilla +++ firefox-153.0.1/js/src/jit/ia64/Lowering-ia64.cpp @@ -0,0 +1,877 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#include "jit/ia64/Lowering-ia64.h" + +#include "mozilla/MathAlgorithms.h" + +#include + +#include "jit/Lowering.h" +#include "jit/MIR-wasm.h" +#include "jit/MIR.h" + +#include "jit/shared/Lowering-shared-inl.h" + +using namespace js; +using namespace js::jit; + +LTableSwitch* LIRGeneratorIa64::newLTableSwitch(const LAllocation& in, + const LDefinition& inputCopy) { + return new (alloc()) LTableSwitch(in, inputCopy, temp()); +} + +LTableSwitchV* LIRGeneratorIa64::newLTableSwitchV(const LBoxAllocation& in) { + return new (alloc()) LTableSwitchV(in, temp(), tempDouble(), temp()); +} + +void LIRGeneratorIa64::lowerForShift(LInstructionHelper<1, 2, 0>* ins, + MDefinition* mir, MDefinition* lhs, + MDefinition* rhs) { + lowerForALU(ins, mir, lhs, rhs); +} + +template +void LIRGeneratorIa64::lowerForShiftInt64(LInstr* ins, MDefinition* mir, + MDefinition* lhs, MDefinition* rhs) { + if constexpr (std::is_same_v) { + ins->setLhs(useInt64RegisterAtStart(lhs)); + ins->setRhs(useRegisterOrConstantAtStart(rhs)); + } else { + ins->setInput(useInt64RegisterAtStart(lhs)); + ins->setCount(useRegisterOrConstantAtStart(rhs)); + } + defineInt64(ins, mir); +} + +template void LIRGeneratorIa64::lowerForShiftInt64(LShiftI64* ins, + MDefinition* mir, + MDefinition* lhs, + MDefinition* rhs); +template void LIRGeneratorIa64::lowerForShiftInt64(LRotateI64* ins, + MDefinition* mir, + MDefinition* lhs, + MDefinition* rhs); + +void LIRGeneratorIa64::lowerForALU(LInstructionHelper<1, 1, 0>* ins, + MDefinition* mir, MDefinition* input) { + ins->setOperand(0, useRegisterAtStart(input)); + define(ins, mir); +} + +void LIRGeneratorIa64::lowerForALU(LInstructionHelper<1, 2, 0>* ins, + MDefinition* mir, MDefinition* lhs, + MDefinition* rhs) { + ins->setOperand(0, useRegisterAtStart(lhs)); + ins->setOperand(1, useRegisterOrConstantAtStart(rhs)); + define(ins, mir); +} + +void LIRGeneratorIa64::lowerForALUInt64( + LInstructionHelper* ins, MDefinition* mir, + MDefinition* input) { + ins->setInt64Operand(0, useInt64RegisterAtStart(input)); + defineInt64(ins, mir); +} + +void LIRGeneratorIa64::lowerForALUInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs) { + ins->setInt64Operand(0, useInt64RegisterAtStart(lhs)); + ins->setInt64Operand(INT64_PIECES, useInt64RegisterOrConstantAtStart(rhs)); + defineInt64(ins, mir); +} + +void LIRGeneratorIa64::lowerForMulInt64(LMulI64* ins, MMul* mir, + MDefinition* lhs, MDefinition* rhs) { + lowerForALUInt64(ins, mir, lhs, rhs); +} + +void LIRGeneratorIa64::lowerForFPU(LInstructionHelper<1, 1, 0>* ins, + MDefinition* mir, MDefinition* input) { + ins->setOperand(0, useRegisterAtStart(input)); + define(ins, mir); +} + +// IA-64 has no floating-point divide instruction, so the binary FPU forms are +// emitted as two-address sequences; reusing the first input as the output keeps +// codegen from having to find a spare FP register. +void LIRGeneratorIa64::lowerForFPU(LInstructionHelper<1, 2, 0>* ins, + MDefinition* mir, MDefinition* lhs, + MDefinition* rhs) { + ins->setOperand(0, useRegisterAtStart(lhs)); + ins->setOperand(1, willHaveDifferentLIRNodes(lhs, rhs) + ? useRegister(rhs) + : useRegisterAtStart(rhs)); + defineReuseInput(ins, mir, 0); +} + +LBoxAllocation LIRGeneratorIa64::useBoxFixed(MDefinition* mir, Register reg1, + Register reg2, bool useAtStart) { + MOZ_ASSERT(mir->type() == MIRType::Value); + + ensureDefined(mir); + return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart)); +} + +LAllocation LIRGeneratorIa64::useByteOpRegister(MDefinition* mir) { + return useRegister(mir); +} + +LAllocation LIRGeneratorIa64::useByteOpRegisterAtStart(MDefinition* mir) { + return useRegisterAtStart(mir); +} + +LAllocation LIRGeneratorIa64::useByteOpRegisterOrNonDoubleConstant( + MDefinition* mir) { + return useRegisterOrNonDoubleConstant(mir); +} + +LDefinition LIRGeneratorIa64::tempByteOpRegister() { return temp(); } +LDefinition LIRGeneratorIa64::tempToUnbox() { return temp(); } + +void LIRGeneratorIa64::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, + LBlock* block, size_t lirIndex) { + lowerTypedPhiInput(phi, inputPosition, block, lirIndex); +} + +void LIRGeneratorIa64::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, + LBlock* block, size_t lirIndex) { + lowerTypedPhiInput(phi, inputPosition, block, lirIndex); +} + +void LIRGeneratorIa64::defineInt64Phi(MPhi* phi, size_t lirIndex) { + defineTypedPhi(phi, lirIndex); +} + +void LIRGeneratorIa64::lowerMulI(MMul* mul, MDefinition* lhs, + MDefinition* rhs) { + LMulI* lir = new (alloc()) LMulI; + if (mul->fallible()) { + assignSnapshot(lir, mul->bailoutKind()); + } + + // The negative zero check reads |lhs| and |rhs| after the output has been + // written, so at-start allocations are not usable there. + if (mul->canBeNegativeZero() && !rhs->isConstant()) { + lir->setOperand(0, useRegister(lhs)); + lir->setOperand(1, useRegister(rhs)); + define(lir, mul); + return; + } + + lowerForALU(lir, mul, lhs, rhs); +} + +void LIRGeneratorIa64::lowerDivI(MDiv* div) { + // IA-64 has no integer divide instruction, so a division that survives to + // codegen becomes a helper call. Strength-reduce the constant cases here. + if (div->rhs()->isConstant()) { + int32_t rhs = div->rhs()->toConstant()->toInt32(); + if (rhs > 0 && std::has_single_bit(mozilla::Abs(rhs))) { + int32_t shift = mozilla::FloorLog2(uint32_t(rhs)); + auto* lir = + new (alloc()) LDivPowTwoI(useRegisterAtStart(div->lhs()), shift); + if (div->fallible()) { + assignSnapshot(lir, div->bailoutKind()); + } + define(lir, div); + return; + } + } + + LAllocation lhs, rhs; + if (!div->canTruncateRemainder()) { + lhs = useRegister(div->lhs()); + rhs = useRegister(div->rhs()); + } else { + lhs = useRegisterAtStart(div->lhs()); + rhs = useRegisterAtStart(div->rhs()); + } + + auto* lir = new (alloc()) LDivI(lhs, rhs, temp()); + if (div->fallible()) { + assignSnapshot(lir, div->bailoutKind()); + } + define(lir, div); +} + +void LIRGeneratorIa64::lowerDivI64(MDiv* div) { + auto* lir = new (alloc()) + LDivI64(useRegisterAtStart(div->lhs()), useRegisterAtStart(div->rhs())); + defineInt64(lir, div); +} + +void LIRGeneratorIa64::lowerModI(MMod* mod) { + if (mod->rhs()->isConstant()) { + int32_t rhs = mod->rhs()->toConstant()->toInt32(); + int32_t shift = mozilla::FloorLog2(uint32_t(rhs)); + if (rhs > 0 && 1 << shift == rhs) { + LModPowTwoI* lir = + new (alloc()) LModPowTwoI(useRegisterAtStart(mod->lhs()), shift); + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + define(lir, mod); + return; + } + if (shift < 31 && (1 << (shift + 1)) - 1 == rhs) { + LModMaskI* lir = new (alloc()) + LModMaskI(useRegister(mod->lhs()), temp(), temp(), shift + 1); + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + define(lir, mod); + return; + } + } + + LAllocation lhs, rhs; + if (mod->canBeNegativeDividend() && !mod->isTruncated()) { + lhs = useRegister(mod->lhs()); + rhs = useRegister(mod->rhs()); + } else { + lhs = useRegisterAtStart(mod->lhs()); + rhs = useRegisterAtStart(mod->rhs()); + } + + auto* lir = new (alloc()) LModI(lhs, rhs); + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + define(lir, mod); +} + +void LIRGeneratorIa64::lowerModI64(MMod* mod) { + auto* lir = new (alloc()) + LModI64(useRegisterAtStart(mod->lhs()), useRegisterAtStart(mod->rhs())); + defineInt64(lir, mod); +} + +void LIRGeneratorIa64::lowerUDiv(MDiv* div) { + LAllocation lhs, rhs; + if (!div->canTruncateRemainder()) { + lhs = useRegister(div->lhs()); + rhs = useRegister(div->rhs()); + } else { + lhs = useRegisterAtStart(div->lhs()); + rhs = useRegisterAtStart(div->rhs()); + } + + auto* lir = new (alloc()) LUDiv(lhs, rhs); + if (div->fallible()) { + assignSnapshot(lir, div->bailoutKind()); + } + define(lir, div); +} + +void LIRGeneratorIa64::lowerUDivI64(MDiv* div) { + auto* lir = new (alloc()) + LUDivI64(useRegisterAtStart(div->lhs()), useRegisterAtStart(div->rhs())); + defineInt64(lir, div); +} + +void LIRGeneratorIa64::lowerUMod(MMod* mod) { + auto* lir = new (alloc()) + LUMod(useRegisterAtStart(mod->lhs()), useRegisterAtStart(mod->rhs())); + if (mod->fallible()) { + assignSnapshot(lir, mod->bailoutKind()); + } + define(lir, mod); +} + +void LIRGeneratorIa64::lowerUModI64(MMod* mod) { + auto* lir = new (alloc()) + LUModI64(useRegisterAtStart(mod->lhs()), useRegisterAtStart(mod->rhs())); + defineInt64(lir, mod); +} + +void LIRGeneratorIa64::lowerUrshD(MUrsh* mir) { + MDefinition* lhs = mir->lhs(); + MDefinition* rhs = mir->rhs(); + + MOZ_ASSERT(lhs->type() == MIRType::Int32); + MOZ_ASSERT(rhs->type() == MIRType::Int32); + + auto* lir = new (alloc()) LUrshD(useRegisterAtStart(lhs), + useRegisterOrConstantAtStart(rhs), temp()); + define(lir, mir); +} + +void LIRGeneratorIa64::lowerPowOfTwoI(MPow* mir) { + int32_t base = mir->input()->toConstant()->toInt32(); + MDefinition* power = mir->power(); + + auto* lir = new (alloc()) LPowOfTwoI(useRegister(power), base); + assignSnapshot(lir, mir->bailoutKind()); + define(lir, mir); +} + +void LIRGeneratorIa64::lowerTruncateDToInt32(MTruncateToInt32* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Double); + + define(new (alloc()) LTruncateDToInt32(useRegister(opd), tempDouble()), ins); +} + +void LIRGeneratorIa64::lowerTruncateFToInt32(MTruncateToInt32* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Float32); + + define(new (alloc()) LTruncateFToInt32(useRegister(opd), tempFloat32()), ins); +} + +void LIRGeneratorIa64::lowerBuiltinInt64ToFloatingPoint( + MBuiltinInt64ToFloatingPoint* ins) { + MOZ_CRASH("We don't use it for this architecture"); +} + +void LIRGeneratorIa64::lowerWasmSelectI(MWasmSelect* select) { + auto* lir = new (alloc()) + LWasmSelect(useRegisterAtStart(select->trueExpr()), + useAny(select->falseExpr()), useRegister(select->condExpr())); + defineReuseInput(lir, select, LWasmSelect::TrueExprIndex); +} + +void LIRGeneratorIa64::lowerWasmSelectI64(MWasmSelect* select) { + auto* lir = new (alloc()) LWasmSelectI64( + useInt64RegisterAtStart(select->trueExpr()), + useInt64(select->falseExpr()), useRegister(select->condExpr())); + defineInt64ReuseInput(lir, select, LWasmSelectI64::TrueExprIndex); +} + +// Fused compare-and-select is not specialized on ia64 yet; the generic +// select lowering above is used instead. +bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect( + MCompare::CompareType compTy, MIRType insTy) { + return false; +} + +void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins, + MDefinition* lhs, + MDefinition* rhs, + MCompare::CompareType compTy, + JSOp jsop) { + MOZ_CRASH("not specialized on ia64"); +} + +void LIRGeneratorIa64::lowerWasmBuiltinTruncateToInt32( + MWasmBuiltinTruncateToInt32* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32); + + if (opd->type() == MIRType::Double) { + define(new (alloc()) LWasmBuiltinTruncateDToInt32( + useRegister(opd), useFixed(ins->instance(), InstanceReg), + LDefinition::BogusTemp()), + ins); + return; + } + + define(new (alloc()) LWasmBuiltinTruncateFToInt32( + useRegister(opd), LAllocation(), LDefinition::BogusTemp()), + ins); +} + +void LIRGeneratorIa64::lowerWasmBuiltinTruncateToInt64( + MWasmBuiltinTruncateToInt64* ins) { + MOZ_CRASH("We don't use it for this architecture"); +} + +void LIRGeneratorIa64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) { + MOZ_CRASH("We don't use runtime div for this architecture"); +} + +void LIRGeneratorIa64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) { + MOZ_CRASH("We don't use runtime mod for this architecture"); +} + +void LIRGeneratorIa64::lowerBigIntPtrLsh(MBigIntPtrLsh* ins) { + auto* lir = new (alloc()) LBigIntPtrLsh( + useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp()); + assignSnapshot(lir, ins->bailoutKind()); + define(lir, ins); +} + +void LIRGeneratorIa64::lowerBigIntPtrRsh(MBigIntPtrRsh* ins) { + auto* lir = new (alloc()) LBigIntPtrRsh( + useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp()); + assignSnapshot(lir, ins->bailoutKind()); + define(lir, ins); +} + +void LIRGeneratorIa64::lowerBigIntPtrDiv(MBigIntPtrDiv* ins) { + auto* lir = new (alloc()) LBigIntPtrDiv(useRegister(ins->lhs()), + useRegister(ins->rhs()), temp(), + temp()); + assignSnapshot(lir, ins->bailoutKind()); + define(lir, ins); +} + +void LIRGeneratorIa64::lowerBigIntPtrMod(MBigIntPtrMod* ins) { + auto* lir = new (alloc()) LBigIntPtrMod(useRegister(ins->lhs()), + useRegister(ins->rhs()), temp(), + temp()); + if (ins->canBeDivideByZero()) { + assignSnapshot(lir, ins->bailoutKind()); + } + define(lir, ins); +} + +void LIRGeneratorIa64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) { + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->storageType()); + + auto* lir = new (alloc()) LAtomicLoad64(elements, index); + defineInt64(lir, ins); +} + +void LIRGeneratorIa64::lowerAtomicStore64(MStoreUnboxedScalar* ins) { + LUse elements = useRegister(ins->elements()); + LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->writeType()); + LInt64Allocation value = useInt64Register(ins->value()); + + add(new (alloc()) LAtomicStore64(elements, index, value), ins); +} + +void LIRGenerator::visitBox(MBox* ins) { + MDefinition* opd = ins->getOperand(0); + + // If the operand is a constant, emit near its uses. + if (opd->isConstant() && ins->canEmitAtUses()) { + emitAtUses(ins); + return; + } + + if (opd->isConstant()) { + define(new (alloc()) LValue(opd->toConstant()->toJSValue()), ins, + LDefinition(LDefinition::BOX)); + } else { + define(new (alloc()) LBox(useRegisterAtStart(opd), opd->type()), ins, + LDefinition(LDefinition::BOX)); + } +} + +void LIRGenerator::visitUnbox(MUnbox* ins) { + MDefinition* box = ins->getOperand(0); + MOZ_ASSERT(box->type() == MIRType::Value); + + LInstructionHelper<1, BOX_PIECES, 0>* lir; + if (IsFloatingPointType(ins->type())) { + MOZ_ASSERT(ins->type() == MIRType::Double); + lir = new (alloc()) LUnboxFloatingPoint(useBoxAtStart(box)); + } else if (ins->fallible()) { + // If the unbox is fallible, load the Value in a register first to + // avoid multiple loads. + lir = new (alloc()) LUnbox(useRegisterAtStart(box)); + } else { + lir = new (alloc()) LUnbox(useAtStart(box)); + } + + if (ins->fallible()) { + assignSnapshot(lir, ins->bailoutKind()); + } + + define(lir, ins); +} + +void LIRGenerator::visitCopySign(MCopySign* ins) { + MDefinition* lhs = ins->lhs(); + MDefinition* rhs = ins->rhs(); + + MOZ_ASSERT(IsFloatingPointType(lhs->type())); + MOZ_ASSERT(lhs->type() == rhs->type()); + MOZ_ASSERT(lhs->type() == ins->type()); + + LInstructionHelper<1, 2, 0>* lir; + if (lhs->type() == MIRType::Double) { + lir = new (alloc()) LCopySignD(); + } else { + lir = new (alloc()) LCopySignF(); + } + + lowerForFPU(lir, ins, lhs, rhs); +} + +void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) { + defineInt64( + new (alloc()) LExtendInt32ToInt64(useRegisterAtStart(ins->input())), ins); +} + +void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) { + defineInt64(new (alloc()) + LSignExtendInt64(useInt64RegisterAtStart(ins->input())), + ins); +} + +void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Int64); + MOZ_ASSERT(IsFloatingPointType(ins->type())); + + define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd)), ins); +} + +void LIRGenerator::visitSubstr(MSubstr* ins) { + LSubstr* lir = new (alloc()) + LSubstr(useRegister(ins->string()), useRegister(ins->begin()), + useRegister(ins->length()), temp(), temp(), temp()); + define(lir, ins); + assignSafepoint(lir, ins); +} + +void LIRGenerator::visitCompareExchangeTypedArrayElement( + MCompareExchangeTypedArrayElement* ins) { + MOZ_ASSERT(!Scalar::isFloatingType(ins->arrayType())); + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + + if (Scalar::isBigIntType(ins->arrayType())) { + LInt64Allocation oldval = useInt64Register(ins->oldval()); + LInt64Allocation newval = useInt64Register(ins->newval()); + + auto* lir = new (alloc()) + LCompareExchangeTypedArrayElement64(elements, index, oldval, newval); + defineInt64(lir, ins); + return; + } + + const LAllocation oldval = useRegister(ins->oldval()); + const LAllocation newval = useRegister(ins->newval()); + + // The cmpxchg destination must be a temp (the loaded old value cannot + // clobber |oldval|/|newval|, which the retry-free sequence still reads), + // so unlike x64 the temp is needed for every element type, not just the + // Uint32 double-result case. + LDefinition outTemp = temp(); + + auto* lir = new (alloc()) + LCompareExchangeTypedArrayElement(elements, index, oldval, newval, + outTemp); + define(lir, ins); +} + +void LIRGenerator::visitAtomicExchangeTypedArrayElement( + MAtomicExchangeTypedArrayElement* ins) { + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + + if (Scalar::isBigIntType(ins->arrayType())) { + LInt64Allocation value = useInt64Register(ins->value()); + + auto* lir = new (alloc()) + LAtomicExchangeTypedArrayElement64(elements, index, value); + defineInt64(lir, ins); + return; + } + + MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32); + + const LAllocation value = useRegister(ins->value()); + + // The xchg destination must be a temp for every element type; see + // visitCompareExchangeTypedArrayElement. + LDefinition outTemp = temp(); + + auto* lir = new (alloc()) + LAtomicExchangeTypedArrayElement(elements, index, value, outTemp); + define(lir, ins); +} + +void LIRGenerator::visitAtomicTypedArrayElementBinop( + MAtomicTypedArrayElementBinop* ins) { + MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped); + MOZ_ASSERT(!Scalar::isFloatingType(ins->arrayType())); + MOZ_ASSERT(ins->elements()->type() == MIRType::Elements); + MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr); + + const LUse elements = useRegister(ins->elements()); + const LAllocation index = + useRegisterOrIndexConstant(ins->index(), ins->arrayType()); + + if (Scalar::isBigIntType(ins->arrayType())) { + LInt64Allocation value = useInt64Register(ins->value()); + LInt64Definition temp = tempInt64(); + + if (ins->isForEffect()) { + auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64( + elements, index, value, temp); + add(lir, ins); + return; + } + + auto* lir = new (alloc()) + LAtomicTypedArrayElementBinop64(elements, index, value, temp); + defineInt64(lir, ins); + return; + } + + const LAllocation value = useRegister(ins->value()); + + if (ins->isForEffect()) { + // The CAS retry loop needs one allocator temp on top of the two + // borrowable reserved scratches (current value, op result, cmpxchg + // destination). + auto* lir = new (alloc()) + LAtomicTypedArrayElementBinopForEffect(elements, index, value, temp()); + add(lir, ins); + return; + } + + // The CAS retry loop needs both temps for every element type: temp1 for + // the op result and temp2 for the cmpxchg destination. + LDefinition tempDef1 = temp(); + LDefinition tempDef2 = temp(); + + auto* lir = new (alloc()) LAtomicTypedArrayElementBinop( + elements, index, value, tempDef1, tempDef2); + define(lir, ins); +} + +void LIRGenerator::visitReturnImpl(MDefinition* def, bool isGenerator) { + MOZ_ASSERT(def->type() == MIRType::Value); + + LReturn* ins = new (alloc()) LReturn(isGenerator); + ins->setOperand(0, useFixed(def, JSReturnReg)); + add(ins); +} + +void LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + + MDefinition* boundsCheckLimit = ins->boundsCheckLimit(); + MOZ_ASSERT_IF(ins->needsBoundsCheck(), + boundsCheckLimit->type() == MIRType::Int32); + + LAllocation baseAlloc = useRegisterAtStart(base); + + LAllocation limitAlloc = ins->needsBoundsCheck() + ? useRegisterAtStart(boundsCheckLimit) + : LAllocation(); + + MOZ_ASSERT(!ins->hasMemoryBase()); + auto* lir = + new (alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc, LAllocation()); + define(lir, ins); +} + +void LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32); + + MDefinition* boundsCheckLimit = ins->boundsCheckLimit(); + MOZ_ASSERT_IF(ins->needsBoundsCheck(), + boundsCheckLimit->type() == MIRType::Int32); + + LAllocation baseAlloc = useRegisterAtStart(base); + + LAllocation limitAlloc = ins->needsBoundsCheck() + ? useRegisterAtStart(boundsCheckLimit) + : LAllocation(); + + MOZ_ASSERT(!ins->hasMemoryBase()); + add(new (alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()), + limitAlloc, LAllocation()), + ins); +} + +void LIRGenerator::visitWasmLoad(MWasmLoad* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64); + + LAllocation memoryBase = + ins->hasMemoryBase() ? LAllocation(useRegisterAtStart(ins->memoryBase())) + : LGeneralReg(HeapReg); + + LAllocation ptr = useRegisterAtStart(base); + + if (ins->type() == MIRType::Int64) { + auto* lir = new (alloc()) LWasmLoadI64(ptr, memoryBase); + defineInt64(lir, ins); + return; + } + + auto* lir = new (alloc()) LWasmLoad(ptr, memoryBase); + define(lir, ins); +} + +void LIRGenerator::visitWasmStore(MWasmStore* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64); + + MDefinition* value = ins->value(); + + LAllocation memoryBase = + ins->hasMemoryBase() ? LAllocation(useRegisterAtStart(ins->memoryBase())) + : LGeneralReg(HeapReg); + + LAllocation baseAlloc = useRegisterAtStart(base); + + if (ins->access().type() == Scalar::Int64) { + LInt64Allocation valueAlloc = useInt64RegisterAtStart(value); + auto* lir = new (alloc()) LWasmStoreI64(baseAlloc, valueAlloc, memoryBase); + add(lir, ins); + return; + } + + LAllocation valueAlloc = useRegisterAtStart(value); + auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc, memoryBase); + add(lir, ins); +} + +void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) { + MDefinition* opd = ins->input(); + MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32); + + defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd)), ins); +} + +void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) { + MOZ_ASSERT(ins->input()->type() == MIRType::Int32); + auto* lir = + new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input())); + define(lir, ins); +} + +void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) { + MOZ_ASSERT(ins->input()->type() == MIRType::Int32); + auto* lir = + new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input())); + define(lir, ins); +} + +void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64); + + LAllocation memoryBase = ins->hasMemoryBase() + ? LAllocation(useRegister(ins->memoryBase())) + : LGeneralReg(HeapReg); + if (ins->access().type() == Scalar::Int64) { + auto* lir = new (alloc()) LWasmCompareExchangeI64( + useRegister(base), useInt64Register(ins->oldValue()), + useInt64Register(ins->newValue()), memoryBase); + defineInt64(lir, ins); + return; + } + + auto* lir = new (alloc()) + LWasmCompareExchangeHeap(useRegister(base), useRegister(ins->oldValue()), + useRegister(ins->newValue()), memoryBase); + define(lir, ins); +} + +void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64); + + LAllocation memoryBase = ins->hasMemoryBase() + ? LAllocation(useRegister(ins->memoryBase())) + : LGeneralReg(HeapReg); + + if (ins->access().type() == Scalar::Int64) { + auto* lir = new (alloc()) LWasmAtomicExchangeI64( + useRegister(base), useInt64Register(ins->value()), memoryBase); + defineInt64(lir, ins); + return; + } + + auto* lir = new (alloc()) LWasmAtomicExchangeHeap( + useRegister(base), useRegister(ins->value()), memoryBase); + define(lir, ins); +} + +void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) { + MDefinition* base = ins->base(); + MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64); + LAllocation memoryBase = ins->hasMemoryBase() + ? LAllocation(useRegister(ins->memoryBase())) + : LGeneralReg(HeapReg); + + if (ins->access().type() == Scalar::Int64) { + auto* lir = new (alloc()) + LWasmAtomicBinopI64(useRegister(base), useInt64Register(ins->value()), + memoryBase, tempInt64()); + defineInt64(lir, ins); + return; + } + + if (!ins->hasUses()) { + auto* lir = new (alloc()) LWasmAtomicBinopHeapForEffect( + useRegister(base), useRegister(ins->value()), memoryBase); + add(lir, ins); + return; + } + + auto* lir = new (alloc()) LWasmAtomicBinopHeap( + useRegister(base), useRegister(ins->value()), memoryBase, temp()); + define(lir, ins); +} + +void LIRGenerator::visitWasmTernarySimd128(MWasmTernarySimd128* ins) { + MOZ_CRASH("ternary SIMD NYI"); +} + +void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) { + MOZ_CRASH("binary SIMD NYI"); +} + +#ifdef ENABLE_WASM_SIMD +bool MWasmTernarySimd128::specializeBitselectConstantMaskAsShuffle( + int8_t shuffle[16]) { + return false; +} +#endif + +bool MWasmBinarySimd128::specializeForConstantRhs() { return false; } + +void LIRGenerator::visitWasmBinarySimd128WithConstant( + MWasmBinarySimd128WithConstant* ins) { + MOZ_CRASH("binary SIMD with constant NYI"); +} + +void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) { + MOZ_CRASH("shift SIMD NYI"); +} + +void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) { + MOZ_CRASH("shuffle SIMD NYI"); +} + +void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) { + MOZ_CRASH("replace-lane SIMD NYI"); +} + +void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) { + MOZ_CRASH("scalar-to-SIMD NYI"); +} + +void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) { + MOZ_CRASH("unary SIMD NYI"); +} + +void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) { + MOZ_CRASH("reduce-SIMD NYI"); +} + +void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) { + MOZ_CRASH("load-lane SIMD NYI"); +} + +void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) { + MOZ_CRASH("store-lane SIMD NYI"); +} diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/ia64/Lowering-ia64.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/ia64/Lowering-ia64.h --- firefox-153.0.1/js/src/jit/ia64/Lowering-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/Lowering-ia64.h @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_Lowering_ia64_h +#define jit_ia64_Lowering_ia64_h + +#include "jit/shared/Lowering-shared.h" + +namespace js { +namespace jit { + +class LIRGeneratorIa64 : public LIRGeneratorShared { + protected: + LIRGeneratorIa64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph) + : LIRGeneratorShared(gen, graph, lirGraph) {} + + LTableSwitch* newLTableSwitch(const LAllocation& in, + const LDefinition& inputCopy); + LTableSwitchV* newLTableSwitchV(const LBoxAllocation& in); + + void lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir, + MDefinition* lhs, MDefinition* rhs); + template + void lowerForShiftInt64(LInstr* ins, MDefinition* mir, MDefinition* lhs, + MDefinition* rhs); + + void lowerForALU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir, + MDefinition* input); + void lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir, + MDefinition* lhs, MDefinition* rhs); + void lowerForALUInt64(LInstructionHelper* ins, + MDefinition* mir, MDefinition* input); + void lowerForALUInt64( + LInstructionHelper* ins, + MDefinition* mir, MDefinition* lhs, MDefinition* rhs); + void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs, + MDefinition* rhs); + + void lowerForFPU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir, + MDefinition* input); + void lowerForFPU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir, + MDefinition* lhs, MDefinition* rhs); + + // Returns a box allocation. reg2 is ignored on 64-bit platforms. + LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2, + bool useAtStart = false); + + LAllocation useByteOpRegister(MDefinition* mir); + LAllocation useByteOpRegisterAtStart(MDefinition* mir); + LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir); + + LDefinition tempByteOpRegister(); + LDefinition tempToUnbox(); + + bool needTempForPostBarrier() { return true; } + + void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block, + size_t lirIndex); + void lowerInt64PhiInput(MPhi*, uint32_t, LBlock*, size_t); + void defineInt64Phi(MPhi*, size_t); + + void lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs); + void lowerDivI(MDiv* div); + void lowerDivI64(MDiv* div); + void lowerModI(MMod* mod); + void lowerModI64(MMod* mod); + void lowerUDiv(MDiv* div); + void lowerUDivI64(MDiv* div); + void lowerUMod(MMod* mod); + void lowerUModI64(MMod* mod); + void lowerUrshD(MUrsh* mir); + void lowerPowOfTwoI(MPow* mir); + void lowerTruncateDToInt32(MTruncateToInt32* ins); + void lowerTruncateFToInt32(MTruncateToInt32* ins); + void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins); + void lowerWasmSelectI(MWasmSelect* select); + void lowerWasmSelectI64(MWasmSelect* select); + void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins); + void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins); + void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div); + void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod); + + void lowerBigIntPtrLsh(MBigIntPtrLsh* ins); + void lowerBigIntPtrRsh(MBigIntPtrRsh* ins); + void lowerBigIntPtrDiv(MBigIntPtrDiv* ins); + void lowerBigIntPtrMod(MBigIntPtrMod* ins); + + void lowerAtomicLoad64(MLoadUnboxedScalar* ins); + void lowerAtomicStore64(MStoreUnboxedScalar* ins); +}; + +using LIRGeneratorSpecific = LIRGeneratorIa64; + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_Lowering_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/ia64/MacroAssembler-ia64-inl.h b/js/src/jit/ia64/MacroAssembler-ia64-inl.h --- firefox-153.0.1/js/src/jit/ia64/MacroAssembler-ia64-inl.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/MacroAssembler-ia64-inl.h @@ -0,0 +1,1916 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_MacroAssembler_ia64_inl_h +#define jit_ia64_MacroAssembler_ia64_inl_h + +#include "jit/ia64/MacroAssembler-ia64.h" + +#include "jit/JitOptions.h" + +namespace js { +namespace jit { + +// Inline MacroAssembler definitions for ia64. A method appears here once ia64 +// has been added to its DEFINED_ON() list in jit/MacroAssembler.h; methods that +// do not list ia64 expand to "= delete" and need no definition. +// +// Every instruction below is emitted through the encoders in +// AssemblerCore-ia64.h, which are verified byte-for-byte against GNU as. +// +// Two IA-64 properties shape all of this code: +// +// * There are no condition codes. A compare writes a pair of predicate +// registers (the relation and its complement) and the following instruction +// is predicated on one of them. p6/p7 are the scratch pair used here, and +// emitCompare()/emitTest() return which of the two to use. Predication also +// gives conditional moves and conditional loads for free. +// +// * There is no base+displacement addressing. Every Address or BaseIndex has +// to be materialised into a register by computeAddress() first. +// +// The low-level helpers below use ScratchReg/SecondScratchReg directly rather +// than through ScratchRegisterScope, because they are called from methods that +// already hold a scope and nesting an AutoRegisterScope is not allowed. + +// =========================================================================== +// Logical and arithmetic operations on 32-bit values. +// +// IA-64 has no 32-bit ALU forms: the registers are 64-bit and the logical +// operations are width-agnostic, so and/or/xor need no truncation. add and sub +// leave a 64-bit result whose low 32 bits are correct, which is all callers of +// the 32-bit forms rely on. + +void MacroAssembler::and32(Register src, Register dest) { + emitM(ia64::And(dest.encoding(), dest.encoding(), src.encoding())); +} + +void MacroAssembler::or32(Register src, Register dest) { + emitM(ia64::Or(dest.encoding(), dest.encoding(), src.encoding())); +} + +void MacroAssembler::xor32(Register src, Register dest) { + emitM(ia64::Xor(dest.encoding(), dest.encoding(), src.encoding())); +} + +void MacroAssembler::add32(Register src, Register dest) { + emitM(ia64::Add(dest.encoding(), dest.encoding(), src.encoding())); +} + +void MacroAssembler::sub32(Register src, Register dest) { + emitM(ia64::Sub(dest.encoding(), dest.encoding(), src.encoding())); +} + +// Pointer-width forms use the same instructions: IA-64 registers are 64 bits +// and add/sub/and/or/xor operate on the full width. + +void MacroAssembler::andPtr(Register src, Register dest) { + emitM(ia64::And(dest.encoding(), dest.encoding(), src.encoding())); +} + +void MacroAssembler::orPtr(Register src, Register dest) { + emitM(ia64::Or(dest.encoding(), dest.encoding(), src.encoding())); +} + +void MacroAssembler::xorPtr(Register src, Register dest) { + emitM(ia64::Xor(dest.encoding(), dest.encoding(), src.encoding())); +} + +void MacroAssembler::addPtr(Register src, Register dest) { + emitM(ia64::Add(dest.encoding(), dest.encoding(), src.encoding())); +} + +void MacroAssembler::subPtr(Register src, Register dest) { + emitM(ia64::Sub(dest.encoding(), dest.encoding(), src.encoding())); +} + +// not32: IA-64 has no NOT, but andcm computes (~src & mask); with an all-ones +// mask this is dest = ~reg. +void MacroAssembler::not32(Register reg) { + emitMovl(ScratchReg.encoding(), uint64_t(-1)); + emitM(ia64::Andcm(reg.encoding(), ScratchReg.encoding(), reg.encoding())); +} + +// neg32: 0 - reg. r0 is hardwired to zero. +void MacroAssembler::neg32(Register reg) { + emitM(ia64::Sub(reg.encoding(), Registers::zero, reg.encoding())); +} + +void MacroAssembler::negPtr(Register reg) { neg32(reg); } + +void MacroAssembler::neg64(Register64 reg) { neg32(reg.reg); } + +// --------------------------------------------------------------------------- +// Immediate forms. +// +// "adds" reaches a signed 14-bit immediate; anything wider, and every logical +// operation, goes through movl into the scratch register. + +inline void MacroAssemblerIa64::ma_li(Register dest, int64_t imm) { + if (imm >= -8192 && imm <= 8191) { + emitM(ia64::Adds(dest.encoding(), imm, Registers::zero)); + return; + } + emitMovl(dest.encoding(), uint64_t(imm)); +} + +void MacroAssembler::add32(Imm32 imm, Register dest) { + if (imm.value >= -8192 && imm.value <= 8191) { + emitM(ia64::Adds(dest.encoding(), imm.value, dest.encoding())); + return; + } + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(imm.value))); + emitM(ia64::Add(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::sub32(Imm32 imm, Register dest) { + // Fold the negation into adds where it still fits. + int64_t neg = -int64_t(imm.value); + if (neg >= -8192 && neg <= 8191) { + emitM(ia64::Adds(dest.encoding(), neg, dest.encoding())); + return; + } + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(imm.value))); + emitM(ia64::Sub(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::and32(Imm32 imm, Register dest) { + emitMovl(ScratchReg.encoding(), uint64_t(uint32_t(imm.value))); + emitM(ia64::And(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::or32(Imm32 imm, Register dest) { + emitMovl(ScratchReg.encoding(), uint64_t(uint32_t(imm.value))); + emitM(ia64::Or(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::xor32(Imm32 imm, Register dest) { + emitMovl(ScratchReg.encoding(), uint64_t(uint32_t(imm.value))); + emitM(ia64::Xor(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::and32(Imm32 imm, Register src, Register dest) { + movePtr(src, dest); + and32(imm, dest); +} + +void MacroAssembler::or32(Imm32 imm, Register src, Register dest) { + movePtr(src, dest); + or32(imm, dest); +} + +void MacroAssembler::xor32(Imm32 imm, Register src, Register dest) { + movePtr(src, dest); + xor32(imm, dest); +} + +void MacroAssembler::add32(Imm32 imm, Register src, Register dest) { + movePtr(src, dest); + add32(imm, dest); +} + +void MacroAssembler::and32(Imm32 imm, const Address& dest) { + computeAddress(dest, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + and32(imm, SecondScratchReg); + computeAddress(dest, ScratchReg); + emitM(ia64::St4(ScratchReg.encoding(), SecondScratchReg.encoding())); +} + +void MacroAssembler::or32(Imm32 imm, const Address& dest) { + computeAddress(dest, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + or32(imm, SecondScratchReg); + computeAddress(dest, ScratchReg); + emitM(ia64::St4(ScratchReg.encoding(), SecondScratchReg.encoding())); +} + +void MacroAssembler::xor32(Imm32 imm, const Address& dest) { + computeAddress(dest, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + xor32(imm, SecondScratchReg); + computeAddress(dest, ScratchReg); + emitM(ia64::St4(ScratchReg.encoding(), SecondScratchReg.encoding())); +} + +void MacroAssembler::add32(Imm32 imm, const Address& dest) { + computeAddress(dest, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + add32(imm, SecondScratchReg); + computeAddress(dest, ScratchReg); + emitM(ia64::St4(ScratchReg.encoding(), SecondScratchReg.encoding())); +} + +void MacroAssembler::and32(const Address& src, Register dest) { + computeAddress(src, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + and32(SecondScratchReg, dest); +} + +void MacroAssembler::xor32(const Address& src, Register dest) { + computeAddress(src, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + xor32(SecondScratchReg, dest); +} + +void MacroAssembler::add32(const Address& src, Register dest) { + computeAddress(src, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + add32(SecondScratchReg, dest); +} + +void MacroAssembler::sub32(const Address& src, Register dest) { + computeAddress(src, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + sub32(SecondScratchReg, dest); +} + +// Pointer-width immediate forms. + +void MacroAssembler::addPtr(Imm32 imm, Register dest) { add32(imm, dest); } + +void MacroAssembler::subPtr(Imm32 imm, Register dest) { sub32(imm, dest); } + +// The movl carries the whole immediate at a known bundle offset, which is what +// makes it patchable after the fact. +CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) { + CodeOffset offset(currentOffset()); + emitMovl(ScratchReg.encoding(), 0); + emitM(ia64::Sub(dest.encoding(), StackPointer.encoding(), + ScratchReg.encoding())); + return offset; +} + +void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) { + WriteMovlImm( + reinterpret_cast(bundleAt(BufferOffset(offset.offset()))), + uint64_t(int64_t(imm.value))); +} + +void MacroAssembler::andPtr(Imm32 imm, Register dest) { + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(imm.value))); + emitM(ia64::And(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::orPtr(Imm32 imm, Register dest) { + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(imm.value))); + emitM(ia64::Or(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::xorPtr(Imm32 imm, Register dest) { + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(imm.value))); + emitM(ia64::Xor(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::andPtr(Imm32 imm, Register src, Register dest) { + movePtr(src, dest); + andPtr(imm, dest); +} + +void MacroAssembler::orPtr(Imm32 imm, Register src, Register dest) { + movePtr(src, dest); + orPtr(imm, dest); +} + +void MacroAssembler::xorPtr(Imm32 imm, Register src, Register dest) { + movePtr(src, dest); + xorPtr(imm, dest); +} + +void MacroAssembler::addPtr(ImmWord imm, Register dest) { + emitMovl(ScratchReg.encoding(), imm.value); + emitM(ia64::Add(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::addPtr(Imm32 imm, const Address& dest) { + computeAddress(dest, SecondScratchReg); + emitM(ia64::Ld8(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + addPtr(imm, SecondScratchReg); + computeAddress(dest, ScratchReg); + emitM(ia64::St8(ScratchReg.encoding(), SecondScratchReg.encoding())); +} + +void MacroAssembler::addPtr(const Address& src, Register dest) { + computeAddress(src, SecondScratchReg); + emitM(ia64::Ld8(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + addPtr(SecondScratchReg, dest); +} + +void MacroAssembler::subPtr(const Address& addr, Register dest) { + computeAddress(addr, SecondScratchReg); + emitM(ia64::Ld8(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + subPtr(SecondScratchReg, dest); +} + +void MacroAssembler::subPtr(Register src, const Address& dest) { + computeAddress(dest, SecondScratchReg); + emitM(ia64::Ld8(ScratchReg.encoding(), SecondScratchReg.encoding())); + emitM(ia64::Sub(ScratchReg.encoding(), ScratchReg.encoding(), + src.encoding())); + emitM(ia64::St8(SecondScratchReg.encoding(), ScratchReg.encoding())); +} + +// 64-bit forms. On a 64-bit target Register64 wraps a single register, so all +// of these are the pointer-width operations. + +void MacroAssembler::add64(Register64 src, Register64 dest) { + addPtr(src.reg, dest.reg); +} +void MacroAssembler::add64(Imm32 imm, Register64 dest) { + addPtr(imm, dest.reg); +} +void MacroAssembler::add64(Imm64 imm, Register64 dest) { + addPtr(ImmWord(imm.value), dest.reg); +} +void MacroAssembler::sub64(Register64 src, Register64 dest) { + subPtr(src.reg, dest.reg); +} +void MacroAssembler::sub64(Imm64 imm, Register64 dest) { + emitMovl(ScratchReg.encoding(), imm.value); + emitM(ia64::Sub(dest.reg.encoding(), dest.reg.encoding(), + ScratchReg.encoding())); +} +void MacroAssembler::and64(Register64 src, Register64 dest) { + andPtr(src.reg, dest.reg); +} +void MacroAssembler::or64(Register64 src, Register64 dest) { + orPtr(src.reg, dest.reg); +} +void MacroAssembler::xor64(Register64 src, Register64 dest) { + xorPtr(src.reg, dest.reg); +} +void MacroAssembler::and64(Imm64 imm, Register64 dest) { + emitMovl(ScratchReg.encoding(), imm.value); + emitM(ia64::And(dest.reg.encoding(), dest.reg.encoding(), + ScratchReg.encoding())); +} +void MacroAssembler::or64(Imm64 imm, Register64 dest) { + emitMovl(ScratchReg.encoding(), imm.value); + emitM(ia64::Or(dest.reg.encoding(), dest.reg.encoding(), + ScratchReg.encoding())); +} +void MacroAssembler::xor64(Imm64 imm, Register64 dest) { + emitMovl(ScratchReg.encoding(), imm.value); + emitM(ia64::Xor(dest.reg.encoding(), dest.reg.encoding(), + ScratchReg.encoding())); +} + +void MacroAssembler::inc64(AbsoluteAddress dest) { + computeAddress(dest, SecondScratchReg); + emitM(ia64::Ld8(ScratchReg.encoding(), SecondScratchReg.encoding())); + emitM(ia64::Adds(ScratchReg.encoding(), 1, ScratchReg.encoding())); + emitM(ia64::St8(SecondScratchReg.encoding(), ScratchReg.encoding())); +} + +void MacroAssembler::move64(Imm64 imm, Register64 dest) { + movePtr(ImmWord(imm.value), dest.reg); +} +void MacroAssembler::move64(Register64 src, Register64 dest) { + movePtr(src.reg, dest.reg); +} +void MacroAssembler::move64To32(Register64 src, Register dest) { + ma_sxt(dest, src.reg, 4); +} +void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) { + ma_zxt(dest.reg, src, 4); +} +void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) { + ma_zxt(dest, src, 4); +} +void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) { + ma_sxt(dest.reg, src, 1); +} +void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) { + ma_sxt(dest.reg, src, 2); +} +void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) { + ma_sxt(dest.reg, src, 4); +} +void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) { + load32(src, dest); +} + +// Multiplication has no ALU form on IA-64; it goes through the floating-point +// unit (setf.sig / xma.l / getf.sig), which ma_mul() encapsulates. + +void MacroAssembler::mul32(Register rhs, Register srcDest) { + ma_mul(srcDest, srcDest, rhs); +} +void MacroAssembler::mul32(Imm32 imm, Register srcDest) { + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(imm.value))); + ma_mul(srcDest, srcDest, ScratchReg); +} +void MacroAssembler::mulPtr(Register rhs, Register srcDest) { + ma_mul(srcDest, srcDest, rhs); +} +void MacroAssembler::mulPtr(ImmWord rhs, Register srcDest) { + emitMovl(ScratchReg.encoding(), rhs.value); + ma_mul(srcDest, srcDest, ScratchReg); +} +void MacroAssembler::mul64(const Register64& rhs, const Register64& srcDest) { + ma_mul(srcDest.reg, srcDest.reg, rhs.reg); +} +void MacroAssembler::mul64(const Register64& src, const Register64& dest, + const Register temp) { + ma_mul(dest.reg, dest.reg, src.reg); +} +void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) { + // Both operands are 32-bit magnitudes zero-extended into 64-bit registers, + // so their product fits entirely in ma_mul's 64-bit result; the "high + // unsigned 32 bits of a 32x32 multiply" this API wants is just that + // product's top half, with no need for xma.h/hu's 64x64->128 range. + ma_zxt(ScratchReg, src, 4); + emitMovl(SecondScratchReg.encoding(), uint64_t(uint32_t(imm.value))); + ma_mul(dest, ScratchReg, SecondScratchReg); + emitI(ia64::ShrUImm(dest.encoding(), dest.encoding(), 32)); +} + +// Integer divide is a software sequence too; see ma_divmod(). +void MacroAssembler::quotient32(Register lhs, Register rhs, Register dest, + bool isUnsigned) { + ma_divmod(dest, InvalidReg, lhs, rhs, isUnsigned, 32); +} +void MacroAssembler::quotient64(Register lhs, Register rhs, Register dest, + bool isUnsigned) { + ma_divmod(dest, InvalidReg, lhs, rhs, isUnsigned, 64); +} +void MacroAssembler::remainder32(Register lhs, Register rhs, Register dest, + bool isUnsigned) { + ma_divmod(InvalidReg, dest, lhs, rhs, isUnsigned, 32); +} +void MacroAssembler::remainder64(Register lhs, Register rhs, Register dest, + bool isUnsigned) { + ma_divmod(InvalidReg, dest, lhs, rhs, isUnsigned, 64); +} + +void MacroAssembler::min32(Register lhs, Register rhs, Register result) { + // Stage through the scratch register so that result may alias either input. + uint32_t p = emitCompare(Assembler::LessThan, lhs, rhs); + emitM(ia64::MovReg(SecondScratchReg.encoding(), rhs.encoding())); + emitM(ia64::Adds(SecondScratchReg.encoding(), 0, lhs.encoding(), p)); + emitM(ia64::MovReg(result.encoding(), SecondScratchReg.encoding())); +} +void MacroAssembler::min32(Register lhs, Imm32 rhs, Register result) { + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(rhs.value))); + min32(lhs, ScratchReg, result); +} +void MacroAssembler::max32(Register lhs, Register rhs, Register result) { + uint32_t p = emitCompare(Assembler::GreaterThan, lhs, rhs); + emitM(ia64::MovReg(SecondScratchReg.encoding(), rhs.encoding())); + emitM(ia64::Adds(SecondScratchReg.encoding(), 0, lhs.encoding(), p)); + emitM(ia64::MovReg(result.encoding(), SecondScratchReg.encoding())); +} +void MacroAssembler::max32(Register lhs, Imm32 rhs, Register result) { + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(rhs.value))); + max32(lhs, ScratchReg, result); +} + +// =========================================================================== +// Shifts and rotates. + +void MacroAssembler::lshift32(Imm32 shift, Register srcDest) { + ma_lsl(srcDest, srcDest, shift); +} +void MacroAssembler::lshift32(Imm32 shift, Register src, Register dest) { + ma_lsl(dest, src, shift); +} +void MacroAssembler::rshift32(Imm32 shift, Register srcDest) { + ma_zxt(srcDest, srcDest, 4); + ma_lsr(srcDest, srcDest, shift); +} +void MacroAssembler::rshift32(Imm32 shift, Register src, Register dest) { + ma_zxt(dest, src, 4); + ma_lsr(dest, dest, shift); +} +void MacroAssembler::rshift32Arithmetic(Imm32 shift, Register srcDest) { + ma_asr(srcDest, srcDest, shift); +} +void MacroAssembler::rshift32Arithmetic(Imm32 shift, Register src, + Register dest) { + ma_asr(dest, src, shift); +} +void MacroAssembler::lshift32(Register shift, Register srcDest) { + // JS shift counts are taken modulo 32. + emitM(ia64::Adds(ScratchReg.encoding(), 31, Registers::zero)); + emitM(ia64::And(ScratchReg.encoding(), shift.encoding(), + ScratchReg.encoding())); + ma_lsl(srcDest, srcDest, ScratchReg); +} +void MacroAssembler::rshift32(Register shift, Register srcDest) { + emitM(ia64::Adds(ScratchReg.encoding(), 31, Registers::zero)); + emitM(ia64::And(ScratchReg.encoding(), shift.encoding(), + ScratchReg.encoding())); + ma_zxt(srcDest, srcDest, 4); + ma_lsr(srcDest, srcDest, ScratchReg); +} +void MacroAssembler::rshift32Arithmetic(Register shift, Register srcDest) { + emitM(ia64::Adds(ScratchReg.encoding(), 31, Registers::zero)); + emitM(ia64::And(ScratchReg.encoding(), shift.encoding(), + ScratchReg.encoding())); + ma_sxt(srcDest, srcDest, 4); + ma_asr(srcDest, srcDest, ScratchReg); +} + +void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) { + ma_lsl(dest, dest, imm); +} +void MacroAssembler::lshiftPtr(Imm32 imm, Register src, Register dest) { + ma_lsl(dest, src, imm); +} +void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) { + ma_lsr(dest, dest, imm); +} +void MacroAssembler::rshiftPtr(Imm32 imm, Register src, Register dest) { + ma_lsr(dest, src, imm); +} +void MacroAssembler::lshiftPtr(Register shift, Register srcDest) { + ma_lsl(srcDest, srcDest, shift); +} +void MacroAssembler::rshiftPtr(Register shift, Register srcDest) { + ma_lsr(srcDest, srcDest, shift); +} + +void MacroAssembler::lshift64(Imm32 imm, Register64 dest) { + ma_lsl(dest.reg, dest.reg, imm); +} +void MacroAssembler::rshift64(Imm32 imm, Register64 dest) { + ma_lsr(dest.reg, dest.reg, imm); +} +void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) { + ma_asr(dest.reg, dest.reg, imm); +} +void MacroAssembler::lshift64(Register shift, Register64 srcDest) { + ma_lsl(srcDest.reg, srcDest.reg, shift); +} +void MacroAssembler::rshift64(Register shift, Register64 srcDest) { + ma_lsr(srcDest.reg, srcDest.reg, shift); +} +void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) { + ma_asr(srcDest.reg, srcDest.reg, shift); +} + +void MacroAssembler::flexibleLshift32(Register shift, Register srcDest) { + lshift32(shift, srcDest); +} +void MacroAssembler::flexibleRshift32(Register shift, Register srcDest) { + rshift32(shift, srcDest); +} +void MacroAssembler::flexibleRshift32Arithmetic(Register shift, + Register srcDest) { + rshift32Arithmetic(shift, srcDest); +} +void MacroAssembler::flexibleLshiftPtr(Register shift, Register srcDest) { + lshiftPtr(shift, srcDest); +} + +void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) { + ma_rol(dest, input, count, 32); +} +void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) { + ma_ror(dest, input, count, 32); +} +void MacroAssembler::rotateLeft(Register count, Register input, Register dest) { + ma_rol(dest, input, count, 32); +} +void MacroAssembler::rotateRight(Register count, Register input, Register dest) { + ma_ror(dest, input, count, 32); +} +void MacroAssembler::rotateLeft64(Imm32 count, Register64 input, + Register64 dest, Register temp) { + ma_rol(dest.reg, input.reg, count, 64); +} +void MacroAssembler::rotateRight64(Imm32 count, Register64 input, + Register64 dest, Register temp) { + ma_ror(dest.reg, input.reg, count, 64); +} +void MacroAssembler::rotateLeft64(Register count, Register64 input, + Register64 dest, Register temp) { + ma_rol(dest.reg, input.reg, count, 64); +} +void MacroAssembler::rotateRight64(Register count, Register64 input, + Register64 dest, Register temp) { + ma_ror(dest.reg, input.reg, count, 64); +} + +// =========================================================================== +// Comparisons producing a boolean in a register. The compare writes a +// predicate pair; a predicated "adds dest = 1, r0" then materialises the bit. + +template +void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) { + uint32_t p = emitCompare32(cond, lhs, rhs); + emitM(ia64::MovReg(dest.encoding(), Registers::zero)); + emitM(ia64::Adds(dest.encoding(), 1, Registers::zero, p)); +} + +template +void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) { + uint32_t p = emitComparePtr(cond, lhs, rhs); + emitM(ia64::MovReg(dest.encoding(), Registers::zero)); + emitM(ia64::Adds(dest.encoding(), 1, Registers::zero, p)); +} + +void MacroAssembler::cmp64Set(Condition cond, Register64 lhs, Register64 rhs, + Register dest) { + cmp32Set(cond, lhs.reg, rhs.reg, dest); +} +void MacroAssembler::cmp64Set(Condition cond, Register64 lhs, Imm64 rhs, + Register dest) { + cmp32Set(cond, lhs.reg, ImmWord(uint64_t(rhs.value)), dest); +} +void MacroAssembler::cmp64Set(Condition cond, Address lhs, Register64 rhs, + Register dest) { + computeAddress(lhs, SecondScratchReg); + emitM(ia64::Ld8(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + cmp32Set(cond, SecondScratchReg, rhs.reg, dest); +} +void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs, + Register dest) { + computeAddress(lhs, SecondScratchReg); + emitM(ia64::Ld8(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + cmp32Set(cond, SecondScratchReg, ImmWord(uint64_t(rhs.value)), dest); +} + +void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs, + Register dest) { + bool isSigned = cond != Above && cond != AboveOrEqual && cond != Below && + cond != BelowOrEqual; + if (isSigned) { + load8SignExtend(lhs, SecondScratchReg); + } else { + load8ZeroExtend(lhs, SecondScratchReg); + } + cmp32Set(cond, SecondScratchReg, rhs, dest); +} + +void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs, + Register dest) { + bool isSigned = cond != Above && cond != AboveOrEqual && cond != Below && + cond != BelowOrEqual; + if (isSigned) { + load16SignExtend(lhs, SecondScratchReg); + } else { + load16ZeroExtend(lhs, SecondScratchReg); + } + cmp32Set(cond, SecondScratchReg, rhs, dest); +} + +// =========================================================================== +// Conditional moves and loads. Predication makes these a single predicated +// instruction after the compare. + +void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs, + Register src, Register dest) { + uint32_t p = emitCompare(cond, lhs, rhs); + emitM(ia64::Adds(dest.encoding(), 0, src.encoding(), p)); +} +void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Imm32 rhs, + Register src, Register dest) { + uint32_t p = emitCompare(cond, lhs, rhs); + emitM(ia64::Adds(dest.encoding(), 0, src.encoding(), p)); +} +void MacroAssembler::cmp32Move32(Condition cond, Register lhs, + const Address& rhs, Register src, + Register dest) { + computeAddress(rhs, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + ma_sxt(SecondScratchReg, SecondScratchReg, 4); + cmp32Move32(cond, lhs, SecondScratchReg, src, dest); +} +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs, + Register src, Register dest) { + cmp32Move32(cond, lhs, rhs, src, dest); +} +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Imm32 rhs, + Register src, Register dest) { + cmp32Move32(cond, lhs, rhs, src, dest); +} +void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, + const Address& rhs, Register src, + Register dest) { + computeAddress(rhs, SecondScratchReg); + emitM(ia64::Ld8(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + cmp32Move32(cond, lhs, SecondScratchReg, src, dest); +} +void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs, + Register src, Register dest) { + cmp32Move32(cond, lhs, rhs, src, dest); +} +void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs, + const Address& src, Register dest) { + computeAddress(lhs, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + ma_sxt(SecondScratchReg, SecondScratchReg, 4); + uint32_t p = emitCompare(cond, SecondScratchReg, rhs); + computeAddress(src, SecondScratchReg); + emitM(ia64::Ld8(dest.encoding(), SecondScratchReg.encoding(), p)); +} +void MacroAssembler::test32MovePtr(Condition cond, Register operand, Imm32 mask, + Register src, Register dest) { + uint32_t p = emitTest(cond, operand, mask, 32); + emitM(ia64::Adds(dest.encoding(), 0, src.encoding(), p)); +} +void MacroAssembler::test32MovePtr(Condition cond, const Address& addr, + Imm32 mask, Register src, Register dest) { + computeAddress(addr, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + uint32_t p = emitTest(cond, SecondScratchReg, mask, 32); + emitM(ia64::Adds(dest.encoding(), 0, src.encoding(), p)); +} +void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr, + Imm32 mask, const Address& src, + Register dest) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreStringMitigations); + computeAddress(addr, SecondScratchReg); + emitM(ia64::Ld4(SecondScratchReg.encoding(), SecondScratchReg.encoding())); + uint32_t p = emitTest(cond, SecondScratchReg, mask, 32); + computeAddress(src, SecondScratchReg); + emitM(ia64::Ld8(dest.encoding(), SecondScratchReg.encoding(), p)); +} + +// Spectre index masking needs a conditional move fed by the flags of a *prior* +// compare; IA-64 predicates do not survive an intervening compare, so the +// masking variants are not supported and the bounds checks fall back to a +// plain branch, as on the other flag-less backends. +// On x86 these re-check a *just-executed* branch's condition by reusing the +// flags it left behind, as a second, speculation-safe enforcement of an +// outcome a real branch already established architecturally -- defense in +// depth against the branch being mispredicted speculatively, not a +// correctness requirement. IA-64 has no persistent flags register (a +// predicate would have to be threaded through from whichever compare the +// caller already did, which this narrow two-register signature has no way +// to express), so there is no cheap way to redo that check here; every +// caller reaches this only after its own real branch/compare already set +// `dest` to the architecturally-correct value, so doing nothing is +// functionally correct, just without the extra speculative-execution +// hardening. Spectre mitigations default to off (see JitOptions.cpp), so +// these are unreached in a default configuration regardless. +void MacroAssembler::spectreMovePtr(Condition cond, Register src, + Register dest) {} +void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch, + Register dest) {} +void MacroAssembler::spectreBoundsCheck32(Register index, Register length, + Register maybeScratch, + Label* failure) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking); + branch32(Assembler::BelowOrEqual, length, index, failure); +} +void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length, + Register maybeScratch, + Label* failure) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking); + load32(length, SecondScratchReg); + branch32(Assembler::BelowOrEqual, SecondScratchReg, index, failure); +} +void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length, + Register maybeScratch, + Label* failure) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking); + branchPtr(Assembler::BelowOrEqual, length, index, failure); +} +void MacroAssembler::spectreBoundsCheckPtr(Register index, + const Address& length, + Register maybeScratch, + Label* failure) { + MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking); + loadPtr(length, SecondScratchReg); + branchPtr(Assembler::BelowOrEqual, SecondScratchReg, index, failure); +} + +// =========================================================================== +// Branches. + +void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs, + Label* label) { + uint32_t pred = emitCompare(cond, lhs, rhs); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs, + Label* label) { + branch32(cond, lhs, rhs, label); +} + +void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + uint32_t pred = emitCompare(cond, lhs, rhs); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + branch32(cond, lhs, rhs, label); +} + +void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs, + Label* label) { + load32(lhs, SecondScratchReg); + branch32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs, + Label* label) { + load32(lhs, SecondScratchReg); + branch32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs, + Label* label) { + load32(lhs, SecondScratchReg); + branch32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + load32(lhs, SecondScratchReg); + branch32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + load32(lhs, SecondScratchReg); + branch32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs, + Label* label) { + bool isSigned = cond != Above && cond != AboveOrEqual && cond != Below && + cond != BelowOrEqual; + if (isSigned) { + load8SignExtend(lhs, SecondScratchReg); + } else { + load8ZeroExtend(lhs, SecondScratchReg); + } + branch32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs, + Label* label) { + bool isSigned = cond != Above && cond != AboveOrEqual && cond != Below && + cond != BelowOrEqual; + if (isSigned) { + load16SignExtend(lhs, SecondScratchReg); + } else { + load16ZeroExtend(lhs, SecondScratchReg); + } + branch32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs, + Label* label) { + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs, + Register rhs, Label* label) { + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs, + ImmWord rhs, Label* label) { + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + Register rhs, Label* label) { + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs, + ImmWord rhs, Label* label) { + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, rhs, label); +} + +// Immediate-operand branch overloads. Pointer-sized immediates always need a +// movl; ImmGCPtr additionally records a relocation for the GC to trace. + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs, + Label* label) { + uint32_t pred = emitCompare(cond, lhs, rhs); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs, + Label* label) { + branchPtr(cond, lhs, ImmWord(uintptr_t(rhs.value)), label); +} + +void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs, + Label* label) { + movePtr(rhs, SecondScratchReg); + branchPtr(cond, lhs, SecondScratchReg, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs, + Label* label) { + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs, + Label* label) { + branchPtr(cond, lhs, ImmWord(uintptr_t(rhs.value)), label); +} + +void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs, + Label* label) { + loadPtr(lhs, SecondScratchReg); + movePtr(rhs, ScratchReg); + branchPtr(cond, SecondScratchReg, ScratchReg, label); +} + +void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs, + Register rhs, Label* label) { + branchPtr(cond, lhs, rhs, label); +} + +// 64-bit branches. Register64 is a single register here, so a "success/fail" +// pair collapses to one compare plus the two branches. + +void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs, + Label* success, Label* fail) { + branchPtr(cond, lhs.reg, rhs.reg, success); + if (fail) { + jump(fail); + } +} +void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val, + Label* success, Label* fail) { + branchPtr(cond, lhs.reg, ImmWord(uint64_t(val.value)), success); + if (fail) { + jump(fail); + } +} +void MacroAssembler::branch64(Condition cond, const Address& lhs, + Register64 rhs, Label* success, Label* fail) { + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, rhs.reg, success); + if (fail) { + jump(fail); + } +} +void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val, + Label* success, Label* fail) { + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, ImmWord(uint64_t(val.value)), success); + if (fail) { + jump(fail); + } +} +void MacroAssembler::branch64(Condition cond, const Address& lhs, + const Address& rhs, Register scratch, + Label* label) { + MOZ_ASSERT(scratch != lhs.base && scratch != rhs.base); + loadPtr(rhs, scratch); + loadPtr(lhs, SecondScratchReg); + branchPtr(cond, SecondScratchReg, scratch, label); +} + +// --------------------------------------------------------------------------- +// Bit tests. "test" is an AND that discards its result and branches on whether +// the result was zero; IA-64 has no flags, so compute the AND into the scratch +// register and compare it against r0 (hardwired zero). + +void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs, + Label* label) { + uint32_t pred = emitTest(cond, lhs, rhs, 32); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + uint32_t pred = emitTest(cond, lhs, rhs, 32); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs, + Label* label) { + load32(lhs, SecondScratchReg); + branchTest32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs, + Imm32 rhs, Label* label) { + load32(lhs, SecondScratchReg); + branchTest32(cond, SecondScratchReg, rhs, label); +} + +void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs, + Label* label) { + uint32_t pred = emitTest(cond, lhs, rhs, 64); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + uint32_t pred = emitTest(cond, lhs, rhs, 64); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchTestPtr(Condition cond, Register lhs, ImmWord rhs, + Label* label) { + emitMovl(ScratchReg.encoding(), rhs.value); + uint32_t pred = emitTest(cond, lhs, ScratchReg, 64); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs, + Imm32 rhs, Label* label) { + loadPtr(lhs, SecondScratchReg); + uint32_t pred = emitTest(cond, SecondScratchReg, rhs, 64); + emitBranchToLabel(&ia64::BrCondRel, pred, label); +} + +void MacroAssembler::branchTest64(Condition cond, Register64 lhs, + Register64 rhs, Register temp, Label* success, + Label* fail) { + branchTestPtr(cond, lhs.reg, rhs.reg, success); + if (fail) { + jump(fail); + } +} + +void MacroAssembler::branchTest64(Condition cond, Register64 lhs, Imm64 rhs, + Label* success, Label* fail) { + emitMovl(ScratchReg.encoding(), uint64_t(rhs.value)); + branchTestPtr(cond, lhs.reg, ScratchReg, success); + if (fail) { + jump(fail); + } +} + +// --------------------------------------------------------------------------- +// Overflow-checking arithmetic. IA-64 has no overflow flag, so the checks are +// done explicitly on the operands and the result. Every GPR value is kept +// sign-extended (see load32's comment), so a 32-bit signed add/sub/mul +// overflows exactly when the full 64-bit result doesn't equal its own +// sign-extension back from 32 bits. + +static Register MaterializeSrc(MacroAssembler& masm, Register src) { + return src; +} +static Register MaterializeSrc(MacroAssembler& masm, Imm32 src) { + masm.move32(src, ScratchReg); + return ScratchReg; +} +static Register MaterializeSrc(MacroAssembler& masm, ImmWord src) { + masm.movePtr(src, ScratchReg); + return ScratchReg; +} +static Register MaterializeSrc(MacroAssembler& masm, ImmPtr src) { + masm.movePtr(src, ScratchReg); + return ScratchReg; +} + +template +void MacroAssembler::branchAdd32(Condition cond, T src, Register dest, + Label* label) { + Register s = MaterializeSrc(*this, src); + if (cond == Assembler::CarrySet) { + // Unsigned 32-bit carry: zero-extend both operands (the sign-extended + // register convention doesn't apply to an unsigned interpretation) and + // check whether their sum overflows 32 bits. + ma_zxt(SecondScratchReg, dest, 4); + ma_zxt(ScratchReg, s, 4); + emitM(ia64::Add(SecondScratchReg.encoding(), SecondScratchReg.encoding(), + ScratchReg.encoding())); + emitI(ia64::ShrUImm(ScratchReg.encoding(), SecondScratchReg.encoding(), 32)); + ma_zxt(dest, SecondScratchReg, 4); + uint32_t p = emitCompare(Assembler::NotEqual, ScratchReg, Imm32(0)); + emitBranchToLabel(&ia64::BrCondRel, p, label); + return; + } + emitM(ia64::Add(dest.encoding(), dest.encoding(), s.encoding())); + MOZ_ASSERT(cond == Assembler::Overflow); + ma_sxt(ScratchReg, dest, 4); + uint32_t p = emitCompare(Assembler::NotEqual, ScratchReg, dest); + ma_sxt(dest, dest, 4); + emitBranchToLabel(&ia64::BrCondRel, p, label); +} + +template +void MacroAssembler::branchSub32(Condition cond, T src, Register dest, + Label* label) { + Register s = MaterializeSrc(*this, src); + emitM(ia64::Sub(dest.encoding(), dest.encoding(), s.encoding())); + if (cond == Assembler::Overflow) { + ma_sxt(ScratchReg, dest, 4); + uint32_t p = emitCompare(Assembler::NotEqual, ScratchReg, dest); + ma_sxt(dest, dest, 4); + emitBranchToLabel(&ia64::BrCondRel, p, label); + return; + } + ma_sxt(dest, dest, 4); + MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero || + cond == Assembler::Signed || cond == Assembler::NotSigned); + branchTest32(cond, dest, dest, label); +} + +template +void MacroAssembler::branchMul32(Condition cond, T src, Register dest, + Label* label) { + MOZ_ASSERT(cond == Assembler::Overflow); + Register s = MaterializeSrc(*this, src); + ma_mul(dest, dest, s); + ma_sxt(ScratchReg, dest, 4); + uint32_t p = emitCompare(Assembler::NotEqual, ScratchReg, dest); + ma_sxt(dest, dest, 4); + emitBranchToLabel(&ia64::BrCondRel, p, label); +} + +template +void MacroAssembler::branchRshift32(Condition cond, T src, Register dest, + Label* label) { + MOZ_ASSERT(cond == Zero || cond == NonZero); + rshift32(src, dest); + branchTest32(cond, dest, dest, label); +} + +// 64-bit pointer multiply: overflow means the true mathematical product +// doesn't fit in 64 bits, which xma.l's 64-bit truncated result can't reveal +// by itself; xma.h gives the high 64 bits of the full 128-bit product, and +// for a signed result that's fully determined by (and must equal) the sign +// extension of the low 64 bits when there is no overflow. +void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest, + Label* label) { + MOZ_ASSERT(cond == Assembler::Overflow); + const uint32_t f0 = 2, f1 = 3, fHi = 4, fLo = 5; + emitM(ia64::SetfSig(f0, dest.encoding())); + emitM(ia64::SetfSig(f1, src.encoding())); + emitF(ia64::XmaH(fHi, f0, f1, ia64::fpZero)); + emitF(ia64::XmaL(fLo, f0, f1, ia64::fpZero)); + emitM(ia64::GetfSig(dest.encoding(), fLo)); + emitM(ia64::GetfSig(ScratchReg.encoding(), fHi)); + // Expected high half if the low half's sign accounts for the whole + // product: all 0s (low half non-negative) or all 1s (negative). + uint32_t pNeg = + emitCompare(Assembler::LessThan, dest, Register{Registers::zero}); + emitM(ia64::MovReg(SecondScratchReg.encoding(), Registers::zero)); + emitM(ia64::Adds(SecondScratchReg.encoding(), -1, Registers::zero, pNeg)); + uint32_t p = + emitCompare(Assembler::NotEqual, ScratchReg, SecondScratchReg); + emitBranchToLabel(&ia64::BrCondRel, p, label); +} + +// --------------------------------------------------------------------------- +// Addressing. IA-64 has no base+displacement addressing mode: every load and +// store takes a bare register. + +inline void MacroAssemblerIa64::computeAddress(const Address& addr, Register out) { + if (addr.offset == 0) { + emitM(ia64::MovReg(out.encoding(), addr.base.encoding())); + return; + } + if (addr.offset >= -8192 && addr.offset <= 8191) { + emitM(ia64::Adds(out.encoding(), addr.offset, addr.base.encoding())); + return; + } + emitMovl(out.encoding(), uint64_t(int64_t(addr.offset))); + emitM(ia64::Add(out.encoding(), out.encoding(), addr.base.encoding())); +} + +inline void MacroAssemblerIa64::computeAddress(const AbsoluteAddress& addr, + Register out) { + emitMovl(out.encoding(), uint64_t(uintptr_t(addr.addr))); +} + +// =========================================================================== +// Value tags. See the comment on ValueTagMask in MacroAssembler-ia64.h: the +// "tag" of a Value is kept in place rather than shifted down, so every test is +// a comparison against a JSVAL_SHIFTED_TAG_* constant. + +inline void MacroAssemblerIa64::emitTagOf(const ValueOperand& value, Register tag) { + emitMovl(ScratchReg.encoding(), ValueTagMask); + emitM(ia64::And(tag.encoding(), value.valueReg().encoding(), + ScratchReg.encoding())); +} + +inline uint32_t MacroAssemblerIa64::testTagEqual(Assembler::Condition cond, + Register tag, uint64_t tagBits) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + return emitCompare(cond, tag, ImmWord(tagBits)); +} + +inline uint32_t MacroAssemblerIa64::testTagRange(Assembler::Condition cond, + Register tag, uint64_t bound, + bool inclusiveBelow) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + Assembler::Condition c = + inclusiveBelow ? Assembler::BelowOrEqual : Assembler::AboveOrEqual; + if (cond == Assembler::NotEqual) { + c = inclusiveBelow ? Assembler::Above : Assembler::Below; + } + return emitCompare(c, tag, ImmWord(bound)); +} + +#define IA64_TAG_BRANCH(Name, TagBits) \ + void MacroAssembler::branchTest##Name(Condition cond, Register tag, \ + Label* label) { \ + uint32_t p = testTagEqual(cond, tag, TagBits); \ + emitBranchToLabel(&ia64::BrCondRel, p, label); \ + } \ + void MacroAssembler::branchTest##Name(Condition cond, \ + const ValueOperand& value, \ + Label* label) { \ + emitTagOf(value, SecondScratchReg); \ + branchTest##Name(cond, SecondScratchReg, label); \ + } \ + void MacroAssembler::branchTest##Name(Condition cond, \ + const Address& address, \ + Label* label) { \ + loadPtr(address, SecondScratchReg); \ + branchTest##Name(cond, ValueOperand(SecondScratchReg), label); \ + } \ + void MacroAssembler::branchTest##Name(Condition cond, \ + const BaseIndex& address, \ + Label* label) { \ + loadPtr(address, SecondScratchReg); \ + branchTest##Name(cond, ValueOperand(SecondScratchReg), label); \ + } + +IA64_TAG_BRANCH(Undefined, JSVAL_SHIFTED_TAG_UNDEFINED) +IA64_TAG_BRANCH(Int32, JSVAL_SHIFTED_TAG_INT32) +IA64_TAG_BRANCH(Boolean, JSVAL_SHIFTED_TAG_BOOLEAN) +IA64_TAG_BRANCH(Null, JSVAL_SHIFTED_TAG_NULL) +IA64_TAG_BRANCH(String, JSVAL_SHIFTED_TAG_STRING) +IA64_TAG_BRANCH(Symbol, JSVAL_SHIFTED_TAG_SYMBOL) +IA64_TAG_BRANCH(BigInt, JSVAL_SHIFTED_TAG_BIGINT) +IA64_TAG_BRANCH(Object, JSVAL_SHIFTED_TAG_OBJECT) +IA64_TAG_BRANCH(Magic, JSVAL_SHIFTED_TAG_MAGIC) + +#undef IA64_TAG_BRANCH + +// Range tests: double is "tag <= MAX_DOUBLE", number is "tag <= INT32", +// primitive is "tag < OBJECT" and GC thing is "tag >= STRING". + +#define IA64_TAG_RANGE_BRANCH(Name, Bound, CondEq, CondNe) \ + void MacroAssembler::branchTest##Name(Condition cond, Register tag, \ + Label* label) { \ + MOZ_ASSERT(cond == Equal || cond == NotEqual); \ + uint32_t p = \ + emitCompare(cond == Equal ? CondEq : CondNe, tag, ImmWord(Bound)); \ + emitBranchToLabel(&ia64::BrCondRel, p, label); \ + } \ + void MacroAssembler::branchTest##Name( \ + Condition cond, const ValueOperand& value, Label* label) { \ + emitTagOf(value, SecondScratchReg); \ + branchTest##Name(cond, SecondScratchReg, label); \ + } + +IA64_TAG_RANGE_BRANCH(Double, + uint64_t(JSVAL_TAG_MAX_DOUBLE) << JSVAL_TAG_SHIFT, + BelowOrEqual, Above) +IA64_TAG_RANGE_BRANCH(Number, JSVAL_SHIFTED_TAG_INT32, BelowOrEqual, Above) +IA64_TAG_RANGE_BRANCH(Primitive, JSVAL_SHIFTED_TAG_OBJECT, Below, AboveOrEqual) + +#undef IA64_TAG_RANGE_BRANCH + +// branchTestGCThing has no bare-tag form in the shared interface, so its three +// overloads are written out. +void MacroAssembler::branchTestGCThing(Condition cond, + const ValueOperand& value, + Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + emitTagOf(value, SecondScratchReg); + uint32_t p = emitCompare(cond == Equal ? AboveOrEqual : Below, + SecondScratchReg, + ImmWord(JSVAL_SHIFTED_TAG_STRING)); + emitBranchToLabel(&ia64::BrCondRel, p, label); +} + +void MacroAssembler::branchTestGCThing(Condition cond, const Address& address, + Label* label) { + loadPtr(address, SecondScratchReg); + branchTestGCThing(cond, ValueOperand(SecondScratchReg), label); +} + +void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address, + Label* label) { + loadPtr(address, SecondScratchReg); + branchTestGCThing(cond, ValueOperand(SecondScratchReg), label); +} + +void MacroAssembler::branchTestDouble(Condition cond, const Address& address, + Label* label) { + loadPtr(address, SecondScratchReg); + branchTestDouble(cond, ValueOperand(SecondScratchReg), label); +} + +void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address, + Label* label) { + loadPtr(address, SecondScratchReg); + branchTestDouble(cond, ValueOperand(SecondScratchReg), label); +} + +void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, + JSWhyMagic why, Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + Label notMagic; + if (cond == Equal) { + branchTestMagic(NotEqual, valaddr, ¬Magic); + } else { + branchTestMagic(NotEqual, valaddr, label); + } + load32(valaddr, SecondScratchReg); + branch32(cond, SecondScratchReg, Imm32(int32_t(why)), label); + bind(¬Magic); +} + +void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& valaddr, + JSWhyMagic why, Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + Label notMagic; + if (cond == Equal) { + branchTestMagic(NotEqual, valaddr, ¬Magic); + } else { + branchTestMagic(NotEqual, valaddr, label); + } + load32(valaddr, SecondScratchReg); + branch32(cond, SecondScratchReg, Imm32(int32_t(why)), label); + bind(¬Magic); +} + +template +void MacroAssembler::branchTestValue(Condition cond, const T& lhs, + const ValueOperand& rhs, Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + loadValue(lhs, ValueOperand(SecondScratchReg)); + branchPtr(cond, SecondScratchReg, rhs.valueReg(), label); +} + +// Truthiness tests. + +void MacroAssembler::branchTestInt32Truthy(bool truthy, + const ValueOperand& value, + Label* label) { + unboxInt32(value, SecondScratchReg); + branchTest32(truthy ? NonZero : Zero, SecondScratchReg, SecondScratchReg, + label); +} + +void MacroAssembler::branchTestBooleanTruthy(bool truthy, + const ValueOperand& value, + Label* label) { + unboxBoolean(value, SecondScratchReg); + branchTest32(truthy ? NonZero : Zero, SecondScratchReg, SecondScratchReg, + label); +} + +void MacroAssembler::branchTestStringTruthy(bool truthy, + const ValueOperand& value, + Label* label) { + unboxString(value, SecondScratchReg); + load32(Address(SecondScratchReg, JSString::offsetOfLength()), SecondScratchReg); + branchTest32(truthy ? NonZero : Zero, SecondScratchReg, SecondScratchReg, + label); +} + +void MacroAssembler::branchTestBigIntTruthy(bool truthy, + const ValueOperand& value, + Label* label) { + unboxBigInt(value, SecondScratchReg); + load32(Address(SecondScratchReg, BigInt::offsetOfDigitLength()), SecondScratchReg); + branchTest32(truthy ? NonZero : Zero, SecondScratchReg, SecondScratchReg, + label); +} + +void MacroAssembler::branchTestDoubleTruthy(bool truthy, FloatRegister reg, + Label* label) { + // ToBoolean(number) is false exactly for +-0 and NaN. + uint32_t p1 = 8, p2 = 9; + emitF(ia64::Fclass(p1, p2, reg.encoding(), + ia64::FclassZero | ia64::FclassSignBoth | ia64::FclassNaN, + 0)); + emitBranchToLabel(&ia64::BrCondRel, truthy ? p2 : p1, label); +} + +void MacroAssembler::branchToComputedAddress(const BaseIndex& address) { + loadPtr(address, SecondScratchReg); + jump(SecondScratchReg); +} + +void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest, + JSValueType type, Label* fail) { + MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING || + type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT); + emitTagOf(src, SecondScratchReg); + uint32_t p = emitCompare(NotEqual, SecondScratchReg, + ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type))); + emitBranchToLabel(&ia64::BrCondRel, p, fail); + unboxNonDouble(src, dest, type); +} + +// The Value has to be staged in |dest|, not in SecondScratchReg: the +// ValueOperand form below tags it via emitTagOf(src, SecondScratchReg), so a +// Value living there would be overwritten by its own tag, and the subsequent +// unbox would then xor the tag with itself and produce a null pointer. +void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest, + JSValueType type, Label* fail) { + loadPtr(src, dest); + fallibleUnboxPtr(ValueOperand(dest), dest, type, fail); +} + +// =========================================================================== +// Miscellaneous. + +void MacroAssembler::loadAbiReturnAddress(Register dest) { + emitI(ia64::MovFromBr(dest.encoding(), 0)); +} + +void MacroAssembler::memoryBarrier(MemoryBarrier barrier) { + // Emitted as a stop-bit-terminated bundle boundary; the real "mf" encoding is + // not available yet, and this backend is single-threaded for now. + if (barrier.isNone()) { + return; + } + emitM(ia64::NopM()); +} + +// Floating point is not supported yet: SupportsFloatingPoint() returns false, +// so these are only reachable through code paths that should be unreachable. + +// srcDest += src, in place, matching the x86-style two-operand form other +// callers of these particular overloads expect: dest is the *second* +// argument here (see MacroAssembler.h's DEFINED_ON declarations). +void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) { + emitF(ia64::FaddD(dest.encoding(), dest.encoding(), src.encoding(), ia64::sf0)); +} +void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) { + emitF(ia64::FsubD(dest.encoding(), dest.encoding(), src.encoding(), ia64::sf0)); +} +void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) { + emitF(ia64::FaddD(dest.encoding(), dest.encoding(), src.encoding(), ia64::sf0)); +} +void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) { + emitF(ia64::FsubD(dest.encoding(), dest.encoding(), src.encoding(), ia64::sf0)); +} +void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) { + emitF(ia64::FmpyD(dest.encoding(), dest.encoding(), src.encoding(), ia64::sf0)); +} +void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) { + ma_fdiv(dest, dest, src, /* isSingle = */ false); +} +void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) { + // fmerge.s dest = f0, src: sign from f0 (+0.0, i.e. positive), magnitude + // from src. + emitF(ia64::FmergeS(dest.encoding(), 0, src.encoding())); +} +void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + ma_fminmax(srcDest, other, srcDest, handleNaN, /* isMax = */ false, + /* isSingle = */ false); +} +void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest, + bool handleNaN) { + ma_fminmax(srcDest, other, srcDest, handleNaN, /* isMax = */ true, + /* isSingle = */ false); +} +void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp, + FloatRegister dest) { + movePtr(imm, temp); + ScratchDoubleScope scratch(*this); + loadDouble(Address(temp, 0), scratch); + mulDouble(scratch, dest); +} +void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs, + FloatRegister rhs, Label* label) { + uint32_t p = emitCompareDouble(cond, lhs, rhs); + emitBranchToLabel(&ia64::BrCondRel, p, label); +} +void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs, + FloatRegister rhs, Label* label) { + uint32_t p = emitCompareDouble(cond, lhs, rhs); + emitBranchToLabel(&ia64::BrCondRel, p, label); +} +void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src, + Register dest, + Label* fail) { + // Match the x86 "MaybeMod" contract: truncate and let the result wrap + // silently (mod 2^32) rather than bailing out, since ToInt32 semantics + // only need the low 32 bits and the wasm callers that need a real failure + // path use branchTruncateDoubleToInt32 instead. + // + // The wrap is only valid while the 64-bit conversion itself was exact. + // fcvt.fx.trunc yields the integer indefinite (INT64_MIN) for NaN, the + // infinities and every magnitude >= 2^63, and its low 32 bits are zero + // rather than the mod-2^32 answer, so those inputs go to |fail| -- the + // callers' slow path recomputes them with JS::ToInt32. + ScratchDoubleScope scratch(*this); + emitF(ia64::FcvtFxTrunc(scratch.encoding(), src.encoding(), ia64::sf0)); + emitM(ia64::GetfSig(dest.encoding(), scratch.encoding())); + emitMovl(ScratchReg.encoding(), uint64_t(INT64_MIN)); + uint32_t p = emitCompare(Assembler::Equal, dest, ScratchReg); + ma_sxt(dest, dest, 4); + emitBranchToLabel(&ia64::BrCondRel, p, fail); +} +void MacroAssembler::moveFloat16ToGPR(FloatRegister, Register) { + MOZ_CRASH("ia64: no floating point yet"); +} +void MacroAssembler::moveGPRToFloat16(Register, FloatRegister) { + MOZ_CRASH("ia64: no floating point yet"); +} +void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) { + emitM(ia64::GetfS(dest.encoding(), src.encoding())); +} +void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) { + emitM(ia64::SetfS(dest.encoding(), src.encoding())); +} +void MacroAssembler::moveLowDoubleToGPR(FloatRegister, Register) { + // Dead on every 64-bit backend (mips64/riscv64/loong64 all crash here too): + // the only caller is MacroAssembler.cpp's 32-bit hash path. + MOZ_CRASH("Not supported for this target"); +} +// MacroAssembler redeclares these four (unlike loadDouble/loadFloat32, which +// it inherits directly from MacroAssemblerIa64), so the real bodies in +// MacroAssemblerIa64 need an explicit qualification here to avoid recursing +// into this same overload. +FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src, + const Address& dest) { + return MacroAssemblerIa64::storeDouble(src, dest); +} +FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src, + const BaseIndex& dest) { + return MacroAssemblerIa64::storeDouble(src, dest); +} +FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src, + const Address& dest) { + return MacroAssemblerIa64::storeFloat32(src, dest); +} +FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src, + const BaseIndex& dest) { + return MacroAssemblerIa64::storeFloat32(src, dest); +} +FaultingCodeOffset MacroAssembler::storeFloat16(FloatRegister, const Address&, + Register) { + MOZ_CRASH("ia64: no floating point yet"); +} +FaultingCodeOffset MacroAssembler::storeFloat16(FloatRegister, + const BaseIndex&, Register) { + MOZ_CRASH("ia64: no floating point yet"); +} + +void MacroAssembler::clampIntToUint8(Register reg) { + // 0 if negative, 255 if above 255, unchanged otherwise. + uint32_t p = emitCompare(LessThan, reg, Imm32(0)); + emitM(ia64::MovReg(reg.encoding(), Registers::zero, p)); + p = emitCompare(GreaterThan, reg, Imm32(255)); + emitM(ia64::Adds(reg.encoding(), 255, Registers::zero, p)); +} + + +// =========================================================================== +// Operations that IA-64 cannot express yet: floating point (the F-unit +// encoders are still being added) and the wasm-only helpers. All of these are +// unreachable while SupportsFloatingPoint() returns false. + +void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) { + emitF(ia64::FmpyD(dest.encoding(), dest.encoding(), src.encoding(), ia64::sf0)); +} + +void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) { + ma_fdiv(dest, dest, src, /* isSingle = */ true); +} + +void MacroAssembler::negateFloat(FloatRegister reg) { + // fmerge.ns dest = src,src: sign is the complement of src's own sign, + // magnitude from src. + emitF(ia64::FmergeNs(reg.encoding(), reg.encoding(), reg.encoding())); +} + +void MacroAssembler::negateDouble(FloatRegister reg) { + emitF(ia64::FmergeNs(reg.encoding(), reg.encoding(), reg.encoding())); +} + +void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) { + ma_fsqrt(dest, src, /* isSingle = */ false); +} + +void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest, bool handleNaN) { + ma_fminmax(srcDest, srcDest, other, handleNaN, /* isMax = */ false, + /* isSingle = */ true); +} + +void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest, bool handleNaN) { + ma_fminmax(srcDest, srcDest, other, handleNaN, /* isMax = */ true, + /* isSingle = */ true); +} + +void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src, Register dest, Label* fail) { + ScratchDoubleScope scratch(*this); + emitF(ia64::FcvtFxTrunc(scratch.encoding(), src.encoding(), ia64::sf0)); + emitM(ia64::GetfSig(dest.encoding(), scratch.encoding())); + ma_sxt(dest, dest, 4); +} + +void MacroAssembler::wasmMulI64WideHI64(Register lhs, Register rhs, Register output, bool isSigned) { + MOZ_CRASH("ia64: wasmMulI64WideHI64 is not supported"); +} + +// 64-bit add/sub overflow via the classic sign trick: for add, overflow iff +// the two addends share a sign and the result doesn't ((a^r)&(b^r)) < 0); +// for sub, overflow iff the operands differ in sign and the result doesn't +// match the minuend's (((a^b)&(a^r)) < 0). +template +inline void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest, + Label* label) { + // An immediate |src| is materialized into ScratchReg, so |a| has to be + // staged elsewhere and |s| consumed before ScratchReg is reused -- stashing + // |a| in ScratchReg here would overwrite the addend with |dest| and turn + // the add into dest+dest. + Register s = MaterializeSrc(*this, src); + MOZ_ASSERT(dest != SecondScratchReg); + emitM(ia64::MovReg(SecondScratchReg.encoding(), dest.encoding())); // a + emitM(ia64::Add(dest.encoding(), dest.encoding(), s.encoding())); + if (cond == Assembler::Overflow) { + emitM(ia64::Xor(SecondScratchReg.encoding(), SecondScratchReg.encoding(), + dest.encoding())); // a^r + emitM(ia64::Xor(ScratchReg.encoding(), s.encoding(), dest.encoding())); // s^r + emitM(ia64::And(ScratchReg.encoding(), ScratchReg.encoding(), + SecondScratchReg.encoding())); + uint32_t p = emitCompare(Assembler::LessThan, ScratchReg, + Register{Registers::zero}); + emitBranchToLabel(&ia64::BrCondRel, p, label); + return; + } + MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned); + branchTestPtr(cond, dest, dest, label); +} + +// =========================================================================== +// Byte/half extends, and the remaining scalar helpers. + +void MacroAssembler::move8ZeroExtend(Register src, Register dest) { + ma_zxt(dest, src, 1); +} +void MacroAssembler::move8SignExtend(Register src, Register dest) { + ma_sxt(dest, src, 1); +} +void MacroAssembler::move16SignExtend(Register src, Register dest) { + ma_sxt(dest, src, 2); +} +void MacroAssembler::move8SignExtendToPtr(Register src, Register dest) { + ma_sxt(dest, src, 1); +} +void MacroAssembler::move16SignExtendToPtr(Register src, Register dest) { + ma_sxt(dest, src, 2); +} +void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) { + ma_sxt(dest, src, 4); +} + +void MacroAssembler::notPtr(Register reg) { not32(reg); } + +void MacroAssembler::mulBy3(Register src, Register dest) { + emitM(ia64::Add(ScratchReg.encoding(), src.encoding(), src.encoding())); + emitM(ia64::Add(dest.encoding(), ScratchReg.encoding(), src.encoding())); +} + +void MacroAssembler::abs32(Register src, Register dest) { + movePtr(src, dest); + uint32_t p = emitCompare(LessThan, dest, Imm32(0)); + emitM(ia64::Sub(dest.encoding(), Registers::zero, dest.encoding(), p)); +} + +void MacroAssembler::minPtr(Register lhs, Register rhs, Register result) { + min32(lhs, rhs, result); +} +void MacroAssembler::minPtr(Register lhs, ImmWord rhs, Register result) { + emitMovl(ScratchReg.encoding(), rhs.value); + min32(lhs, ScratchReg, result); +} +void MacroAssembler::maxPtr(Register lhs, Register rhs, Register result) { + max32(lhs, rhs, result); +} +void MacroAssembler::maxPtr(Register lhs, ImmWord rhs, Register result) { + emitMovl(ScratchReg.encoding(), rhs.value); + max32(lhs, ScratchReg, result); +} + +void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) { + ma_asr(dest, dest, imm); +} +void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register src, + Register dest) { + ma_asr(dest, src, imm); +} +void MacroAssembler::rshiftPtrArithmetic(Register shift, Register srcDest) { + ma_asr(srcDest, srcDest, shift); +} +void MacroAssembler::flexibleRshiftPtrArithmetic(Register shift, + Register srcDest) { + ma_asr(srcDest, srcDest, shift); +} + +void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs, + Label* label) { + bool isSigned = cond != Above && cond != AboveOrEqual && cond != Below && + cond != BelowOrEqual; + if (isSigned) { + load8SignExtend(lhs, SecondScratchReg); + } else { + load8ZeroExtend(lhs, SecondScratchReg); + } + branch32(cond, SecondScratchReg, rhs, label); +} + +// Register64 is pointer-width here, so an int64 always fits in an intptr. +void MacroAssembler::branchInt64NotInPtrRange(Register64 src, Label* label) {} +void MacroAssembler::branchUInt64NotInPtrRange(Register64 src, Label* label) {} + +void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs, + Label* label) { + subPtr(rhs, lhs); + branchPtr(cond, lhs, Imm32(0), label); +} + +// Staged in |dest| for the same reason as the Address form above. +void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest, + JSValueType type, Label* fail) { + loadPtr(src, dest); + fallibleUnboxPtr(ValueOperand(dest), dest, type, fail); +} + +void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Imm32 rhs, + const Address& src, Register dest) { + // p6/p7 are complements of one another, so xor-ing the low bit of the + // predicate number gives the inverse condition. + uint32_t p = emitCompare(cond, lhs, rhs); + Label skip; + emitBranchToLabel(&ia64::BrCondRel, p ^ 1, &skip); + load32(src, dest); + bind(&skip); +} + +// Value tag tests that leave a boolean in a register. + +#define IA64_TAG_SET(Name, Bound, CondEq, CondNe) \ + template \ + void MacroAssembler::Name(Condition cond, const T& src, Register dest) { \ + MOZ_ASSERT(cond == Equal || cond == NotEqual); \ + emitTagOf(src, SecondScratchReg); \ + cmp32Set(cond == Equal ? CondEq : CondNe, SecondScratchReg, \ + ImmWord(Bound), dest); \ + } + +IA64_TAG_SET(testNumberSet, JSVAL_SHIFTED_TAG_INT32, BelowOrEqual, Above) +IA64_TAG_SET(testBooleanSet, JSVAL_SHIFTED_TAG_BOOLEAN, Equal, NotEqual) +IA64_TAG_SET(testStringSet, JSVAL_SHIFTED_TAG_STRING, Equal, NotEqual) +IA64_TAG_SET(testSymbolSet, JSVAL_SHIFTED_TAG_SYMBOL, Equal, NotEqual) +IA64_TAG_SET(testBigIntSet, JSVAL_SHIFTED_TAG_BIGINT, Equal, NotEqual) + +#undef IA64_TAG_SET + +// Overflow-checking and bit-counting forms still need instructions IA-64 +// expresses differently (czx/popcnt/mux1) or an explicit overflow test. + +template +void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest, + Label* label) { + // See branchAddPtr: |a| must not be staged in ScratchReg, which an + // immediate |src| already occupies. + Register s = MaterializeSrc(*this, src); + MOZ_ASSERT(dest != SecondScratchReg); + emitM(ia64::MovReg(SecondScratchReg.encoding(), dest.encoding())); // a + emitM(ia64::Sub(dest.encoding(), dest.encoding(), s.encoding())); + if (cond == Assembler::Overflow) { + emitM(ia64::Xor(ScratchReg.encoding(), SecondScratchReg.encoding(), + s.encoding())); // a^b + emitM(ia64::Xor(SecondScratchReg.encoding(), SecondScratchReg.encoding(), + dest.encoding())); // a^r + emitM(ia64::And(ScratchReg.encoding(), ScratchReg.encoding(), + SecondScratchReg.encoding())); + uint32_t p = emitCompare(Assembler::LessThan, ScratchReg, + Register{Registers::zero}); + emitBranchToLabel(&ia64::BrCondRel, p, label); + return; + } + MOZ_ASSERT(cond == Assembler::Signed || cond == Assembler::NotSigned); + branchTestPtr(cond, dest, dest, label); +} +// -MIN doesn't fit back in the same width, so that's the only value whose +// negation overflows; test for it before clobbering the register. +void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) { + MOZ_ASSERT(cond == Assembler::Overflow); + uint32_t p = emitCompare(Assembler::Equal, reg, Imm32(INT32_MIN)); + neg32(reg); + emitBranchToLabel(&ia64::BrCondRel, p, label); +} +void MacroAssembler::branchNegPtr(Condition cond, Register reg, Label* label) { + MOZ_ASSERT(cond == Assembler::Overflow); + emitMovl(ScratchReg.encoding(), uint64_t(INT64_MIN)); + uint32_t p = emitCompare(Assembler::Equal, reg, ScratchReg); + negPtr(reg); + emitBranchToLabel(&ia64::BrCondRel, p, label); +} +// mux1 @rev reverses all 8 bytes of the register; a 16/32-bit swap of the +// low bytes then falls out of the high end of that result, so a logical +// shift brings it back down to bit 0. +void MacroAssembler::byteSwap16SignExtend(Register reg) { + emitI(ia64::Mux1(ScratchReg.encoding(), reg.encoding(), ia64::kMux1Rev)); + emitI(ia64::ShrImm(reg.encoding(), ScratchReg.encoding(), 48)); +} +void MacroAssembler::byteSwap16ZeroExtend(Register reg) { + emitI(ia64::Mux1(ScratchReg.encoding(), reg.encoding(), ia64::kMux1Rev)); + emitI(ia64::ShrUImm(reg.encoding(), ScratchReg.encoding(), 48)); +} +void MacroAssembler::byteSwap32(Register reg) { + emitI(ia64::Mux1(ScratchReg.encoding(), reg.encoding(), ia64::kMux1Rev)); + emitI(ia64::ShrUImm(reg.encoding(), ScratchReg.encoding(), 32)); +} +void MacroAssembler::byteSwap64(Register64 reg) { + emitI(ia64::Mux1(reg.reg.encoding(), reg.reg.encoding(), ia64::kMux1Rev)); +} + +// clz via the exponent trick: setf.sig places the raw bit pattern in +// register format as an (always non-negative, per the ISA's fixed integer +// exponent) fixed-point value; normalizing it with fnorm re-floats it, and +// the resulting biased exponent (bias 0xffff for register format) directly +// gives the bit position of the highest set bit. x == 0 has to be special- +// cased since it normalizes to an exponent of 0, not 0xffff-width. +static void Clz(MacroAssembler& masm, Register dest, Register src, + unsigned width) { + const uint32_t f = 2; + // A 32-bit value is kept sign-extended in its 64-bit register (see + // load32's comment), which would make the exponent trick see phantom + // high bits for negative-looking 32-bit values; zero-extend first so only + // the intended width's bits are ever visible to it. + Register magnitude = src; + if (width == 32) { + masm.ma_zxt(ScratchReg, src, 4); + magnitude = ScratchReg; + } + masm.emitM(ia64::SetfSig(f, magnitude.encoding())); + masm.emitF(ia64::Fnorm(f, f, ia64::sf1)); + masm.emitM(ia64::GetfExp(dest.encoding(), f)); + masm.emitMovl(SecondScratchReg.encoding(), 0xffff + (width - 1)); + uint32_t pZero = masm.emitCompare(Assembler::Equal, src, Imm32(0)); + masm.emitM(ia64::Sub(dest.encoding(), SecondScratchReg.encoding(), + dest.encoding())); + masm.emitM(ia64::Adds(dest.encoding(), int64_t(width), Registers::zero, pZero)); +} +void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) { + Clz(*this, dest, src, 32); +} +void MacroAssembler::clz64(Register64 src, Register64 dest) { + Clz(*this, dest.reg, src.reg, 64); +} + +// ctz(x) = popcnt((x & -x) - 1): x & -x isolates the lowest set bit, and +// subtracting 1 from a lone bit at position k turns it into k ones. +static void Ctz(MacroAssembler& masm, Register dest, Register src, + unsigned width) { + masm.emitM(ia64::Sub(ScratchReg.encoding(), Registers::zero, src.encoding())); + masm.emitM(ia64::And(ScratchReg.encoding(), src.encoding(), ScratchReg.encoding())); + masm.emitM(ia64::Adds(ScratchReg.encoding(), -1, ScratchReg.encoding())); + masm.emitI(ia64::Popcnt(dest.encoding(), ScratchReg.encoding())); + uint32_t pZero = masm.emitCompare(Assembler::Equal, src, Imm32(0)); + masm.emitM(ia64::Adds(dest.encoding(), int64_t(width), Registers::zero, pZero)); +} +void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) { + Ctz(*this, dest, src, 32); +} +void MacroAssembler::ctz64(Register64 src, Register64 dest) { + Ctz(*this, dest.reg, src.reg, 64); +} + +void MacroAssembler::popcnt32(Register src, Register dest, Register temp) { + ma_zxt(ScratchReg, src, 4); + emitI(ia64::Popcnt(dest.encoding(), ScratchReg.encoding())); +} +void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register temp) { + emitI(ia64::Popcnt(dest.reg.encoding(), src.reg.encoding())); +} +void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) { + emitM(ia64::GetfD(dest.reg.encoding(), src.encoding())); +} +void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) { + emitM(ia64::SetfD(dest.encoding(), src.reg.encoding())); +} +void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) { + emitF(ia64::FmergeS(dest.encoding(), 0, src.encoding())); +} +void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) { + ma_fsqrt(dest, src, /* isSingle = */ true); +} +void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src, + Register dest, Label* fail) { + ma_truncateToInt32(src, dest, fail); +} +void MacroAssembler::wasmAddSubI128HI64(Register lhsLo, Register lhsHi, + Register rhsLo, Register rhsHi, + Register output, bool isAdd) { + MOZ_CRASH("ia64: wasm is not supported"); +} + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_MacroAssembler_ia64_inl_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/firefox-153.0.1/js/src/jit/ia64/MacroAssembler-ia64.cpp b/firefox-153.0.1/js/src/jit/ia64/MacroAssembler-ia64.cpp --- firefox-153.0.1/js/src/jit/ia64/MacroAssembler-ia64.cpp.vanilla +++ firefox-153.0.1/js/src/jit/ia64/MacroAssembler-ia64.cpp @@ -0,0 +1,3213 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#include "jit/ia64/MacroAssembler-ia64.h" + +#include "mozilla/Maybe.h" + +#include "gc/Marking.h" +#include "jit/AutoWritableJitCode.h" +#include "jit/BaselineFrame.h" +#include "jit/ExecutableAllocator.h" +#include "jit/FlushICache.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/MoveEmitter.h" +#include "util/PortableMath.h" // js::GetBiggestNumberLessThan +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "wasm/WasmStubs.h" // js::wasm::GenerateJumpToCatchHandler + +#include "jit/MacroAssembler-inl.h" + +namespace js { +namespace jit { + +MacroAssembler& MacroAssemblerIa64::asMasm() { + return *static_cast(this); +} + +const MacroAssembler& MacroAssemblerIa64::asMasm() const { + return *static_cast(this); +} + +// =========================================================================== +// Relocations. ReadMovlImm()/WriteMovlImm() live in MacroAssembler-ia64.h. + +void MacroAssemblerIa64::writeDataRelocation(ImmGCPtr ptr) { + if (!ptr.value) { + return; + } + if (gc::IsInsideNursery(ptr.value)) { + embedsNurseryPointers_ = true; + } + dataRelocations_.writeUnsigned(m_buffer.nextOffset().getOffset()); +} + +void MacroAssemblerIa64::writeJumpRelocation() { + jumpRelocations_.writeUnsigned(m_buffer.nextOffset().getOffset()); +} + +uintptr_t Assembler::GetPointer(uint8_t* addr) { + return uintptr_t(ReadMovlImm(addr)); +} + +void Assembler::PatchDataWithValueCheck(CodeLocationLabel label, + ImmWord newValue, + ImmWord expectedValue) { + uint8_t* addr = label.raw(); + MOZ_ASSERT(ReadMovlImm(addr) == expectedValue.value); + WriteMovlImm(addr, newValue.value); + FlushICache(addr, sizeof(ia64::Bundle)); +} + +void Assembler::PatchWrite_NearCall(CodeLocationLabel start, + CodeLocationLabel toCall) { + // The only caller is Ion invalidation, which patches over an OsiPoint. What + // sits there is ordinary code (or ensureOsiSpace()'s nop padding), not a + // call sequence, so all three bundles have to be built from scratch -- + // read-modify-writing a movl that isn't there scatters the immediate across + // whatever instructions were in slots 1 and 2. + uint8_t* addr = start.raw(); + ia64::Bundle* bundles = reinterpret_cast(addr); + bundles[0] = ia64::MovlBundle(ScratchReg.encoding(), + uint64_t(uintptr_t(toCall.raw())), 0); + bundles[1] = ia64::BundleI(ia64::MovToBr(6, ScratchReg.encoding())); + bundles[2] = ia64::BundleB(ia64::BrCall(0, 6)); + FlushICache(addr, 3 * sizeof(ia64::Bundle)); +} + +void MacroAssemblerIa64::TraceDataRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + mozilla::Maybe awjc; + while (reader.more()) { + size_t offset = reader.readUnsigned(); + uint8_t* bundle = code->raw() + offset; + uint64_t word = ReadMovlImm(bundle); + if (!word) { + continue; + } + gc::Cell* cell = reinterpret_cast(uintptr_t(word)); + gc::Cell* prior = cell; + TraceManuallyBarrieredGenericPointerEdge(trc, &cell, "jit-masm-ptr"); + if (cell != prior) { + if (awjc.isNothing()) { + awjc.emplace(code); + } + WriteMovlImm(bundle, uint64_t(uintptr_t(cell))); + } + } +} + +void MacroAssemblerIa64::TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + mozilla::Maybe awjc; + while (reader.more()) { + size_t offset = reader.readUnsigned(); + uint8_t* bundle = code->raw() + offset; + uint64_t word = ReadMovlImm(bundle); + if (!word) { + continue; + } + JitCode* child = + JitCode::FromExecutable(reinterpret_cast(uintptr_t(word))); + TraceManuallyBarrieredEdge(trc, &child, "jitcode"); + if (uintptr_t(child->raw()) != uintptr_t(word)) { + if (awjc.isNothing()) { + awjc.emplace(code); + } + WriteMovlImm(bundle, uint64_t(uintptr_t(child->raw()))); + } + } +} + +void MacroAssemblerIa64::movePtr(ImmGCPtr imm, Register dest) { + writeDataRelocation(imm); + emitMovl(dest.encoding(), uint64_t(uintptr_t(imm.value))); +} + +// =========================================================================== +// Compares and tests. Predicates p6 (the relation) and p7 (its complement) are +// the scratch pair; the returned value says which of the two the caller should +// predicate on. + +static const uint32_t kP = 6; +static const uint32_t kQ = 7; + +uint32_t MacroAssemblerIa64::emitCompare(Assembler::Condition cond, + Register lhs, Register rhs) { + uint32_t a = lhs.encoding(); + uint32_t b = rhs.encoding(); + + switch (cond) { + case Assembler::Equal: + emitI(ia64::CmpEq(kP, kQ, a, b)); + return kP; + case Assembler::NotEqual: + emitI(ia64::CmpEq(kP, kQ, a, b)); + return kQ; + case Assembler::LessThan: + emitI(ia64::CmpLt(kP, kQ, a, b)); + return kP; + case Assembler::GreaterThanOrEqual: + emitI(ia64::CmpLt(kP, kQ, a, b)); + return kQ; + case Assembler::GreaterThan: + // a > b <=> b < a + emitI(ia64::CmpLt(kP, kQ, b, a)); + return kP; + case Assembler::LessThanOrEqual: + emitI(ia64::CmpLt(kP, kQ, b, a)); + return kQ; + case Assembler::Below: + emitI(ia64::CmpLtu(kP, kQ, a, b)); + return kP; + case Assembler::AboveOrEqual: + emitI(ia64::CmpLtu(kP, kQ, a, b)); + return kQ; + case Assembler::Above: + emitI(ia64::CmpLtu(kP, kQ, b, a)); + return kP; + case Assembler::BelowOrEqual: + emitI(ia64::CmpLtu(kP, kQ, b, a)); + return kQ; + // In a two-operand compare, Zero/NonZero are the x86-shared spellings of + // Equal/NotEqual against the right-hand side (generic code writes e.g. + // branchPtr(NonZero, addr, ImmPtr(nullptr))). The and-style tests, where + // they mean something else, go through emitTest() instead. + case Assembler::Zero: + emitI(ia64::CmpEq(kP, kQ, a, b)); + return kP; + case Assembler::NonZero: + emitI(ia64::CmpEq(kP, kQ, a, b)); + return kQ; + default: + MOZ_CRASH("unsupported condition for ia64 compare"); + } +} + +uint32_t MacroAssemblerIa64::emitCompare(Assembler::Condition cond, + Register lhs, Imm32 rhs) { + if (rhs.value == 0) { + return emitCompare(cond, lhs, Register{Registers::zero}); + } + emitMovl(ScratchReg.encoding(), uint64_t(int64_t(rhs.value))); + return emitCompare(cond, lhs, ScratchReg); +} + +uint32_t MacroAssemblerIa64::emitCompare(Assembler::Condition cond, + Register lhs, ImmWord rhs) { + if (rhs.value == 0) { + return emitCompare(cond, lhs, Register{Registers::zero}); + } + emitMovl(ScratchReg.encoding(), rhs.value); + return emitCompare(cond, lhs, ScratchReg); +} + +uint32_t MacroAssemblerIa64::emitTest(Assembler::Condition cond, Register lhs, + Register rhs, unsigned bits) { + MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero || + cond == Assembler::Signed || cond == Assembler::NotSigned); + MOZ_ASSERT(bits == 32 || bits == 64); + emitM(ia64::And(ScratchReg.encoding(), lhs.encoding(), rhs.encoding())); + // A 32-bit operand is not always sign-extended into its register -- a uint32 + // produced by rshift32() is zero-extended -- so bit 63 is not a copy of bit + // 31 and both Zero and Signed have to be read off the low half alone. + if (bits == 32) { + ma_sxt(ScratchReg, ScratchReg, 4); + } + if (cond == Assembler::Zero || cond == Assembler::NonZero) { + emitI(ia64::CmpEq(kP, kQ, ScratchReg.encoding(), Registers::zero)); + return cond == Assembler::Zero ? kP : kQ; + } + emitI(ia64::CmpLt(kP, kQ, ScratchReg.encoding(), Registers::zero)); + return cond == Assembler::Signed ? kP : kQ; +} + +uint32_t MacroAssemblerIa64::emitTest(Assembler::Condition cond, Register lhs, + Imm32 rhs, unsigned bits) { + MOZ_ASSERT(bits == 32 || bits == 64); + emitMovl(ScratchReg.encoding(), uint64_t(uint32_t(rhs.value))); + emitM( + ia64::And(ScratchReg.encoding(), lhs.encoding(), ScratchReg.encoding())); + if (bits == 32) { + ma_sxt(ScratchReg, ScratchReg, 4); + } + if (cond == Assembler::Zero || cond == Assembler::NonZero) { + emitI(ia64::CmpEq(kP, kQ, ScratchReg.encoding(), Registers::zero)); + return cond == Assembler::Zero ? kP : kQ; + } + emitI(ia64::CmpLt(kP, kQ, ScratchReg.encoding(), Registers::zero)); + return cond == Assembler::Signed ? kP : kQ; +} + +// Every base (non-.unc) fcmp already treats "unordered" as false for its +// primary relation p1, and true for its complement p2 -- e.g. fcmp.lt's p2 is +// exactly "not less than", which for IEEE comparisons is ">= or unordered". +// That covers every DoubleCondition except an ordered-only "not equal" (kQ of +// fcmp.eq folds in "or unordered", which JS's DoubleNotEqual must exclude) +// and "equal or unordered" (the OR of two conditions computed by different +// instructions, which no single fcmp predicate gives directly). +uint32_t MacroAssemblerIa64::emitCompareDouble(Assembler::DoubleCondition cond, + FloatRegister lhs, + FloatRegister rhs, + uint32_t sf) { + uint32_t a = lhs.encoding(); + uint32_t b = rhs.encoding(); + + switch (cond) { + case Assembler::DoubleEqual: + emitF(ia64::FcmpEq(kP, kQ, a, b, sf)); + return kP; + case Assembler::DoubleNotEqualOrUnordered: + emitF(ia64::FcmpEq(kP, kQ, a, b, sf)); + return kQ; + case Assembler::DoubleLessThan: + emitF(ia64::FcmpLt(kP, kQ, a, b, sf)); + return kP; + case Assembler::DoubleGreaterThanOrEqualOrUnordered: + emitF(ia64::FcmpLt(kP, kQ, a, b, sf)); + return kQ; + case Assembler::DoubleLessThanOrEqual: + emitF(ia64::FcmpLe(kP, kQ, a, b, sf)); + return kP; + case Assembler::DoubleGreaterThanOrUnordered: + emitF(ia64::FcmpLe(kP, kQ, a, b, sf)); + return kQ; + // a > b <=> b < a, and a >= b <=> b <= a; the base relation is + // already false-on-unordered, so no compound step is needed here either. + case Assembler::DoubleGreaterThan: + emitF(ia64::FcmpLt(kP, kQ, b, a, sf)); + return kP; + case Assembler::DoubleGreaterThanOrEqual: + emitF(ia64::FcmpLe(kP, kQ, b, a, sf)); + return kP; + case Assembler::DoubleLessThanOrUnordered: + emitF(ia64::FcmpLe(kP, kQ, b, a, sf)); + return kQ; + case Assembler::DoubleLessThanOrEqualOrUnordered: + emitF(ia64::FcmpLt(kP, kQ, b, a, sf)); + return kQ; + case Assembler::DoubleUnordered: + emitF(ia64::FcmpUnord(kP, kQ, a, b, sf)); + return kP; + case Assembler::DoubleOrdered: + emitF(ia64::FcmpUnord(kP, kQ, a, b, sf)); + return kQ; + case Assembler::DoubleNotEqual: { + // kP/kQ = unordered/ordered; qualify the second compare on kQ so an + // unordered pair leaves p9 at 0 rather than the stale != result. + emitF(ia64::FcmpUnord(kP, kQ, a, b, sf)); + emitF(ia64::FcmpEqUnc(8, 9, a, b, sf, kQ)); + return 9; + } + case Assembler::DoubleEqualOrUnordered: { + // No single fcmp predicate is "eq or unordered" (kP of fcmp.eq excludes + // unordered; kQ includes != as well), so OR the two disjoint cases + // through a GR: 1 if unordered, 1 if ordered-and-equal, 0 otherwise. + emitF(ia64::FcmpUnord(kP, kQ, a, b, sf)); + emitF(ia64::FcmpEqUnc(8, 9, a, b, sf, kQ)); + emitM(ia64::MovReg(ScratchReg.encoding(), Registers::zero)); + emitM(ia64::Adds(ScratchReg.encoding(), 1, Registers::zero, kP)); + emitM(ia64::Adds(ScratchReg.encoding(), 1, Registers::zero, 8)); + emitI(ia64::CmpNe(kP, kQ, ScratchReg.encoding(), Registers::zero)); + return kP; + } + default: + MOZ_CRASH("unsupported double condition for ia64 fcmp"); + } +} + +// =========================================================================== +// Floating point divide and sqrt. IA-64 has no divide or sqrt instruction; +// both go through frcpa/frsqrta plus Newton-Raphson refinement, copied +// verbatim (register-renamed) from the sequences libgcc/gcc ship for +// __divdf3/__divsf3 and sqrtdf2/sqrtsf2 -- see +// /tmp/gcc-16.1.0/libgcc/config/ia64/lib1funcs.S and +// /tmp/gcc-16.1.0/gcc/config/ia64/div.md. The Newton steps use sf1 so their +// intermediate rounding/exceptions don't disturb the user-visible sf0 flags; +// only the final rounding step writes under sf0. +// +// f2-f5 are reserved (see Architecture-ia64.h) exactly so these sequences +// always have enough live temporaries regardless of what's allocated +// elsewhere. + +void MacroAssemblerIa64::ma_fdiv(FloatRegister dest, FloatRegister lhs, + FloatRegister rhs, bool isSingle) { + uint32_t a = lhs.encoding(), b = rhs.encoding(), d = dest.encoding(); + const uint32_t t0 = 2, t1 = 3, t2 = 4, t3 = 5; + const uint32_t p6 = 6, p7 = 7, pDiscard = 8; + + emitI(ia64::CmpEq(p7, pDiscard, Registers::zero, Registers::zero)); + emitF(ia64::Frcpa(t0, p6, a, b, ia64::sf0)); + emitI(ia64::CmpNe(p7, pDiscard, Registers::zero, Registers::zero, p6)); + + if (isSingle) { + emitF(ia64::Fma(t1, a, t0, ia64::fpZero, ia64::sf1, p6)); // t1=a*t0 + emitF(ia64::Fnma(t2, b, t0, ia64::fpOne, ia64::sf1, p6)); // t2=1-b*t0 + emitF(ia64::Fma(t1, t2, t1, t1, ia64::sf1, p6)); + emitF(ia64::Fma(t2, t2, t2, ia64::fpZero, ia64::sf1, p6)); // t2=t2*t2 + emitF(ia64::Fma(t1, t2, t1, t1, ia64::sf1, p6)); + emitF(ia64::Fma(t2, t2, t2, ia64::fpZero, ia64::sf1, p6)); + emitF(ia64::FmaD(t0, t2, t1, t1, ia64::sf1, p6)); + emitF(ia64::FmaS(d, t0, ia64::fpOne, ia64::fpZero, ia64::sf0, p6)); // fnorm.s + emitF(ia64::FmergeS(d, t0, t0, p7)); + return; + } + + emitF(ia64::Fma(t1, a, t0, ia64::fpZero, ia64::sf1, p6)); // t1 = a*t0 + emitF(ia64::Fnma(t2, b, t0, ia64::fpOne, ia64::sf1, p6)); // t2 = 1-b*t0 + emitF(ia64::Fma(t1, t2, t1, t1, ia64::sf1, p6)); + emitF(ia64::Fma(t3, t2, t2, ia64::fpZero, ia64::sf1, p6)); // t3 = t2*t2 + emitF(ia64::Fma(t0, t2, t0, t0, ia64::sf1, p6)); + emitF(ia64::Fma(t1, t3, t1, t1, ia64::sf1, p6)); + emitF(ia64::Fma(t2, t3, t3, ia64::fpZero, ia64::sf1, p6)); // t2 = t3*t3 + emitF(ia64::Fma(t0, t3, t0, t0, ia64::sf1, p6)); + emitF(ia64::FmaD(t1, t2, t1, t1, ia64::sf1, p6)); + emitF(ia64::Fma(t0, t2, t0, t0, ia64::sf1, p6)); + emitF(ia64::FnmaD(t3, b, t1, a, ia64::sf1, p6)); // t3 = a - b*t1 (reuses t3) + emitF(ia64::FmaD(d, t3, t0, t1, ia64::sf0, p6)); + emitF(ia64::FmergeS(d, t0, t0, p7)); +} + +// IA-64 has no sqrt instruction. frsqrta approximates 1/sqrt(a) in t0, with +// p6 flagging whether Newton-Raphson refinement of t0 (on f(y) = 1/y^2 - a) +// is needed; every Newton step below is predicated on p6, so it is simply +// skipped -- t0 unchanged -- when frsqrta's initial estimate is already +// exact (which covers NaN, negative numbers, and exact powers of the +// radix). sqrt(a) = a*t0 then holds unconditionally, EXCEPT for a = +-0 +// (0 * infinity, handled by a separate pre-check since sqrt(+-0) == +-0 +// bit-for-bit anyway) and a = +infinity (0 * infinity the other way around; +// frsqrta(-infinity) instead correctly yields NaN, so only +infinity needs +// its own fixup). +void MacroAssemblerIa64::ma_fsqrt(FloatRegister dest, FloatRegister src, + bool isSingle) { + uint32_t a = src.encoding(), d = dest.encoding(); + const uint32_t t0 = 2, terr = 3, half = 4; + const uint32_t p6 = 6, p7 = 7, p8 = 8, p9 = 9; + Label done; + + emitF(ia64::Fclass(p8, p9, a, ia64::FclassAnySignZero, 0)); + emitF(ia64::FmergeS(d, a, a, p8)); // sqrt(+-0) == that same +-0 + emitBranchToLabel(&ia64::BrCondRel, p8, &done); + + // Newton-Raphson on f(y) = 1/y^2 - a converges as y += y*e/2, where + // e = 1 - a*y^2. The halving is not optional: with the full e the error + // term maps to roughly -e each step, so the iteration oscillates around + // the root instead of converging and the result keeps frsqrta's initial + // ~8-bit accuracy. IA-64 hardwires only f0 = 0.0 and f1 = 1.0, so 0.5 has + // to be materialised. Intermediates stay in register format (no .d) so the + // extra precision is not thrown away between steps. + emitMovl(ScratchReg.encoding(), 0x3fe0000000000000ULL); + emitM(ia64::SetfD(half, ScratchReg.encoding())); + + emitF(ia64::Frsqrta(t0, p6, a, ia64::sf0)); + for (int i = 0; i < 5; i++) { + emitF(ia64::Fma(terr, t0, t0, ia64::fpZero, ia64::sf1, p6)); // t0^2 + emitF(ia64::Fnma(terr, a, terr, ia64::fpOne, ia64::sf1, p6)); // 1-a*t0^2 + emitF(ia64::Fma(terr, terr, half, ia64::fpZero, ia64::sf1, p6)); // e/2 + emitF(ia64::Fma(t0, t0, terr, t0, ia64::sf1, p6)); // t0 += t0*e/2 + } + if (isSingle) { + emitF(ia64::FmaS(d, a, t0, ia64::fpZero, ia64::sf0)); + } else { + emitF(ia64::FmaD(d, a, t0, ia64::fpZero, ia64::sf0)); + } + emitF(ia64::Fclass(p8, p9, a, ia64::FclassPosInf, 0)); + emitF(ia64::FmergeS(d, a, a, p8)); // fix up the 0*inf mishap for +infinity + + bind(&done); +} + +// JS Math.min/max: NaN propagates (handleNaN), and -0 sorts below +0 even +// though fcmp treats them as equal. fcmp.lt already gives the right answer +// whenever the operands compare unequal; the only case it can't distinguish +// is the +-0 pair, where a bitwise AND of the two patterns yields +0 and a +// bitwise OR yields -0 -- and both are identities when the patterns match. +// +// emitCompareDouble()'s results always land in the same fixed pair of +// predicate registers (kP/kQ), so each compare has to be fully consumed +// before the next one is emitted. The result is accumulated in a scratch +// because |dest| aliases |rhs| for the srcDest callers while both inputs stay +// live to the end. +void MacroAssemblerIa64::ma_fminmax(FloatRegister dest, FloatRegister lhs, + FloatRegister rhs, bool handleNaN, + bool isMax, bool isSingle) { + uint32_t a = lhs.encoding(), b = rhs.encoding(); + + ScratchDoubleScope acc(asMasm()); + uint32_t t = acc.encoding(); + + // Default to b, then take a when it strictly wins. + emitF(ia64::FmergeS(t, b, b, 0)); + uint32_t pAWins = emitCompareDouble( + isMax ? Assembler::DoubleGreaterThan : Assembler::DoubleLessThan, lhs, + rhs); + emitF(ia64::FmergeS(t, a, a, pAWins)); + + // Resolve the +-0 tie by merging the raw bit patterns. + uint32_t pEq = emitCompareDouble(Assembler::DoubleEqual, lhs, rhs); + emitM(ia64::GetfD(ScratchReg.encoding(), a, pEq)); + emitM(ia64::GetfD(SecondScratchReg.encoding(), b, pEq)); + if (isMax) { + emitM(ia64::And(ScratchReg.encoding(), ScratchReg.encoding(), + SecondScratchReg.encoding(), pEq)); + } else { + emitM(ia64::Or(ScratchReg.encoding(), ScratchReg.encoding(), + SecondScratchReg.encoding(), pEq)); + } + emitM(ia64::SetfD(t, ScratchReg.encoding(), pEq)); + + // A NaN operand overrides everything computed above, so test for it last. + if (handleNaN) { + uint32_t p8 = 8, p9 = 9; + emitF(ia64::Fclass(p8, p9, a, ia64::FclassNaN, 0)); + emitF(ia64::FmergeS(t, a, a, p8)); + emitF(ia64::Fclass(p8, p9, b, ia64::FclassNaN, 0)); + emitF(ia64::FmergeS(t, b, b, p8)); + } + + emitF(ia64::FmergeS(dest.encoding(), t, t, 0)); +} + +// Truncate-and-verify: convert to int32 via fcvt.fx.trunc, convert that +// int32 back to the source's precision, and compare against the original -- +// any information lost in the round trip (fraction, out-of-i32-range +// magnitude, NaN) shows up as an unequal compare. When negativeZeroCheck is +// set, -0 must also fail even though it trivially round-trips through 0. +static void ConvertFloatingToInt32(MacroAssembler& masm, FloatRegister src, + Register dest, Label* fail, + bool negativeZeroCheck, bool isSingle) { + ScratchDoubleScope scratch(masm); + masm.emitF(ia64::FcvtFxTrunc(scratch.encoding(), src.encoding(), ia64::sf0)); + masm.emitM(ia64::GetfSig(dest.encoding(), scratch.encoding())); + masm.ma_sxt(dest, dest, 4); + + { + SecondScratchDoubleScope roundTripped(masm); + if (isSingle) { + masm.emitM(ia64::SetfSig(roundTripped.encoding(), dest.encoding())); + masm.emitF(ia64::FcvtXf(roundTripped.encoding(), roundTripped.encoding())); + masm.emitF(ia64::FmaS(roundTripped.encoding(), roundTripped.encoding(), + ia64::fpOne, ia64::fpZero, ia64::sf0)); + } else { + masm.emitM(ia64::SetfSig(roundTripped.encoding(), dest.encoding())); + masm.emitF(ia64::FcvtXf(roundTripped.encoding(), roundTripped.encoding())); + } + // NotEqualOrUnordered also catches the NaN-source case, which has no + // valid int32 representation either. + uint32_t pFail = masm.emitCompareDouble( + Assembler::DoubleNotEqualOrUnordered, src, roundTripped); + masm.emitBranchToLabel(&ia64::BrCondRel, pFail, fail); + } + + if (negativeZeroCheck) { + Label notZero; + uint32_t pNonzero = masm.emitCompare(Assembler::NotEqual, dest, Imm32(0)); + masm.emitBranchToLabel(&ia64::BrCondRel, pNonzero, ¬Zero); + masm.emitM(ia64::GetfD(ScratchReg.encoding(), src.encoding())); + uint32_t pNeg = masm.emitCompare(Assembler::LessThan, ScratchReg, + Register{Registers::zero}); + masm.emitBranchToLabel(&ia64::BrCondRel, pNeg, fail); + masm.bind(¬Zero); + } +} + +void MacroAssemblerIa64::convertDoubleToInt32(FloatRegister src, + Register dest, Label* fail, + bool negativeZeroCheck) { + ConvertFloatingToInt32(asMasm(), src, dest, fail, negativeZeroCheck, false); +} + +void MacroAssemblerIa64::convertFloat32ToInt32(FloatRegister src, + Register dest, Label* fail, + bool negativeZeroCheck) { + ConvertFloatingToInt32(asMasm(), src, dest, fail, negativeZeroCheck, true); +} + +// Genuine truncation (drop any fraction, matching x86's cvttsd2si): unlike +// ConvertFloatingToInt32 above, fractional inputs are not a failure, so this +// skips the round-trip-through-double exactness check and only rejects +// magnitudes that don't fit in an int32 (which also catches NaN: fcvt.fx's +// "integer indefinite" result for an invalid input is architecturally a +// value whose sign-extended low 32 bits don't match the full result). +void MacroAssemblerIa64::ma_truncateToInt32(FloatRegister src, Register dest, + Label* fail) { + ScratchDoubleScope scratch(asMasm()); + emitF(ia64::FcvtFxTrunc(scratch.encoding(), src.encoding(), ia64::sf0)); + emitM(ia64::GetfSig(dest.encoding(), scratch.encoding())); + ma_sxt(ScratchReg, dest, 4); + uint32_t p = emitCompare(Assembler::NotEqual, ScratchReg, dest); + ma_sxt(dest, dest, 4); + emitBranchToLabel(&ia64::BrCondRel, p, fail); +} + +// =========================================================================== +// Addressing. + +void MacroAssemblerIa64::computeAddress(const BaseIndex& addr, Register out) { + // Any temporary here has to be one of the two scratch registers, and callers + // legitimately hold a live value in the other one (a store passes the value + // to be stored), so scale into |out| itself wherever possible. Only |out| == + // base forces a temporary, since the shift would destroy base first. + Register tmp = (out == ScratchReg) ? SecondScratchReg : ScratchReg; + uint32_t shift = ScaleToShift(addr.scale); + if (shift == 0) { + emitM( + ia64::Add(out.encoding(), addr.base.encoding(), addr.index.encoding())); + } else if (out != addr.base) { + emitI(ia64::ShlImm(out.encoding(), addr.index.encoding(), shift)); + emitM(ia64::Add(out.encoding(), addr.base.encoding(), out.encoding())); + } else { + emitI(ia64::ShlImm(tmp.encoding(), addr.index.encoding(), shift)); + emitM(ia64::Add(out.encoding(), addr.base.encoding(), tmp.encoding())); + } + if (addr.offset == 0) { + return; + } + if (addr.offset >= -8192 && addr.offset <= 8191) { + emitM(ia64::Adds(out.encoding(), addr.offset, out.encoding())); + return; + } + emitMovl(tmp.encoding(), uint64_t(int64_t(addr.offset))); + emitM(ia64::Add(out.encoding(), out.encoding(), tmp.encoding())); +} + +// =========================================================================== +// Shifts, extends and multiply. +// +// The shift, extend and multiply encoders are still being added to +// AssemblerCore-ia64.h. Everything that can be synthesised from the encoders +// that already exist is synthesised here so that the rest of the backend is +// complete; what is left is one MOZ_CRASH per operation, each of which becomes +// a single instruction once its encoding lands. + +// A handful of sequences (variable-count rotate, the atomic CAS retry loops) +// need a register more than the two assembler scratches provide. It has to come +// from the reserved pair: taking one from the allocatable set would silently +// clobber whatever live value the register allocator had parked there, which is +// invisible to it and only shows up as wrong results under register pressure. +// Two are reserved because ma_ror()/ma_rol() and atomicEffectOpJS()/ +// AtomicFetchOpJS() each nest one borrow inside another, and the |avoid| +// arguments are what keeps those two apart. +static Register BorrowScratch(Register avoid1, Register avoid2 = InvalidReg) { + const Register candidates[] = {ThirdScratchReg, FourthScratchReg}; + for (Register r : candidates) { + if (r != avoid1 && r != avoid2) { + return r; + } + } + MOZ_CRASH("no register left to borrow"); +} + +void MacroAssemblerIa64::ma_lsl(Register dest, Register src, Imm32 shift) { + uint32_t n = uint32_t(shift.value) & 63; + emitI(ia64::ShlImm(dest.encoding(), src.encoding(), n)); +} + +void MacroAssemblerIa64::ma_lsr(Register dest, Register src, Imm32 shift) { + uint32_t n = uint32_t(shift.value) & 63; + emitI(ia64::ShrUImm(dest.encoding(), src.encoding(), n)); +} + +void MacroAssemblerIa64::ma_asr(Register dest, Register src, Imm32 shift) { + uint32_t n = uint32_t(shift.value) & 63; + emitI(ia64::ShrImm(dest.encoding(), src.encoding(), n)); +} + +void MacroAssemblerIa64::ma_lsl(Register dest, Register src, Register shift) { + emitI(ia64::Shl(dest.encoding(), src.encoding(), shift.encoding())); +} + +void MacroAssemblerIa64::ma_lsr(Register dest, Register src, Register shift) { + emitI(ia64::ShrU(dest.encoding(), src.encoding(), shift.encoding())); +} + +void MacroAssemblerIa64::ma_asr(Register dest, Register src, Register shift) { + emitI(ia64::Shr(dest.encoding(), src.encoding(), shift.encoding())); +} + +// Rotates are a shift pair plus an or; IA-64's shrp would do it in one +// instruction but that encoding is not available yet. +void MacroAssemblerIa64::ma_rol(Register dest, Register src, Imm32 count, + unsigned width) { + MOZ_ASSERT(width == 32 || width == 64); + uint32_t n = uint32_t(count.value) & (width - 1); + if (width == 32) { + emitI(ia64::Zxt4(ScratchReg.encoding(), src.encoding())); + if (n == 0) { + emitM(ia64::MovReg(dest.encoding(), ScratchReg.encoding())); + return; + } + emitI(ia64::ShlImm(dest.encoding(), ScratchReg.encoding(), n)); + emitI(ia64::ShrUImm(ScratchReg.encoding(), ScratchReg.encoding(), 32 - n)); + emitM(ia64::Or(dest.encoding(), dest.encoding(), ScratchReg.encoding())); + emitI(ia64::Zxt4(dest.encoding(), dest.encoding())); + return; + } + if (n == 0) { + emitM(ia64::MovReg(dest.encoding(), src.encoding())); + return; + } + emitI(ia64::ShlImm(ScratchReg.encoding(), src.encoding(), n)); + emitI(ia64::ShrUImm(dest.encoding(), src.encoding(), 64 - n)); + emitM(ia64::Or(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssemblerIa64::ma_ror(Register dest, Register src, Imm32 count, + unsigned width) { + MOZ_ASSERT(width == 32 || width == 64); + uint32_t n = uint32_t(count.value) & (width - 1); + ma_rol(dest, src, Imm32(int32_t((width - n) & (width - 1))), width); +} + +// Variable-count rotate: shrp's count field is a compile-time immediate, so +// this needs the shift-pair-plus-or form with the shift amounts computed at +// runtime. comp = (-count) & (width-1) is congruent to (width-count) mod +// width but skips needing a materialised |width| constant, and naturally +// handles count%width == 0 (comp becomes 0 too). +void MacroAssemblerIa64::ma_rol(Register dest, Register src, Register count, + unsigned width) { + MOZ_ASSERT(width == 32 || width == 64); + const uint32_t mask = width - 1; + const uint32_t n = ScratchReg.encoding(); + const uint32_t comp = SecondScratchReg.encoding(); + const uint32_t srcZxt = BorrowScratch(dest, count).encoding(); + + if (width == 32) { + emitI(ia64::Zxt4(srcZxt, src.encoding())); + } else { + emitM(ia64::MovReg(srcZxt, src.encoding())); + } + emitM(ia64::AndImm(n, mask, count.encoding())); + emitM(ia64::Sub(comp, Registers::zero, count.encoding())); + emitM(ia64::AndImm(comp, mask, comp)); + + emitI(ia64::Shl(n, srcZxt, n)); + emitI(ia64::ShrU(comp, srcZxt, comp)); + emitM(ia64::Or(dest.encoding(), n, comp)); + if (width == 32) { + emitI(ia64::Zxt4(dest.encoding(), dest.encoding())); + } +} + +void MacroAssemblerIa64::ma_ror(Register dest, Register src, Register count, + unsigned width) { + // rotate_right(src, count) == rotate_left(src, -count): ma_rol computes + // n = c & mask and comp = -c & mask internally, so handing it a + // pre-negated count swaps those two into exactly the right shift amounts. + Register negated = BorrowScratch(dest, count); + emitM(ia64::Sub(negated.encoding(), Registers::zero, count.encoding())); + ma_rol(dest, src, negated, width); +} + +void MacroAssemblerIa64::ma_sxt(Register dest, Register src, unsigned bytes) { + switch (bytes) { + case 1: + emitI(ia64::Sxt1(dest.encoding(), src.encoding())); + return; + case 2: + emitI(ia64::Sxt2(dest.encoding(), src.encoding())); + return; + case 4: + emitI(ia64::Sxt4(dest.encoding(), src.encoding())); + return; + default: + MOZ_CRASH("bad width for ia64 sign extend"); + } +} + +void MacroAssemblerIa64::ma_zxt(Register dest, Register src, unsigned bytes) { + switch (bytes) { + case 1: + emitI(ia64::Zxt1(dest.encoding(), src.encoding())); + return; + case 2: + emitI(ia64::Zxt2(dest.encoding(), src.encoding())); + return; + case 4: + emitI(ia64::Zxt4(dest.encoding(), src.encoding())); + return; + default: + MOZ_CRASH("bad width for ia64 zero extend"); + } +} + +// IA-64 has no integer multiply; xma.l (fused 64x64->64 multiply-add on the +// F unit, adding zero) computes it via setf.sig/getf.sig round-tripping the +// operands through the significand of an FR, exactly as the divide/sqrt +// sequences below round-trip through register-format floats. +void MacroAssemblerIa64::ma_mul(Register dest, Register lhs, Register rhs) { + const uint32_t t0 = 2, t1 = 3; + emitM(ia64::SetfSig(t0, lhs.encoding())); + emitM(ia64::SetfSig(t1, rhs.encoding())); + emitF(ia64::XmaL(t0, t0, t1, ia64::fpZero)); + emitM(ia64::GetfSig(dest.encoding(), t0)); +} + +// libgcc's __divdi3/__udivdi3/__moddi3/__umoddi3 (lib1funcs.S): convert both +// operands to register-format float, refine a reciprocal approximation with +// 3 rounds of Newton-Raphson on f(y) = 1/d - y (all sf1), then truncate back +// to an integer. Signed division is turned into unsigned-magnitude division +// by negating negative operands up front (so the conversion below is always +// applied to a non-negative 64-bit magnitude) and reapplying the sign +// afterwards; only a genuinely-unsigned 64-bit magnitude can exceed +// INT64_MAX, which is why the conversion/truncation opcodes differ in that +// one case (fcvt.xuf is, per the assembler, just fnorm -- setf.sig already +// represents the bit pattern as an unsigned fixed-point value, and fcvt.xf's +// extra work is exactly the sign correction fnorm skips). +void MacroAssemblerIa64::ma_divmod(Register divOutput, Register remOutput, + Register lhs, Register rhs, bool isUnsigned, + unsigned width) { + MOZ_ASSERT(width == 32 || width == 64); + MOZ_ASSERT(divOutput != InvalidReg || remOutput != InvalidReg); + MOZ_ASSERT(divOutput != remOutput || divOutput == InvalidReg); + + // The remainder needs a register throughout; when only the quotient is + // wanted its output register does that job and is overwritten at the end. + Register acc = remOutput != InvalidReg ? remOutput : divOutput; + MOZ_ASSERT(acc != ScratchReg && acc != SecondScratchReg); + + const uint32_t n = ScratchReg.encoding(); // dividend magnitude + const uint32_t d = SecondScratchReg.encoding(); // divisor magnitude + const uint32_t r = acc.encoding(); // remainder, then quotient + const uint32_t z = Registers::zero; + + const uint32_t kNegDividend = 10; + const uint32_t kNegQuotient = 11; + const bool wideUnsigned = isUnsigned && width == 64; + + if (width == 32) { + if (isUnsigned) { + emitI(ia64::Zxt4(n, lhs.encoding())); + emitI(ia64::Zxt4(d, rhs.encoding())); + } else { + emitI(ia64::Sxt4(n, lhs.encoding())); + emitI(ia64::Sxt4(d, rhs.encoding())); + } + } else { + emitM(ia64::MovReg(n, lhs.encoding())); + emitM(ia64::MovReg(d, rhs.encoding())); + } + + if (!isUnsigned) { + // The quotient is negative when the operand signs differ; |r| is still + // free at this point, so it can hold the xor that tests that. + emitM(ia64::Xor(r, n, d)); + emitI(ia64::CmpLt(kNegQuotient, 12, r, z)); + emitI(ia64::CmpLt(kNegDividend, 12, n, z)); + emitM(ia64::Sub(n, z, n, kNegDividend)); + emitI(ia64::CmpLt(12, 13, d, z)); + emitM(ia64::Sub(d, z, d, 12)); + } + + // f2/f3 hold the converted dividend/divisor; f4-f7 are the Newton state. + const uint32_t fn = 2, fd = 3, f10 = 4, f11 = 5, f12 = 6, f13 = 7; + const uint32_t p6 = 6; + + emitM(ia64::SetfSig(fn, n)); + emitM(ia64::SetfSig(fd, d)); + if (wideUnsigned) { + emitF(ia64::Fnorm(fn, fn, ia64::sf1)); + emitF(ia64::Fnorm(fd, fd, ia64::sf1)); + } else { + emitF(ia64::FcvtXf(fn, fn)); + emitF(ia64::FcvtXf(fd, fd)); + } + + emitF(ia64::Frcpa(f10, p6, fn, fd, ia64::sf1)); + emitF(ia64::Fnma(f11, fd, f10, ia64::fpOne, ia64::sf1, p6)); + emitF(ia64::Fma(f12, fn, f10, ia64::fpZero, ia64::sf1, p6)); + emitF(ia64::Fma(f13, f11, f11, ia64::fpZero, ia64::sf1, p6)); + emitF(ia64::Fma(f12, f11, f12, f12, ia64::sf1, p6)); + emitF(ia64::Fma(f10, f11, f10, f10, ia64::sf1, p6)); + emitF(ia64::Fma(f11, f13, f12, f12, ia64::sf1, p6)); + emitF(ia64::Fma(f10, f13, f10, f10, ia64::sf1, p6)); + emitF(ia64::Fnma(f12, fd, f11, fn, ia64::sf1, p6)); + emitF(ia64::Fma(f10, f12, f10, f11, ia64::sf1, p6)); + if (wideUnsigned) { + emitF(ia64::FcvtFxuTrunc(f10, f10, ia64::sf1)); + } else { + emitF(ia64::FcvtFxTrunc(f10, f10, ia64::sf1)); + } + // fcvt.fx.trunc leaves the quotient in f10 in integer (setf.sig) form. + + if (remOutput != InvalidReg) { + // remainder = dividend - quotient*divisor. xma.l multiplies the two + // significands, so it needs the *integer* divisor: fd was overwritten by + // fcvt.xf above and now holds the float form, whose significand is the + // divisor normalised (11 becomes 11 << 60), which silently yields a + // product of zero for most inputs. libgcc's __moddi3 re-materialises the + // integer divisor with setf.sig for the same reason, right before its + // "xma.l f10 = f10, f9, f14". + emitM(ia64::SetfSig(f11, d)); + emitF(ia64::XmaL(f11, f10, f11, ia64::fpZero)); + emitM(ia64::GetfSig(d, f11)); + emitM(ia64::Sub(r, n, d)); + } + if (divOutput != InvalidReg) { + emitM(ia64::GetfSig(divOutput.encoding(), f10)); + } + + if (!isUnsigned) { + // The remainder takes the sign of the dividend, the quotient that of the + // xor of the two operand signs. + if (remOutput != InvalidReg) { + emitM(ia64::Sub(r, z, r, kNegDividend)); + } + if (divOutput != InvalidReg) { + emitM(ia64::Sub(divOutput.encoding(), z, divOutput.encoding(), + kNegQuotient)); + } + } + + if (width == 32) { + if (remOutput != InvalidReg) { + emitI(isUnsigned ? ia64::Zxt4(r, r) : ia64::Sxt4(r, r)); + } + if (divOutput != InvalidReg) { + emitI(isUnsigned ? ia64::Zxt4(divOutput.encoding(), divOutput.encoding()) + : ia64::Sxt4(divOutput.encoding(), + divOutput.encoding())); + } + } +} + +void MacroAssemblerIa64::ma_cmov(Assembler::Condition cond, Register src, + Register dest) { + MOZ_CRASH("ia64: ma_cmov needs the compare that produced |cond|"); +} + +// =========================================================================== +// Jumps. + +void MacroAssemblerIa64::jump(Label* label) { + // p0 is hardwired true, so an unpredicated br.cond is an unconditional jump. + emitBranchToLabel(&ia64::BrCondRel, 0, label); +} + +void MacroAssemblerIa64::jump(Register reg) { + emitI(ia64::MovToBr(6, reg.encoding())); + emitB(ia64::BrCond(6)); +} + +void MacroAssemblerIa64::jump(ImmPtr ptr) { + movePtr(ptr, ScratchReg); + jump(ScratchReg); +} + +void MacroAssemblerIa64::jump(TrampolinePtr code) { jump(ImmPtr(code.value)); } + +void MacroAssemblerIa64::jump(JitCode* code) { + writeJumpRelocation(); + emitMovl(ScratchReg.encoding(), uint64_t(uintptr_t(code->raw()))); + jump(ScratchReg); +} + +void MacroAssemblerIa64::jump(const Address& addr) { + loadPtr(addr, ScratchReg); + jump(ScratchReg); +} + +// =========================================================================== +// Loads and stores. IA-64 addresses memory through a bare register, so every +// Address or BaseIndex costs a computeAddress first. + +FaultingCodeOffset MacroAssemblerIa64::loadPtr(const Address& addr, + Register dest) { + computeAddress(addr, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ld8(dest.encoding(), ScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::loadPtr(const BaseIndex& addr, + Register dest) { + computeAddress(addr, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ld8(dest.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::loadPtr(AbsoluteAddress addr, + Register dest) { + computeAddress(addr, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ld8(dest.encoding(), ScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::loadPtr(ImmPtr addr, Register dest) { + return loadPtr(AbsoluteAddress(addr.value), dest); +} + +FaultingCodeOffset MacroAssemblerIa64::load32(const Address& addr, + Register dest) { + computeAddress(addr, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ld4(dest.encoding(), ScratchReg.encoding())); + // ld4 zero-extends; the JIT keeps int32 values sign-extended in registers so + // that the 64-bit compares used by branch32() stay correct. + ma_sxt(dest, dest, 4); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::load32(const BaseIndex& addr, + Register dest) { + computeAddress(addr, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ld4(dest.encoding(), SecondScratchReg.encoding())); + ma_sxt(dest, dest, 4); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::load32(AbsoluteAddress addr, + Register dest) { + computeAddress(addr, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ld4(dest.encoding(), ScratchReg.encoding())); + ma_sxt(dest, dest, 4); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::load32(ImmPtr addr, Register dest) { + return load32(AbsoluteAddress(addr.value), dest); +} + +// Itanium handles a misaligned access in hardware as long as it stays within +// one 8-byte chunk, and traps into the kernel's fixup handler when it crosses +// one -- thousands of cycles either side of a 37.5% chance for a 4-byte load +// at an arbitrary address. Assembling the value a byte at a time is far +// cheaper, and every ld1 is trivially in bounds. Little-endian, matching every +// other 64-bit backend. +void MacroAssemblerIa64::loadUnalignedBytes(Register addr, Register tmp, + Register out, uint32_t bytes) { + MOZ_ASSERT(out != addr); + MOZ_ASSERT(out != tmp); + MOZ_ASSERT(addr != tmp); + emitM(ia64::Ld1(out.encoding(), addr.encoding())); + for (uint32_t i = 1; i < bytes; i++) { + emitM(ia64::Adds(addr.encoding(), 1, addr.encoding())); + emitM(ia64::Ld1(tmp.encoding(), addr.encoding())); + emitI(ia64::ShlImm(tmp.encoding(), tmp.encoding(), i * 8)); + emitM(ia64::Or(out.encoding(), out.encoding(), tmp.encoding())); + } +} + +void MacroAssemblerIa64::loadUnalignedValue(const Address& src, + ValueOperand dest) { + computeAddress(src, ScratchReg); + loadUnalignedBytes(ScratchReg, SecondScratchReg, dest.valueReg(), 8); +} + +// ld1 zero-extends and the shifts stay inside |Bytes| * 8 bits, so the +// assembled value is already zero-extended; only the sign-extending forms need +// a fixup. load32Unaligned sign-extends to match load32(), which keeps int32 +// values sign-extended in registers so the 64-bit compares stay correct. +#define IA64_DEFINE_UNALIGNED_LOAD(Name, Bytes, Extend) \ + void MacroAssemblerIa64::Name(const Address& addr, Register dest) { \ + computeAddress(addr, ScratchReg); \ + loadUnalignedBytes(ScratchReg, SecondScratchReg, dest, Bytes); \ + Extend; \ + } \ + void MacroAssemblerIa64::Name(const BaseIndex& addr, Register dest) { \ + computeAddress(addr, SecondScratchReg); \ + loadUnalignedBytes(SecondScratchReg, ScratchReg, dest, Bytes); \ + Extend; \ + } + +IA64_DEFINE_UNALIGNED_LOAD(load32Unaligned, 4, ma_sxt(dest, dest, 4)) +IA64_DEFINE_UNALIGNED_LOAD(load16UnalignedSignExtend, 2, ma_sxt(dest, dest, 2)) +IA64_DEFINE_UNALIGNED_LOAD(load16UnalignedZeroExtend, 2, (void)0) + +#undef IA64_DEFINE_UNALIGNED_LOAD + +void MacroAssemblerIa64::load64Unaligned(const Address& addr, + Register64 dest) { + computeAddress(addr, ScratchReg); + loadUnalignedBytes(ScratchReg, SecondScratchReg, dest.reg, 8); +} + +void MacroAssemblerIa64::load64Unaligned(const BaseIndex& addr, + Register64 dest) { + computeAddress(addr, SecondScratchReg); + loadUnalignedBytes(SecondScratchReg, ScratchReg, dest.reg, 8); +} + +#define IA64_DEFINE_NARROW_LOAD(Name, LoadInsn, Bytes, Extend) \ + FaultingCodeOffset MacroAssemblerIa64::Name(const Address& addr, \ + Register dest) { \ + computeAddress(addr, ScratchReg); \ + FaultingCodeOffset fco(currentOffset()); \ + emitM(ia64::LoadInsn(dest.encoding(), ScratchReg.encoding())); \ + Extend(dest, dest, Bytes); \ + return fco; \ + } \ + FaultingCodeOffset MacroAssemblerIa64::Name(const BaseIndex& addr, \ + Register dest) { \ + computeAddress(addr, SecondScratchReg); \ + FaultingCodeOffset fco(currentOffset()); \ + emitM(ia64::LoadInsn(dest.encoding(), SecondScratchReg.encoding())); \ + Extend(dest, dest, Bytes); \ + return fco; \ + } + +IA64_DEFINE_NARROW_LOAD(load8SignExtend, Ld1, 1, ma_sxt) +IA64_DEFINE_NARROW_LOAD(load8ZeroExtend, Ld1, 1, ma_zxt) +IA64_DEFINE_NARROW_LOAD(load16SignExtend, Ld2, 2, ma_sxt) +IA64_DEFINE_NARROW_LOAD(load16ZeroExtend, Ld2, 2, ma_zxt) + +#undef IA64_DEFINE_NARROW_LOAD + +// |src| is frequently one of the two scratch registers (callWithABIPre stores +// b0 from ScratchReg, storeValue tags into SecondScratchReg), so the address +// must be synthesised into the *other* one -- otherwise computing it overwrites +// the value being stored and the store silently writes the address instead. +#define IA64_DEFINE_STORE(Name, StoreInsn) \ + FaultingCodeOffset MacroAssemblerIa64::Name(Register src, \ + const Address& dest) { \ + Register tmp = (src == ScratchReg) ? SecondScratchReg : ScratchReg; \ + computeAddress(dest, tmp); \ + FaultingCodeOffset fco(currentOffset()); \ + emitM(ia64::StoreInsn(tmp.encoding(), src.encoding())); \ + return fco; \ + } \ + FaultingCodeOffset MacroAssemblerIa64::Name(Register src, \ + const BaseIndex& dest) { \ + Register tmp = \ + (src == SecondScratchReg) ? ScratchReg : SecondScratchReg; \ + computeAddress(dest, tmp); \ + FaultingCodeOffset fco(currentOffset()); \ + emitM(ia64::StoreInsn(tmp.encoding(), src.encoding())); \ + return fco; \ + } + +IA64_DEFINE_STORE(storePtr, St8) +IA64_DEFINE_STORE(store32, St4) +IA64_DEFINE_STORE(store16, St2) +IA64_DEFINE_STORE(store8, St1) + +#undef IA64_DEFINE_STORE + +// =========================================================================== +// Floating point loads, stores and moves. + +FaultingCodeOffset MacroAssemblerIa64::loadFloat32(const Address& addr, + FloatRegister dest) { + computeAddress(addr, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ldfs(dest.encoding(), ScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::loadFloat32(const BaseIndex& addr, + FloatRegister dest) { + computeAddress(addr, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ldfs(dest.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::loadDouble(const Address& addr, + FloatRegister dest) { + computeAddress(addr, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ldfd(dest.encoding(), ScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::loadDouble(const BaseIndex& addr, + FloatRegister dest) { + computeAddress(addr, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Ldfd(dest.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::storeFloat32(FloatRegister src, + const Address& dest) { + computeAddress(dest, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Stfs(ScratchReg.encoding(), src.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::storeFloat32(FloatRegister src, + const BaseIndex& dest) { + computeAddress(dest, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Stfs(SecondScratchReg.encoding(), src.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::storeDouble(FloatRegister src, + const Address& dest) { + computeAddress(dest, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Stfd(ScratchReg.encoding(), src.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::storeDouble(FloatRegister src, + const BaseIndex& dest) { + computeAddress(dest, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::Stfd(SecondScratchReg.encoding(), src.encoding())); + return fco; +} + +// fmerge.s picks the sign from its second operand and the magnitude from its +// third; with both the same register this is a plain register-format copy, +// exact for both Float32 and Double (no rounding, unlike fnorm). +void MacroAssemblerIa64::moveFloat32(FloatRegister src, FloatRegister dest) { + if (src != dest) { + emitF(ia64::FmergeS(dest.encoding(), src.encoding(), src.encoding())); + } +} + +void MacroAssemblerIa64::moveDouble(FloatRegister src, FloatRegister dest) { + if (src != dest) { + emitF(ia64::FmergeS(dest.encoding(), src.encoding(), src.encoding())); + } +} + +void MacroAssemblerIa64::push(FloatRegister src) { + emitM(ia64::Adds(StackPointer.encoding(), -int64_t(sizeof(double)), + StackPointer.encoding())); + emitM(ia64::Stfd(StackPointer.encoding(), src.encoding())); +} + +void MacroAssemblerIa64::pop(FloatRegister dest) { + emitM(ia64::Ldfd(dest.encoding(), StackPointer.encoding())); + emitM(ia64::Adds(StackPointer.encoding(), sizeof(double), + StackPointer.encoding())); +} + +// JS_PUNBOX64 stores double values as their literal IEEE bit pattern (the +// NaN-boxing scheme reserves the non-canonical NaN space for pointer tags, so +// every real double round-trips byte-for-byte); setf.d/getf.d move that +// pattern between a GR and an FR without touching the NaN payload, which is +// exactly what boxing/unboxing needs. +void MacroAssemblerIa64::boxDouble(FloatRegister src, ValueOperand dest) { + emitM(ia64::GetfD(dest.valueReg().encoding(), src.encoding())); +} + +void MacroAssemblerIa64::unboxDouble(const ValueOperand& src, + FloatRegister dest) { + emitM(ia64::SetfD(dest.encoding(), src.valueReg().encoding())); +} + +void MacroAssemblerIa64::unboxDouble(const Address& src, FloatRegister dest) { + loadDouble(src, dest); +} + +void MacroAssemblerIa64::unboxDouble(const BaseIndex& src, + FloatRegister dest) { + loadDouble(src, dest); +} + +void MacroAssemblerIa64::loadConstantDouble(double d, FloatRegister dest) { + uint64_t bits; + memcpy(&bits, &d, sizeof(bits)); + emitMovl(ScratchReg.encoding(), bits); + emitM(ia64::SetfD(dest.encoding(), ScratchReg.encoding())); +} + +void MacroAssemblerIa64::loadConstantFloat32(float f, FloatRegister dest) { + uint32_t bits; + memcpy(&bits, &f, sizeof(bits)); + emitMovl(ScratchReg.encoding(), uint64_t(bits)); + emitM(ia64::SetfS(dest.encoding(), ScratchReg.encoding())); +} + +FaultingCodeOffset MacroAssemblerIa64::storePtr(Register src, + AbsoluteAddress dest) { + computeAddress(dest, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St8(ScratchReg.encoding(), src.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::store32(Register src, + AbsoluteAddress dest) { + computeAddress(dest, ScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St4(ScratchReg.encoding(), src.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::storePtr(ImmWord imm, + const Address& dest) { + computeAddress(dest, ScratchReg); + emitMovl(SecondScratchReg.encoding(), imm.value); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St8(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::storePtr(ImmWord imm, + const BaseIndex& dest) { + computeAddress(dest, ScratchReg); + emitMovl(SecondScratchReg.encoding(), imm.value); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St8(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::storePtr(ImmPtr imm, + const Address& dest) { + return storePtr(ImmWord(uintptr_t(imm.value)), dest); +} + +FaultingCodeOffset MacroAssemblerIa64::storePtr(ImmPtr imm, + const BaseIndex& dest) { + return storePtr(ImmWord(uintptr_t(imm.value)), dest); +} + +FaultingCodeOffset MacroAssemblerIa64::storePtr(ImmGCPtr imm, + const Address& dest) { + computeAddress(dest, ScratchReg); + movePtr(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St8(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::storePtr(ImmGCPtr imm, + const BaseIndex& dest) { + computeAddress(dest, ScratchReg); + movePtr(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St8(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::store32(Imm32 imm, + const Address& dest) { + computeAddress(dest, ScratchReg); + move32(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St4(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::store32(Imm32 imm, + const BaseIndex& dest) { + computeAddress(dest, ScratchReg); + move32(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St4(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::store32(Imm32 imm, + AbsoluteAddress dest) { + computeAddress(dest, ScratchReg); + move32(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St4(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::store16(Imm32 imm, + const Address& dest) { + computeAddress(dest, ScratchReg); + move32(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St2(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::store16(Imm32 imm, + const BaseIndex& dest) { + computeAddress(dest, ScratchReg); + move32(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St2(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::store8(Imm32 imm, const Address& dest) { + computeAddress(dest, ScratchReg); + move32(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St1(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +FaultingCodeOffset MacroAssemblerIa64::store8(Imm32 imm, + const BaseIndex& dest) { + computeAddress(dest, ScratchReg); + move32(imm, SecondScratchReg); + FaultingCodeOffset fco(currentOffset()); + emitM(ia64::St1(ScratchReg.encoding(), SecondScratchReg.encoding())); + return fco; +} + +// =========================================================================== +// Raw stack pushes and pops. + +void MacroAssemblerIa64::push(Register reg) { + if (reg == StackPointer) { + // Like x86's `push %rsp`, this must store the value sp had *before* the + // decrement: BaselineIC's call fallback uses push(getStackPointer()) to + // hand the VM a |vp| pointing at the arguments it just pushed. + emitM(ia64::MovReg(ScratchReg.encoding(), StackPointer.encoding())); + emitM(ia64::Adds(StackPointer.encoding(), -int64_t(sizeof(intptr_t)), + StackPointer.encoding())); + emitM(ia64::St8(StackPointer.encoding(), ScratchReg.encoding())); + return; + } + emitM(ia64::Adds(StackPointer.encoding(), -int64_t(sizeof(intptr_t)), + StackPointer.encoding())); + emitM(ia64::St8(StackPointer.encoding(), reg.encoding())); +} + +void MacroAssemblerIa64::push(Imm32 imm) { + move32(imm, ScratchReg); + push(ScratchReg); +} + +void MacroAssemblerIa64::push(ImmWord imm) { + movePtr(imm, ScratchReg); + push(ScratchReg); +} + +void MacroAssemblerIa64::push(ImmPtr imm) { + push(ImmWord(uintptr_t(imm.value))); +} + +void MacroAssemblerIa64::push(ImmGCPtr imm) { + movePtr(imm, ScratchReg); + push(ScratchReg); +} + +void MacroAssemblerIa64::push(const Address& addr) { + loadPtr(addr, SecondScratchReg); + push(SecondScratchReg); +} + +void MacroAssemblerIa64::pop(Register reg) { + emitM(ia64::Ld8(reg.encoding(), StackPointer.encoding())); + emitM(ia64::Adds(StackPointer.encoding(), sizeof(intptr_t), + StackPointer.encoding())); +} + +void MacroAssemblerIa64::pop(const ValueOperand& val) { pop(val.valueReg()); } + +void MacroAssemblerIa64::retn(Imm32 n) { + // The return address is at [sp]; load it before sp moves, since n covers the + // whole frame including that slot. + emitM(ia64::Ld8(ScratchReg.encoding(), StackPointer.encoding())); + emitM(ia64::Adds(StackPointer.encoding(), n.value, StackPointer.encoding())); + emitI(ia64::MovToBr(0, ScratchReg.encoding())); + emitB(ia64::BrRet(0)); +} + +// =========================================================================== +// Values. JS_PUNBOX64 keeps a Value in one 64-bit register. + +void MacroAssembler::moveValue(const ValueOperand& src, + const ValueOperand& dest) { + if (src.valueReg() != dest.valueReg()) { + movePtr(src.valueReg(), dest.valueReg()); + } +} + +void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) { + if (!src.isGCThing()) { + movePtr(ImmWord(src.asRawBits()), dest.valueReg()); + return; + } + // A GC-thing Value must go through ImmGCPtr so the pointer is traced; the + // tag is or'ed back on afterwards. + movePtr(ImmGCPtr(src.toGCThing()), dest.valueReg()); + emitMovl(ScratchReg.encoding(), uint64_t(src.asRawBits()) & ValueTagMask); + emitM(ia64::Or(dest.valueReg().encoding(), dest.valueReg().encoding(), + ScratchReg.encoding())); +} + +void MacroAssemblerIa64::loadValue(const Address& src, ValueOperand val) { + loadPtr(src, val.valueReg()); +} + +void MacroAssemblerIa64::loadValue(const BaseIndex& src, ValueOperand val) { + loadPtr(src, val.valueReg()); +} + +void MacroAssemblerIa64::storeValue(ValueOperand val, const Address& dest) { + storePtr(val.valueReg(), dest); +} + +void MacroAssemblerIa64::storeValue(ValueOperand val, const BaseIndex& dest) { + storePtr(val.valueReg(), dest); +} + +void MacroAssemblerIa64::storeValue(const Value& val, const Address& dest) { + asMasm().moveValue(val, ValueOperand(SecondScratchReg)); + storePtr(SecondScratchReg, dest); +} + +void MacroAssemblerIa64::storeValue(const Value& val, const BaseIndex& dest) { + asMasm().moveValue(val, ValueOperand(SecondScratchReg)); + storePtr(SecondScratchReg, dest); +} + +void MacroAssemblerIa64::storeValue(JSValueType type, Register reg, + const Address& dest) { + tagValue(type, reg, ValueOperand(SecondScratchReg)); + storePtr(SecondScratchReg, dest); +} + +void MacroAssemblerIa64::storeValue(JSValueType type, Register reg, + const BaseIndex& dest) { + tagValue(type, reg, ValueOperand(SecondScratchReg)); + storePtr(SecondScratchReg, dest); +} + +void MacroAssemblerIa64::pushValue(ValueOperand val) { push(val.valueReg()); } + +void MacroAssemblerIa64::pushValue(const Value& val) { + asMasm().moveValue(val, ValueOperand(SecondScratchReg)); + push(SecondScratchReg); +} + +void MacroAssemblerIa64::pushValue(const Address& addr) { push(addr); } + +void MacroAssemblerIa64::pushValue(JSValueType type, Register reg) { + tagValue(type, reg, ValueOperand(SecondScratchReg)); + push(SecondScratchReg); +} + +void MacroAssemblerIa64::popValue(ValueOperand val) { pop(val.valueReg()); } + +void MacroAssemblerIa64::tagValue(JSValueType type, Register payload, + ValueOperand dest) { + Register out = dest.valueReg(); + if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { + ma_zxt(out, payload, 4); + } else { + emitMovl(ScratchReg.encoding(), JS::detail::ValueGCThingPayloadMask); + emitM( + ia64::And(out.encoding(), payload.encoding(), ScratchReg.encoding())); + } + emitMovl(ScratchReg.encoding(), JSVAL_TYPE_TO_SHIFTED_TAG(type)); + emitM(ia64::Or(out.encoding(), out.encoding(), ScratchReg.encoding())); +} + +// Like tagValue(), but the type tag is a runtime value rather than a +// compile-time constant -- used by baseline's generic compare-switch fast +// path. dest = ((type | JSVAL_TAG_MAX_DOUBLE) << JSVAL_TAG_SHIFT) | src, +// matching x64/x86's boxValue(Register, Register, Register). +void MacroAssemblerIa64::boxNonDouble(Register type, Register src, + ValueOperand dest) { + Register out = dest.valueReg(); + MOZ_ASSERT(src != out); + if (type != out) { + movePtr(type, out); + } + emitMovl(ScratchReg.encoding(), JSVAL_TAG_MAX_DOUBLE); + emitM(ia64::Or(out.encoding(), out.encoding(), ScratchReg.encoding())); + ma_lsl(out, out, Imm32(JSVAL_TAG_SHIFT)); + // The payload must be zero-extended before it goes under the tag. x64 and + // arm64 get that for free because the caller builds it in a 32-bit register + // (movsbl/ldrsb leave bits 32-63 clear), but IA-64 has no 32-bit registers, + // so LoadConstantCompareOperand()'s load8SignExtend() sets all 64 bits and a + // negative constant would otherwise smear ones across the tag -- making the + // bitwise branch64() in emitConstantStrictEq() compare unequal to a properly + // boxed value. The only caller is that one, whose ConstantCompareOperand is + // always Int32/Boolean/Null/Undefined, so 32 bits is always enough. + ma_zxt(SecondScratchReg, src, 4); + emitM(ia64::Or(out.encoding(), out.encoding(), SecondScratchReg.encoding())); +} + +void MacroAssemblerIa64::unboxValue(const ValueOperand& src, AnyRegister dest, + JSValueType type) { + if (dest.isFloat()) { + Label notInt32, end; + asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); + convertInt32ToDouble(src.valueReg(), dest.fpu()); + jump(&end); + bind(¬Int32); + unboxDouble(src, dest.fpu()); + bind(&end); + } else { + unboxNonDouble(src, dest.gpr(), type); + } +} + +void MacroAssemblerIa64::emitTagOf(const Address& value, Register tag) { + loadPtr(value, tag); + emitMovl(ScratchReg.encoding(), ValueTagMask); + emitM(ia64::And(tag.encoding(), tag.encoding(), ScratchReg.encoding())); +} + +void MacroAssemblerIa64::emitTagOf(const BaseIndex& value, Register tag) { + loadPtr(value, tag); + emitMovl(ScratchReg.encoding(), ValueTagMask); + emitM(ia64::And(tag.encoding(), tag.encoding(), ScratchReg.encoding())); +} + +void MacroAssemblerIa64::unboxNonDouble(const ValueOperand& src, Register dest, + JSValueType type) { + MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); + if (type == JSVAL_TYPE_INT32) { + ma_sxt(dest, src.valueReg(), 4); + return; + } + if (type == JSVAL_TYPE_BOOLEAN) { + ma_zxt(dest, src.valueReg(), 4); + return; + } + emitMovl(ScratchReg.encoding(), JSVAL_TYPE_TO_SHIFTED_TAG(type)); + emitM(ia64::Xor(dest.encoding(), src.valueReg().encoding(), + ScratchReg.encoding())); +} + +void MacroAssemblerIa64::unboxNonDouble(const Address& src, Register dest, + JSValueType type) { + loadPtr(src, dest); + unboxNonDouble(ValueOperand(dest), dest, type); +} + +void MacroAssemblerIa64::unboxNonDouble(const BaseIndex& src, Register dest, + JSValueType type) { + loadPtr(src, dest); + unboxNonDouble(ValueOperand(dest), dest, type); +} + +void MacroAssemblerIa64::unboxGCThingForGCBarrier(const ValueOperand& src, + Register dest) { + emitMovl(ScratchReg.encoding(), JS::detail::ValueGCThingPayloadMask); + emitM(ia64::And(dest.encoding(), src.valueReg().encoding(), + ScratchReg.encoding())); +} + +void MacroAssemblerIa64::unboxGCThingForGCBarrier(const Address& src, + Register dest) { + loadPtr(src, dest); + unboxGCThingForGCBarrier(ValueOperand(dest), dest); +} + +void MacroAssemblerIa64::unboxWasmAnyRefGCThingForGCBarrier( + const Address& src, Register dest) { + MOZ_ASSERT(ScratchReg != dest); + emitMovl(ScratchReg.encoding(), uint64_t(wasm::AnyRef::GCThingMask)); + loadPtr(src, dest); + emitM(ia64::And(dest.encoding(), dest.encoding(), ScratchReg.encoding())); +} + +// b0 isn't a GPR (see JS_USE_LINK_REGISTER's comment in Assembler-shared.h), +// so every "pop the return address and branch" spot below is +// popReturnAddress()+abiret() rather than the single ret() other +// link-register backends use -- exactly the pair pushReturnAddress() uses +// on entry. +void MacroAssemblerIa64::handleFailureWithHandlerTail( + Label* profilerExitTail, Label* bailoutTail, + uint32_t* returnValueCheckOffset) { + int size = (sizeof(ResumeFromException) + ABIStackAlignment - 1) & + ~(ABIStackAlignment - 1); + asMasm().subPtr(Imm32(size), StackPointer); + movePtr(StackPointer, IntArgReg0); + + using Fn = void (*)(ResumeFromException* rfe); + asMasm().setupUnalignedABICall(IntArgReg1); + asMasm().passABIArg(IntArgReg0); + asMasm().callWithABI( + ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + *returnValueCheckOffset = asMasm().currentOffset(); + + Label entryFrame; + Label catch_; + Label finally; + Label returnBaseline; + Label returnIon; + Label bailout; + Label wasmInterpEntry; + Label wasmCatch; + + load32(Address(StackPointer, ResumeFromException::offsetOfKind()), + IntArgReg0); + asMasm().branch32(Assembler::Equal, IntArgReg0, + Imm32(ExceptionResumeKind::EntryFrame), &entryFrame); + asMasm().branch32(Assembler::Equal, IntArgReg0, + Imm32(ExceptionResumeKind::Catch), &catch_); + asMasm().branch32(Assembler::Equal, IntArgReg0, + Imm32(ExceptionResumeKind::Finally), &finally); + asMasm().branch32(Assembler::Equal, IntArgReg0, + Imm32(ExceptionResumeKind::ForcedReturnBaseline), + &returnBaseline); + asMasm().branch32(Assembler::Equal, IntArgReg0, + Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon); + asMasm().branch32(Assembler::Equal, IntArgReg0, + Imm32(ExceptionResumeKind::Bailout), &bailout); + asMasm().branch32(Assembler::Equal, IntArgReg0, + Imm32(ExceptionResumeKind::WasmInterpEntry), + &wasmInterpEntry); + asMasm().branch32(Assembler::Equal, IntArgReg0, + Imm32(ExceptionResumeKind::WasmCatch), &wasmCatch); + + breakpoint(); // Invalid kind. + + // No exception handler. Load the error value, restore state and return + // from the entry frame. + bind(&entryFrame); + asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), + FramePointer); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), + StackPointer); + asMasm().popReturnAddress(); + abiret(); + + // If we found a catch handler, this must be a baseline frame. Restore + // state and jump to the catch block. + bind(&catch_); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), + IntArgReg0); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), + FramePointer); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), + StackPointer); + jump(IntArgReg0); + + // If we found a finally block, this must be a baseline frame. Push three + // values expected by the finally block: the exception, the exception + // stack, and BooleanValue(true). + bind(&finally); + ValueOperand exception = ValueOperand(IntArgReg1); + loadValue(Address(StackPointer, ResumeFromException::offsetOfException()), + exception); + + ValueOperand exceptionStack = ValueOperand(IntArgReg2); + loadValue( + Address(StackPointer, ResumeFromException::offsetOfExceptionStack()), + exceptionStack); + + loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), + IntArgReg0); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), + FramePointer); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), + StackPointer); + + pushValue(exception); + pushValue(exceptionStack); + pushValue(BooleanValue(true)); + jump(IntArgReg0); + + // Return BaselineFrame->returnValue() to the caller. + // Used in debug mode and for GeneratorReturn. + Label profilingInstrumentation; + bind(&returnBaseline); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), + FramePointer); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), + StackPointer); + loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()), + JSReturnOperand); + jump(&profilingInstrumentation); + + // Return the given value to the caller. + bind(&returnIon); + loadValue(Address(StackPointer, ResumeFromException::offsetOfException()), + JSReturnOperand); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), + FramePointer); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), + StackPointer); + + // If profiling is enabled, then update the lastProfilingFrame to refer to + // caller frame before returning. This code is shared by ForcedReturnIon + // and ForcedReturnBaseline. + bind(&profilingInstrumentation); + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled( + asMasm().runtime()->geckoProfiler().addressOfEnabled()); + asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + jump(profilerExitTail); + bind(&skipProfilingInstrumentation); + } + + movePtr(FramePointer, StackPointer); + pop(FramePointer); + asMasm().popReturnAddress(); + abiret(); + + // If we are bailing out to baseline to handle an exception, jump to + // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success. + bind(&bailout); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfBailoutInfo()), + CallTempReg1); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), + StackPointer); + move32(Imm32(1), ReturnReg); + jump(bailoutTail); + + // Reset SP and FP; SP is pointing to the unwound return address to the + // wasm interpreter entry, so we can just pop it and return. + bind(&wasmInterpEntry); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()), + FramePointer); + loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()), + StackPointer); + movePtr(ImmWord(wasm::InterpFailInstanceReg), InstanceReg); + asMasm().popReturnAddress(); + abiret(); + + // Found a wasm catch handler, restore state and jump to it. + bind(&wasmCatch); + wasm::GenerateJumpToCatchHandler(asMasm(), StackPointer, IntArgReg1, + IntArgReg2, IntArgReg3); +} + +#define IA64_DEFINE_UNBOX(Name, Type) \ + void MacroAssemblerIa64::Name(const ValueOperand& src, Register dest) { \ + unboxNonDouble(src, dest, Type); \ + } \ + void MacroAssemblerIa64::Name(const Address& src, Register dest) { \ + unboxNonDouble(src, dest, Type); \ + } + +IA64_DEFINE_UNBOX(unboxString, JSVAL_TYPE_STRING) +IA64_DEFINE_UNBOX(unboxSymbol, JSVAL_TYPE_SYMBOL) +IA64_DEFINE_UNBOX(unboxBigInt, JSVAL_TYPE_BIGINT) + +#undef IA64_DEFINE_UNBOX + +#define IA64_DEFINE_UNBOX3(Name, Type) \ + void MacroAssemblerIa64::Name(const ValueOperand& src, Register dest) { \ + unboxNonDouble(src, dest, Type); \ + } \ + void MacroAssemblerIa64::Name(const Address& src, Register dest) { \ + unboxNonDouble(src, dest, Type); \ + } \ + void MacroAssemblerIa64::Name(const BaseIndex& src, Register dest) { \ + unboxNonDouble(src, dest, Type); \ + } + +IA64_DEFINE_UNBOX3(unboxInt32, JSVAL_TYPE_INT32) +IA64_DEFINE_UNBOX3(unboxBoolean, JSVAL_TYPE_BOOLEAN) +IA64_DEFINE_UNBOX3(unboxObject, JSVAL_TYPE_OBJECT) + +#undef IA64_DEFINE_UNBOX3 + +void MacroAssemblerIa64::incrementInt32Value(const Address& addr) { + computeAddress(addr, SecondScratchReg); + emitM(ia64::Ld4(ScratchReg.encoding(), SecondScratchReg.encoding())); + emitM(ia64::Adds(ScratchReg.encoding(), 1, ScratchReg.encoding())); + emitM(ia64::St4(SecondScratchReg.encoding(), ScratchReg.encoding())); +} + +Assembler::Condition MacroAssemblerIa64::testInt32Truthy( + bool truthy, const ValueOperand& value) { + MOZ_CRASH("ia64: testInt32Truthy leaves no condition code behind"); +} + +Assembler::Condition MacroAssemblerIa64::testStringTruthy( + bool truthy, const ValueOperand& value) { + MOZ_CRASH("ia64: testStringTruthy leaves no condition code behind"); +} + +Assembler::Condition MacroAssemblerIa64::testBigIntTruthy( + bool truthy, const ValueOperand& value) { + MOZ_CRASH("ia64: testBigIntTruthy leaves no condition code behind"); +} + +void MacroAssemblerIa64::testNullSet(Condition cond, ValueOperand value, + Register dest) { + emitTagOf(value, SecondScratchReg); + asMasm().cmp32Set(cond, SecondScratchReg, ImmWord(JSVAL_SHIFTED_TAG_NULL), + dest); +} + +void MacroAssemblerIa64::testObjectSet(Condition cond, ValueOperand value, + Register dest) { + emitTagOf(value, SecondScratchReg); + asMasm().cmp32Set(cond, SecondScratchReg, ImmWord(JSVAL_SHIFTED_TAG_OBJECT), + dest); +} + +void MacroAssemblerIa64::testUndefinedSet(Condition cond, ValueOperand value, + Register dest) { + emitTagOf(value, SecondScratchReg); + asMasm().cmp32Set(cond, SecondScratchReg, + ImmWord(JSVAL_SHIFTED_TAG_UNDEFINED), dest); +} + +// =========================================================================== +// MacroAssembler: stack management. + +void MacroAssembler::flush() {} + +void MacroAssembler::comment(const char* msg) {} + +void MacroAssembler::subFromStackPtr(Imm32 imm32) { + if (imm32.value) { + subPtr(imm32, StackPointer); + } +} + +void MacroAssembler::freeStackTo(uint32_t framePushed) { + MOZ_ASSERT(framePushed <= framePushed_); + // sp = FramePointer - framePushed. This has to be recomputed from the frame + // pointer rather than adjusted relative to the current sp: the callers of + // freeStackTo are join points where the real sp and the assembler's static + // framePushed_ have deliberately diverged. + movePtr(FramePointer, StackPointer); + if (framePushed) { + subPtr(Imm32(framePushed), StackPointer); + } + framePushed_ = framePushed; +} + +size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) { + return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes(); +} + +void MacroAssembler::PushRegsInMask(LiveRegisterSet set) { + int32_t diff = + int32_t(set.gprs().size() * sizeof(intptr_t) + + set.fpus().getPushSizeInBytes()); + reserveStack(diff); + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diff -= sizeof(intptr_t); + storePtr(*iter, Address(StackPointer, diff)); + } + // Singles and doubles alias the same architectural register, so + // reduceSetForPush collapses each pair to one double-sized slot. + for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); + iter.more(); ++iter) { + diff -= sizeof(double); + storeDouble(*iter, Address(StackPointer, diff)); + } + MOZ_ASSERT(diff == 0); +} + +// |dest| points at the *end* of the save area -- the first register goes at +// dest.offset - sizeof(register) -- so the layout has to be built downwards +// from there to match the one PushRegsInMask leaves behind. +void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest, + Register scratch) { + FloatRegisterSet fpuSet(set.fpus().reduceSetForPush()); + int32_t diff = int32_t(set.gprs().size() * sizeof(intptr_t) + + fpuSet.getPushSizeInBytes()); + MOZ_ASSERT(dest.offset >= diff); + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diff -= sizeof(intptr_t); + dest.offset -= sizeof(intptr_t); + storePtr(*iter, dest); + } + for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) { + diff -= sizeof(double); + dest.offset -= sizeof(double); + storeDouble(*iter, dest); + } + MOZ_ASSERT(diff == 0); +} + +void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set, + LiveRegisterSet ignore) { + int32_t diff = + int32_t(set.gprs().size() * sizeof(intptr_t) + + set.fpus().getPushSizeInBytes()); + const int32_t reserved = diff; + for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) { + diff -= sizeof(intptr_t); + if (!ignore.has(*iter)) { + loadPtr(Address(StackPointer, diff), *iter); + } + } + for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush()); + iter.more(); ++iter) { + diff -= sizeof(double); + if (!ignore.has(*iter)) { + loadDouble(Address(StackPointer, diff), *iter); + } + } + MOZ_ASSERT(diff == 0); + freeStack(reserved); +} + +void MacroAssembler::Push(Register reg) { + push(reg); + adjustFrame(sizeof(intptr_t)); +} + +void MacroAssembler::Push(const Imm32 imm) { + push(imm); + adjustFrame(sizeof(intptr_t)); +} + +void MacroAssembler::Push(const ImmWord imm) { + push(imm); + adjustFrame(sizeof(intptr_t)); +} + +void MacroAssembler::Push(const ImmPtr imm) { + push(ImmWord(uintptr_t(imm.value))); + adjustFrame(sizeof(intptr_t)); +} + +void MacroAssembler::Push(const ImmGCPtr ptr) { + push(ptr); + adjustFrame(sizeof(intptr_t)); +} + +void MacroAssembler::Push(FloatRegister reg) { + push(reg); + adjustFrame(sizeof(double)); +} + +void MacroAssembler::PushBoxed(FloatRegister reg) { + subFromStackPtr(Imm32(sizeof(double))); + boxDouble(reg, Address(getStackPointer(), 0)); + adjustFrame(sizeof(double)); +} + +void MacroAssembler::Pop(Register reg) { + pop(reg); + implicitPop(sizeof(intptr_t)); +} + +void MacroAssembler::Pop(FloatRegister reg) { + pop(reg); + implicitPop(sizeof(double)); +} + +void MacroAssembler::PopStackPtr() { + loadPtr(Address(StackPointer, 0), StackPointer); + adjustFrame(-int32_t(sizeof(intptr_t))); +} + +void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch, + ExitFrameType type) { + enterFakeExitFrame(cxreg, scratch, type); +} + +void MacroAssembler::Pop(const ValueOperand& val) { + popValue(val); + implicitPop(sizeof(Value)); +} + +// =========================================================================== +// MacroAssembler: calls. +// +// A call goes through a branch register: "mov b6 = target" then +// "br.call.sptk.many b0 = b6", which leaves the return address in b0. + +CodeOffset MacroAssembler::call(Register reg) { + emitI(ia64::MovToBr(6, reg.encoding())); + emitB(ia64::BrCall(0, 6)); + return CodeOffset(currentOffset()); +} + +// IA-64 psABI: a C function pointer is the address of a 2-word descriptor +// {entry@0, gp@8}, not a code entry -- a plain call(Register) would branch +// straight into that descriptor's data instead of the callee. Save our GP +// across the call (the callee's descriptor GP need not be ours, and nothing +// else restores it once the callee returns), load the callee's entry and GP +// from the descriptor, call, then restore. +CodeOffset MacroAssemblerIa64::callABIDescriptorIA64(Register descriptor) { + // Our own GP is saved/restored around the whole call by callWithABIPre/ + // callWithABIPost (in the same reserved stack slot used for b0) -- doing + // it here instead, via push/pop, would shift SP between argument setup + // and the branch and misalign any stack-passed arguments already written + // relative to the SP callWithABIPre established. + Register entry = (descriptor == ScratchReg) ? SecondScratchReg : ScratchReg; + loadPtr(Address(descriptor, 8), GpReg); + loadPtr(Address(descriptor, 0), entry); + emitI(ia64::MovToBr(6, entry.encoding())); + emitB(ia64::BrCall(0, 6)); + return CodeOffset(currentOffset()); +} + +static ia64::Insn BrCallRelB0(int32_t bundleDisp, uint32_t qp) { + return ia64::BrCallRel(0, bundleDisp, qp); +} + +CodeOffset MacroAssembler::call(Label* label) { + emitBranchToLabel(&BrCallRelB0, 0, label); + return CodeOffset(currentOffset()); +} + +CodeOffset MacroAssembler::call(const Address& addr) { + loadPtr(addr, ScratchReg); + return call(ScratchReg); +} + +void MacroAssembler::call(ImmWord imm) { + movePtr(imm, ScratchReg); + call(ScratchReg); +} + +void MacroAssembler::call(ImmPtr imm) { call(ImmWord(uintptr_t(imm.value))); } + +CodeOffset MacroAssembler::call(wasm::SymbolicAddress imm) { + MOZ_CRASH("ia64: wasm is not supported"); +} + +void MacroAssembler::call(JitCode* c) { + writeJumpRelocation(); + emitMovl(ScratchReg.encoding(), uint64_t(uintptr_t(c->raw()))); + call(ScratchReg); +} + +CodeOffset MacroAssembler::callWithPatch() { + emitB(BrCallRelB0(0, 0)); + return CodeOffset(currentOffset()); +} + +void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) { + // callerOffset is the return address, i.e. just past the br.call bundle. + BufferOffset site(callerOffset - sizeof(ia64::Bundle)); + int32_t disp = BundleIndex(BufferOffset(calleeOffset)) - BundleIndex(site); + MOZ_RELEASE_ASSERT(ia64::BranchDispInRange(disp)); + ia64::PatchBranchDisp(bundleAt(site), disp); +} + +// Unreachable now that JS_USE_LINK_REGISTER is defined for ia64 (see +// shared/Assembler-shared.h): the one generic caller (BaselineCodeGen's +// genStart trampoline) takes the #ifdef JS_USE_LINK_REGISTER branch instead, +// using a real call()/pushReturnAddress() pair that matches how b0 already +// works everywhere else in this backend. +void MacroAssembler::callAndPushReturnAddress(Register reg) { + MOZ_CRASH("ia64: unreachable, JS_USE_LINK_REGISTER is defined"); +} + +void MacroAssembler::callAndPushReturnAddress(Label* label) { + MOZ_CRASH("ia64: unreachable, JS_USE_LINK_REGISTER is defined"); +} + +void MacroAssembler::pushReturnAddress() { + emitI(ia64::MovFromBr(ScratchReg.encoding(), 0)); + push(ScratchReg); +} + +void MacroAssembler::popReturnAddress() { + pop(ScratchReg); + emitI(ia64::MovToBr(0, ScratchReg.encoding())); +} + +bool MacroAssemblerIa64::buildOOLFakeExitFrame(void* fakeReturnAddr) { + asMasm().Push(FrameDescriptor(FrameType::IonJS)); + asMasm().Push(ImmPtr(fakeReturnAddr)); + asMasm().Push(FramePointer); + return true; +} + +uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) { + CodeLabel cl; + + mov(&cl, scratch); + Push(scratch); + bind(&cl); + uint32_t retAddr = currentOffset(); + + addCodeLabel(cl); + return retAddr; +} + +// A movl holding the absolute target, moved into b6 and branched through. The +// returned offset is that of the movl bundle, which is what patchFarJump() +// rewrites. +CodeOffset MacroAssembler::farJumpWithPatch() { + CodeOffset offset(currentOffset()); + emitMovl(ScratchReg.encoding(), 0); + emitI(ia64::MovToBr(6, ScratchReg.encoding())); + emitB(ia64::BrCond(6)); + return offset; +} + +void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) { + MOZ_CRASH("ia64: an in-buffer far jump has no address to patch in yet"); +} + +// static +void MacroAssembler::patchFarJump(uint8_t* farJump, uint8_t* target) { + WriteMovlImm(farJump, uint64_t(uintptr_t(target))); + FlushICache(farJump, sizeof(ia64::Bundle)); +} + +// Reserves exactly the 3 bundles a near call needs (movl/MovToBr/BrCall, see +// call(Register)/PatchWrite_NearCallSize), so patchNopToCall can fill them +// in later. The returned offset is the *end* of the reserved region, and +// patchNopToCall's |callsite| is that same offset -- matching every other +// backend's nopPatchableToCall/patchNopToCall pairing. +CodeOffset MacroAssembler::nopPatchableToCall() { + emitM(ia64::NopM()); + emitM(ia64::NopM()); + emitM(ia64::NopM()); + return CodeOffset(currentOffset()); +} + +void MacroAssembler::patchNopToCall(uint8_t* callsite, uint8_t* target) { + // Each slot currently holds a nop, not a valid movl, so the movl bundle + // has to be built fresh rather than read-modify-written via WriteMovlImm. + uint8_t* start = callsite - 3 * sizeof(ia64::Bundle); + ia64::Bundle* bundles = reinterpret_cast(start); + bundles[0] = + ia64::MovlBundle(ScratchReg.encoding(), uint64_t(uintptr_t(target)), 0); + bundles[1] = ia64::BundleI(ia64::MovToBr(6, ScratchReg.encoding())); + bundles[2] = ia64::BundleB(ia64::BrCall(0, 6)); + FlushICache(start, 3 * sizeof(ia64::Bundle)); +} + +void MacroAssembler::patchCallToNop(uint8_t* callsite) { + uint8_t* start = callsite - 3 * sizeof(ia64::Bundle); + ia64::Bundle nop = ia64::BundleM(ia64::NopM()); + for (int i = 0; i < 3; i++) { + reinterpret_cast(start)[i] = nop; + } + FlushICache(start, 3 * sizeof(ia64::Bundle)); +} + +// Same movl-placeholder shape as farJumpWithPatch: a same-buffer address +// that isn't known yet (e.g. a jump table emitted later in the same +// function) gets a 0 placeholder now and the real value written in once +// it's known. +CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) { + CodeOffset offset(currentOffset()); + emitMovl(dest.encoding(), 0); + return offset; +} + +void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc, + CodeLocationLabel target) { + WriteMovlImm(loc.raw(), uint64_t(uintptr_t(target.raw()))); + FlushICache(loc.raw(), sizeof(ia64::Bundle)); +} + +// =========================================================================== +// MacroAssembler: the C ABI. + +void MacroAssembler::setupUnalignedABICall(Register scratch) { + MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls"); + setupNativeABICall(); + dynamicAlignment_ = true; + + movePtr(StackPointer, scratch); + subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + andPtr(Imm32(~(ABIStackAlignment - 1)), StackPointer); + storePtr(scratch, Address(StackPointer, 0)); +} + +void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) { + MOZ_ASSERT(inCall_); + uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar(); + + // Room for the saved return-branch register, plus our own GP -- a call + // through callABIDescriptorIA64's descriptor dance clobbers GP with the + // callee's, and nothing else restores it before callWithABIPost does. Both + // go at the very top of the frame, above the outgoing stack arguments and + // well above the callee's scratch area (see ABIArgGenerator). + stackForCall += 2 * sizeof(intptr_t); + + if (dynamicAlignment_) { + stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment); + } else { + uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0; + stackForCall += ComputeByteAlignment( + stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment); + } + + *stackAdjust = stackForCall; + reserveStack(stackForCall); + + // br.call overwrites b0, so stash it for callWithABIPost. Keep it in + // SecondScratchReg: storePtr(Address) synthesises the address in ScratchReg. + emitI(ia64::MovFromBr(SecondScratchReg.encoding(), 0)); + storePtr(SecondScratchReg, + Address(StackPointer, stackForCall - sizeof(intptr_t))); + storePtr(GpReg, Address(StackPointer, stackForCall - 2 * sizeof(intptr_t))); + + { + enoughMemory_ &= moveResolver_.resolve(); + if (!enoughMemory_) { + return; + } + MoveEmitter emitter(*this); + emitter.emit(moveResolver_); + emitter.finish(); + } +} + +void MacroAssembler::callWithABIPost(uint32_t stackAdjust, ABIType result) { + loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ScratchReg); + emitI(ia64::MovToBr(0, ScratchReg.encoding())); + loadPtr(Address(StackPointer, stackAdjust - 2 * sizeof(intptr_t)), GpReg); + + if (dynamicAlignment_) { + loadPtr(Address(StackPointer, stackAdjust), StackPointer); + adjustFrame(-int32_t(stackAdjust)); + } else { + freeStack(stackAdjust); + } + +#ifdef DEBUG + MOZ_ASSERT(inCall_); + inCall_ = false; +#endif +} + +void MacroAssembler::callWithABINoProfiler(Register fun, ABIType result) { + // Move the callee somewhere the argument setup below cannot clobber. + movePtr(fun, CallTempReg5); + + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + // |fun| is a C function pointer, i.e. the address of a {entry, gp} + // descriptor, not a code entry -- see callABIDescriptorIA64. + callABIDescriptorIA64(CallTempReg5); + callWithABIPost(stackAdjust, result); +} + +void MacroAssembler::callWithABINoProfiler(const Address& fun, ABIType result) { + loadPtr(fun, CallTempReg5); + + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + callABIDescriptorIA64(CallTempReg5); + callWithABIPost(stackAdjust, result); +} + +void MacroAssembler::callWithABIJitCode(Register fun, ABIType result) { + AutoProfilerCallInstrumentation profiler(*this); + movePtr(fun, CallTempReg5); + + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + // |fun| is a raw JIT entry rather than a {entry, gp} descriptor, so branch to + // it directly. Our own GP is still saved and restored by callWithABIPre/Post, + // and r1 is non-allocatable, so the callee cannot clobber it either. + call(CallTempReg5); + callWithABIPost(stackAdjust, result); +} + +// =========================================================================== +// MacroAssembler: GC support. + +void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, + Register temp, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + MOZ_ASSERT(ptr != temp); + MOZ_ASSERT(temp != InvalidReg); + + movePtr(ptr, temp); + andPtr(Imm32(int32_t(~gc::ChunkMask)), temp); + branchPtr(cond == Assembler::Equal ? Assembler::NotEqual : Assembler::Equal, + Address(temp, gc::ChunkStoreBufferOffset), ImmWord(0), label); +} + +void MacroAssembler::branchValueIsNurseryCell(Condition cond, + const Address& address, + Register temp, Label* label) { + branchValueIsNurseryCellImpl(cond, address, temp, label); +} + +void MacroAssembler::branchValueIsNurseryCell(Condition cond, + ValueOperand value, Register temp, + Label* label) { + branchValueIsNurseryCellImpl(cond, value, temp, label); +} + +template +void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond, + const T& value, Register temp, + Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + MOZ_ASSERT(temp != InvalidReg); + Label done; + branchTestGCThing(Assembler::NotEqual, value, + cond == Assembler::Equal ? &done : label); + + unboxGCThingForGCBarrier(value, temp); + andPtr(Imm32(int32_t(~gc::ChunkMask)), temp); + loadPtr(Address(temp, gc::ChunkStoreBufferOffset), temp); + branchPtr(cond == Assembler::Equal ? Assembler::NotEqual : Assembler::Equal, + temp, ImmWord(0), label); + + bind(&done); +} + +void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs, + const Value& rhs, Label* label) { + MOZ_ASSERT(cond == Equal || cond == NotEqual); + if (!rhs.isGCThing()) { + branchPtr(cond, lhs.valueReg(), ImmWord(rhs.asRawBits()), label); + return; + } + MOZ_ASSERT(lhs.valueReg() != SecondScratchReg); + moveValue(rhs, ValueOperand(SecondScratchReg)); + branchPtr(cond, lhs.valueReg(), SecondScratchReg, label); +} + +void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift, + Register pointer) { + if (IsShiftInScaleRange(shift)) { + computeEffectiveAddress( + BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer); + return; + } + lshift32(Imm32(shift), indexTemp32); + addPtr(indexTemp32, pointer); +} + +// =========================================================================== +// MacroAssembler: operations that still need encodings. +// +// Integer divide is a software sequence on IA-64 and every floating-point +// operation needs the F-unit encodings; SupportsFloatingPoint() returns false, +// so the floating-point paths should not be reachable. + +void MacroAssembler::flexibleDivMod32(Register lhs, Register rhs, + Register divOutput, Register remOutput, + bool isUnsigned, const LiveRegisterSet&) { + ma_divmod(divOutput, remOutput, lhs, rhs, isUnsigned, 32); +} + +void MacroAssembler::flexibleQuotient32(Register lhs, Register rhs, + Register dest, bool isUnsigned, + const LiveRegisterSet&) { + ma_divmod(dest, InvalidReg, lhs, rhs, isUnsigned, 32); +} + +void MacroAssembler::flexibleRemainder32(Register lhs, Register rhs, + Register dest, bool isUnsigned, + const LiveRegisterSet&) { + ma_divmod(InvalidReg, dest, lhs, rhs, isUnsigned, 32); +} + +CodeOffset MacroAssembler::move32WithPatch(Register dest) { + CodeOffset offset(currentOffset()); + emitMovl(dest.encoding(), 0); + return offset; +} + +void MacroAssembler::patchMove32(CodeOffset offset, Imm32 n) { + WriteMovlImm( + reinterpret_cast(bundleAt(BufferOffset(offset.offset()))), + uint64_t(uint32_t(n.value))); +} + +CodeOffset MacroAssembler::sub32FromMemAndBranchIfNegativeWithPatch( + Address address, Label* label) { + computeAddress(address, SecondScratchReg); + emitM(ia64::Ld4(ScratchReg.encoding(), SecondScratchReg.encoding())); + // The "adds" below carries the immediate; it is emitted as a no-op add and + // rewritten by patchSub32FromMemAndBranchIfNegative(). + CodeOffset patchPoint(currentOffset()); + emitM(ia64::Adds(ScratchReg.encoding(), 0, ScratchReg.encoding())); + emitM(ia64::St4(SecondScratchReg.encoding(), ScratchReg.encoding())); + emitI(ia64::Sxt4(ScratchReg.encoding(), ScratchReg.encoding())); + emitI(ia64::CmpLt(kP, kQ, ScratchReg.encoding(), Registers::zero)); + emitBranchToLabel(&ia64::BrCondRel, kP, label); + return patchPoint; +} + +void MacroAssembler::patchSub32FromMemAndBranchIfNegative(CodeOffset offset, + Imm32 imm) { + MOZ_RELEASE_ASSERT(imm.value >= 1 && imm.value <= 127); + WriteAddsImm14(bundleAt(BufferOffset(offset.offset())), -int64_t(imm.value)); +} + +bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; } + +// fcvt.fx takes its rounding mode from the status field rather than the +// instruction, and HasRoundInstruction() returns false precisely so generic +// code never asks for anything but the default round-to-nearest-even (sf0), +// which is what NearestTiesToEven wants directly; the other RoundingModes +// are unreachable here for the same reason. +void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src, + FloatRegister dest) { + MOZ_ASSERT(mode == RoundingMode::TowardsNegativeInfinity || + mode == RoundingMode::NearestTiesToEven); + ScratchDoubleScope scratch(*this); + emitF(ia64::FcvtFx(scratch.encoding(), src.encoding(), ia64::sf0)); + emitF(ia64::FcvtXf(dest.encoding(), scratch.encoding())); +} + +void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src, + FloatRegister dest) { + nearbyIntDouble(mode, src, dest); +} + +// Clamp to [0, 255], rounding to nearest-even in between (Uint8Clamped's +// defined behaviour), matching convertDoubleToInt32-style truncation but +// with saturation instead of failure. +void MacroAssembler::clampDoubleToUint8(FloatRegister src, Register dest) { + // ToUint8Clamp: NaN and x <= 0 give 0, x >= 255 gives 255, and everything + // between rounds half-to-even -- which is what fcvt.fx does in sf0. + ScratchDoubleScope scratch(*this); + emitF(ia64::FcvtFx(scratch.encoding(), src.encoding(), ia64::sf0)); + emitM(ia64::GetfSig(dest.encoding(), scratch.encoding())); + + // Keep all 64 bits: narrowing to 32 here would fold large inputs back into + // the clamped range (2^31 reads as negative, 2^32 as zero). + uint32_t pHigh = emitCompare(Assembler::GreaterThan, dest, Imm32(255)); + emitM(ia64::Adds(dest.encoding(), 255, Registers::zero, pHigh)); + + // fcvt.fx delivers the integer indefinite (INT64_MIN) for NaN, the + // infinities and anything too big for 64 bits. A negative result therefore + // means either a genuinely negative input or a positive overflow; clamp it + // to 255 here and let the sign test below rewrite the former to 0. + uint32_t pOverflow = emitCompare(Assembler::LessThan, dest, Imm32(0)); + emitM(ia64::Adds(dest.encoding(), 255, Registers::zero, pOverflow)); + + // f0 is hardwired +0.0, and this condition is false-on-unordered's + // complement, so NaN lands here too. + FloatRegister zero = + FloatRegister(FloatRegister::Code(FloatRegisters::f0), FloatRegister::Double); + uint32_t pNotPos = emitCompareDouble( + Assembler::DoubleLessThanOrEqualOrUnordered, src, zero); + emitM(ia64::MovReg(dest.encoding(), Registers::zero, pNotPos)); +} + +FaultingCodeOffset MacroAssembler::wasmTrapInstruction() { + MOZ_CRASH("ia64: wasm is not supported"); +} + +void MacroAssembler::wasmMarkCallAsSlow() { + MOZ_CRASH("ia64: wasm is not supported"); +} + +void MacroAssembler::wasmCheckSlowCallsite(Register, Label*, Register, + Register) { + MOZ_CRASH("ia64: wasm is not supported"); +} + +CodeOffset MacroAssembler::wasmMarkedSlowCall(const wasm::CallSiteDesc&, + const Register) { + MOZ_CRASH("ia64: wasm is not supported"); +} + + +// =========================================================================== +// Atomics, wasm memory access and the remaining floating-point conversions all +// need encodings that AssemblerCore-ia64.h does not provide yet. + +void MacroAssembler::flexibleRemainderPtr(Register lhs, Register rhs, Register dest, bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) { + ma_divmod(InvalidReg, dest, lhs, rhs, isUnsigned, 64); +} + +void MacroAssembler::flexibleQuotientPtr(Register lhs, Register rhs, Register dest, bool isUnsigned, const LiveRegisterSet& volatileLiveRegs) { + ma_divmod(dest, InvalidReg, lhs, rhs, isUnsigned, 64); +} + +// Same setf.sig+fcvt.xf exactness as convertInt32ToDouble, just fed the full +// 64-bit register (no need to isolate 32 bits first); unsigned magnitudes +// that exceed INT64_MAX go through Fnorm instead of FcvtXf for the same +// "fcvt.xuf is just fnorm on setf.sig's already-unsigned representation" +// reason ma_divmod's unsigned 64-bit path does. +void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) { + emitM(ia64::SetfSig(dest.encoding(), src.reg.encoding())); + emitF(ia64::FcvtXf(dest.encoding(), dest.encoding())); +} + +void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) { + convertInt64ToDouble(src, dest); +} + +void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, Register temp) { + emitM(ia64::SetfSig(dest.encoding(), src.reg.encoding())); + emitF(ia64::Fnorm(dest.encoding(), dest.encoding(), ia64::sf0)); +} + +void MacroAssembler::convertUInt64ToFloat32(Register64 src, FloatRegister dest, Register temp) { + convertUInt64ToDouble(src, dest, temp); +} + +void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index, Register boundsCheckLimit, Label* label) { + MOZ_CRASH("ia64: wasmBoundsCheck32 is not supported"); +} + +void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr, AnyRegister output) { + MOZ_CRASH("ia64: wasmLoad is not supported"); +} + +void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr, Register64 output) { + MOZ_CRASH("ia64: wasmLoadI64 is not supported"); +} + +void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Register memoryBase, Register ptr) { + MOZ_CRASH("ia64: wasmStore is not supported"); +} + +void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase, Register ptr) { + MOZ_CRASH("ia64: wasmStoreI64 is not supported"); +} + +void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, Register output, bool isSaturating, Label* oolEntry) { + MOZ_CRASH("ia64: wasmTruncateDoubleToUInt32 is not supported"); +} + +void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input, Register output, bool isSaturating, Label* oolEntry) { + MOZ_CRASH("ia64: wasmTruncateDoubleToInt32 is not supported"); +} + +void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input, Register output, TruncFlags flags, const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) { + MOZ_CRASH("ia64: oolWasmTruncateCheckF64ToI32 is not supported"); +} + +void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, Register output, bool isSaturating, Label* oolEntry) { + MOZ_CRASH("ia64: wasmTruncateFloat32ToUInt32 is not supported"); +} + +void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input, Register output, bool isSaturating, Label* oolEntry) { + MOZ_CRASH("ia64: wasmTruncateFloat32ToInt32 is not supported"); +} + +void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input, Register output, TruncFlags flags, const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) { + MOZ_CRASH("ia64: oolWasmTruncateCheckF32ToI32 is not supported"); +} + +void MacroAssembler::wasmTruncateDoubleToInt64(FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) { + MOZ_CRASH("ia64: wasmTruncateDoubleToInt64 is not supported"); +} + +void MacroAssembler::wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) { + MOZ_CRASH("ia64: wasmTruncateDoubleToUInt64 is not supported"); +} + +void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input, Register64 output, TruncFlags flags, const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) { + MOZ_CRASH("ia64: oolWasmTruncateCheckF64ToI64 is not supported"); +} + +void MacroAssembler::wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) { + MOZ_CRASH("ia64: wasmTruncateFloat32ToInt64 is not supported"); +} + +void MacroAssembler::wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) { + MOZ_CRASH("ia64: wasmTruncateFloat32ToUInt64 is not supported"); +} + +void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input, Register64 output, TruncFlags flags, const wasm::TrapSiteDesc& trapSiteDesc, Label* rejoin) { + MOZ_CRASH("ia64: oolWasmTruncateCheckF32ToI64 is not supported"); +} + +// The comparand goes through ar.ccv unmodified, and for a full 64-bit access +// the sign-extended-register convention already matches the memory width +// exactly, so unlike the 32-bit case there is no separate zero-extension +// step needed here. +template +static void CompareExchange64(MacroAssembler& masm, const T& mem, + Register64 expected, Register64 replacement, + Register64 output) { + masm.computeAddress(mem, SecondScratchReg); + masm.emitM(ia64::MovToArCcv(expected.reg.encoding())); + masm.emitM(ia64::Cmpxchg8Acq(output.reg.encoding(), + SecondScratchReg.encoding(), + replacement.reg.encoding())); +} + +void MacroAssembler::compareExchange64(Synchronization sync, const Address& mem, Register64 expected, Register64 replacement, Register64 output) { + CompareExchange64(*this, mem, expected, replacement, output); +} + +void MacroAssembler::compareExchange64(Synchronization sync, const BaseIndex& mem, Register64 expected, Register64 replacement, Register64 output) { + CompareExchange64(*this, mem, expected, replacement, output); +} + +template +static void AtomicExchange64(MacroAssembler& masm, const T& mem, + Register64 value, Register64 output) { + masm.computeAddress(mem, SecondScratchReg); + masm.emitM(ia64::Xchg8(output.reg.encoding(), SecondScratchReg.encoding(), + value.reg.encoding())); +} + +void MacroAssembler::atomicExchange64(Synchronization sync, const Address& mem, Register64 value, Register64 output) { + AtomicExchange64(*this, mem, value, output); +} + +void MacroAssembler::atomicExchange64(Synchronization sync, const BaseIndex& mem, Register64 value, Register64 output) { + AtomicExchange64(*this, mem, value, output); +} + +// IA-64's fetchadd only covers a fixed set of +-1/2/4/8/16 increments, so a +// general fetch-op (any AtomicOp, any value) has to be a CAS retry loop: +// read the current value, compute the new one, try to install it, and loop +// on failure (cmpxchg leaves the memory's actual current value in output +// when the compare fails, which is exactly the retry loop's next guess). +template +static void AtomicFetchOp64(MacroAssembler& masm, AtomicOp op, Register64 value, + const T& mem, Register64 temp, Register64 output) { + masm.computeAddress(mem, SecondScratchReg); + masm.emitM(ia64::Ld8(output.reg.encoding(), SecondScratchReg.encoding())); + Label retry; + masm.bind(&retry); + masm.emitM(ia64::MovReg(temp.reg.encoding(), output.reg.encoding())); + switch (op) { + case AtomicOp::Add: + masm.emitM(ia64::Add(temp.reg.encoding(), temp.reg.encoding(), + value.reg.encoding())); + break; + case AtomicOp::Sub: + masm.emitM(ia64::Sub(temp.reg.encoding(), temp.reg.encoding(), + value.reg.encoding())); + break; + case AtomicOp::And: + masm.emitM(ia64::And(temp.reg.encoding(), temp.reg.encoding(), + value.reg.encoding())); + break; + case AtomicOp::Or: + masm.emitM(ia64::Or(temp.reg.encoding(), temp.reg.encoding(), + value.reg.encoding())); + break; + case AtomicOp::Xor: + masm.emitM(ia64::Xor(temp.reg.encoding(), temp.reg.encoding(), + value.reg.encoding())); + break; + default: + MOZ_CRASH("unexpected AtomicOp"); + } + masm.emitM(ia64::MovToArCcv(output.reg.encoding())); + masm.emitM(ia64::Cmpxchg8Acq(ScratchReg.encoding(), + SecondScratchReg.encoding(), + temp.reg.encoding())); + uint32_t p = + masm.emitCompare(Assembler::NotEqual, ScratchReg, output.reg); + masm.emitM(ia64::MovReg(output.reg.encoding(), ScratchReg.encoding())); + masm.emitBranchToLabel(&ia64::BrCondRel, p, &retry); +} + +void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op, Register64 value, const Address& mem, Register64 temp, Register64 output) { + AtomicFetchOp64(*this, op, value, mem, temp, output); +} + +void MacroAssembler::atomicFetchOp64(Synchronization sync, AtomicOp op, Register64 value, const BaseIndex& mem, Register64 temp, Register64 output) { + AtomicFetchOp64(*this, op, value, mem, temp, output); +} + +// AtomicFetchOp64 needs a third register beyond ScratchReg/SecondScratchReg +// (already spoken for by the cmpxchg's transient result and the address) and +// |temp|, so borrow one for the discarded "current value" role. +template +static void AtomicEffectOp64(MacroAssembler& masm, AtomicOp op, + Register64 value, const T& mem, Register64 temp) { + Register64 discard(BorrowScratch(temp.reg, value.reg)); + AtomicFetchOp64(masm, op, value, mem, temp, discard); +} + +void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value, const Address& mem, Register64 temp) { + AtomicEffectOp64(*this, op, value, mem, temp); +} + +void MacroAssembler::atomicEffectOp64(Synchronization sync, AtomicOp op, Register64 value, const BaseIndex& mem, Register64 temp) { + AtomicEffectOp64(*this, op, value, mem, temp); +} + +void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access, const BaseIndex& mem, Register expected, Register replacement, Register output) { + MOZ_CRASH("ia64: wasmCompareExchange is not supported"); +} + +void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access, const BaseIndex& mem, Register value, Register output) { + MOZ_CRASH("ia64: wasmAtomicExchange is not supported"); +} + +void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Register value, const BaseIndex& mem, Register temp, Register output) { + MOZ_CRASH("ia64: wasmAtomicFetchOp is not supported"); +} + +void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op, Register value, const BaseIndex& mem, Register temp) { + MOZ_CRASH("ia64: wasmAtomicEffectOp is not supported"); +} + +void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access, const BaseIndex& mem, Register64 expected, Register64 replacement, Register64 output) { + MOZ_CRASH("ia64: wasmCompareExchange64 is not supported"); +} + +// xchg8 is a single instruction with acquire semantics, so both forms are just +// an address computation away. +template +static void WasmAtomicExchange64(MacroAssembler& masm, + const wasm::MemoryAccessDesc& access, + const T& mem, Register64 value, + Register64 output) { + masm.computeAddress(mem, SecondScratchReg); + masm.append(access, wasm::TrapMachineInsn::Atomic, + FaultingCodeOffset(masm.currentOffset())); + masm.emitM(ia64::Xchg8(output.reg.encoding(), SecondScratchReg.encoding(), + value.reg.encoding())); +} + +void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, const Address& mem, Register64 value, Register64 output) { + WasmAtomicExchange64(*this, access, mem, value, output); +} + +void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access, const BaseIndex& mem, Register64 value, Register64 output) { + WasmAtomicExchange64(*this, access, mem, value, output); +} + +void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op, Register64 value, const BaseIndex& mem, Register64 temp, Register64 output) { + MOZ_CRASH("ia64: wasmAtomicFetchOp64 is not supported"); +} + +// IA-64 has native cmpxchg1/2/4/8 and xchg1/2/4/8, so narrow (<64-bit) +// typed-array atomics need no masked-word emulation, just picking the right +// encoder for the element width. +static uint32_t ScalarWidthLog2(Scalar::Type type) { + switch (type) { + case Scalar::Int8: + case Scalar::Uint8: + case Scalar::Uint8Clamped: + return 0; + case Scalar::Int16: + case Scalar::Uint16: + return 1; + case Scalar::Int32: + case Scalar::Uint32: + return 2; + default: + MOZ_CRASH("unexpected Scalar::Type for a narrow JS atomic"); + } +} + +// ar.ccv compares the exact bit pattern, so the comparand must be masked +// down to the access width regardless of how it's sign/zero-extended in the +// register (see [[ia64-llvm-cmpxchg-ccv-missing-zext]]). +static void MaskToWidth(MacroAssembler& masm, Register dest, Register src, + uint32_t widthLog2) { + switch (widthLog2) { + case 0: + masm.ma_zxt(dest, src, 1); + break; + case 1: + masm.ma_zxt(dest, src, 2); + break; + case 2: + masm.ma_zxt(dest, src, 4); + break; + default: + MOZ_CRASH("unexpected width"); + } +} + +static void EmitCmpxchgAcq(MacroAssembler& masm, uint32_t widthLog2, + uint32_t r1, uint32_t r3, uint32_t r2) { + switch (widthLog2) { + case 0: + masm.emitM(ia64::Cmpxchg1Acq(r1, r3, r2)); + break; + case 1: + masm.emitM(ia64::Cmpxchg2Acq(r1, r3, r2)); + break; + case 2: + masm.emitM(ia64::Cmpxchg4Acq(r1, r3, r2)); + break; + default: + MOZ_CRASH("unexpected width"); + } +} + +static void EmitXchg(MacroAssembler& masm, uint32_t widthLog2, uint32_t r1, + uint32_t r3, uint32_t r2) { + switch (widthLog2) { + case 0: + masm.emitM(ia64::Xchg1(r1, r3, r2)); + break; + case 1: + masm.emitM(ia64::Xchg2(r1, r3, r2)); + break; + case 2: + masm.emitM(ia64::Xchg4(r1, r3, r2)); + break; + default: + MOZ_CRASH("unexpected width"); + } +} + +// Sign/zero-extend the raw memory-width result per the element type's JS +// semantics; Uint32 alone may not fit in an int32 Value, so it becomes a +// double in the float half of the output AnyRegister instead. +static void FinishNarrowAtomic(MacroAssembler& masm, Scalar::Type arrayType, + Register raw, AnyRegister output) { + if (arrayType == Scalar::Uint32) { + masm.convertUInt32ToDouble(raw, output.fpu()); + return; + } + switch (arrayType) { + case Scalar::Int8: + masm.ma_sxt(output.gpr(), raw, 1); + break; + case Scalar::Uint8: + case Scalar::Uint8Clamped: + masm.ma_zxt(output.gpr(), raw, 1); + break; + case Scalar::Int16: + masm.ma_sxt(output.gpr(), raw, 2); + break; + case Scalar::Uint16: + masm.ma_zxt(output.gpr(), raw, 2); + break; + case Scalar::Int32: + masm.ma_sxt(output.gpr(), raw, 4); + break; + default: + MOZ_CRASH("unexpected Scalar::Type for a narrow JS atomic"); + } +} + +template +static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, + const T& mem, Register expected, + Register replacement, Register temp, + AnyRegister output) { + uint32_t width = ScalarWidthLog2(arrayType); + masm.computeAddress(mem, SecondScratchReg); + MaskToWidth(masm, ScratchReg, expected, width); + masm.emitM(ia64::MovToArCcv(ScratchReg.encoding())); + EmitCmpxchgAcq(masm, width, temp.encoding(), SecondScratchReg.encoding(), + replacement.encoding()); + FinishNarrowAtomic(masm, arrayType, temp, output); +} + +void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, Synchronization sync, const Address& mem, Register expected, Register replacement, Register temp, AnyRegister output) { + CompareExchangeJS(*this, arrayType, mem, expected, replacement, temp, output); +} + +void MacroAssembler::compareExchangeJS(Scalar::Type arrayType, Synchronization sync, const BaseIndex& mem, Register expected, Register replacement, Register temp, AnyRegister output) { + CompareExchangeJS(*this, arrayType, mem, expected, replacement, temp, output); +} + +template +static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType, + const T& mem, Register value, Register temp, + AnyRegister output) { + uint32_t width = ScalarWidthLog2(arrayType); + masm.computeAddress(mem, SecondScratchReg); + EmitXchg(masm, width, temp.encoding(), SecondScratchReg.encoding(), + value.encoding()); + FinishNarrowAtomic(masm, arrayType, temp, output); +} + +void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, Synchronization sync, const Address& mem, Register value, Register temp, AnyRegister output) { + AtomicExchangeJS(*this, arrayType, mem, value, temp, output); +} + +void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType, Synchronization sync, const BaseIndex& mem, Register value, Register temp, AnyRegister output) { + AtomicExchangeJS(*this, arrayType, mem, value, temp, output); +} + +// Same CAS-retry-loop shape as AtomicFetchOp64, at the element's own width: +// IA-64 has no general-op fetch primitive narrower than 64 bits either. +template +static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType, + AtomicOp op, Register value, const T& mem, + Register temp1, Register temp2, AnyRegister output, + bool wantResult = true) { + uint32_t width = ScalarWidthLog2(arrayType); + Register current = BorrowScratch(temp1, temp2); + masm.computeAddress(mem, SecondScratchReg); + switch (width) { + case 0: + masm.emitM(ia64::Ld1(current.encoding(), SecondScratchReg.encoding())); + break; + case 1: + masm.emitM(ia64::Ld2(current.encoding(), SecondScratchReg.encoding())); + break; + case 2: + masm.emitM(ia64::Ld4(current.encoding(), SecondScratchReg.encoding())); + break; + default: + MOZ_CRASH("unexpected width"); + } + Label retry; + masm.bind(&retry); + masm.emitM(ia64::MovReg(temp1.encoding(), current.encoding())); + switch (op) { + case AtomicOp::Add: + masm.emitM(ia64::Add(temp1.encoding(), temp1.encoding(), value.encoding())); + break; + case AtomicOp::Sub: + masm.emitM(ia64::Sub(temp1.encoding(), temp1.encoding(), value.encoding())); + break; + case AtomicOp::And: + masm.emitM(ia64::And(temp1.encoding(), temp1.encoding(), value.encoding())); + break; + case AtomicOp::Or: + masm.emitM(ia64::Or(temp1.encoding(), temp1.encoding(), value.encoding())); + break; + case AtomicOp::Xor: + masm.emitM(ia64::Xor(temp1.encoding(), temp1.encoding(), value.encoding())); + break; + default: + MOZ_CRASH("unexpected AtomicOp"); + } + MaskToWidth(masm, temp1, temp1, width); + MaskToWidth(masm, ScratchReg, current, width); + masm.emitM(ia64::MovToArCcv(ScratchReg.encoding())); + EmitCmpxchgAcq(masm, width, temp2.encoding(), SecondScratchReg.encoding(), + temp1.encoding()); + uint32_t p = masm.emitCompare(Assembler::NotEqual, temp2, ScratchReg); + masm.emitM(ia64::MovReg(current.encoding(), temp2.encoding())); + masm.emitBranchToLabel(&ia64::BrCondRel, p, &retry); + if (wantResult) { + FinishNarrowAtomic(masm, arrayType, current, output); + } +} + +void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp op, Register value, const Address& mem, Register temp1, Register temp2, AnyRegister output) { + AtomicFetchOpJS(*this, arrayType, op, value, mem, temp1, temp2, output); +} + +void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp op, Register value, const BaseIndex& mem, Register temp1, Register temp2, AnyRegister output) { + AtomicFetchOpJS(*this, arrayType, op, value, mem, temp1, temp2, output); +} + +void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp op, Register value, const Address& mem, Register temp) { + Register temp2 = BorrowScratch(temp, value); + AtomicFetchOpJS(*this, arrayType, op, value, mem, temp, temp2, + AnyRegister(temp2), /* wantResult = */ false); +} + +void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType, Synchronization sync, AtomicOp op, Register value, const BaseIndex& mem, Register temp) { + Register temp2 = BorrowScratch(temp, value); + AtomicFetchOpJS(*this, arrayType, op, value, mem, temp, temp2, + AnyRegister(temp2), /* wantResult = */ false); +} + + +// =========================================================================== + +void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) { + movePtr(ptr, buffer); + andPtr(Imm32(int32_t(~gc::ChunkMask)), buffer); + loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer); +} + +void MacroAssembler::widenInt32(Register r) { ma_sxt(r, r, 4); } + +// IA-64 orders memory through stop bits and the mf instruction; there is no +// separate speculation barrier, and no pause hint, so both are a nop. +void MacroAssembler::speculationBarrier() { nop(); } + +void MacroAssembler::atomicPause() { nop(); } + +template +void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, const T& dest) { + MOZ_ASSERT(valueType < MIRType::Value); + + if (valueType == MIRType::Double) { + boxDouble(value.reg().typedReg().fpu(), dest); + return; + } + + if (value.constant()) { + storeValue(value.value(), dest); + } else { + storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(), + dest); + } +} + +template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, + MIRType valueType, + const Address& dest); +template void MacroAssembler::storeUnboxedValue( + const ConstantOrRegister& value, MIRType valueType, + const BaseObjectElementIndex& dest); + +void MacroAssembler::branchTestNaNValue(Condition cond, const ValueOperand& val, + Register temp, Label* label) { + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + ScratchDoubleScope scratch(*this); + unboxDouble(val, scratch); + uint32_t p8 = 8, p9 = 9; + emitF(ia64::Fclass(p8, p9, scratch.encoding(), ia64::FclassNaN, 0)); + emitBranchToLabel(&ia64::BrCondRel, cond == Assembler::Equal ? p8 : p9, + label); +} + +// Math.trunc/floor/ceil of anything in (-1, -0] is -0, which has no int32 +// representation, so those inputs have to take the bailout path. Whenever the +// int32 result is zero the source is in (-1, 1), and there the sign bit alone +// decides: set means the true result is -0. +static void FailOnNegativeZeroResult(MacroAssembler& masm, FloatRegister src, + Register dest, Label* fail) { + Label notZero; + uint32_t pNonzero = masm.emitCompare(Assembler::NotEqual, dest, Imm32(0)); + masm.emitBranchToLabel(&ia64::BrCondRel, pNonzero, ¬Zero); + masm.emitM(ia64::GetfD(ScratchReg.encoding(), src.encoding())); + uint32_t pNeg = masm.emitCompare(Assembler::LessThan, ScratchReg, + Register{Registers::zero}); + masm.emitBranchToLabel(&ia64::BrCondRel, pNeg, fail); + masm.bind(¬Zero); +} + +void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + ma_truncateToInt32(src, dest, fail); + FailOnNegativeZeroResult(*this, src, dest, fail); +} +void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + ma_truncateToInt32(src, dest, fail); + FailOnNegativeZeroResult(*this, src, dest, fail); +} + +// floor/ceil have no dedicated instruction; fcvt.fx.trunc plus fcvt.xf gives +// the truncated integer and its exact float value back, and the two only +// disagree when the source had a fractional part -- exactly when floor/ceil +// need to step one further in their respective direction. -0 is handled for +// free: truncating -0.5 gives 0, whose round-trip (0.0) differs from -0.5, +// so floor correctly steps to -1. A -0 result itself has no int32 form and +// goes to |fail| via FailOnNegativeZeroResult. +static void RoundToInt32(MacroAssembler& masm, FloatRegister src, + Register dest, Label* fail, bool isSingle, + bool towardPositive) { + ScratchDoubleScope scratch(masm); + masm.emitF(ia64::FcvtFxTrunc(scratch.encoding(), src.encoding(), ia64::sf0)); + masm.emitM(ia64::GetfSig(dest.encoding(), scratch.encoding())); + masm.ma_sxt(ScratchReg, dest, 4); + uint32_t pOverflow = masm.emitCompare(Assembler::NotEqual, ScratchReg, dest); + masm.emitBranchToLabel(&ia64::BrCondRel, pOverflow, fail); + masm.ma_sxt(dest, dest, 4); + + SecondScratchDoubleScope roundTripped(masm); + masm.emitF(ia64::FcvtXf(roundTripped.encoding(), scratch.encoding())); + uint32_t pExact = masm.emitCompareDouble(Assembler::DoubleEqual, src, + roundTripped); + Label done; + masm.emitBranchToLabel(&ia64::BrCondRel, pExact, &done); + // Had a fraction: step toward +inf (ceil, when src > truncated) or -inf + // (floor, when src < truncated). + uint32_t p = masm.emitCompareDouble( + towardPositive ? Assembler::DoubleGreaterThan : Assembler::DoubleLessThan, + src, roundTripped); + masm.emitM(ia64::Adds(dest.encoding(), towardPositive ? 1 : -1, + dest.encoding(), p)); + masm.bind(&done); + FailOnNegativeZeroResult(masm, src, dest, fail); + (void)isSingle; +} + +void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + RoundToInt32(*this, src, dest, fail, true, false); +} +void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + RoundToInt32(*this, src, dest, fail, false, false); +} +void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest, + Label* fail) { + RoundToInt32(*this, src, dest, fail, true, true); +} +void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest, + Label* fail) { + RoundToInt32(*this, src, dest, fail, false, true); +} + +// JS Math.round is round-half-up (toward +infinity), unlike IEEE +// round-half-to-even. Positive inputs get there by adding the biggest double +// below 0.5 and truncating toward zero -- adding a true 0.5 would push +// 0.49999999999999994 all the way up to 1. Negative inputs add 0.5 and floor, +// after rejecting the [-0.5, -0.0] range whose result is -0. +static void RoundHalfUpToInt32(MacroAssembler& masm, FloatRegister src, + Register dest, FloatRegister temp, Label* fail, + bool isSingle) { + static constexpr FloatRegister kZero{FloatRegisters::f0, + FloatRegister::Double}; + Label negativeOrZero, negative, end; + + masm.branchDouble(Assembler::DoubleLessThanOrEqual, src, kZero, + &negativeOrZero); + + // Strictly positive, or NaN -- which ma_truncateToInt32 rejects. + if (isSingle) { + masm.loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp); + masm.addFloat32(src, temp); + } else { + masm.loadConstantDouble(GetBiggestNumberLessThan(0.5), temp); + masm.addDouble(src, temp); + } + masm.ma_truncateToInt32(temp, dest, fail); + masm.jump(&end); + + masm.bind(&negativeOrZero); + masm.branchDouble(Assembler::DoubleLessThan, src, kZero, &negative); + + // +-0. Math.round(-0) is -0, which no int32 can represent. + masm.emitM(ia64::GetfD(ScratchReg.encoding(), src.encoding())); + { + uint32_t pNeg = masm.emitCompare(Assembler::LessThan, ScratchReg, + Register{Registers::zero}); + masm.emitBranchToLabel(&ia64::BrCondRel, pNeg, fail); + } + masm.move32(Imm32(0), dest); + masm.jump(&end); + + masm.bind(&negative); + { + ScratchDoubleScope half(masm); + masm.loadConstantDouble(-0.5, half); + masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, src, half, fail); + } + if (isSingle) { + masm.loadConstantFloat32(0.5f, temp); + masm.addFloat32(src, temp); + } else { + masm.loadConstantDouble(0.5, temp); + masm.addDouble(src, temp); + } + RoundToInt32(masm, temp, dest, fail, isSingle, /* towardPositive = */ false); + + masm.bind(&end); +} +void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest, + FloatRegister temp, Label* fail) { + RoundHalfUpToInt32(*this, src, dest, temp, fail, true); +} +void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest, + FloatRegister temp, Label* fail) { + RoundHalfUpToInt32(*this, src, dest, temp, fail, false); +} + +// fmerge.s dest = sign_source, magnitude_source: sign comes from its second +// operand, magnitude from its third. +void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs, + FloatRegister output) { + emitF(ia64::FmergeS(output.encoding(), rhs.encoding(), lhs.encoding())); +} +void MacroAssembler::copySignFloat32(FloatRegister lhs, FloatRegister rhs, + FloatRegister output) { + emitF(ia64::FmergeS(output.encoding(), rhs.encoding(), lhs.encoding())); +} +void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) { + convertInt64RegToFloat(src, dest); +} +void MacroAssembler::wasmBoundsCheck32(Condition, Register, Address, Label*) { + MOZ_CRASH("ia64: wasm is not supported"); +} +void MacroAssembler::wasmBoundsCheck64(Condition, Register64, Register64, + Label*) { + MOZ_CRASH("ia64: wasm is not supported"); +} +void MacroAssembler::wasmBoundsCheck64(Condition, Register64, Address, Label*) { + MOZ_CRASH("ia64: wasm is not supported"); +} + +void MacroAssemblerIa64::profilerEnterFrame(Register framePtr, + Register scratch) { + asMasm().loadJSContext(scratch); + loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), + scratch); + storePtr(framePtr, + Address(scratch, JitActivation::offsetOfLastProfilingFrame())); + storePtr(ImmPtr(nullptr), + Address(scratch, JitActivation::offsetOfLastProfilingCallSite())); +} + +void MacroAssemblerIa64::profilerExitFrame() { + jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail()); +} + +} // namespace jit +} // namespace js diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/ia64/MacroAssembler-ia64.h b/js/src/jit/ia64/MacroAssembler-ia64.h --- firefox-153.0.1/js/src/jit/ia64/MacroAssembler-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/MacroAssembler-ia64.h @@ -0,0 +1,1150 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_MacroAssembler_ia64_h +#define jit_ia64_MacroAssembler_ia64_h + +#include + +#include "jit/CompactBuffer.h" +#include "jit/ia64/Assembler-ia64.h" +#include "jit/MoveResolver.h" +#include "jit/shared/IonAssemblerBuffer.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmTypeDecls.h" + +using js::wasm::FaultingCodeOffset; +using js::wasm::FaultingCodeOffsetPair; + +namespace js { +namespace jit { + +class CompactBufferReader; +class MacroAssembler; + +// On JS_PUNBOX64 a Value is a single 64-bit register. IA-64 has no cheap way to +// shift the tag down to bit 0, so the "tag register" used by the testXxx() +// helpers below holds the value with its payload bits masked off instead: +// +// tag = value & ~JS::detail::ValueGCThingPayloadMask +// +// Every tag test then becomes a comparison against the corresponding +// JSVAL_SHIFTED_TAG_* constant, which needs no shift at all. +static const uint64_t ValueTagMask = ~JS::detail::ValueGCThingPayloadMask; + +// movl immediate access. A movl occupies the L and X slots of one MLX bundle +// with its 64-bit immediate scattered across both; MovlBundle() builds one and +// the two helpers below read and rewrite the immediate of one already emitted. +// Relocation processing and every *WithPatch() sequence go through these. + +static const uint64_t kMovlSlotMask = (uint64_t(1) << 41) - 1; + +inline uint64_t ReadMovlImm(const uint8_t* bundle) { + uint64_t lo, hi; + memcpy(&lo, bundle, sizeof(lo)); + memcpy(&hi, bundle + sizeof(lo), sizeof(hi)); + uint64_t l = ((lo >> 46) | (hi << 18)) & kMovlSlotMask; + uint64_t x = (hi >> 23) & kMovlSlotMask; + + uint64_t imm = l << 22; + imm |= ((x >> 13) & 0x7f); + imm |= ((x >> 27) & 0x1ff) << 7; + imm |= ((x >> 22) & 0x1f) << 16; + imm |= ((x >> 21) & 0x1) << 21; + imm |= ((x >> 36) & 0x1) << 63; + return imm; +} + +inline void WriteMovlImm(uint8_t* bundle, uint64_t imm) { + uint64_t lo, hi; + memcpy(&lo, bundle, sizeof(lo)); + memcpy(&hi, bundle + sizeof(lo), sizeof(hi)); + uint64_t x = (hi >> 23) & kMovlSlotMask; + // r1 and qp stay where they are; only the immediate fields are rewritten. + uint64_t keep = ~((uint64_t(0x7f) << 13) | (uint64_t(0x1ff) << 27) | + (uint64_t(0x1f) << 22) | (uint64_t(1) << 21) | + (uint64_t(1) << 36)); + x &= keep; + x |= ((imm >> 0) & 0x7f) << 13; + x |= ((imm >> 7) & 0x1ff) << 27; + x |= ((imm >> 16) & 0x1f) << 22; + x |= ((imm >> 21) & 0x1) << 21; + x |= ((imm >> 63) & 0x1) << 36; + + uint64_t l = (imm >> 22) & kMovlSlotMask; + lo = (lo & ((uint64_t(1) << 46) - 1)) | (l << 46); + hi = (l >> 18) | (x << 23); + memcpy(bundle, &lo, sizeof(lo)); + memcpy(bundle + sizeof(lo), &hi, sizeof(hi)); +} + +// Rewrite the imm14 field of an A4 "adds" sitting in slot 0 of |b|, which is +// where emitM() puts it. Used by the patchable decrement sequence. +inline void WriteAddsImm14(ia64::Bundle* b, int64_t imm) { + uint64_t slot0 = (b->lo >> 5) & kMovlSlotMask; + uint64_t fields = (uint64_t(0x7f) << 13) | (uint64_t(0x3f) << 27) | + (uint64_t(1) << 36); + slot0 = (slot0 & ~fields) | (uint64_t(ia64::fImm14(imm)) & fields); + b->lo = (b->lo & ~(kMovlSlotMask << 5)) | (slot0 << 5); +} + +class ScratchTagScope { + Register reg_; + + public: + ScratchTagScope(MacroAssembler&, const ValueOperand) : reg_(SecondScratchReg) {} + operator Register() { return reg_; } + void release() {} + void reacquire() {} +}; + +class ScratchTagScopeRelease { + public: + explicit ScratchTagScopeRelease(ScratchTagScope*) {} +}; + +class MacroAssemblerIa64 : public Assembler { + public: + // Emit a compare for |cond| and return the predicate register to branch on. + uint32_t emitCompare(Assembler::Condition cond, Register lhs, Register rhs); + uint32_t emitCompare(Assembler::Condition cond, Register lhs, Imm32 rhs); + uint32_t emitCompare(Assembler::Condition cond, Register lhs, ImmWord rhs); + uint32_t emitCompare(Assembler::Condition cond, Register lhs, ImmPtr rhs) { + return emitCompare(cond, lhs, ImmWord(uintptr_t(rhs.value))); + } + uint32_t emitCompare(Assembler::Condition cond, Register lhs, ImmGCPtr rhs) { + movePtr(rhs, ScratchReg); + return emitCompare(cond, lhs, ScratchReg); + } + // Address operands for the cmp32Set()/cmpPtrSet() templates. The load width + // has to be picked by the caller rather than deduced from the right-hand + // side: a Register right-hand side says nothing about it, and cmp32Set() on + // an Ion int32 spill slot must not read 8 bytes -- such a slot is only + // 4-byte aligned, so the wide load both traps and takes half its result from + // the neighbouring slot. + template + uint32_t emitCompare32(Assembler::Condition cond, Register lhs, S rhs) { + return emitCompare(cond, lhs, rhs); + } + template + uint32_t emitCompare32(Assembler::Condition cond, const Address& lhs, S rhs) { + load32(lhs, SecondScratchReg); + return emitCompare(cond, SecondScratchReg, rhs); + } + template + uint32_t emitComparePtr(Assembler::Condition cond, Register lhs, S rhs) { + return emitCompare(cond, lhs, rhs); + } + template + uint32_t emitComparePtr(Assembler::Condition cond, const Address& lhs, + S rhs) { + loadPtr(lhs, SecondScratchReg); + return emitCompare(cond, SecondScratchReg, rhs); + } + + // Emit the "and, then compare against zero" sequence shared by every test. + // |bits| is the width of the test: a 32-bit test has to narrow the AND to + // its low half first, because Signed then means bit 31 rather than bit 63. + uint32_t emitTest(Assembler::Condition cond, Register lhs, Register rhs, + unsigned bits); + uint32_t emitTest(Assembler::Condition cond, Register lhs, Imm32 rhs, + unsigned bits); + + // As emitCompare, but for fcmp. Every DoubleCondition maps to a single + // fcmp's p1 or p2 except DoubleNotEqual and DoubleEqualOrUnordered, which + // need a second, .unc-qualified instruction to fold in the ordered/ + // unordered distinction (see the .cpp for why). + uint32_t emitCompareDouble(Assembler::DoubleCondition cond, + FloatRegister lhs, FloatRegister rhs, + uint32_t sf = ia64::sf0); + + // IA-64 psABI: a C function pointer is the address of a 2-word descriptor + // {entry, gp}, not a code entry -- calling it with a plain call(Register) + // would branch straight into that descriptor's data. Used for every call + // from JIT code into a native C++ helper (see callWithABINoProfiler and + // MacroAssembler::callWithABINoProfiler(void*, ...)). + CodeOffset callABIDescriptorIA64(Register descriptor); + + // IA-64 has no divide or sqrt instruction; both go through the libgcc/gcc + // frcpa+Newton-Raphson sequences (see MacroAssembler-ia64.cpp). + void ma_fdiv(FloatRegister dest, FloatRegister lhs, FloatRegister rhs, + bool isSingle); + void ma_fsqrt(FloatRegister dest, FloatRegister src, bool isSingle); + void ma_fminmax(FloatRegister dest, FloatRegister lhs, FloatRegister rhs, + bool handleNaN, bool isMax, bool isSingle); + + // IA-64 loads and stores have no displacement field, so an Address must be + // materialised into a register first. |out| receives base + offset. + void computeAddress(const Address& addr, Register out); + void computeAddress(const BaseIndex& addr, Register out); + void computeAddress(const AbsoluteAddress& addr, Register out); + + // Shifts, extends and multiplies. IA-64 provides these on the I and F units; + // they are funnelled through these helpers so that the rest of the backend + // needs no knowledge of which encodings exist yet. + void ma_lsl(Register dest, Register src, Imm32 shift); + void ma_lsr(Register dest, Register src, Imm32 shift); + void ma_asr(Register dest, Register src, Imm32 shift); + void ma_lsl(Register dest, Register src, Register shift); + void ma_lsr(Register dest, Register src, Register shift); + void ma_asr(Register dest, Register src, Register shift); + void ma_rol(Register dest, Register src, Imm32 count, unsigned width); + void ma_ror(Register dest, Register src, Imm32 count, unsigned width); + void ma_rol(Register dest, Register src, Register count, unsigned width); + void ma_ror(Register dest, Register src, Register count, unsigned width); + void ma_sxt(Register dest, Register src, unsigned bytes); + void ma_zxt(Register dest, Register src, unsigned bytes); + void ma_mul(Register dest, Register lhs, Register rhs); + + // Software divide; |width| is 32 or 64. Either output may be InvalidReg, but + // not both, and the two must not be the same register. + void ma_divmod(Register divOutput, Register remOutput, Register lhs, + Register rhs, bool isUnsigned, unsigned width); + + // Materialise |imm| into |dest|, using the short "adds" form where possible. + void ma_li(Register dest, int64_t imm); + + // Conditional register move: dest = cond ? src : dest. + void ma_cmov(Assembler::Condition cond, Register src, Register dest); + + // tag = value & ValueTagMask + void emitTagOf(const ValueOperand& value, Register tag); + void emitTagOf(const Address& value, Register tag); + void emitTagOf(const BaseIndex& value, Register tag); + + // Tag tests. Each takes the masked tag produced by emitTagOf() and leaves a + // comparison result in the predicate pair; the returned predicate is the one + // to branch on. + uint32_t testTagEqual(Assembler::Condition cond, Register tag, uint64_t tagBits); + uint32_t testTagRange(Assembler::Condition cond, Register tag, uint64_t bound, + bool inclusiveBelow); + + public: + MacroAssemblerIa64() = default; + + MoveResolver moveResolver_; + + CompactBufferWriter dataRelocations_; + CompactBufferWriter jumpRelocations_; + + // Record that the movl about to be emitted embeds a GC pointer, so that the + // GC can trace and update it. + void writeDataRelocation(ImmGCPtr ptr); + void writeJumpRelocation(); + + size_t size() const { return m_buffer.size(); } + size_t bytesNeeded() const { + return size() + jumpRelocationTableBytes() + dataRelocationTableBytes(); + } + size_t jumpRelocationTableBytes() const { return jumpRelocations_.length(); } + size_t dataRelocationTableBytes() const { return dataRelocations_.length(); } + size_t preBarrierTableBytes() const { return 0; } + + size_t numCodeLabels() const { return codeLabels_.length(); } + CodeLabel codeLabel(size_t i) { return codeLabels_[i]; } + + bool reserve(size_t size) { return !oom(); } + bool appendRawCode(const uint8_t* code, size_t numBytes) { MOZ_CRASH(); } + bool swapBuffer(wasm::Bytes& bytes) { MOZ_CRASH(); } + + void assertNoGCThings() const { + MOZ_ASSERT(dataRelocations_.length() == 0); + } + + static void TraceJumpRelocations(JSTracer*, JitCode*, CompactBufferReader&); + static void TraceDataRelocations(JSTracer*, JitCode*, CompactBufferReader&); + + static bool SupportsFloatingPoint() { return true; } + static bool SupportsUnalignedAccesses() { return false; } + static bool SupportsFastUnalignedFPAccesses() { return false; } + static bool SupportsFloat64To16() { return false; } + static bool SupportsFloat32To16() { return false; } + + void executableCopy(void* buffer, bool = true) { + memcpy(buffer, m_buffer.data(), m_buffer.size()); + } + void copyJumpRelocationTable(uint8_t* dest) { + if (jumpRelocations_.length()) { + memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length()); + } + } + void copyDataRelocationTable(uint8_t* dest) { + if (dataRelocations_.length()) { + memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length()); + } + } + void copyPreBarrierTable(uint8_t*) {} + // No JS_CODELABEL_LINKMODE on ia64 (see shared/Assembler-shared.h): every + // CodeLabel patch site is the bare pointer written by writeCodePointer(). + static void Bind(uint8_t* rawCode, const CodeLabel& label) { + if (label.patchAt().bound()) { + intptr_t offset = label.patchAt().offset(); + intptr_t target = label.target().offset(); + if (label.linkMode() == CodeLabel::MoveImmediate) { + // A movl's 64-bit immediate is scattered across non-contiguous + // bundle bits, so it needs the bit-scatter writer, not a plain + // pointer store. + WriteMovlImm(rawCode + offset, uint64_t(uintptr_t(rawCode + target))); + } else { + *reinterpret_cast(rawCode + offset) = rawCode + target; + } + } + } + void processCodeLabels(uint8_t* rawCode) { + for (const CodeLabel& label : codeLabels_) { + Bind(rawCode, label); + } + } + + void flushBuffer() {} + + void bind(Label* label) { Assembler::bind(label); } + void bind(CodeLabel* label) { label->target()->bind(currentOffset()); } + template || + !std::is_base_of_v< + Label, + std::remove_pointer_t>>> + void bind(T) { + MOZ_CRASH(); + } + template + void j(Condition, T) { + MOZ_CRASH(); + } + + void jump(Label* label); + void jump(Register reg); + void jump(ImmPtr ptr); + void jump(TrampolinePtr code); + void jump(JitCode* code); + void jump(const Address& addr); + template || + !std::is_base_of_v< + Label, + std::remove_pointer_t>>> + void jump(T) { + MOZ_CRASH(); + } + + // Exactly one pointer, because jump tables are indexed with ScalePointer and + // Assembler::Bind patches each site with a single 8-byte store. That is half + // a bundle, so a table with an odd number of entries leaves the buffer out of + // step: call alignToBundle() once the table is complete. + void writeCodePointer(CodeLabel* label) { + label->patchAt()->bind(currentOffset()); + label->setLinkMode(CodeLabel::RawPointer); + uint64_t zero = 0; + m_buffer.putBytes(sizeof(zero), &zero); + } + + // Restore bundle alignment after a run of writeCodePointer()s. Bundles must + // start on a 16-byte boundary -- the low 4 bits of an instruction address are + // the slot number, so the hardware simply cannot fetch a bundle from a + // half-aligned address -- and BundleIndex() derives every branch displacement + // from offset / sizeof(Bundle), so anything emitted past an odd-length table + // would otherwise be both unfetchable and mis-targeted. + void alignToBundle() { + while (currentOffset() % sizeof(ia64::Bundle) != 0) { + uint64_t zero = 0; + m_buffer.putBytes(sizeof(zero), &zero); + } + } + void haltingAlign(size_t) {} + void nopAlign(size_t) {} + void checkStackAlignment() {} + uint32_t currentOffset() { return m_buffer.nextOffset().getOffset(); } + + void nop() { emitM(ia64::NopM()); } + void breakpoint() { emitM(ia64::NopM()); } + void abiret() { emitB(ia64::BrRet(0)); } + + // Like every other link-register backend, ret() pops the return address off + // the stack into the link register first; abiret() is the variant that + // returns through b0 as it stands. + BufferOffset ret() { + pop(ScratchReg); + emitI(ia64::MovToBr(0, ScratchReg.encoding())); + return emitB(ia64::BrRet(0)); + } + + // Read b0 into a GPR without touching the stack -- mirrors ppc64's + // xs_mflr(), used by generic test code that needs the return address in a + // normal register right after a call(). + void moveFromLinkRegister(Register dest) { + emitI(ia64::MovFromBr(dest.encoding(), 0)); + } + + // A branch (or, once toggled off, an inert nop.b carrying the same target + // bits -- see ia64::ToggleSlot2Op) to a same-buffer label. + CodeOffset toggledJump(Label* label) { + emitBranchToLabel(&ia64::BrCondRel, 0, label); + return CodeOffset(currentOffset()); + } + // A fixed 3-bundle movl/MovToBr/BrCall sequence, matching call(Register) + // and PatchWrite_NearCallSize exactly. The disabled state must still carry + // BrCall's own b1/pa/whc bits (not a generic nop's), since ToggleCall only + // flips the 4-bit op field and leaves everything else alone -- toggling a + // bare nop.b back on would branch through garbage instead of b6. + CodeOffset toggledCall(JitCode* target, bool enabled) { + CodeOffset offset(currentOffset()); + writeJumpRelocation(); + emitMovl(ScratchReg.encoding(), uint64_t(uintptr_t(target->raw()))); + emitI(ia64::MovToBr(6, ScratchReg.encoding())); + BufferOffset callSite = emitB(ia64::BrCall(0, 6)); + if (!enabled) { + ia64::ToggleSlot2Op(bundleAt(callSite), 2); + } + return offset; + } + static size_t ToggledCallSize(uint8_t*) { return 3 * sizeof(ia64::Bundle); } + + void finish() {} + + // ---- Values (JS_PUNBOX64: one 64-bit register) ------------------------- + + template + void moveValue(T, S) { + MOZ_CRASH(); + } + template + void moveValue(T, S, U) { + MOZ_CRASH(); + } + + void storeValue(ValueOperand val, const Address& dest); + void storeValue(ValueOperand val, const BaseIndex& dest); + void storeValue(const Value& val, const Address& dest); + void storeValue(const Value& val, const BaseIndex& dest); + void storeValue(JSValueType type, Register reg, const Address& dest); + void storeValue(JSValueType type, Register reg, const BaseIndex& dest); + template >> + void storeValue(const T&, const S&) { + MOZ_CRASH(); + } + void storeValue(const Address& src, const Address& dest, Register temp) { + loadPtr(src, temp); + storePtr(temp, dest); + } + template >> + void storeValue(T, S, U) { + MOZ_CRASH(); + } + void storePrivateValue(Register src, const Address& dest) { + storePtr(src, dest); + } + void storePrivateValue(ImmGCPtr imm, const Address& dest) { + storePtr(imm, dest); + } + template + void storePrivateValue(const T&, const S&) { + MOZ_CRASH(); + } + + void loadValue(const Address& src, ValueOperand val); + void loadValue(const BaseIndex& src, ValueOperand val); + template >> + void loadValue(T, S) { + MOZ_CRASH(); + } + // IA-64 traps to the kernel's slow unaligned-access fixup handler on any + // misaligned load, so this cannot be a plain ld8 like the other 64-bit + // backends' loadValue()-based loadUnalignedValue(). Assemble the value + // byte-by-byte instead, matching the technique used by the interpreter's + // GET_UINT16/24/32 bytecode-operand readers. + void loadUnalignedValue(const Address& src, ValueOperand dest); + + // Assemble |bytes| little-endian bytes at [addr] into |out|, using |tmp| as + // a staging register. |addr| is advanced as a side effect. + void loadUnalignedBytes(Register addr, Register tmp, Register out, + uint32_t bytes); + + void pushValue(ValueOperand val); + void pushValue(const Value& val); + void pushValue(const Address& addr); + void pushValue(JSValueType type, Register reg); + template + void pushValue(const T&) { + MOZ_CRASH(); + } + void pushValue(const BaseIndex& addr, Register scratch) { + loadValue(addr, ValueOperand(scratch)); + pushValue(ValueOperand(scratch)); + } + template >> + void pushValue(T, S) { + MOZ_CRASH(); + } + void popValue(ValueOperand val); + void tagValue(JSValueType type, Register payload, ValueOperand dest); + void retn(Imm32 n); + + // ---- Raw stack pushes/pops -------------------------------------------- + // + // The JIT's stack discipline moves the stack pointer one 8-byte slot at a + // time. IA-64 hardware only requires natural alignment for each access, so an + // 8-byte-aligned sp is fine inside JIT code; the 16-byte ABI alignment is + // re-established at ABI call boundaries by setupUnalignedABICall(). + + void push(Register reg); + void push(Imm32 imm); + void push(ImmWord imm); + void push(ImmPtr imm); + void push(ImmGCPtr imm); + void push(const Address& addr); + void push(FloatRegister src); + template + void push(const T&) { + MOZ_CRASH(); + } + + void pop(Register reg); + void pop(const ValueOperand& val); + void pop(FloatRegister dest); + template + void pop(T) { + MOZ_CRASH(); + } + + template + void Push(T) { + MOZ_CRASH(); + } + template + void Pop(T) { + MOZ_CRASH(); + } + // The returned offset is that of the movl bundle (matching farJumpWithPatch + // and friends), which is what PatchDataWithValueCheck/WriteMovlImm rewrite. + CodeOffset pushWithPatch(ImmWord imm) { + CodeOffset offset(currentOffset()); + emitMovl(ScratchReg.encoding(), imm.value); + push(ScratchReg); + return offset; + } + CodeOffset pushWithPatch(ImmPtr imm) { + return pushWithPatch(ImmWord(uintptr_t(imm.value))); + } + template + CodeOffset pushWithPatch(T) { + MOZ_CRASH(); + } + + void testNullSet(Condition, ValueOperand, Register); + void testObjectSet(Condition, ValueOperand, Register); + void testUndefinedSet(Condition, ValueOperand, Register); + + // Compare-then-predicated-set as one unit: Assembler::Condition carries no + // predicate of its own on ia64, so this can't consume a condition produced + // elsewhere the way x86's setCC can -- it has to emit the compare itself, + // via the same emitCompare() overload set branch32/branchPtr use. + template + void cmpPtrSet(Condition cond, T lhs, S rhs, Register dest) { + uint32_t p = emitComparePtr(cond, lhs, rhs); + emitM(ia64::MovReg(dest.encoding(), Registers::zero)); + emitM(ia64::Adds(dest.encoding(), 1, Registers::zero, p)); + } + // No generic caller uses these narrow widths (grepped: dead on every + // backend that doesn't have a native byte/half compare). + void cmp8Set(Condition, Address, Imm32, Register) { MOZ_CRASH(); } + void cmp16Set(Condition, Address, Imm32, Register) { MOZ_CRASH(); } + template + void cmp32Set(Condition cond, T lhs, S rhs, Register dest) { + uint32_t p = emitCompare32(cond, lhs, rhs); + emitM(ia64::MovReg(dest.encoding(), Registers::zero)); + emitM(ia64::Adds(dest.encoding(), 1, Registers::zero, p)); + } + void cmp64Set(Condition cond, Address lhs, Imm64 rhs, Register dest) { + loadPtr(lhs, SecondScratchReg); + uint32_t p = emitCompare(cond, SecondScratchReg, ImmWord(rhs.value)); + emitM(ia64::MovReg(dest.encoding(), Registers::zero)); + emitM(ia64::Adds(dest.encoding(), 1, Registers::zero, p)); + } + + // Register-to-register moves are "adds dest = 0, src"; immediates go through + // movl, which carries a full 64-bit value in one MLX bundle. + void movePtr(Register src, Register dest) { + if (src != dest) { + emitM(ia64::MovReg(dest.encoding(), src.encoding())); + } + } + void movePtr(ImmWord imm, Register dest) { + emitMovl(dest.encoding(), imm.value); + } + void movePtr(ImmPtr imm, Register dest) { + movePtr(ImmWord(uintptr_t(imm.value)), dest); + } + void movePtr(wasm::SymbolicAddress, Register) { MOZ_CRASH(); } + void move32(Register src, Register dest) { movePtr(src, dest); } + void move32(Imm32 imm, Register dest) { + emitMovl(dest.encoding(), uint64_t(int64_t(imm.value))); + } + void mov(Register src, Register dest) { movePtr(src, dest); } + void mov(ImmWord imm, Register dest) { movePtr(imm, dest); } + void mov(ImmPtr imm, Register dest) { movePtr(imm, dest); } + void mov(Imm32 imm, Register dest) { move32(imm, dest); } + // A same-buffer address that isn't known until link time (the label may + // not be bound yet, e.g. a return address recorded before the code that + // follows it is emitted): a movl placeholder now, patched via Bind() once + // the label's target is known. + void mov(CodeLabel* label, Register dest) { + label->patchAt()->bind(currentOffset()); + label->setLinkMode(CodeLabel::MoveImmediate); + emitMovl(dest.encoding(), 0); + } + + // An embedded GC pointer needs a data relocation so the GC can trace and + // update it; the relocation records the offset of the movl bundle. + void movePtr(ImmGCPtr imm, Register dest); + + template + void movePtr(T, Register) { + MOZ_CRASH(); + } + template + void move32(const T&, Register) { + MOZ_CRASH(); + } + template + void mov(T, Register) { + MOZ_CRASH(); + } + template + void movq(T, S) { + MOZ_CRASH(); + } + void moveFloat32(FloatRegister src, FloatRegister dest); + void moveDouble(FloatRegister src, FloatRegister dest); + template + void move64(T, S) { + MOZ_CRASH(); + } + // As for pushWithPatch(), the offset is that of the movl bundle itself, which + // is what PatchDataWithValueCheck()/WriteMovlImm() rewrite. + CodeOffset movWithPatch(ImmWord imm, Register dest) { + CodeOffset offset(currentOffset()); + emitMovl(dest.encoding(), imm.value); + return offset; + } + CodeOffset movWithPatch(ImmPtr imm, Register dest) { + return movWithPatch(ImmWord(uintptr_t(imm.value)), dest); + } + template + CodeOffset movWithPatch(T, Register) { + MOZ_CRASH(); + } + + // ---- Loads and stores -------------------------------------------------- + + FaultingCodeOffset loadPtr(const Address& addr, Register dest); + FaultingCodeOffset loadPtr(const BaseIndex& addr, Register dest); + FaultingCodeOffset loadPtr(AbsoluteAddress addr, Register dest); + FaultingCodeOffset loadPtr(ImmPtr addr, Register dest); + template >> + FaultingCodeOffset loadPtr(T, Register) { + MOZ_CRASH(); + } + + FaultingCodeOffset load32(const Address& addr, Register dest); + FaultingCodeOffset load32(const BaseIndex& addr, Register dest); + FaultingCodeOffset load32(AbsoluteAddress addr, Register dest); + FaultingCodeOffset load32(ImmPtr addr, Register dest); + template >> + FaultingCodeOffset load32(T, Register) { + MOZ_CRASH(); + } + + void load32Unaligned(const Address& addr, Register dest); + void load32Unaligned(const BaseIndex& addr, Register dest); + template >> + void load32Unaligned(T, Register) { + MOZ_CRASH(); + } + template + FaultingCodeOffset loadFloat16(T, FloatRegister, Register) { + MOZ_CRASH(); + } + FaultingCodeOffset loadFloat32(const Address& addr, FloatRegister dest); + FaultingCodeOffset loadFloat32(const BaseIndex& addr, FloatRegister dest); + FaultingCodeOffset loadDouble(const Address& addr, FloatRegister dest); + FaultingCodeOffset loadDouble(const BaseIndex& addr, FloatRegister dest); + template >> + FaultingCodeOffset loadFloat32(T, FloatRegister) { + MOZ_CRASH(); + } + template >> + FaultingCodeOffset loadDouble(T, FloatRegister) { + MOZ_CRASH(); + } + void loadPrivate(const Address& addr, Register dest) { loadPtr(addr, dest); } + template + void loadPrivate(T, Register) { + MOZ_CRASH(); + } + + FaultingCodeOffset load8SignExtend(const Address& addr, Register dest); + FaultingCodeOffset load8SignExtend(const BaseIndex& addr, Register dest); + template >> + FaultingCodeOffset load8SignExtend(T, Register) { + MOZ_CRASH(); + } + FaultingCodeOffset load8ZeroExtend(const Address& addr, Register dest); + FaultingCodeOffset load8ZeroExtend(const BaseIndex& addr, Register dest); + template >> + FaultingCodeOffset load8ZeroExtend(T, Register) { + MOZ_CRASH(); + } + FaultingCodeOffset load16SignExtend(const Address& addr, Register dest); + FaultingCodeOffset load16SignExtend(const BaseIndex& addr, Register dest); + template >> + FaultingCodeOffset load16SignExtend(T, Register) { + MOZ_CRASH(); + } + FaultingCodeOffset load16ZeroExtend(const Address& addr, Register dest); + FaultingCodeOffset load16ZeroExtend(const BaseIndex& addr, Register dest); + template >> + FaultingCodeOffset load16ZeroExtend(T, Register) { + MOZ_CRASH(); + } + void load16UnalignedSignExtend(const Address& addr, Register dest); + void load16UnalignedSignExtend(const BaseIndex& addr, Register dest); + template >> + void load16UnalignedSignExtend(T, Register) { + MOZ_CRASH(); + } + void load16UnalignedZeroExtend(const Address& addr, Register dest); + void load16UnalignedZeroExtend(const BaseIndex& addr, Register dest); + template >> + void load16UnalignedZeroExtend(T, Register) { + MOZ_CRASH(); + } + FaultingCodeOffset load64(const Address& addr, Register64 dest) { + return loadPtr(addr, dest.reg); + } + FaultingCodeOffset load64(const BaseIndex& addr, Register64 dest) { + return loadPtr(addr, dest.reg); + } + template >> + FaultingCodeOffset load64(T, Register64) { + MOZ_CRASH(); + } + void load64Unaligned(const Address& addr, Register64 dest); + void load64Unaligned(const BaseIndex& addr, Register64 dest); + template >> + void load64Unaligned(T, Register64) { + MOZ_CRASH(); + } + + FaultingCodeOffset storePtr(Register src, const Address& dest); + FaultingCodeOffset storePtr(Register src, const BaseIndex& dest); + FaultingCodeOffset storePtr(Register src, AbsoluteAddress dest); + FaultingCodeOffset storePtr(ImmWord imm, const Address& dest); + FaultingCodeOffset storePtr(ImmWord imm, const BaseIndex& dest); + FaultingCodeOffset storePtr(ImmPtr imm, const Address& dest); + FaultingCodeOffset storePtr(ImmPtr imm, const BaseIndex& dest); + FaultingCodeOffset storePtr(ImmGCPtr imm, const Address& dest); + FaultingCodeOffset storePtr(ImmGCPtr imm, const BaseIndex& dest); + template && + !std::is_base_of_v>> + FaultingCodeOffset storePtr(const T&, S) { + MOZ_CRASH(); + } + + FaultingCodeOffset store32(Register src, const Address& dest); + FaultingCodeOffset store32(Register src, const BaseIndex& dest); + FaultingCodeOffset store32(Register src, AbsoluteAddress dest); + FaultingCodeOffset store32(Imm32 imm, const Address& dest); + FaultingCodeOffset store32(Imm32 imm, const BaseIndex& dest); + FaultingCodeOffset store32(Imm32 imm, AbsoluteAddress dest); + template && + !std::is_base_of_v>> + FaultingCodeOffset store32(T, S) { + MOZ_CRASH(); + } + + template + void store32Unaligned(T t, S s) { + store32(t, s); + } + FaultingCodeOffset storeFloat32(FloatRegister src, const Address& dest); + FaultingCodeOffset storeFloat32(FloatRegister src, const BaseIndex& dest); + FaultingCodeOffset storeDouble(FloatRegister src, const Address& dest); + FaultingCodeOffset storeDouble(FloatRegister src, const BaseIndex& dest); + template >> + void storeFloat32(T, S) { + MOZ_CRASH(); + } + template >> + void storeDouble(T, S) { + MOZ_CRASH(); + } + + FaultingCodeOffset store8(Register src, const Address& dest); + FaultingCodeOffset store8(Register src, const BaseIndex& dest); + FaultingCodeOffset store8(Imm32 imm, const Address& dest); + FaultingCodeOffset store8(Imm32 imm, const BaseIndex& dest); + template && + !std::is_base_of_v>> + FaultingCodeOffset store8(T, S) { + MOZ_CRASH(); + } + FaultingCodeOffset store16(Register src, const Address& dest); + FaultingCodeOffset store16(Register src, const BaseIndex& dest); + FaultingCodeOffset store16(Imm32 imm, const Address& dest); + FaultingCodeOffset store16(Imm32 imm, const BaseIndex& dest); + template && + !std::is_base_of_v>> + FaultingCodeOffset store16(T, S) { + MOZ_CRASH(); + } + template + void store16Unaligned(T t, S s) { + store16(t, s); + } + FaultingCodeOffset store64(Register64 src, const Address& dest) { + return storePtr(src.reg, dest); + } + FaultingCodeOffset store64(Register64 src, const BaseIndex& dest) { + return storePtr(src.reg, dest); + } + FaultingCodeOffset store64(Imm64 imm, const Address& dest) { + return storePtr(ImmWord(imm.value), dest); + } + FaultingCodeOffset store64(Imm64 imm, const BaseIndex& dest) { + return storePtr(ImmWord(imm.value), dest); + } + template >> + FaultingCodeOffset store64(T, S) { + MOZ_CRASH(); + } + template + void store64Unaligned(T t, S s) { + store64(t, s); + } + + void computeEffectiveAddress(const Address& addr, Register dest) { + computeAddress(addr, dest); + } + void computeEffectiveAddress(const BaseIndex& addr, Register dest) { + computeAddress(addr, dest); + } + template >> + void computeEffectiveAddress(T, Register) { + MOZ_CRASH(); + } + + void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) { + emitTagOf(value, Register(tag)); + } + + void boxDouble(FloatRegister src, ValueOperand dest); + // The scratch FloatRegister argument is only used by architectures that + // lack a direct FR-to-GR transfer (none here; setf.d/getf.d do it in one + // instruction). + void boxDouble(FloatRegister src, ValueOperand dest, FloatRegister) { + boxDouble(src, dest); + } + void boxNonDouble(JSValueType type, Register src, ValueOperand dest) { + tagValue(type, src, dest); + } + void boxNonDouble(Register type, Register src, ValueOperand dest); + template + void boxDouble(FloatRegister src, const T& dest) { + MOZ_CRASH(); + } + + void unboxInt32(const ValueOperand& src, Register dest); + void unboxInt32(const Address& src, Register dest); + void unboxInt32(const BaseIndex& src, Register dest); + template >> + void unboxInt32(T, Register) { + MOZ_CRASH(); + } + void unboxBoolean(const ValueOperand& src, Register dest); + void unboxBoolean(const Address& src, Register dest); + void unboxBoolean(const BaseIndex& src, Register dest); + template >> + void unboxBoolean(T, Register) { + MOZ_CRASH(); + } + void unboxString(const ValueOperand& src, Register dest); + void unboxString(const Address& src, Register dest); + template + void unboxString(T, Register) { + MOZ_CRASH(); + } + void unboxSymbol(const ValueOperand& src, Register dest); + void unboxSymbol(const Address& src, Register dest); + template + void unboxSymbol(T, Register) { + MOZ_CRASH(); + } + void unboxBigInt(const ValueOperand& src, Register dest); + void unboxBigInt(const Address& src, Register dest); + template + void unboxBigInt(T, Register) { + MOZ_CRASH(); + } + void unboxObject(const ValueOperand& src, Register dest); + void unboxObject(const Address& src, Register dest); + void unboxObject(const BaseIndex& src, Register dest); + template >> + void unboxObject(T, Register) { + MOZ_CRASH(); + } + void unboxDouble(const ValueOperand& src, FloatRegister dest); + void unboxDouble(const Address& src, FloatRegister dest); + void unboxDouble(const BaseIndex& src, FloatRegister dest); + template >> + void unboxDouble(T, FloatRegister) { + MOZ_CRASH(); + } + void unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType type); + void unboxNonDouble(const ValueOperand& src, Register dest, JSValueType type); + void unboxNonDouble(const Address& src, Register dest, JSValueType type); + void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type); + void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest); + void unboxGCThingForGCBarrier(const Address& src, Register dest); + template + void unboxGCThingForGCBarrier(const T&, Register) { + MOZ_CRASH(); + } + + void unboxWasmAnyRefGCThingForGCBarrier(const Address& src, Register dest); + template + void unboxWasmAnyRefGCThingForGCBarrier(const T&, Register) { + MOZ_CRASH(); + } + + void getWasmAnyRefGCThingChunk(Register, Register) { MOZ_CRASH(); } + + void notBoolean(ValueOperand val) { + emitMovl(ScratchReg.encoding(), 1); + emitM(ia64::Xor(val.valueReg().encoding(), val.valueReg().encoding(), + ScratchReg.encoding())); + } + [[nodiscard]] Register extractObject(const Address& addr, Register scratch) { + unboxObject(addr, scratch); + return scratch; + } + [[nodiscard]] Register extractObject(const ValueOperand& val, + Register scratch) { + unboxObject(val, scratch); + return scratch; + } + [[nodiscard]] Register extractSymbol(const ValueOperand& val, + Register scratch) { + unboxSymbol(val, scratch); + return scratch; + } + [[nodiscard]] Register extractInt32(const ValueOperand& val, + Register scratch) { + unboxInt32(val, scratch); + return scratch; + } + [[nodiscard]] Register extractBoolean(const ValueOperand& val, + Register scratch) { + unboxBoolean(val, scratch); + return scratch; + } + [[nodiscard]] Register extractTag(const ValueOperand& val, Register scratch) { + emitTagOf(val, scratch); + return scratch; + } + [[nodiscard]] Register extractTag(const Address& addr, Register scratch) { + emitTagOf(addr, scratch); + return scratch; + } + [[nodiscard]] Register extractTag(const BaseIndex& addr, Register scratch) { + emitTagOf(addr, scratch); + return scratch; + } + template >> + [[nodiscard]] Register extractTag(T, Register) { + MOZ_CRASH(); + } + + void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true); + void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true); + // Genuine truncation (drop the fraction, only fail on overflow/NaN); see + // the .cpp for how this differs from convertDoubleToInt32's exact check. + void ma_truncateToInt32(FloatRegister src, Register dest, Label* fail); + void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail, + bool negativeZeroCheck = true) { + convertDoubleToInt32(src, dest, fail, negativeZeroCheck); + } + void convertBoolToInt32(Register src, Register dest) { + ma_zxt(dest, src, 1); + } + + // The register format is shared between Float32 and Double (only memory + // stores and GR extraction round); double->float32 narrowing therefore + // needs an explicit round, but float32->double widening is exact and is + // just moveDouble. + void convertDoubleToFloat32(FloatRegister src, FloatRegister dest) { + emitF(ia64::FmaS(dest.encoding(), src.encoding(), ia64::fpOne, + ia64::fpZero, ia64::sf0)); + } + // setf.sig + fcvt.xf converts a 64-bit signed integer to register-format + // float exactly (the 64-bit significand has no precision loss); the + // eventual single/double distinction is applied later, at store or GR + // extraction time. + void convertInt64RegToFloat(Register src, FloatRegister dest) { + emitM(ia64::SetfSig(dest.encoding(), src.encoding())); + emitF(ia64::FcvtXf(dest.encoding(), dest.encoding())); + } + + // Only the low 32 bits are the int32: convertInt32ValueToDouble() passes a + // whole boxed Value in here and relies on the upper bits being discarded + // (x64 uses a 32-bit vcvtsi2sd operand, arm64 an ARMRegister(src, 32)). + // Feeding all 64 bits to setf.sig instead turns the JSVAL_TAG_INT32 box into + // a huge negative double. + void convertInt32ToFloat32(Register src, FloatRegister dest) { + ma_sxt(ScratchReg, src, 4); + convertInt64RegToFloat(ScratchReg, dest); + } + + void convertInt32ToDouble(Register src, FloatRegister dest) { + ma_sxt(ScratchReg, src, 4); + convertInt64RegToFloat(ScratchReg, dest); + } + void convertInt32ToDouble(const Address& src, FloatRegister dest) { + load32(src, ScratchReg); + convertInt32ToDouble(ScratchReg, dest); + } + void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest) { + load32(src, ScratchReg); + convertInt32ToDouble(ScratchReg, dest); + } + template >> + void convertInt32ToDouble(T, FloatRegister) { + MOZ_CRASH(); + } + void convertFloat32ToDouble(FloatRegister src, FloatRegister dest) { + moveDouble(src, dest); + } + + void convertDoubleToFloat16(FloatRegister, FloatRegister) { MOZ_CRASH(); } + void convertFloat16ToDouble(FloatRegister, FloatRegister) { MOZ_CRASH(); } + void convertFloat32ToFloat16(FloatRegister, FloatRegister) { MOZ_CRASH(); } + void convertFloat16ToFloat32(FloatRegister, FloatRegister) { MOZ_CRASH(); } + void convertInt32ToFloat16(Register, FloatRegister) { MOZ_CRASH(); } + + void loadConstantDouble(double d, FloatRegister dest); + void loadConstantFloat32(float f, FloatRegister dest); + Condition testInt32Truthy(bool, const ValueOperand&); + Condition testStringTruthy(bool, const ValueOperand&); + Condition testBigIntTruthy(bool, const ValueOperand&); + + template + void loadInt32OrDouble(const T& src, FloatRegister dest) { + Label notInt32, end; + asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32); + convertInt32ToDouble(src, dest); + jump(&end); + bind(¬Int32); + unboxDouble(src, dest); + bind(&end); + } + template + void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) { + if (dest.isFloat()) { + loadInt32OrDouble(src, dest.fpu()); + } else { + unboxNonDouble(src, dest.gpr(), ValueTypeFromMIRType(type)); + } + } + template + void storeUnboxedValue(const ConstantOrRegister&, MIRType, T) { + MOZ_CRASH(); + } + + void convertUInt32ToDouble(Register src, FloatRegister dest) { + ma_zxt(ScratchReg, src, 4); + convertInt64RegToFloat(ScratchReg, dest); + } + void convertUInt32ToFloat32(Register src, FloatRegister dest) { + ma_zxt(ScratchReg, src, 4); + convertInt64RegToFloat(ScratchReg, dest); + } + void incrementInt32Value(const Address& addr); + void handleFailureWithHandlerTail(Label* profilerExitTail, + Label* bailoutTail, + uint32_t* returnValueCheckOffset); + + void buildFakeExitFrame(Register, uint32_t*) { MOZ_CRASH(); } + bool buildOOLFakeExitFrame(void* fakeReturnAddr); + + void setPrinter(Sprinter*) {} + Operand ToPayload(Operand base) { MOZ_CRASH(); } + Address ToPayload(Address addr) { return addr; } + + Register getStackPointer() const { return StackPointer; } + + // Instrumentation for entering and leaving the profiler. + void profilerEnterFrame(Register framePtr, Register scratch); + void profilerExitFrame(); + +#ifdef JS_NUNBOX32 + Address ToType(Address) { MOZ_CRASH(); } +#endif + + protected: + MacroAssembler& asMasm(); + const MacroAssembler& asMasm() const; +}; + +using MacroAssemblerSpecific = MacroAssemblerIa64; + +static inline bool GetTempRegForIntArg(uint32_t usedIntArgs, uint32_t, + Register* out) { + if (usedIntArgs >= NumCallTempNonArgRegs) { + return false; + } + *out = CallTempNonArgRegs[usedIntArgs]; + return true; +} + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_MacroAssembler_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/ia64/MoveEmitter-ia64.cpp b/js/src/jit/ia64/MoveEmitter-ia64.cpp --- firefox-153.0.1/js/src/jit/ia64/MoveEmitter-ia64.cpp.vanilla +++ firefox-153.0.1/js/src/jit/ia64/MoveEmitter-ia64.cpp @@ -0,0 +1,316 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#include "jit/ia64/MoveEmitter-ia64.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +// Memory-to-memory floating point moves need a staging register. Use the +// canonical f6 scratch (ScratchDoubleReg/ScratchFloat32Reg from +// Assembler-ia64.h) rather than f2-f5, which are reserved as internal +// temporaries for the multi-step divide/sqrt/multiply sequences in +// MacroAssembler-ia64.cpp. +static constexpr FloatRegister Ia64ScratchDouble = ScratchDoubleReg; +static constexpr FloatRegister Ia64ScratchFloat32 = ScratchFloat32Reg; + +void MoveEmitterIa64::breakCycle(const MoveOperand& from, const MoveOperand& to, + MoveOp::Type type, uint32_t slotId) { + // There is some pattern: + // (A -> B) + // (B -> A) + // + // This case handles (A -> B), which we reach first. We save B, then allow + // the original move to continue. + switch (type) { + case MoveOp::FLOAT32: + if (to.isMemory()) { + masm.loadFloat32(getAdjustedAddress(to), Ia64ScratchFloat32); + masm.storeFloat32(Ia64ScratchFloat32, cycleSlot(slotId)); + } else { + masm.storeFloat32(to.floatReg(), cycleSlot(slotId)); + } + break; + case MoveOp::DOUBLE: + if (to.isMemory()) { + masm.loadDouble(getAdjustedAddress(to), Ia64ScratchDouble); + masm.storeDouble(Ia64ScratchDouble, cycleSlot(slotId)); + } else { + masm.storeDouble(to.floatReg(), cycleSlot(slotId)); + } + break; + case MoveOp::INT32: + if (to.isMemory()) { + SecondScratchRegisterScope temp(masm); + Register scratch2 = temp; + masm.load32(getAdjustedAddress(to), scratch2); + masm.store32(scratch2, cycleSlot(0)); + } else { + masm.store32(to.reg(), cycleSlot(0)); + } + break; + case MoveOp::GENERAL: + if (to.isMemory()) { + SecondScratchRegisterScope temp(masm); + Register scratch2 = temp; + masm.loadPtr(getAdjustedAddress(to), scratch2); + masm.storePtr(scratch2, cycleSlot(0)); + } else { + masm.storePtr(to.reg(), cycleSlot(0)); + } + break; + default: + MOZ_CRASH("Unexpected move type"); + } +} + +void MoveEmitterIa64::completeCycle(const MoveOperand& from, + const MoveOperand& to, MoveOp::Type type, + uint32_t slotId) { + // This case handles (B -> A), which we reach last. We emit a move from the + // saved value of B, to A. + switch (type) { + case MoveOp::FLOAT32: + if (to.isMemory()) { + masm.loadFloat32(cycleSlot(slotId), Ia64ScratchFloat32); + masm.storeFloat32(Ia64ScratchFloat32, getAdjustedAddress(to)); + } else { + masm.loadFloat32(cycleSlot(slotId), to.floatReg()); + } + break; + case MoveOp::DOUBLE: + if (to.isMemory()) { + masm.loadDouble(cycleSlot(slotId), Ia64ScratchDouble); + masm.storeDouble(Ia64ScratchDouble, getAdjustedAddress(to)); + } else { + masm.loadDouble(cycleSlot(slotId), to.floatReg()); + } + break; + case MoveOp::INT32: + MOZ_ASSERT(slotId == 0); + if (to.isMemory()) { + SecondScratchRegisterScope temp(masm); + Register scratch2 = temp; + masm.load32(cycleSlot(0), scratch2); + masm.store32(scratch2, getAdjustedAddress(to)); + } else { + masm.load32(cycleSlot(0), to.reg()); + } + break; + case MoveOp::GENERAL: + MOZ_ASSERT(slotId == 0); + if (to.isMemory()) { + SecondScratchRegisterScope temp(masm); + Register scratch2 = temp; + masm.loadPtr(cycleSlot(0), scratch2); + masm.storePtr(scratch2, getAdjustedAddress(to)); + } else { + masm.loadPtr(cycleSlot(0), to.reg()); + } + break; + default: + MOZ_CRASH("Unexpected move type"); + } +} + +void MoveEmitterIa64::emit(const MoveResolver& moves) { + if (moves.numCycles()) { + masm.reserveStack(moves.numCycles() * sizeof(double)); + pushedAtCycle_ = masm.framePushed(); + } + + for (size_t i = 0; i < moves.numMoves(); i++) { + emit(moves.getMove(i)); + } +} + +void MoveEmitterIa64::emit(const MoveOp& move) { + const MoveOperand& from = move.from(); + const MoveOperand& to = move.to(); + + if (move.isCycleEnd() && move.isCycleBegin()) { + // A cycle can end exactly where another begins. + breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot()); + completeCycle(from, to, move.type(), move.cycleEndSlot()); + return; + } + + if (move.isCycleEnd()) { + MOZ_ASSERT(inCycle_); + completeCycle(from, to, move.type(), move.cycleEndSlot()); + MOZ_ASSERT(inCycle_ > 0); + inCycle_--; + return; + } + + if (move.isCycleBegin()) { + breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot()); + inCycle_++; + } + + switch (move.type()) { + case MoveOp::FLOAT32: + emitFloat32Move(from, to); + break; + case MoveOp::DOUBLE: + emitDoubleMove(from, to); + break; + case MoveOp::INT32: + emitInt32Move(from, to); + break; + case MoveOp::GENERAL: + emitMove(from, to); + break; + default: + MOZ_CRASH("Unexpected move type"); + } +} + +void MoveEmitterIa64::emitMove(const MoveOperand& from, const MoveOperand& to) { + if (from.isGeneralReg()) { + if (to.isGeneralReg()) { + masm.movePtr(from.reg(), to.reg()); + } else if (to.isMemory()) { + masm.storePtr(from.reg(), getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitMove arguments."); + } + } else if (from.isMemory()) { + if (to.isGeneralReg()) { + masm.loadPtr(getAdjustedAddress(from), to.reg()); + } else if (to.isMemory()) { + SecondScratchRegisterScope temp(masm); + Register scratch2 = temp; + masm.loadPtr(getAdjustedAddress(from), scratch2); + masm.storePtr(scratch2, getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitMove arguments."); + } + } else if (from.isEffectiveAddress()) { + if (to.isGeneralReg()) { + masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg()); + } else if (to.isMemory()) { + SecondScratchRegisterScope temp(masm); + Register scratch2 = temp; + masm.computeEffectiveAddress(getAdjustedAddress(from), scratch2); + masm.storePtr(scratch2, getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitMove arguments."); + } + } else { + MOZ_CRASH("Invalid emitMove arguments."); + } +} + +void MoveEmitterIa64::emitInt32Move(const MoveOperand& from, + const MoveOperand& to) { + if (from.isGeneralReg()) { + if (to.isGeneralReg()) { + masm.move32(from.reg(), to.reg()); + } else if (to.isMemory()) { + masm.store32(from.reg(), getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitInt32Move arguments."); + } + } else if (from.isMemory()) { + if (to.isGeneralReg()) { + masm.load32(getAdjustedAddress(from), to.reg()); + } else if (to.isMemory()) { + SecondScratchRegisterScope temp(masm); + Register scratch2 = temp; + masm.load32(getAdjustedAddress(from), scratch2); + masm.store32(scratch2, getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitInt32Move arguments."); + } + } else if (from.isEffectiveAddress()) { + if (to.isGeneralReg()) { + masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg()); + } else if (to.isMemory()) { + SecondScratchRegisterScope temp(masm); + Register scratch2 = temp; + masm.computeEffectiveAddress(getAdjustedAddress(from), scratch2); + masm.store32(scratch2, getAdjustedAddress(to)); + } else { + MOZ_CRASH("Invalid emitInt32Move arguments."); + } + } else { + MOZ_CRASH("Invalid emitInt32Move arguments."); + } +} + +// The IA-64 psABI passes floating point arguments in f8-f15 and integers in the +// r32-r39 window, so a float never has to be shuffled into a general register +// the way it does on the soft-float and register-pair ABIs. + +void MoveEmitterIa64::emitFloat32Move(const MoveOperand& from, + const MoveOperand& to) { + if (from.isFloatReg()) { + if (to.isFloatReg()) { + masm.moveFloat32(from.floatReg(), to.floatReg()); + } else { + MOZ_ASSERT(to.isMemory()); + masm.storeFloat32(from.floatReg(), getAdjustedAddress(to)); + } + } else if (to.isFloatReg()) { + MOZ_ASSERT(from.isMemory()); + masm.loadFloat32(getAdjustedAddress(from), to.floatReg()); + } else { + MOZ_ASSERT(from.isMemory()); + MOZ_ASSERT(to.isMemory()); + masm.loadFloat32(getAdjustedAddress(from), Ia64ScratchFloat32); + masm.storeFloat32(Ia64ScratchFloat32, getAdjustedAddress(to)); + } +} + +void MoveEmitterIa64::emitDoubleMove(const MoveOperand& from, + const MoveOperand& to) { + if (from.isFloatReg()) { + if (to.isFloatReg()) { + masm.moveDouble(from.floatReg(), to.floatReg()); + } else { + MOZ_ASSERT(to.isMemory()); + masm.storeDouble(from.floatReg(), getAdjustedAddress(to)); + } + } else if (to.isFloatReg()) { + MOZ_ASSERT(from.isMemory()); + masm.loadDouble(getAdjustedAddress(from), to.floatReg()); + } else { + MOZ_ASSERT(from.isMemory()); + MOZ_ASSERT(to.isMemory()); + masm.loadDouble(getAdjustedAddress(from), Ia64ScratchDouble); + masm.storeDouble(Ia64ScratchDouble, getAdjustedAddress(to)); + } +} + +Address MoveEmitterIa64::cycleSlot(uint32_t slot, uint32_t subslot) const { + int32_t offset = masm.framePushed() - pushedAtCycle_; + return Address(StackPointer, offset + slot * sizeof(double) + subslot); +} + +int32_t MoveEmitterIa64::getAdjustedOffset(const MoveOperand& operand) { + MOZ_ASSERT(operand.isMemoryOrEffectiveAddress()); + if (operand.base() != StackPointer) { + return operand.disp(); + } + + // Adjust the offset if the stack pointer has moved since construction. + return operand.disp() + masm.framePushed() - pushedAtStart_; +} + +Address MoveEmitterIa64::getAdjustedAddress(const MoveOperand& operand) { + return Address(operand.base(), getAdjustedOffset(operand)); +} + +void MoveEmitterIa64::assertDone() { MOZ_ASSERT(inCycle_ == 0); } + +void MoveEmitterIa64::finish() { + assertDone(); + + masm.freeStack(masm.framePushed() - pushedAtStart_); +} diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/ia64/MoveEmitter-ia64.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/ia64/MoveEmitter-ia64.h --- firefox-153.0.1/js/src/jit/ia64/MoveEmitter-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/MoveEmitter-ia64.h @@ -0,0 +1,68 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_MoveEmitter_ia64_h +#define jit_ia64_MoveEmitter_ia64_h + +#include "jit/MacroAssembler.h" +#include "jit/MoveResolver.h" + +namespace js { +namespace jit { + +class MoveEmitterIa64 { + void emitDoubleMove(const MoveOperand& from, const MoveOperand& to); + void breakCycle(const MoveOperand& from, const MoveOperand& to, + MoveOp::Type type, uint32_t slot); + void completeCycle(const MoveOperand& from, const MoveOperand& to, + MoveOp::Type type, uint32_t slot); + + protected: + uint32_t inCycle_; + MacroAssembler& masm; + + // Stack depth at construction, used to rebase sp-relative operands after + // stack space has been reserved for cycle temporaries. + uint32_t pushedAtStart_; + + // Stack depth at which the cycle spill slots were allocated, or -1 if no + // cycle slots were needed. + int32_t pushedAtCycle_; + + void assertDone(); + Address cycleSlot(uint32_t slot, uint32_t subslot = 0) const; + int32_t getAdjustedOffset(const MoveOperand& operand); + Address getAdjustedAddress(const MoveOperand& operand); + + void emitMove(const MoveOperand& from, const MoveOperand& to); + void emitInt32Move(const MoveOperand& from, const MoveOperand& to); + void emitFloat32Move(const MoveOperand& from, const MoveOperand& to); + void emit(const MoveOp& move); + + public: + explicit MoveEmitterIa64(MacroAssembler& masm) + : inCycle_(0), + masm(masm), + pushedAtStart_(masm.framePushed()), + pushedAtCycle_(-1) {} + + ~MoveEmitterIa64() { assertDone(); } + + void emit(const MoveResolver& moves); + void finish(); + + // Part of the cross-architecture MoveEmitter interface; ia64 always takes its + // temporary from the reserved assembler scratch registers, so there is + // nothing to configure. + void setScratchRegister(Register reg) {} +}; + +using MoveEmitter = MoveEmitterIa64; + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_MoveEmitter_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/ia64/SharedICHelpers-ia64-inl.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/ia64/SharedICHelpers-ia64-inl.h --- firefox-153.0.1/js/src/jit/ia64/SharedICHelpers-ia64-inl.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/SharedICHelpers-ia64-inl.h @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_SharedICHelpers_ia64_inl_h +#define jit_ia64_SharedICHelpers_ia64_inl_h + +#include "jit/BaselineFrame.h" +#include "jit/SharedICHelpers.h" + +#include "jit/MacroAssembler-inl.h" + +namespace js { +namespace jit { + +inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm, + uint32_t argSize) { +#ifdef DEBUG + // The return address is in b0, not on the stack, so the frame size is + // simply FramePointer - StackPointer, minus the VMFunction arguments. + Register scratch = R2.scratchReg(); + masm.movePtr(FramePointer, scratch); + masm.subPtr(StackPointer, scratch); + masm.subPtr(Imm32(argSize), scratch); + + Address frameSizeAddr(FramePointer, + BaselineFrame::reverseOffsetOfDebugFrameSize()); + masm.store32(scratch, frameSizeAddr); +#endif + + // Push the frame descriptor and perform the tail call. The return address + // is in b0; the VM wrapper spills it with pushReturnAddress(). + masm.push(FrameDescriptor(FrameType::BaselineJS)); + masm.jump(target); +} + +inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) { + masm.push(FrameDescriptor(FrameType::BaselineStub)); + masm.call(target); +} + +inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) { + MOZ_ASSERT(scratch != ICTailCallReg); + +#ifdef DEBUG + // The return address is in b0, not on the stack, so the frame size is + // simply FramePointer - StackPointer. + masm.movePtr(FramePointer, scratch); + masm.subPtr(StackPointer, scratch); + + Address frameSizeAddr(FramePointer, + BaselineFrame::reverseOffsetOfDebugFrameSize()); + masm.store32(scratch, frameSizeAddr); +#endif + + // Push the frame descriptor, then spill b0 on top of it. + masm.Push(FrameDescriptor(FrameType::BaselineJS)); + masm.pushReturnAddress(); + + // Save old frame pointer, stack pointer and stub reg. + masm.Push(FramePointer); + masm.movePtr(StackPointer, FramePointer); + masm.Push(ICStubReg); + + // Stack should remain aligned. + masm.assertStackAlignment(sizeof(Value), 0); +} + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_SharedICHelpers_ia64_inl_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/ia64/SharedICHelpers-ia64.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/ia64/SharedICHelpers-ia64.h --- firefox-153.0.1/js/src/jit/ia64/SharedICHelpers-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/SharedICHelpers-ia64.h @@ -0,0 +1,78 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_SharedICHelpers_ia64_h +#define jit_ia64_SharedICHelpers_ia64_h + +#include "jit/BaselineIC.h" +#include "jit/JitFrames.h" +#include "jit/MacroAssembler.h" +#include "jit/SharedICRegisters.h" + +namespace js { +namespace jit { + +// IA-64 is a JS_USE_LINK_REGISTER platform: br.call leaves the return address +// in b0, and the callee spills it with pushReturnAddress(). Nothing pushes a +// return address caller-side, so there is none on the stack below an IC stub. + +// Distance from the stack top to the top Value inside an IC stub (no return +// address on the stack on ia64). +static const size_t ICStackValueOffset = 0; + +inline void EmitRestoreTailCallReg(MacroAssembler& masm) { + // No-op: b0 still holds the return address. +} + +inline void EmitRepushTailCallReg(MacroAssembler& masm) { + // No-op: b0 still holds the return address. +} + +inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) { + // The stub pointer must already be in ICStubReg. + masm.call(Address(ICStubReg, ICStub::offsetOfStubCode())); + *callOffset = CodeOffset(masm.currentOffset()); +} + +inline void EmitReturnFromIC(MacroAssembler& masm) { + // Return through b0, still holding the return address left by EmitCallIC. + masm.abiret(); +} + +inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) { + Address stubAddr(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP); + masm.loadPtr(stubAddr, ICStubReg); + + masm.movePtr(FramePointer, StackPointer); + masm.Pop(FramePointer); + + // The return address is on top of the stack, followed by the frame + // descriptor. Restore it to b0 and discard the descriptor. + masm.popReturnAddress(); + masm.freeStack(sizeof(uintptr_t)); +} + +template +inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr, + MIRType type) { + // b0 is clobbered by the call inside guardedCallPreBarrier. Save it first. + masm.pushReturnAddress(); + masm.guardedCallPreBarrier(addr, type); + masm.popReturnAddress(); +} + +inline void EmitStubGuardFailure(MacroAssembler& masm) { + // Load next stub into ICStubReg. + masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg); + + // The return address is still in b0; just jump to the next stub code. + masm.jump(Address(ICStubReg, ICStub::offsetOfStubCode())); +} + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_SharedICHelpers_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/ia64/SharedICRegisters-ia64.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/ia64/SharedICRegisters-ia64.h --- firefox-153.0.1/js/src/jit/ia64/SharedICRegisters-ia64.h.vanilla +++ firefox-153.0.1/js/src/jit/ia64/SharedICRegisters-ia64.h @@ -0,0 +1,60 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#ifndef jit_ia64_SharedICRegisters_ia64_h +#define jit_ia64_SharedICRegisters_ia64_h + +#include "jit/ia64/MacroAssembler-ia64.h" +#include "jit/Registers.h" +#include "jit/RegisterSets.h" + +namespace js { +namespace jit { + +// All of these come from the static register file r0-r31; the register-stack +// window r32-r39 is reserved for outgoing ABI arguments and is never handed to +// the register allocator (see [SMDOC] in Architecture-ia64.h). +// +// Already spoken for by Assembler-ia64.h: r0-r3 (zero/gp/scratch), r4 +// (FramePointer), r7 (HeapReg), r8 (ReturnReg), r12 (sp), r13 (tp), r15 +// (OsrFrameReg), r16 (PreBarrierReg), r17 (InterpreterPCReg), r18-r23 +// (CallTempReg0-5). + +// ValueOperands R0, R1 and R2. On PUNBOX64 a Value is a single 64-bit +// register. Shared IC code static_asserts R0 == JSReturnOperand, so R0 is +// defined in terms of it rather than picked here; Assembler-ia64.h still needs +// to give JSReturnReg a real register (r24 is free). R1 is callee-saved so its +// value survives calls; R2 is the volatile scratch pair. +static constexpr ValueOperand R0 = JSReturnOperand; +static constexpr ValueOperand R1(Register{Registers::r5}); +static constexpr ValueOperand R2(Register{Registers::r26}); + +// ICTailCallReg and ICStubReg use volatile static registers. Unlike the +// register-based ABIs, an IA-64 C call takes its arguments in the r32-r39 +// window, so no ABI call can clobber these by argument passing; they are only +// at risk across a call, and the stub frame spills them there. +// +// The JIT return address itself lives in branch register b0 (written by +// br.call), not in ICTailCallReg; b0 is not a general register, so +// ICTailCallReg is the GPR used to shuffle it to and from the stack. See +// SharedICHelpers-ia64.h. +static constexpr Register ICTailCallReg{Registers::r31}; +static constexpr Register ICStubReg{Registers::r30}; + +// FloatReg0 must be equal to ReturnDoubleReg. +static constexpr FloatRegister FloatReg0 = {FloatRegisters::f8, + FloatRegister::Double}; +static constexpr FloatRegister FloatReg1 = {FloatRegisters::f9, + FloatRegister::Double}; +static constexpr FloatRegister FloatReg2 = {FloatRegisters::f10, + FloatRegister::Double}; +static constexpr FloatRegister FloatReg3 = {FloatRegisters::f11, + FloatRegister::Double}; + +} // namespace jit +} // namespace js + +#endif /* jit_ia64_SharedICRegisters_ia64_h */ diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/ia64/Trampoline-ia64.cpp b/js/src/jit/ia64/Trampoline-ia64.cpp --- firefox-153.0.1/js/src/jit/ia64/Trampoline-ia64.cpp.vanilla +++ firefox-153.0.1/js/src/jit/ia64/Trampoline-ia64.cpp @@ -0,0 +1,551 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* Copyright (C) 2026 René Rebe */ + +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/BaselineIC.h" +#include "jit/CalleeToken.h" +#include "jit/ia64/SharedICRegisters-ia64.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/PerfSpewer.h" +#include "jit/VMFunctions.h" +#include "vm/JitActivation.h" +#include "vm/JSContext.h" +#include "vm/Realm.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +static_assert(sizeof(uintptr_t) == sizeof(uint64_t), "Not 64-bit clean."); + +// The register dump covers every architecturally visible register, including +// the r32-r39 outgoing window, because RegisterDump is indexed by register +// code. IA-64 has no packed SIMD in the FP file, so there is no separate +// vector set to worry about. +static const LiveRegisterSet AllRegs = + LiveRegisterSet(GeneralRegisterSet(Registers::AllMask), + FloatRegisterSet(FloatRegisters::AllMask)); + +// Restrict the wrapper and volatile sets to registers the JIT may actually +// touch: Registers::VolatileMask nominally contains r0 (zero), gp, the two +// assembler scratches, sp and tp, none of which may be spilled or handed out. +static constexpr Registers::SetType SafeVolatileMask = + Registers::VolatileMask & Registers::AllocatableMask; +static constexpr FloatRegisters::SetType SafeFloatVolatileMask = + FloatRegisters::VolatileMask & FloatRegisters::AllocatableMask; + +// Registers holding EnterJitCode's arguments once they have been copied out of +// the incoming register-stack window, plus the two temporaries the prologue +// needs for ar.pfs and b0. All are static (r0-r31) registers, so `alloc` does +// not disturb them. +static constexpr Register reg_code{Registers::r14}; +static constexpr Register reg_argc{Registers::r24}; +static constexpr Register reg_argv{Registers::r25}; +static constexpr Register reg_token{Registers::r26}; +static constexpr Register reg_chain{Registers::r27}; +static constexpr Register reg_values{Registers::r28}; +static constexpr Register reg_vp{Registers::r29}; +static constexpr Register reg_pfs{Registers::r30}; +static constexpr Register reg_b0{Registers::r31}; + +static constexpr Register Reg5{Registers::r5}; +static constexpr Register Reg6{Registers::r6}; +static constexpr Register Reg7{Registers::r7}; + +static FloatRegister Fp(uint32_t code) { + return FloatRegister(FloatRegisters::Code(code), FloatRegister::Double); +} + +// Callee-saved state saved by the C entry trampoline. f2-f5 and f16-f31 are +// preserved by the psABI; f2/f3 are additionally the JIT's floating point +// scratch, so they have to be saved here as well. alignas(16) keeps sizeof a +// multiple of 16, which is what keeps sp 16-byte aligned after reserveStack. +struct alignas(16) EnterJITRegs { + double f31; + double f30; + double f29; + double f28; + double f27; + double f26; + double f25; + double f24; + double f23; + double f22; + double f21; + double f20; + double f19; + double f18; + double f17; + double f16; + double f5; + double f4; + double f3; + double f2; + + uint64_t r7; + uint64_t r6; + uint64_t r5; + uint64_t r4; // FramePointer + uint64_t gp; // r1 + uint64_t pfs; // ar.pfs, as saved by the prologue's alloc + uint64_t b0; // return branch register + uint64_t vp; // EnterJitCode's |vp|, needed again after the JIT call +}; + +static_assert((sizeof(EnterJITRegs) % 16) == 0, + "EnterJITRegs must keep sp 16-byte aligned"); + +static void GeneratePrologue(MacroAssembler& masm) { + // EnterJitCode's eight arguments arrive in the caller's outgoing register + // window, which this procedure sees as r32-r39 (IntArgReg0-7) until it runs + // its own `alloc`. That alloc redefines r32-r39 as *this* frame's outgoing + // window and discards the incoming values, so copy them into static + // registers first. + masm.movePtr(IntArgReg0, reg_code); + masm.movePtr(IntArgReg1, reg_argc); + masm.movePtr(IntArgReg2, reg_argv); + masm.movePtr(IntArgReg3, OsrFrameReg); + masm.movePtr(IntArgReg4, reg_token); + masm.movePtr(IntArgReg5, reg_chain); + masm.movePtr(IntArgReg6, reg_values); + masm.movePtr(IntArgReg7, reg_vp); + + // Create the outgoing-argument window out0-out7 used for every C call made + // from JIT code, and capture the caller's frame marker. ins = locals = 0, so + // a later br.call leaves CFM unchanged (new sof = sof - sol = 8) and the + // window stays valid across nested calls. + masm.emitM(ia64::Alloc(reg_pfs.encoding(), 0, 0, 8, 0)); + + // b0 holds the return address written by the caller's br.call; it is not a + // general register, so move it before it can be spilled. + masm.emitI(ia64::MovFromBr(reg_b0.encoding(), 0)); + + masm.reserveStack(sizeof(EnterJITRegs)); + + masm.storePtr(reg_pfs, Address(StackPointer, offsetof(EnterJITRegs, pfs))); + masm.storePtr(reg_b0, Address(StackPointer, offsetof(EnterJITRegs, b0))); + masm.storePtr(GpReg, Address(StackPointer, offsetof(EnterJITRegs, gp))); + masm.storePtr(FramePointer, Address(StackPointer, offsetof(EnterJITRegs, r4))); + masm.storePtr(Reg5, Address(StackPointer, offsetof(EnterJITRegs, r5))); + masm.storePtr(Reg6, Address(StackPointer, offsetof(EnterJITRegs, r6))); + masm.storePtr(Reg7, Address(StackPointer, offsetof(EnterJITRegs, r7))); + masm.storePtr(reg_vp, Address(StackPointer, offsetof(EnterJITRegs, vp))); + + masm.storeDouble(Fp(FloatRegisters::f2), + Address(StackPointer, offsetof(EnterJITRegs, f2))); + masm.storeDouble(Fp(FloatRegisters::f3), + Address(StackPointer, offsetof(EnterJITRegs, f3))); + masm.storeDouble(Fp(FloatRegisters::f4), + Address(StackPointer, offsetof(EnterJITRegs, f4))); + masm.storeDouble(Fp(FloatRegisters::f5), + Address(StackPointer, offsetof(EnterJITRegs, f5))); + for (uint32_t i = 0; i < 16; i++) { + size_t offset = offsetof(EnterJITRegs, f16) - i * sizeof(double); + masm.storeDouble(Fp(FloatRegisters::f16 + i), + Address(StackPointer, offset)); + } +} + +static void GenerateReturn(MacroAssembler& masm) { + MOZ_ASSERT(masm.framePushed() == sizeof(EnterJITRegs)); + + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, gp)), GpReg); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, r4)), FramePointer); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, r5)), Reg5); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, r6)), Reg6); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, r7)), Reg7); + + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f2)), + Fp(FloatRegisters::f2)); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f3)), + Fp(FloatRegisters::f3)); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f4)), + Fp(FloatRegisters::f4)); + masm.loadDouble(Address(StackPointer, offsetof(EnterJITRegs, f5)), + Fp(FloatRegisters::f5)); + for (uint32_t i = 0; i < 16; i++) { + size_t offset = offsetof(EnterJITRegs, f16) - i * sizeof(double); + masm.loadDouble(Address(StackPointer, offset), Fp(FloatRegisters::f16 + i)); + } + + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, pfs)), reg_pfs); + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, b0)), reg_b0); + + masm.freeStack(sizeof(EnterJITRegs)); + + // br.ret restores CFM from ar.pfs, so ar.pfs has to be back in place first; + // that is what unwinds this frame's register-stack window. + masm.emitI(ia64::MovToPfs(reg_pfs.encoding())); + masm.emitI(ia64::MovToBr(0, reg_b0.encoding())); + masm.emitB(ia64::BrRet(0)); +} + +void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) { + AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT"); + + enterJITOffset_ = startTrampolineCode(masm); + + // The IA-64 return address is in b0, not on the stack, so sp is already + // fully aligned on entry. + masm.assertStackAlignment(ABIStackAlignment, 0); + + GeneratePrologue(masm); + + // The saved-register block doubles as this trampoline's frame. + masm.movePtr(StackPointer, FramePointer); + + generateEnterJitShared(masm, reg_argc, reg_argv, reg_token, CallTempReg0, + CallTempReg1, CallTempReg2); + + // Push the descriptor. + masm.unboxInt32(Address(reg_vp, 0), reg_argc); + masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, reg_argc, reg_argc); + + CodeLabel returnLabel; + Label oomReturnLabel; + { + // Handle Interpreter -> Baseline OSR. + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + regs.takeUnchecked(FramePointer); + regs.takeUnchecked(OsrFrameReg); + regs.takeUnchecked(reg_code); + + Label notOsr; + masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr); + + Register numStackValues = reg_values; + regs.takeUnchecked(numStackValues); + Register scratch = regs.takeAny(); + + // Push the return address, then the previous frame pointer. + masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + masm.mov(&returnLabel, scratch); + masm.storePtr(scratch, Address(StackPointer, 0)); + + masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer); + masm.storePtr(FramePointer, Address(StackPointer, 0)); + + // Reserve the BaselineFrame. + Register framePtr = FramePointer; + masm.movePtr(StackPointer, framePtr); + masm.subPtr(Imm32(BaselineFrame::Size()), StackPointer); + + Register framePtrScratch = regs.takeAny(); + masm.movePtr(StackPointer, framePtrScratch); + + // Reserve space for locals and stack values. + masm.movePtr(numStackValues, scratch); + masm.lshiftPtr(Imm32(3), scratch); + masm.subPtr(scratch, StackPointer); + + // Enter the exit frame: descriptor, fake return address, frame pointer. + masm.reserveStack(3 * sizeof(uintptr_t)); + masm.storePtr(ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)), + Address(StackPointer, 2 * sizeof(uintptr_t))); + masm.storePtr(ImmPtr(nullptr), Address(StackPointer, sizeof(uintptr_t))); + masm.storePtr(FramePointer, Address(StackPointer, 0)); + + // No GC things to mark, push a bare token. + masm.loadJSContext(scratch); + masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare); + + masm.reserveStack(2 * sizeof(uintptr_t)); + masm.storePtr(framePtr, Address(StackPointer, sizeof(uintptr_t))); + masm.storePtr(reg_code, Address(StackPointer, 0)); + + using Fn = void (*)(BaselineFrame* frame, InterpreterFrame* interpFrame, + uint32_t numStackValues); + masm.setupUnalignedABICall(scratch); + masm.passABIArg(framePtrScratch); + masm.passABIArg(OsrFrameReg); + masm.passABIArg(numStackValues); + masm.callWithABI( + ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + regs.add(OsrFrameReg); + Register jitcode = regs.takeAny(); + masm.loadPtr(Address(StackPointer, 0), jitcode); + masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), framePtr); + masm.freeStack(2 * sizeof(uintptr_t)); + + masm.freeStack(ExitFrameLayout::SizeWithFooter()); + + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled( + cx->runtime()->geckoProfiler().addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.profilerEnterFrame(framePtr, scratch); + masm.bind(&skipProfilingInstrumentation); + } + + masm.jump(jitcode); + + masm.bind(¬Osr); + MOZ_ASSERT(R1.scratchReg() != reg_code); + masm.movePtr(reg_chain, R1.scratchReg()); + } + + // The call pushes the return address and the frame pointer, so check that + // the stack will be aligned once the call is complete. + masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t)); + + masm.callJitNoProfiler(reg_code); + + { + // Interpreter -> Baseline OSR returns here. + masm.bind(&returnLabel); + masm.addCodeLabel(returnLabel); + masm.bind(&oomReturnLabel); + } + + // Discard arguments and padding; sp goes back to the EnterJITRegs block. + masm.movePtr(FramePointer, StackPointer); + masm.setFramePushed(sizeof(EnterJITRegs)); + + // Store the returned value into vp. + masm.loadPtr(Address(StackPointer, offsetof(EnterJITRegs, vp)), reg_vp); + masm.storeValue(JSReturnOperand, Address(reg_vp, 0)); + + GenerateReturn(masm); +} + +// static +mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState> +JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) { + // The C entry state lives partly in the register stack backing store, which + // the profiler has no way to walk; report nothing rather than something + // wrong. + return mozilla::Nothing{}; +} + +void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator"); + + invalidatorOffset_ = startTrampolineCode(masm); + + // Push all registers so they can be accessed as [base + code]. + masm.PushRegsInMask(AllRegs); + + masm.movePtr(StackPointer, CallTempReg0); + + // Make space for InvalidationBailout's bailoutInfo outparam. + masm.reserveStack(sizeof(void*)); + masm.movePtr(StackPointer, CallTempReg1); + + using Fn = bool (*)(InvalidationBailoutStack* sp, BaselineBailoutInfo** info); + masm.setupUnalignedABICall(CallTempReg2); + masm.passABIArg(CallTempReg0); + masm.passABIArg(CallTempReg1); + masm.callWithABI( + ABIType::General, CheckUnsafeCallWithABI::DontCheckOther); + + masm.pop(CallTempReg1); + + // Pop the machine state and the dead frame. + masm.moveToStackPtr(FramePointer); + + masm.jump(bailoutTail); +} + +static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) { + masm.PushRegsInMask(AllRegs); + masm.movePtr(StackPointer, CallTempReg0); + + // Make space for Bailout's bailoutInfo outparam. + masm.reserveStack(sizeof(void*)); + masm.movePtr(StackPointer, CallTempReg1); + + using Fn = bool (*)(BailoutStack* sp, BaselineBailoutInfo** info); + masm.setupUnalignedABICall(CallTempReg2); + masm.passABIArg(CallTempReg0); + masm.passABIArg(CallTempReg1); + masm.callWithABI(ABIType::General, + CheckUnsafeCallWithABI::DontCheckOther); + + masm.pop(CallTempReg1); + + // Remove both the bailout frame and the topmost Ion frame's stack. + masm.moveToStackPtr(FramePointer); + + masm.jump(bailoutTail); +} + +void JitRuntime::generateBailoutHandler(MacroAssembler& masm, + Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler"); + + bailoutHandlerOffset_ = startTrampolineCode(masm); + + GenerateBailoutThunk(masm, bailoutTail); +} + +bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm, + VMFunctionId id, const VMFunctionData& f, + DynFn nativeFun, uint32_t* wrapperOffset) { + AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper"); + + *wrapperOffset = startTrampolineCode(masm); + + AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask & + Register::Codes::AllocatableMask); + + static_assert( + (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0, + "Wrapper register set must be a superset of Volatile register set"); + + // The context is the first argument. IntArgReg0 lives in the outgoing + // window, which is not part of the allocatable set. + Register cxreg = IntArgReg0; + regs.takeUnchecked(cxreg); + + // On entry b0 holds the return address and the stack is: + // ... frame ... + // +8 [args] + // +0 descriptor + // + // Spill b0 so the stack matches the exit-frame layout: + // +16 [args] / +8 descriptor / +0 returnAddress + masm.pushReturnAddress(); + + // Push the frame pointer to finish the exit frame, then link it up. + masm.Push(FramePointer); + masm.moveStackPtrTo(FramePointer); + masm.loadJSContext(cxreg); + masm.enterExitFrame(cxreg, regs.getAny(), id); + + // Reserve space for the outparameter. + masm.reserveVMFunctionOutParamSpace(f); + + masm.setupUnalignedABICallDontSaveRestoreSP(); + masm.passABIArg(cxreg); + + size_t argDisp = ExitFrameLayout::Size(); + + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) { + switch (f.argProperties(explicitArg)) { + case VMFunctionData::WordByValue: + if (f.argPassedInFloatReg(explicitArg)) { + masm.passABIArg(MoveOperand(FramePointer, argDisp), ABIType::Float64); + } else { + masm.passABIArg(MoveOperand(FramePointer, argDisp), ABIType::General); + } + argDisp += sizeof(void*); + break; + case VMFunctionData::WordByRef: + masm.passABIArg(MoveOperand(FramePointer, argDisp, + MoveOperand::Kind::EffectiveAddress), + ABIType::General); + argDisp += sizeof(void*); + break; + case VMFunctionData::DoubleByValue: + case VMFunctionData::DoubleByRef: + MOZ_CRASH("NYI: ia64 callVM should not be used with 128bit values."); + } + } + + // Copy the implicit outparam, if any. + const int32_t outParamOffset = + -int32_t(ExitFooterFrame::Size()) - f.sizeOfOutParamStackSlot(); + if (f.outParam != Type_Void) { + masm.passABIArg(MoveOperand(FramePointer, outParamOffset, + MoveOperand::Kind::EffectiveAddress), + ABIType::General); + } + + masm.callWithABI(nativeFun, ABIType::General, + CheckUnsafeCallWithABI::DontCheckHasExitFrame); + + switch (f.failType()) { + case Type_Cell: + masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, + masm.failureLabel()); + break; + case Type_Bool: + masm.branchIfFalseBool(ReturnReg, masm.failureLabel()); + break; + case Type_Void: + break; + default: + MOZ_CRASH("unknown failure kind"); + } + + masm.loadVMFunctionOutParam(f, Address(FramePointer, outParamOffset)); + + // Pop the frame and restore the frame pointer. + masm.moveToStackPtr(FramePointer); + masm.pop(FramePointer); + + // Return, subtracting sizeof(void*) for the frame pointer. + masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) + + f.explicitStackSlots() * sizeof(void*) + + f.extraValuesToPop * sizeof(Value))); + + return true; +} + +uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm, + MIRType type) { + AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier"); + + uint32_t offset = startTrampolineCode(masm); + + Register temp1 = CallTempReg0; + Register temp2 = CallTempReg1; + Register temp3 = CallTempReg2; + + // b0 is clobbered by the callWithABI below, so spill it on entry; both exit + // paths restore it before branching. + masm.pushReturnAddress(); + masm.push(temp1); + masm.push(temp2); + masm.push(temp3); + + Label noBarrier; + masm.emitPreBarrierFastPath(type, temp1, temp2, temp3, &noBarrier); + + // Call into C++ to mark this GC thing. + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + + LiveRegisterSet save = + LiveRegisterSet(GeneralRegisterSet(SafeVolatileMask), + FloatRegisterSet(SafeFloatVolatileMask)); + masm.PushRegsInMask(save); + + masm.movePtr(ImmPtr(cx->runtime()), CallTempReg3); + + masm.setupUnalignedABICall(CallTempReg4); + masm.passABIArg(CallTempReg3); + masm.passABIArg(PreBarrierReg); + masm.callWithABI(JitPreWriteBarrier(type)); + + masm.PopRegsInMask(save); + masm.popReturnAddress(); + masm.abiret(); + + masm.bind(&noBarrier); + masm.pop(temp3); + masm.pop(temp2); + masm.pop(temp1); + masm.popReturnAddress(); + masm.abiret(); + + return offset; +} + +void JitRuntime::generateBailoutTailStub(MacroAssembler& masm, + Label* bailoutTail) { + AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub"); + + masm.bind(bailoutTail); + masm.generateBailoutTail(CallTempReg0, CallTempReg1); +} diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jit/moz.build /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jit/moz.build --- firefox-153.0.1/js/src/jit/moz.build.vanilla +++ firefox-153.0.1/js/src/jit/moz.build @@ -240,6 +240,15 @@ elif CONFIG["JS_CODEGEN_PPC64"]: ] if CONFIG["JS_SIMULATOR_PPC64"]: UNIFIED_SOURCES += ["ppc64/Simulator-ppc64.cpp"] +elif CONFIG["JS_CODEGEN_IA64"]: + UNIFIED_SOURCES += [ + "ia64/Architecture-ia64.cpp", + "ia64/CodeGenerator-ia64.cpp", + "ia64/Lowering-ia64.cpp", + "ia64/MacroAssembler-ia64.cpp", + "ia64/MoveEmitter-ia64.cpp", + "ia64/Trampoline-ia64.cpp", + ] elif CONFIG["JS_CODEGEN_RISCV64"]: UNIFIED_SOURCES += [ "riscv64/Architecture-riscv64.cpp", diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/jsapi-tests/testJitABIcalls.cpp /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/jsapi-tests/testJitABIcalls.cpp --- firefox-153.0.1/js/src/jsapi-tests/testJitABIcalls.cpp.vanilla +++ firefox-153.0.1/js/src/jsapi-tests/testJitABIcalls.cpp @@ -721,6 +721,11 @@ class JitABICall final : public jsapitest #elif defined(JS_CODEGEN_PPC64) Register base = r11; regs.take(base); +#elif defined(JS_CODEGEN_IA64) + // generateVMWrapper on ia64 reads the arguments straight off FramePointer + // and never reserves a dedicated base register, and MoveEmitter-ia64 uses + // SecondScratchRegisterScope for its temporaries, so any register will do. + Register base = regs.takeAny(); #else # error "Unknown architecture!" #endif diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jsapi-tests/testsJit.cpp b/js/src/jsapi-tests/testsJit.cpp --- firefox-153.0.1/js/src/jsapi-tests/testsJit.cpp.vanilla +++ firefox-153.0.1/js/src/jsapi-tests/testsJit.cpp @@ -33,6 +33,9 @@ masm.xs_mflr(scratch); masm.as_stdu(scratch, StackPointer, -8); } +#elif defined(JS_CODEGEN_IA64) + // b0 on IA-64 isn't a GPR either; save it to the stack manually. + masm.pushReturnAddress(); #elif defined(JS_USE_LINK_REGISTER) save.add(js::jit::lr); #endif @@ -54,6 +57,8 @@ restore.add(js::jit::ra); #elif defined(JS_CODEGEN_PPC64) // LR will be restored manually after PopRegsInMask. +#elif defined(JS_CODEGEN_IA64) + // b0 will be restored manually after PopRegsInMask. #elif defined(JS_USE_LINK_REGISTER) restore.add(js::jit::lr); #endif @@ -75,6 +80,10 @@ masm.as_addi(StackPointer, StackPointer, 8); } masm.as_blr(); +#elif defined(JS_CODEGEN_IA64) + // Restore b0 from the stack, then return. + masm.popReturnAddress(); + masm.abiret(); #else // Exit the JIT-ed code using the ABI return style. masm.abiret(); diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jsapi-tests/testWasmReturnCalls.cpp b/js/src/jsapi-tests/testWasmReturnCalls.cpp --- firefox-153.0.1/js/src/jsapi-tests/testWasmReturnCalls.cpp.vanilla +++ firefox-153.0.1/js/src/jsapi-tests/testWasmReturnCalls.cpp @@ -35,6 +35,9 @@ # if defined(JS_CODEGEN_PPC64) static constexpr Register ra = ABINonArgReg3; masm.xs_mflr(ra); +# elif defined(JS_CODEGEN_IA64) + static constexpr Register ra = ABINonArgReg3; + masm.moveFromLinkRegister(ra); # elif !defined(JS_CODEGEN_LOONG64) && !defined(JS_CODEGEN_MIPS64) && \ !defined(JS_CODEGEN_RISCV64) static constexpr Register ra = lr; @@ -76,6 +79,9 @@ # if defined(JS_CODEGEN_PPC64) static constexpr Register ra = ABINonArgReg3; masm.xs_mflr(ra); +# elif defined(JS_CODEGEN_IA64) + static constexpr Register ra = ABINonArgReg3; + masm.moveFromLinkRegister(ra); # elif !defined(JS_CODEGEN_LOONG64) && !defined(JS_CODEGEN_MIPS64) && \ !defined(JS_CODEGEN_RISCV64) static constexpr Register ra = lr; diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/util/Poison.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/util/Poison.h --- firefox-153.0.1/js/src/util/Poison.h.vanilla +++ firefox-153.0.1/js/src/util/Poison.h @@ -94,6 +94,9 @@ const uint8_t JS_SCOPE_DATA_TRAILING_NAM 0x29 // illegal sb instruction, crashes in user mode. #elif defined(JS_CODEGEN_PPC64) # define JS_SWEPT_CODE_PATTERN 0x00 // illegal instruction (all zeros) +#elif defined(JS_CODEGEN_IA64) +// An all-zero bundle decodes as "break.m 0", which raises a Break fault. +# define JS_SWEPT_CODE_PATTERN 0x00 #else # error "JS_SWEPT_CODE_PATTERN not defined for this platform" #endif diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmAnyRef.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmAnyRef.h --- firefox-153.0.1/js/src/wasm/WasmAnyRef.h.vanilla +++ firefox-153.0.1/js/src/wasm/WasmAnyRef.h @@ -209,7 +209,8 @@ class AnyRef { // Truncate the value to the 31-bit value size. uintptr_t wideValue = uintptr_t(value & 0x7FFFFFFF); #elif defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) || \ - defined(JS_CODEGEN_RISCV64) || defined(JS_CODEGEN_PPC64) + defined(JS_CODEGEN_RISCV64) || defined(JS_CODEGEN_PPC64) || \ + defined(JS_CODEGEN_IA64) // Sign extend the value to the native pointer size. uintptr_t wideValue = uintptr_t(int64_t((uint64_t(value) << 33)) >> 33); #elif !defined(JS_64BIT) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmBCClass.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmBCClass.h --- firefox-153.0.1/js/src/wasm/WasmBCClass.h.vanilla +++ firefox-153.0.1/js/src/wasm/WasmBCClass.h @@ -14,6 +14,8 @@ * limitations under the License. */ +/* Copyright (C) 2026 René Rebe */ + // This is an INTERNAL header for Wasm baseline compiler: the compiler object // and its supporting types. @@ -1257,9 +1259,9 @@ struct BaseCompiler final { // Some consumers depend on the returned Address not incorporating instance, // as instance may be the scratch register. template - Address prepareAtomicMemoryAccess(MemoryAccessDesc* access, - AccessCheck* check, RegPtr instance, - RegAddressType ptr); + AtomicAddress prepareAtomicMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check, RegPtr instance, + RegAddressType ptr); template void computeEffectiveAddress(MemoryAccessDesc* access); diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmBCDefs.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmBCDefs.h --- firefox-153.0.1/js/src/wasm/WasmBCDefs.h.vanilla +++ firefox-153.0.1/js/src/wasm/WasmBCDefs.h @@ -14,6 +14,8 @@ * limitations under the License. */ +/* Copyright (C) 2026 René Rebe */ + // This is an INTERNAL header for Wasm baseline compiler: common configuration // and simple definitions; all include directives. @@ -47,6 +49,9 @@ #if defined(JS_CODEGEN_PPC64) # include "jit/ppc64/Assembler-ppc64.h" #endif +#if defined(JS_CODEGEN_IA64) +# include "jit/ia64/Assembler-ia64.h" +#endif #include "js/ScalarType.h" #include "util/Memory.h" #include "wasm/WasmCodegenTypes.h" diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmBCMemory.cpp /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmBCMemory.cpp --- firefox-153.0.1/js/src/wasm/WasmBCMemory.cpp.vanilla +++ firefox-153.0.1/js/src/wasm/WasmBCMemory.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +/* Copyright (C) 2026 René Rebe */ + #include "wasm/WasmBCClass.h" #include "wasm/WasmBCDefs.h" #include "wasm/WasmBCRegDefs.h" @@ -1053,10 +1055,10 @@ static inline Register ToRegister(RegI64 // // RegAddressType is RegI32 for Memory32 and RegI64 for Memory64. template -Address BaseCompiler::prepareAtomicMemoryAccess(MemoryAccessDesc* access, - AccessCheck* check, - RegPtr instance, - RegAddressType ptr) { +AtomicAddress BaseCompiler::prepareAtomicMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check, + RegPtr instance, + RegAddressType ptr) { MOZ_ASSERT(needInstanceForAccess(access, *check) == instance.isValid()); prepareMemoryAccess(access, check, instance, ptr); @@ -1076,7 +1078,13 @@ Address BaseCompiler::prepareAtomicMemor // At this point, 64-bit offsets will have been folded away by // prepareMemoryAccess. +#ifdef JS_CODEGEN_IA64 + // r0 reads as zero, so it serves as the (unused) index. + return BaseIndex(ToRegister(ptr), Register::FromCode(Registers::zero), + TimesOne, access->offset32()); +#else return Address(ToRegister(ptr), access->offset32()); +#endif } #ifndef WASM_HAS_HEAPREG @@ -1288,7 +1296,8 @@ static void Deallocate(BaseCompiler* bc, # endif } -#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_IA64) struct Temps { RegI32 t0; @@ -1303,7 +1312,7 @@ static void PopAndAllocate(BaseCompiler* } static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, - Address srcAddr, AtomicOp op, RegI32 rv, RegI32 rd, + AtomicAddress srcAddr, AtomicOp op, RegI32 rv, RegI32 rd, const Temps& temps) { bc->masm.wasmAtomicFetchOp(access, op, rv, srcAddr, temps.t0, rd); } @@ -1522,8 +1531,9 @@ static void Deallocate(BaseCompiler* bc, bc->freeI64(temp); } -#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ - defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_PPC64) +#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_PPC64) || \ + defined(JS_CODEGEN_IA64) static void PopAndAllocate(BaseCompiler* bc, AtomicOp op, RegI64* rd, RegI64* rv, RegI64* temp) { @@ -1533,7 +1543,7 @@ static void PopAndAllocate(BaseCompiler* } static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, - Address srcAddr, AtomicOp op, RegI64 rv, RegI64 temp, + AtomicAddress srcAddr, AtomicOp op, RegI64 rv, RegI64 temp, RegI64 rd) { bc->masm.wasmAtomicFetchOp64(access, op, rv, srcAddr, temp, rd); } @@ -1677,7 +1687,8 @@ static void Perform(BaseCompiler* bc, co static void Deallocate(BaseCompiler* bc, RegI32, const Temps&) {} -#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_IA64) using Temps = Nothing; @@ -1689,7 +1700,7 @@ static void PopAndAllocate(BaseCompiler* } static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, - Address srcAddr, RegI32 rv, RegI32 rd, const Temps&) { + AtomicAddress srcAddr, RegI32 rv, RegI32 rd, const Temps&) { bc->masm.wasmAtomicExchange(access, srcAddr, rv, rd); } @@ -1863,8 +1874,9 @@ static void Deallocate(BaseCompiler* bc, bc->freeI32(bc->specific_.ecx); } -#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ - defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_PPC64) +#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_PPC64) || \ + defined(JS_CODEGEN_IA64) static void PopAndAllocate(BaseCompiler* bc, RegI64* rd, RegI64* rv) { *rv = bc->popI64(); @@ -2008,7 +2020,8 @@ static void Deallocate(BaseCompiler* bc, bc->freeI32(rnew); } -#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_IA64) using Temps = Nothing; @@ -2026,8 +2039,8 @@ static void PopAndAllocate(BaseCompiler* } static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, - Address srcAddr, RegI32 rexpect, RegI32 rnew, RegI32 rd, - const Temps&) { + AtomicAddress srcAddr, RegI32 rexpect, RegI32 rnew, + RegI32 rd, const Temps&) { bc->masm.wasmCompareExchange(access, srcAddr, rexpect, rnew, rd); } @@ -2307,8 +2320,9 @@ static void Deallocate(BaseCompiler* bc, bc->freeI64(rnew); } -#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ - defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_PPC64) +#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_PPC64) || \ + defined(JS_CODEGEN_IA64) template static void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, @@ -2319,7 +2333,8 @@ static void PopAndAllocate(BaseCompiler* } static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, - Address srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd) { + AtomicAddress srcAddr, RegI64 rexpect, RegI64 rnew, + RegI64 rd) { bc->masm.wasmCompareExchange64(access, srcAddr, rexpect, rnew, rd); } diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmBCRegDefs.h /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmBCRegDefs.h --- firefox-153.0.1/js/src/wasm/WasmBCRegDefs.h.vanilla +++ firefox-153.0.1/js/src/wasm/WasmBCRegDefs.h @@ -14,6 +14,8 @@ * limitations under the License. */ +/* Copyright (C) 2026 René Rebe */ + // This is an INTERNAL header for Wasm baseline compiler: definitions of // registers and the register allocator. @@ -29,6 +31,14 @@ struct BaseCompiler; using namespace js::jit; +// The address form used for the wasm atomic memory operations. ia64 implements +// those only for BaseIndex operands, everywhere else a plain Address is used. +#ifdef JS_CODEGEN_IA64 +using AtomicAddress = BaseIndex; +#else +using AtomicAddress = Address; +#endif + ////////////////////////////////////////////////////////////////////////////// // // Scratch register configuration. @@ -125,6 +135,13 @@ static constexpr Register RabaldrScratch static constexpr Register RabaldrScratchI32 = r25; #endif +#ifdef JS_CODEGEN_IA64 +// r2 and r3 are the MacroAssembler's own scratches, so the baseline compiler +// needs a private one; CallTempReg2 (r20) is not used by wasm infrastructure. +# define RABALDR_SCRATCH_I32 +static constexpr Register RabaldrScratchI32 = CallTempReg2; +#endif + #ifdef RABALDR_SCRATCH_F32_ALIASES_F64 # if !defined(RABALDR_SCRATCH_F32) || !defined(RABALDR_SCRATCH_F64) # error "Bad configuration" @@ -395,7 +412,7 @@ struct SpecificRegs { }; #elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) || \ - defined(JS_CODEGEN_PPC64) + defined(JS_CODEGEN_PPC64) || defined(JS_CODEGEN_IA64) struct SpecificRegs { // Required by gcc. SpecificRegs() {} diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmBaselineCompile.cpp /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmBaselineCompile.cpp --- firefox-153.0.1/js/src/wasm/WasmBaselineCompile.cpp.vanilla +++ firefox-153.0.1/js/src/wasm/WasmBaselineCompile.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +/* Copyright (C) 2026 René Rebe */ + /* * [SMDOC] WebAssembly baseline compiler (RabaldrMonkey) * @@ -1410,8 +1412,9 @@ void BaseCompiler::popStackResults(ABIRe Stk& v = stk_.back(); switch (v.kind()) { case Stk::ConstI32: -#if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \ - defined(JS_CODEGEN_RISCV64) || defined(JS_CODEGEN_PPC64) +#if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \ + defined(JS_CODEGEN_RISCV64) || defined(JS_CODEGEN_PPC64) || \ + defined(JS_CODEGEN_IA64) fr.storeImmediatePtrToStack(v.i32val_, resultHeight, temp); #else fr.storeImmediatePtrToStack(uint32_t(v.i32val_), resultHeight, temp); @@ -2396,7 +2399,7 @@ RegI32 BaseCompiler::needRotate64Temp() #elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) || \ - defined(JS_CODEGEN_PPC64) + defined(JS_CODEGEN_PPC64) || defined(JS_CODEGEN_IA64) return RegI32::Invalid(); #else MOZ_CRASH("BaseCompiler platform hook: needRotate64Temp"); @@ -2457,6 +2460,8 @@ void BaseCompiler::popAndAllocateForMulI pop2xI64(r0, r1); #elif defined(JS_CODEGEN_PPC64) pop2xI64(r0, r1); +#elif defined(JS_CODEGEN_IA64) + pop2xI64(r0, r1); #else MOZ_CRASH("BaseCompiler porting interface: popAndAllocateForMulI64"); #endif @@ -2893,6 +2898,9 @@ static RegI32 PopcntTemp(BaseCompiler& b #elif defined(JS_CODEGEN_PPC64) // PPC64 has native popcntd/popcntw; no temp register needed. return RegI32::Invalid(); +#elif defined(JS_CODEGEN_IA64) + // ia64 has popcnt; no temp register needed. + return RegI32::Invalid(); #else MOZ_CRASH("BaseCompiler platform hook: PopcntTemp"); #endif diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmCompile.cpp /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmCompile.cpp --- firefox-153.0.1/js/src/wasm/WasmCompile.cpp.vanilla +++ firefox-153.0.1/js/src/wasm/WasmCompile.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +/* Copyright (C) 2026 René Rebe */ + #include "wasm/WasmCompile.h" #include @@ -74,8 +76,9 @@ uint32_t wasm::ObservedCPUFeatures() { LOONG64 = 0x7, RISCV64 = 0x8, PPC64 = 0x9, + IA64 = 0xa, - LAST = PPC64, + LAST = IA64, ARCH_BITS = 4 }; @@ -107,6 +110,8 @@ uint32_t wasm::ObservedCPUFeatures() { #elif defined(JS_CODEGEN_PPC64) MOZ_ASSERT(jit::GetPPC64Flags() <= (UINT32_MAX >> ARCH_BITS)); return PPC64 | (jit::GetPPC64Flags() << ARCH_BITS); +#elif defined(JS_CODEGEN_IA64) + return IA64; #elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) return 0; #else diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmFrameIter.cpp /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmFrameIter.cpp --- firefox-153.0.1/js/src/wasm/WasmFrameIter.cpp.vanilla +++ firefox-153.0.1/js/src/wasm/WasmFrameIter.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +/* Copyright (C) 2026 René Rebe */ + #include "wasm/WasmFrameIter.h" #include "jit/JitFrames.h" @@ -635,6 +637,15 @@ static const unsigned SetFP = 16; // post-poppedFP window (single-step profiling fires every instruction). static const unsigned PoppedFP = 8; static const unsigned PoppedFPJitEntry = 8; +#elif defined(JS_CODEGEN_IA64) +// One instruction per bundle (16 bytes each): push(FP) is adds+st8, and +// moveStackPtrTo is a single adds. The call sequence pushes the return +// address, as on x86. +static const unsigned PushedRetAddr = 0; +static const unsigned PushedFP = 32; +static const unsigned SetFP = 48; +static const unsigned PoppedFP = 0; +static const unsigned PoppedFPJitEntry = 0; #elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) // Synthetic values to satisfy asserts and avoid compiler warnings. static const unsigned PushedRetAddr = 0; diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps /tmp/claude-0/-srv-t2-src-firefox-ia64-desktop-260731-192730-583159/2ec03345-b130-4ed1-a35a-8809b1a7caca/scratchpad/refjit/firefox-153.0.1/js/src/wasm/WasmTypeDef.cpp /srv/t2/src-firefox.ia64-desktop.260731.192730.583159/firefox-153.0.1/js/src/wasm/WasmTypeDef.cpp --- firefox-153.0.1/js/src/wasm/WasmTypeDef.cpp.vanilla +++ firefox-153.0.1/js/src/wasm/WasmTypeDef.cpp @@ -51,11 +51,11 @@ using mozilla::MallocSizeOf; // freed memory. Instead reserve one low region up front and suballocate from // it, which guarantees distinct addresses and only depends on a single mmap // landing low. +# include + # include # include -# include - namespace js { namespace wasm { --- firefox-153.0.1/config/check_macroassembler_style.py.vanilla +++ firefox-153.0.1/config/check_macroassembler_style.py @@ -25,7 +25,7 @@ import re import sys architecture_independent = set(["generic"]) -all_unsupported_architectures_names = set(["mips64", "mips_shared"]) +all_unsupported_architectures_names = set(["mips64", "mips_shared", "ia64"]) all_architecture_names = set([ "x86", "x64", diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/JitRuntime.h b/js/src/jit/JitRuntime.h --- firefox-153.0.1/js/src/jit/JitRuntime.h.vanilla +++ firefox-153.0.1/js/src/jit/JitRuntime.h @@ -148,6 +148,14 @@ void* env = nullptr; }; mutable EnterJitDescriptor enterJitDesc_; +#elif defined(JS_CODEGEN_IA64) + // IA-64 psABI: a function pointer is a pointer to a 2-word descriptor + // {entry IP, GP}, not a raw code address; see enterJit(). + struct alignas(16) EnterJitDescriptor { + void* entry = nullptr; + void* gp = nullptr; + }; + mutable EnterJitDescriptor enterJitDesc_; #endif // Generic bailout table; used if the bailout table overflows. @@ -409,6 +417,18 @@ enterJitDesc_.entry = trampolineCode(enterJITOffset_).value; } return reinterpret_cast(&enterJitDesc_); +#elif defined(JS_CODEGEN_IA64) + // A raw JIT entry called as a C function pointer would have its + // prologue bytes misread as {entry, GP}; build a synthetic descriptor + // pointing at the trampoline instead, capturing libmozjs's GP (r1) so + // the trampoline can call back into libmozjs C++ with GP valid. + if (!enterJitDesc_.entry) { + void* gp; + asm volatile("mov %0 = gp" : "=r"(gp)); + enterJitDesc_.gp = gp; + enterJitDesc_.entry = trampolineCode(enterJITOffset_).value; + } + return reinterpret_cast(&enterJitDesc_); #else return JS_DATA_TO_FUNC_PTR(EnterJitCode, trampolineCode(enterJITOffset_).value); diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/JitCommon.h b/js/src/jit/JitCommon.h --- firefox-153.0.1/js/src/jit/JitCommon.h.vanilla +++ firefox-153.0.1/js/src/jit/JitCommon.h @@ -83,4 +83,37 @@ } // namespace js #endif +#if defined(JS_CODEGEN_IA64) +namespace js { +namespace jit { + +// IA-64 psABI function descriptor: {entry IP, GP}. See MakeIA64Call. +struct IA64FunctionDescriptor { + void* entry; + void* gp; +}; + +// Fill |desc| with a synthetic IA-64 descriptor for calling raw JIT code +// |entry| as a C function pointer, and return it typed as |Fn|. On IA-64 a +// function pointer is the address of such a descriptor, not a code entry, and +// the compiler emits every indirect call as a descriptor dereference (load +// the entry IP and GP, set gp, branch); a raw entry would be read as +// descriptor data. The captured GP (r1) is libmozjs's, shared by all its +// functions, so the callee can reach C++ data. The trailing inline-asm memory +// clobber forces the descriptor stores to retire before the compiler emits +// the indirect call from the returned pointer. |desc| must outlive the call. +template +inline Fn MakeIA64Call(void* entry, IA64FunctionDescriptor* desc) { + void* gp; + asm volatile("mov %0 = gp" : "=r"(gp)); + desc->entry = entry; + desc->gp = gp; + asm volatile("" : : "r"(desc) : "memory"); + return reinterpret_cast(desc); +} + +} // namespace jit +} // namespace js +#endif + #endif // jit_JitCommon_h diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/Jit.cpp b/js/src/jit/Jit.cpp --- firefox-153.0.1/js/src/jit/Jit.cpp.vanilla +++ firefox-153.0.1/js/src/jit/Jit.cpp @@ -157,6 +157,12 @@ // concurrently), so it lives on the stack. See MakeELFv1Call. ELFv1FunctionDescriptor desc; auto funcPtr = MakeELFv1Call(code, &desc); +#elif defined(JS_CODEGEN_IA64) + // IA-64: |code| is a raw JIT entry, not a function descriptor. The + // descriptor is per-call (the entry differs per script and runtimes run + // concurrently), so it lives on the stack. See MakeIA64Call. + IA64FunctionDescriptor desc; + auto funcPtr = MakeIA64Call(code, &desc); #else auto funcPtr = JS_DATA_TO_FUNC_PTR(EnterTrampolineCodePtr, code); #endif diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp --- firefox-153.0.1/js/src/jit/MacroAssembler.cpp.vanilla +++ firefox-153.0.1/js/src/jit/MacroAssembler.cpp @@ -5253,6 +5253,13 @@ // rather than jumping to the descriptor as code (which call(ImmPtr) would do). movePtr(ImmPtr(fun), SecondScratchReg); callABIDescriptorELFv1(SecondScratchReg); +#elif defined(JS_CODEGEN_IA64) + // IA-64: |fun| is a C function pointer, i.e. the address of a 2-word + // {entry, gp} descriptor, not a code entry. Route through the IA-64 + // descriptor dance rather than jumping to the descriptor as code (which + // call(ImmPtr) would do). + movePtr(ImmPtr(fun), SecondScratchReg); + callABIDescriptorIA64(SecondScratchReg); #else call(ImmPtr(fun)); #endif diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/CacheIRCompiler.cpp b/js/src/jit/CacheIRCompiler.cpp --- firefox-153.0.1/js/src/jit/CacheIRCompiler.cpp.vanilla +++ firefox-153.0.1/js/src/jit/CacheIRCompiler.cpp @@ -10328,6 +10328,11 @@ masm.xs_mflr(r0); masm.push(r0); #endif +#ifdef JS_CODEGEN_IA64 + // Same on IA-64: the return address lives in branch register b0, which + // ICTailCallReg does not shadow, and the inner br.call clobbers it. + masm.pushReturnAddress(); +#endif masm.PushRegsInMask(liveRegs); // The stub expects lhs in CallTempReg0 and rhs in CallTempReg1. @@ -10352,6 +10357,9 @@ masm.pop(r0); masm.xs_mtlr(r0); #endif +#ifdef JS_CODEGEN_IA64 + masm.popReturnAddress(); +#endif masm.jump(&done); masm.bind(&vmCall); @@ -10361,6 +10369,9 @@ masm.pop(r0); masm.xs_mtlr(r0); #endif +#ifdef JS_CODEGEN_IA64 + masm.popReturnAddress(); +#endif } { diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/wasm/WasmSignalHandlers.cpp b/js/src/wasm/WasmSignalHandlers.cpp --- a/js/src/wasm/WasmSignalHandlers.cpp +++ b/js/src/wasm/WasmSignalHandlers.cpp @@ -169,6 +169,17 @@ # define R03_sig(p) ((p)->uc_mcontext.__gregs[3]) # define RFP_sig(p) ((p)->uc_mcontext.__gregs[22]) # endif +# if defined(__linux__) && defined(__ia64__) +// ucontext_t on IA-64 is just the mcontext; ContextToFP() is deliberately left +// crashing since it is only reached after a wasm trap site has been confirmed, +// and wasm is unavailable here (SupportsUnalignedAccesses() is false). Defining +// PC_sig is what matters: without it every SIGSEGV in the process -- including +// each MOZ_CRASH -- reaches ContextToPC() and crashes the handler itself, +// destroying the original fault context. +# define PC_sig(p) ((p)->uc_mcontext.sc_ip) +# define SP_sig(p) ((p)->uc_mcontext.sc_gr[12]) +# define LR_sig(p) ((p)->uc_mcontext.sc_br[0]) +# endif # if defined(__linux__) && defined(__riscv) # define RPC_sig(p) ((p)->uc_mcontext.__gregs[REG_PC]) # define RRA_sig(p) ((p)->uc_mcontext.__gregs[REG_RA]) diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -2528,8 +2528,8 @@ codePointer = temp3; #endif masm.passABIArg(temp2); -#if defined(JS_CODEGEN_PPC64) - // The regexp code pointer is a raw JIT entry, not an ELFv1 function +#if defined(JS_CODEGEN_PPC64) || defined(JS_CODEGEN_IA64) + // The regexp code pointer is a raw JIT entry, not an ELFv1 or IA-64 function // descriptor, so it must not be called through the descriptor path. masm.callWithABIJitCode(codePointer); #else diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/irregexp/RegExpAPI.cpp b/js/src/irregexp/RegExpAPI.cpp --- a/js/src/irregexp/RegExpAPI.cpp +++ b/js/src/irregexp/RegExpAPI.cpp @@ -890,6 +890,13 @@ js::jit::ELFv1FunctionDescriptor desc; auto function = js::jit::MakeELFv1Call(code->raw(), &desc); +#elif defined(JS_CODEGEN_IA64) + // IA-64: same story -- |code->raw()| is a raw JIT entry, not a {entry, GP} + // descriptor. The descriptor is per-call, so it lives on the stack. See + // MakeIA64Call. + js::jit::IA64FunctionDescriptor desc; + auto function = + js::jit::MakeIA64Call(code->raw(), &desc); #else auto function = reinterpret_cast(code->raw()); #endif --- a/js/src/jit/BacktrackingAllocator.cpp +++ b/js/src/jit/BacktrackingAllocator.cpp @@ -3376,6 +3376,29 @@ bool BacktrackingAllocator::chooseBundleSplit(LiveBundle* bundle, bool hasCall, if (success) return true; + // If the bundle has no definition and only a single register use, splitting + // at that use is useless – it creates an identical bundle and leads to an + // infinite loop. Instead we must spill a conflicting bundle. + bool hasDef = false; + for (auto iter = bundle->rangesBegin(); iter; ++iter) { + if ((*iter)->hasDefinition()) { + hasDef = true; + break; + } + } + if (!hasDef) { + size_t numUses = 0; + const UsePosition* singleUse = nullptr; + for (auto iter = bundle->rangesBegin(); iter; ++iter) { + for (auto use = (*iter)->usesBegin(); use; ++use) { + numUses++; + singleUse = use.operator->(); + } + } + if (numUses == 1 && singleUse->usePolicy() == LUse::REGISTER) + return false; + } + // Split at all register uses. SplitPositionVector emptyPositions; return splitAt(bundle, emptyPositions); @@ -3827,10 +3827,20 @@ bool BacktrackingAllocator::processBundle(const MIRGenerator* mir, // We have to split this bundle. break; } } + // A minimal bundle cannot be split any further. If we still can't allocate + // it after all attempts, the register pressure is too high for this + // function. Abort the compilation and fall back to Baseline. + if (minimalBundle(bundle)) { + JitSpew(JitSpew_RegAlloc, + "Aborting because minimal bundle %s cannot be allocated", + bundle->toString().get()); + return false; + } + // A minimal bundle cannot be split any further. If we try to split it // it at this point we will just end up with the same bundle and will // enter an infinite loop. Weights and the initial live ranges must // be constructed so that any minimal bundle is allocatable. MOZ_ASSERT(!minimalBundle(bundle)); diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/js/src/irregexp/RegExpNativeMacroAssembler.cpp b/js/src/irregexp/RegExpNativeMacroAssembler.cpp --- a/js/src/irregexp/RegExpNativeMacroAssembler.cpp 2026-08-05 11:03:21.110576793 +0200 +++ b/js/src/irregexp/RegExpNativeMacroAssembler.cpp 2026-08-05 11:49:47.967895170 +0200 @@ -1119,6 +1119,18 @@ void SMRegExpMacroAssembler::createStack masm_.pushReturnAddress(); #endif +#ifdef JS_CODEGEN_IA64 + // This code is entered directly from C++, not through the EnterJIT + // trampoline, so it must do the trampoline's job itself: establish the + // standard 8-out register-stack window used by ABI calls, and preserve the + // caller's ar.pfs and b0, both of which the br.call issued by + // CheckBacktrackStackLimit() clobbers. br.ret restores the caller's CFM + // from ar.pfs, so losing it unwinds the wrong register-stack frame. + masm_.emitM(js::jit::ia64::Alloc(js::jit::ScratchReg.encoding(), 0, 0, 8, 0)); + masm_.push(js::jit::ScratchReg); + masm_.pushReturnAddress(); +#endif + masm_.Push(js::jit::FramePointer); masm_.moveStackPtrTo(js::jit::FramePointer); @@ -1350,6 +1362,11 @@ void SMRegExpMacroAssembler::exitHandler # ifdef JS_CODEGEN_PPC64 masm_.popReturnAddress(); # endif +# ifdef JS_CODEGEN_IA64 + masm_.popReturnAddress(); + masm_.pop(js::jit::ScratchReg); + masm_.emitI(js::jit::ia64::MovToPfs(js::jit::ScratchReg.encoding())); +# endif masm_.abiret(); #endif diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/firefox-153.0.1/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h b/firefox-153.0.1/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h --- firefox-153.0.1/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h.vanilla +++ firefox-153.0.1/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h @@ -82,6 +82,11 @@ # endif #endif +#ifdef __ia64__ +# define HAS_64BIT_ATOMICS +# define HAS_64BIT_LOCKFREE +#endif + #ifdef JS_CODEGEN_NONE # ifdef JS_64BIT # define HAS_64BIT_ATOMICS diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/firefox-153.0.1/modules/libpref/init/StaticPrefList.yaml b/firefox-153.0.1/modules/libpref/init/StaticPrefList.yaml --- firefox-153.0.1/modules/libpref/init/StaticPrefList.yaml.vanilla +++ firefox-153.0.1/modules/libpref/init/StaticPrefList.yaml @@ -9979,11 +9979,11 @@ set_spidermonkey_pref: startup #endif -#if !defined(JS_CODEGEN_MIPS64) && !defined(JS_CODEGEN_LOONG64) +#if !defined(JS_CODEGEN_MIPS64) && !defined(JS_CODEGEN_LOONG64) && !defined(JS_CODEGEN_IA64) # Spectre security vulnerability mitigations for the JS JITs. # - # NOTE: The MIPS and LoongArch backends do not support these mitigations (and generally - # do not need them). In that case, leave the pref unlisted with its + # NOTE: The MIPS, LoongArch and IA-64 backends do not support these mitigations (and + # generally do not need them). In that case, leave the pref unlisted with its # default value of false. - name: javascript.options.spectre.index_masking type: bool diff -aurpN -x '*.orig' -x '*.rej' -x __pycache__ -x '*.pyc' -x .deps a/firefox-153.0.1/js/xpconnect/src/XPCJSContext.cpp b/firefox-153.0.1/js/xpconnect/src/XPCJSContext.cpp --- firefox-153.0.1/js/xpconnect/src/XPCJSContext.cpp.vanilla +++ firefox-153.0.1/js/xpconnect/src/XPCJSContext.cpp @@ -958,7 +958,7 @@ static void LoadStartupJSPrefs(XPCJSCont #endif #if !defined(JS_CODEGEN_MIPS64) && !defined(JS_CODEGEN_RISCV64) && \ - !defined(JS_CODEGEN_LOONG64) + !defined(JS_CODEGEN_LOONG64) && !defined(JS_CODEGEN_IA64) JS_SetGlobalJitCompilerOption( cx, JSJITCOMPILER_SPECTRE_INDEX_MASKING, StaticPrefs::javascript_options_spectre_index_masking_DoNotUseDirectly());