# Copyright (C) 2019-2020 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1. Redistributions of source code must retain the above copyright

# notice, this list of conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above copyright

# notice, this list of conditions and the following disclaimer in the

# documentation and/or other materials provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''

# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,

# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS

# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR

# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF

# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN

# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF

# THE POSSIBILITY OF SUCH DAMAGE.

# Calling conventions

const CalleeSaveSpaceAsVirtualRegisters = constexpr Wasm::numberOfLLIntCalleeSaveRegisters

const CalleeSaveSpaceStackAligned = (CalleeSaveSpaceAsVirtualRegisters * SlotSize + StackAlignment - 1) & ~StackAlignmentMask

const WasmEntryPtrTag = constexpr WasmEntryPtrTag

if HAVE_FAST_TLS

const WTF_WASM_CONTEXT_KEY = constexpr WTF_WASM_CONTEXT_KEY

if X86_64

const NumberOfWasmArgumentGPRs = 6

elsif ARM64 or ARM64E

const NumberOfWasmArgumentGPRs = 8

error

const NumberOfWasmArgumentFPRs = 8

const NumberOfWasmArguments = NumberOfWasmArgumentGPRs + NumberOfWasmArgumentFPRs

# All callee saves must match the definition in WasmCallee.cpp

# These must match the definition in WasmMemoryInformation.cpp

const wasmInstance = csr0

const memoryBase = csr3

const memorySize = csr4

# This must match the definition in LowLevelInterpreter.asm

if X86_64

const PB = csr2

elsif ARM64 or ARM64E

const PB = csr7

error

macro forEachArgumentGPR(fn)

fn(0 * 8, wa0)

fn(1 * 8, wa1)

fn(2 * 8, wa2)

fn(3 * 8, wa3)

fn(4 * 8, wa4)

fn(5 * 8, wa5)

if ARM64 or ARM64E

fn(6 * 8, wa6)

fn(7 * 8, wa7)

end

macro forEachArgumentFPR(fn)

fn((NumberOfWasmArgumentGPRs + 0) * 8, wfa0)

fn((NumberOfWasmArgumentGPRs + 1) * 8, wfa1)

fn((NumberOfWasmArgumentGPRs + 2) * 8, wfa2)

fn((NumberOfWasmArgumentGPRs + 3) * 8, wfa3)

fn((NumberOfWasmArgumentGPRs + 4) * 8, wfa4)

fn((NumberOfWasmArgumentGPRs + 5) * 8, wfa5)

fn((NumberOfWasmArgumentGPRs + 6) * 8, wfa6)

fn((NumberOfWasmArgumentGPRs + 7) * 8, wfa7)

# Helper macros

# FIXME: Eventually this should be unified with the JS versions

# https://bugs.webkit.org/show_bug.cgi?id=203656

macro wasmDispatch(advanceReg)

addp advanceReg, PC

wasmNextInstruction()

macro wasmDispatchIndirect(offsetReg)

wasmDispatch(offsetReg)

macro wasmNextInstruction()

loadb [PB, PC, 1], t0

leap JSCConfig + constexpr JSC::offsetOfJSCConfigOpcodeMap, t1

jmp NumberOfJSOpcodeIDs * PtrSize[t1, t0, PtrSize], BytecodePtrTag

macro wasmNextInstructionWide16()

loadb OpcodeIDNarrowSize[PB, PC, 1], t0

leap JSCConfig + constexpr JSC::offsetOfJSCConfigOpcodeMapWide16, t1

jmp NumberOfJSOpcodeIDs * PtrSize[t1, t0, PtrSize], BytecodePtrTag

macro wasmNextInstructionWide32()

loadb OpcodeIDNarrowSize[PB, PC, 1], t0

leap JSCConfig + constexpr JSC::offsetOfJSCConfigOpcodeMapWide32, t1

jmp NumberOfJSOpcodeIDs * PtrSize[t1, t0, PtrSize], BytecodePtrTag

macro checkSwitchToJIT(increment, action)

loadp CodeBlock[cfr], ws0

baddis increment, Wasm::FunctionCodeBlock::m_tierUpCounter + Wasm::LLIntTierUpCounter::m_counter[ws0], .continue

action()

.continue:

macro checkSwitchToJITForPrologue(codeBlockRegister)

checkSwitchToJIT(

5,

macro()

move cfr, a0

move PC, a1

move wasmInstance, a2

cCall4(_slow_path_wasm_prologue_osr)

btpz r0, .recover

move r0, ws0

forEachArgumentGPR(macro (offset, gpr)

loadq -offset - 8 - CalleeSaveSpaceAsVirtualRegisters * 8[cfr], gpr

end)

forEachArgumentFPR(macro (offset, fpr)

loadd -offset - 8 - CalleeSaveSpaceAsVirtualRegisters * 8[cfr], fpr

end)

restoreCalleeSavesUsedByWasm()

restoreCallerPCAndCFR()

untagReturnAddress sp

jmp ws0, WasmEntryPtrTag

.recover:

notFunctionCodeBlockGetter(codeBlockRegister)

end)

macro checkSwitchToJITForLoop()

checkSwitchToJIT(

1,

macro()

storei PC, ArgumentCountIncludingThis + TagOffset[cfr]

prepareStateForCCall()

move cfr, a0

move PC, a1

move wasmInstance, a2

cCall4(_slow_path_wasm_loop_osr)

btpz r1, .recover

restoreCalleeSavesUsedByWasm()

restoreCallerPCAndCFR()

untagReturnAddress sp

move r0, a0

jmp r1, WasmEntryPtrTag

.recover:

loadi ArgumentCountIncludingThis + TagOffset[cfr], PC

end)

macro checkSwitchToJITForEpilogue()

checkSwitchToJIT(

10,

macro ()

callWasmSlowPath(_slow_path_wasm_epilogue_osr)

end)

# Wasm specific helpers

macro preserveCalleeSavesUsedByWasm()

# NOTE: We intentionally don't save memoryBase and memorySize here. See the comment

# in restoreCalleeSavesUsedByWasm() below for why.

subp CalleeSaveSpaceStackAligned, sp

if ARM64 or ARM64E

emit "stp x19, x26, [x29, #-16]"

elsif X86_64

storep PB, -0x8[cfr]

storep wasmInstance, -0x10[cfr]

else

error

end

macro restoreCalleeSavesUsedByWasm()

# NOTE: We intentionally don't restore memoryBase and memorySize here. These are saved

# and restored when entering Wasm by the JSToWasm wrapper and changes to them are meant

# to be observable within the same Wasm module.

if ARM64 or ARM64E

emit "ldp x19, x26, [x29, #-16]"

elsif X86_64

loadp -0x8[cfr], PB

loadp -0x10[cfr], wasmInstance

else

error

end

macro loadWasmInstanceFromTLS()

if HAVE_FAST_TLS

tls_loadp WTF_WASM_CONTEXT_KEY, wasmInstance

crash()

macro storeWasmInstanceToTLS(instance)

if HAVE_FAST_TLS

tls_storep instance, WTF_WASM_CONTEXT_KEY

crash()

macro reloadMemoryRegistersFromInstance(instance, scratch1, scratch2)

loadp Wasm::Instance::m_cachedMemory[instance], memoryBase

loadi Wasm::Instance::m_cachedMemorySize[instance], memorySize

cagedPrimitive(memoryBase, memorySize, scratch1, scratch2)

macro throwException(exception)

storei constexpr Wasm::ExceptionType::%exception%, ArgumentCountIncludingThis + PayloadOffset[cfr]

jmp _wasm_throw_from_slow_path_trampoline

macro callWasmSlowPath(slowPath)

prepareStateForCCall()

move cfr, a0

move PC, a1

move wasmInstance, a2

cCall4(slowPath)

restoreStateAfterCCall()

macro callWasmCallSlowPath(slowPath, action)

storei PC, ArgumentCountIncludingThis + TagOffset[cfr]

prepareStateForCCall()

move cfr, a0

move PC, a1

move wasmInstance, a2

cCall4(slowPath)

action(r0, r1)

macro wasmPrologue(codeBlockGetter, codeBlockSetter, loadWasmInstance)

# Set up the call frame and check if we should OSR.

tagReturnAddress sp

preserveCallerPCAndCFR()

preserveCalleeSavesUsedByWasm()

loadWasmInstance()

reloadMemoryRegistersFromInstance(wasmInstance, ws0, ws1)

codeBlockGetter(ws0)

codeBlockSetter(ws0)

# Get new sp in ws1 and check stack height.

loadi Wasm::FunctionCodeBlock::m_numCalleeLocals[ws0], ws1

lshiftp 3, ws1

addp maxFrameExtentForSlowPathCall, ws1

subp cfr, ws1, ws1

bpa ws1, cfr, .stackOverflow

bpbeq Wasm::Instance::m_cachedStackLimit[wasmInstance], ws1, .stackHeightOK

throwException(StackOverflow)

move ws1, sp

forEachArgumentGPR(macro (offset, gpr)

storeq gpr, -offset - 8 - CalleeSaveSpaceAsVirtualRegisters * 8[cfr]

end)

forEachArgumentFPR(macro (offset, fpr)

stored fpr, -offset - 8 - CalleeSaveSpaceAsVirtualRegisters * 8[cfr]

end)

checkSwitchToJITForPrologue(ws0)

# Set up the PC.

loadp Wasm::FunctionCodeBlock::m_instructionsRawPointer[ws0], PB

move 0, PC

loadi Wasm::FunctionCodeBlock::m_numVars[ws0], ws1

subi NumberOfWasmArguments + CalleeSaveSpaceAsVirtualRegisters, ws1

btiz ws1, .zeroInitializeLocalsDone

negi ws1

sxi2q ws1, ws1

leap (NumberOfWasmArguments + CalleeSaveSpaceAsVirtualRegisters + 1) * -8[cfr], ws0

addq 1, ws1

storeq 0, [ws0, ws1, 8]

btqz ws1, .zeroInitializeLocalsDone

jmp .zeroInitializeLocalsLoop

macro traceExecution()

if TRACING

callWasmSlowPath(_slow_path_wasm_trace)

end

# Less convenient, but required for opcodes that collide with reserved instructions (e.g. wasm_nop)

macro unprefixedWasmOp(opcodeName, opcodeStruct, fn)

commonOp(opcodeName, traceExecution, macro(size)

fn(macro(fn2)

fn2(opcodeName, opcodeStruct, size)

end)

end)

macro wasmOp(opcodeName, opcodeStruct, fn)

unprefixedWasmOp(wasm_%opcodeName%, opcodeStruct, fn)

# Same as unprefixedWasmOp, necessary for e.g. wasm_call

macro unprefixedSlowWasmOp(opcodeName)

unprefixedWasmOp(opcodeName, unusedOpcodeStruct, macro(ctx)

callWasmSlowPath(_slow_path_%opcodeName%)

dispatch(ctx)

end)

macro slowWasmOp(opcodeName)

unprefixedSlowWasmOp(wasm_%opcodeName%)

# Macro version of load operations: mload[suffix]

# loads field from the instruction stream and performs load[suffix] to dst

macro firstConstantRegisterIndex(ctx, fn)

ctx(macro(opcodeName, opcodeStruct, size)

size(FirstConstantRegisterIndexNarrow, FirstConstantRegisterIndexWide16, FirstConstantRegisterIndexWide32, fn)

end)

macro loadConstantOrVariable(ctx, index, loader)

firstConstantRegisterIndex(ctx, macro (firstConstantIndex)

bpgteq index, firstConstantIndex, .constant

loader([cfr, index, 8])

jmp .done

.constant:

loadp CodeBlock[cfr], t6

loadp Wasm::FunctionCodeBlock::m_constants + VectorBufferOffset[t6], t6

subp firstConstantIndex, index

loader([t6, index, 8])

.done:

end)

macro mloadq(ctx, field, dst)

wgets(ctx, field, dst)

loadConstantOrVariable(ctx, dst, macro (from)

loadq from, dst

end)

macro mloadi(ctx, field, dst)

wgets(ctx, field, dst)

loadConstantOrVariable(ctx, dst, macro (from)

loadi from, dst

end)

macro mloadp(ctx, field, dst)

wgets(ctx, field, dst)

loadConstantOrVariable(ctx, dst, macro (from)

loadp from, dst

end)

macro mloadf(ctx, field, dst)

wgets(ctx, field, t5)

loadConstantOrVariable(ctx, t5, macro (from)

loadf from, dst

end)

macro mloadd(ctx, field, dst)

wgets(ctx, field, t5)

loadConstantOrVariable(ctx, t5, macro (from)

loadd from, dst

end)

# Typed returns

macro returnq(ctx, value)

wgets(ctx, m_dst, t5)

storeq value, [cfr, t5, 8]

dispatch(ctx)

macro returni(ctx, value)

wgets(ctx, m_dst, t5)

storei value, [cfr, t5, 8]

dispatch(ctx)

macro returnf(ctx, value)

wgets(ctx, m_dst, t5)

storef value, [cfr, t5, 8]

dispatch(ctx)

macro returnd(ctx, value)

wgets(ctx, m_dst, t5)

stored value, [cfr, t5, 8]

dispatch(ctx)

# Wasm wrapper of get/getu that operate on ctx

macro wgets(ctx, field, dst)

ctx(macro(opcodeName, opcodeStruct, size)

size(getOperandNarrow, getOperandWide16, getOperandWide32, macro (get)

get(opcodeStruct, field, dst)

end)

end)

macro wgetu(ctx, field, dst)

ctx(macro(opcodeName, opcodeStruct, size)

size(getuOperandNarrow, getuOperandWide16, getuOperandWide32, macro (getu)

getu(opcodeStruct, field, dst)

end)

end)

# Control flow helpers

macro dispatch(ctx)

ctx(macro(opcodeName, opcodeStruct, size)

genericDispatchOp(wasmDispatch, size, opcodeName)

end)

macro jump(ctx, target)

wgets(ctx, target, t0)

btiz t0, .outOfLineJumpTarget

wasmDispatchIndirect(t0)

callWasmSlowPath(_slow_path_wasm_out_of_line_jump_target)

wasmNextInstruction()

macro doReturn()

restoreCalleeSavesUsedByWasm()

restoreCallerPCAndCFR()

ret

# Entry point

macro wasmCodeBlockGetter(targetRegister)

loadp Callee[cfr], targetRegister

andp ~3, targetRegister

loadp Wasm::LLIntCallee::m_codeBlock[targetRegister], targetRegister

op(wasm_function_prologue, macro ()

if not WEBASSEMBLY or not JSVALUE64 or C_LOOP or C_LOOP_WIN

error

end

wasmPrologue(wasmCodeBlockGetter, functionCodeBlockSetter, loadWasmInstanceFromTLS)

wasmNextInstruction()

end)

op(wasm_function_prologue_no_tls, macro ()

if not WEBASSEMBLY or not JSVALUE64 or C_LOOP or C_LOOP_WIN

error

end

wasmPrologue(wasmCodeBlockGetter, functionCodeBlockSetter, macro () end)

wasmNextInstruction()

end)

op(wasm_throw_from_slow_path_trampoline, macro ()

loadp Wasm::Instance::m_pointerToTopEntryFrame[wasmInstance], t5

loadp [t5], t5

copyCalleeSavesToEntryFrameCalleeSavesBuffer(t5)

move cfr, a0

addp PB, PC, a1

move wasmInstance, a2

# Slow paths and the throwException macro store the exception code in the ArgumentCountIncludingThis slot

loadi ArgumentCountIncludingThis + PayloadOffset[cfr], a3

cCall4(_slow_path_wasm_throw_exception)

jmp r0, ExceptionHandlerPtrTag

end)

# Disable wide version of narrow-only opcodes

noWide(wasm_enter)

noWide(wasm_wide16)

noWide(wasm_wide32)

# Opcodes that always invoke the slow path

slowWasmOp(ref_func)

slowWasmOp(table_get)

slowWasmOp(table_set)

slowWasmOp(table_size)

slowWasmOp(table_fill)

slowWasmOp(table_grow)

slowWasmOp(set_global_ref)

slowWasmOp(set_global_ref_portable_binding)

wasmOp(grow_memory, WasmGrowMemory, macro(ctx)

callWasmSlowPath(_slow_path_wasm_grow_memory)

reloadMemoryRegistersFromInstance(wasmInstance, ws0, ws1)

dispatch(ctx)

end)

# Opcodes that should eventually be shared with JS llint

wasmNextInstructionWide16()

wasmNextInstructionWide32()

traceExecution()

checkStackPointerAlignment(t2, 0xdead00e1)

loadp CodeBlock[cfr], t2 // t2<CodeBlock> = cfr.CodeBlock

loadi Wasm::FunctionCodeBlock::m_numVars[t2], t2 // t2<size_t> = t2<CodeBlock>.m_numVars

subq CalleeSaveSpaceAsVirtualRegisters + NumberOfWasmArguments, t2

btiz t2, .opEnterDone

move cfr, t1

subq CalleeSaveSpaceAsVirtualRegisters * 8 + NumberOfWasmArguments * 8, t1

negi t2

sxi2q t2, t2

storeq 0, [t1, t2, 8]

addq 1, t2

btqnz t2, .opEnterLoop

wasmDispatchIndirect(1)

unprefixedWasmOp(wasm_nop, WasmNop, macro(ctx)

dispatch(ctx)

end)

wasmOp(loop_hint, WasmLoopHint, macro(ctx)

checkSwitchToJITForLoop()

dispatch(ctx)

end)

wasmOp(mov, WasmMov, macro(ctx)

mloadq(ctx, m_src, t0)

returnq(ctx, t0)

end)

wasmOp(jtrue, WasmJtrue, macro(ctx)

mloadi(ctx, m_condition, t0)

btiz t0, .continue

jump(ctx, m_targetLabel)

dispatch(ctx)

end)

wasmOp(jfalse, WasmJfalse, macro(ctx)

mloadi(ctx, m_condition, t0)

btinz t0, .continue

jump(ctx, m_targetLabel)

dispatch(ctx)

end)

wasmOp(switch, WasmSwitch, macro(ctx)

mloadi(ctx, m_scrutinee, t0)

wgetu(ctx, m_tableIndex, t1)

loadp CodeBlock[cfr], t2

loadp Wasm::FunctionCodeBlock::m_jumpTables + VectorBufferOffset[t2], t2

muli sizeof Wasm::FunctionCodeBlock::JumpTable, t1

addp t1, t2

loadi VectorSizeOffset[t2], t3

bib t0, t3, .inBounds

subi t3, 1, t0

loadp VectorBufferOffset[t2], t2

muli sizeof Wasm::FunctionCodeBlock::JumpTableEntry, t0

loadi Wasm::FunctionCodeBlock::JumpTableEntry::startOffset[t2, t0], t1

loadi Wasm::FunctionCodeBlock::JumpTableEntry::dropCount[t2, t0], t3

loadi Wasm::FunctionCodeBlock::JumpTableEntry::keepCount[t2, t0], t5

dropKeep(t1, t3, t5)

loadis Wasm::FunctionCodeBlock::JumpTableEntry::target[t2, t0], t3

assert(macro(ok) btinz t3, .ok end)

wasmDispatchIndirect(t3)

end)

unprefixedWasmOp(wasm_jmp, WasmJmp, macro(ctx)

jump(ctx, m_targetLabel)

end)

unprefixedWasmOp(wasm_ret, WasmRet, macro(ctx)

checkSwitchToJITForEpilogue()

forEachArgumentGPR(macro (offset, gpr)

loadq -offset - 8 - CalleeSaveSpaceAsVirtualRegisters * 8[cfr], gpr

end)

forEachArgumentFPR(macro (offset, fpr)

loadd -offset - 8 - CalleeSaveSpaceAsVirtualRegisters * 8[cfr], fpr

end)

doReturn()

end)

# Wasm specific bytecodes

wasmOp(unreachable, WasmUnreachable, macro(ctx)

throwException(Unreachable)

end)

wasmOp(ret_void, WasmRetVoid, macro(ctx)

checkSwitchToJITForEpilogue()

doReturn()

end)

wasmOp(ref_is_null, WasmRefIsNull, macro(ctx)

mloadp(ctx, m_ref, t0)

cqeq t0, ValueNull, t0

returni(ctx, t0)

end)

wasmOp(get_global, WasmGetGlobal, macro(ctx)

loadp Wasm::Instance::m_globals[wasmInstance], t0

wgetu(ctx, m_globalIndex, t1)

loadq [t0, t1, 8], t0

returnq(ctx, t0)

end)

wasmOp(set_global, WasmSetGlobal, macro(ctx)

loadp Wasm::Instance::m_globals[wasmInstance], t0

wgetu(ctx, m_globalIndex, t1)

mloadq(ctx, m_value, t2)

storeq t2, [t0, t1, 8]

dispatch(ctx)

end)

wasmOp(get_global_portable_binding, WasmGetGlobalPortableBinding, macro(ctx)

loadp Wasm::Instance::m_globals[wasmInstance], t0

wgetu(ctx, m_globalIndex, t1)

loadq [t0, t1, 8], t0

loadq [t0], t0

returnq(ctx, t0)

end)

wasmOp(set_global_portable_binding, WasmSetGlobalPortableBinding, macro(ctx)

loadp Wasm::Instance::m_globals[wasmInstance], t0

wgetu(ctx, m_globalIndex, t1)

mloadq(ctx, m_value, t2)

loadq [t0, t1, 8], t0

storeq t2, [t0]

dispatch(ctx)

end)

macro slowPathForWasmCall(ctx, slowPath, storeWasmInstance)

callWasmCallSlowPath(

slowPath,

# callee is r0 and targetWasmInstance is r1

macro (callee, targetWasmInstance)

move callee, ws0

loadi ArgumentCountIncludingThis + TagOffset[cfr], PC

# the call might throw (e.g. indirect call with bad signature)

btpz targetWasmInstance, .throw

wgetu(ctx, m_stackOffset, ws1)

lshifti 3, ws1

subp cfr, ws1, sp

wgetu(ctx, m_numberOfStackArgs, ws1)

# Preserve the current instance

move wasmInstance, PB

storeWasmInstance(targetWasmInstance)

reloadMemoryRegistersFromInstance(targetWasmInstance, wa0, wa1)

# Load registers from stack

forEachArgumentGPR(macro (offset, gpr)

loadq CallFrameHeaderSize + offset[sp, ws1, 8], gpr

end)

forEachArgumentFPR(macro (offset, fpr)

loadd CallFrameHeaderSize + offset[sp, ws1, 8], fpr

end)

addp CallerFrameAndPCSize, sp

call ws0, SlowPathPtrTag

loadp CodeBlock[cfr], ws1

loadi Wasm::FunctionCodeBlock::m_numCalleeLocals[ws1], ws1

lshiftp 3, ws1

addp maxFrameExtentForSlowPathCall, ws1

subp cfr, ws1, sp

# We need to set PC to load information from the instruction stream, but we

# need to preserve its current value since it might contain a return value

move PC, memoryBase

move PB, wasmInstance

loadi ArgumentCountIncludingThis + TagOffset[cfr], PC

loadp CodeBlock[cfr], PB

loadp Wasm::FunctionCodeBlock::m_instructionsRawPointer[PB], PB

wgetu(ctx, m_stackOffset, ws1)

lshifti 3, ws1

negi ws1

sxi2q ws1, ws1

addp cfr, ws1

# Argument registers are also return registers, so they must be stored to the stack

# in case they contain return values.

wgetu(ctx, m_numberOfStackArgs, ws0)

move memoryBase, PC

forEachArgumentGPR(macro (offset, gpr)

storeq gpr, CallFrameHeaderSize + offset[ws1, ws0, 8]

end)

forEachArgumentFPR(macro (offset, fpr)

stored fpr, CallFrameHeaderSize + offset[ws1, ws0, 8]

end)

loadi ArgumentCountIncludingThis + TagOffset[cfr], PC

storeWasmInstance(wasmInstance)

reloadMemoryRegistersFromInstance(wasmInstance, ws0, ws1)

# Restore stack limit

loadp Wasm::Instance::m_pointerToActualStackLimit[wasmInstance], t5

loadp [t5], t5

storep t5, Wasm::Instance::m_cachedStackLimit[wasmInstance]

dispatch(ctx)

.throw:

restoreStateAfterCCall()

dispatch(ctx)

end)

unprefixedWasmOp(wasm_call, WasmCall, macro(ctx)

slowPathForWasmCall(ctx, _slow_path_wasm_call, storeWasmInstanceToTLS)

end)

unprefixedWasmOp(wasm_call_no_tls, WasmCallNoTls, macro(ctx)

slowPathForWasmCall(ctx, _slow_path_wasm_call_no_tls, macro(targetInstance) move targetInstance, wasmInstance end)

end)

wasmOp(call_indirect, WasmCallIndirect, macro(ctx)

slowPathForWasmCall(ctx, _slow_path_wasm_call_indirect, storeWasmInstanceToTLS)

end)

wasmOp(call_indirect_no_tls, WasmCallIndirectNoTls, macro(ctx)

slowPathForWasmCall(ctx, _slow_path_wasm_call_indirect_no_tls, macro(targetInstance) move targetInstance, wasmInstance end)

end)

wasmOp(current_memory, WasmCurrentMemory, macro(ctx)

loadp Wasm::Instance::m_cachedMemorySize[wasmInstance], t0

urshiftq 16, t0

returnq(ctx, t0)

end)

wasmOp(select, WasmSelect, macro(ctx)

mloadi(ctx, m_condition, t0)

btiz t0, .isZero

mloadq(ctx, m_nonZero, t0)

returnq(ctx, t0)

mloadq(ctx, m_zero, t0)

returnq(ctx, t0)

end)

# uses offset as scratch and returns result on pointer

macro emitCheckAndPreparePointer(ctx, pointer, offset, size)

leap size - 1[pointer, offset], t5

bpb t5, memorySize, .continuation

throwException(OutOfBoundsMemoryAccess)

addp memoryBase, pointer

macro wasmLoadOp(name, struct, size, fn)

wasmOp(name, struct, macro(ctx)

mloadi(ctx, m_pointer, t0)

wgetu(ctx, m_offset, t1)

emitCheckAndPreparePointer(ctx, t0, t1, size)

fn([t0, t1], t2)

returnq(ctx, t2)

end)

wasmLoadOp(load8_u, WasmLoad8U, 1, macro(mem, dst) loadb mem, dst end)

wasmLoadOp(load16_u, WasmLoad16U, 2, macro(mem, dst) loadh mem, dst end)

wasmLoadOp(load32_u, WasmLoad32U, 4, macro(mem, dst) loadi mem, dst end)

wasmLoadOp(load64_u, WasmLoad64U, 8, macro(mem, dst) loadq mem, dst end)

wasmLoadOp(i32_load8_s, WasmI32Load8S, 1, macro(mem, dst) loadbsi mem, dst end)

wasmLoadOp(i64_load8_s, WasmI64Load8S, 1, macro(mem, dst) loadbsq mem, dst end)

wasmLoadOp(i32_load16_s, WasmI32Load16S, 2, macro(mem, dst) loadhsi mem, dst end)

wasmLoadOp(i64_load16_s, WasmI64Load16S, 2, macro(mem, dst) loadhsq mem, dst end)

wasmLoadOp(i64_load32_s, WasmI64Load32S, 4, macro(mem, dst) loadis mem, dst end)

macro wasmStoreOp(name, struct, size, fn)

wasmOp(name, struct, macro(ctx)

mloadi(ctx, m_pointer, t0)

wgetu(ctx, m_offset, t1)

emitCheckAndPreparePointer(ctx, t0, t1, size)

mloadq(ctx, m_value, t2)

fn(t2, [t0, t1])

dispatch(ctx)

end)

wasmStoreOp(store8, WasmStore8, 1, macro(value, mem) storeb value, mem end)

wasmStoreOp(store16, WasmStore16, 2, macro(value, mem) storeh value, mem end)

wasmStoreOp(store32, WasmStore32, 4, macro(value, mem) storei value, mem end)

wasmStoreOp(store64, WasmStore64, 8, macro(value, mem) storeq value, mem end)

# Opcodes that don't have the `b3op` entry in wasm.json. This should be kept in sync

wasmOp(i32_div_s, WasmI32DivS, macro (ctx)

mloadi(ctx, m_lhs, t0)

mloadi(ctx, m_rhs, t1)

btiz t1, .throwDivisionByZero

bineq t1, -1, .safe

bieq t0, constexpr INT32_MIN, .throwIntegerOverflow

if X86_64

# FIXME: Add a way to static_asset that t0 is rax and t2 is rdx

# https://bugs.webkit.org/show_bug.cgi?id=203692

cdqi

idivi t1

elsif ARM64 or ARM64E

divis t1, t0

else

error

end

returni(ctx, t0)

throwException(DivisionByZero)

throwException(IntegerOverflow)

end)

wasmOp(i32_div_u, WasmI32DivU, macro (ctx)

mloadi(ctx, m_lhs, t0)

mloadi(ctx, m_rhs, t1)

btiz t1, .throwDivisionByZero

if X86_64

xori t2, t2

udivi t1

elsif ARM64 or ARM64E

divi t1, t0

else

error

end

returni(ctx, t0)

throwException(DivisionByZero)

end)

wasmOp(i32_rem_s, WasmI32RemS, macro (ctx)

mloadi(ctx, m_lhs, t0)

mloadi(ctx, m_rhs, t1)

btiz t1, .throwDivisionByZero

bineq t1, -1, .safe

bineq t0, constexpr INT32_MIN, .safe

move 0, t2

jmp .return

if X86_64

# FIXME: Add a way to static_asset that t0 is rax and t2 is rdx

# https://bugs.webkit.org/show_bug.cgi?id=203692

cdqi

idivi t1

elsif ARM64 or ARM64E

divis t1, t0, t2

muli t1, t2

subi t0, t2, t2

else

error

end

returni(ctx, t2)

throwException(DivisionByZero)

end)

wasmOp(i32_rem_u, WasmI32RemU, macro (ctx)

mloadi(ctx, m_lhs, t0)

mloadi(ctx, m_rhs, t1)

btiz t1, .throwDivisionByZero

if X86_64

xori t2, t2

udivi t1

elsif ARM64 or ARM64E

divi t1, t0, t2

muli t1, t2

subi t0, t2, t2

else

error

end

returni(ctx, t2)

throwException(DivisionByZero)

end)

wasmOp(i32_ctz, WasmI32Ctz, macro (ctx)

mloadq(ctx, m_operand, t0)

tzcnti t0, t0

returni(ctx, t0)

end)

wasmOp(i32_popcnt, WasmI32Popcnt, macro (ctx)

mloadi(ctx, m_operand, a1)

prepareStateForCCall()

move PC, a0

cCall2(_slow_path_wasm_popcount)

restoreStateAfterCCall()

returni(ctx, r1)

end)

wasmOp(i64_div_s, WasmI64DivS, macro (ctx)

mloadq(ctx, m_lhs, t0)

mloadq(ctx, m_rhs, t1)

btqz t1, .throwDivisionByZero

bqneq t1, -1, .safe

bqeq t0, constexpr INT64_MIN, .throwIntegerOverflow

if X86_64

# FIXME: Add a way to static_asset that t0 is rax and t2 is rdx

# https://bugs.webkit.org/show_bug.cgi?id=203692

cqoq

idivq t1

elsif ARM64 or ARM64E

divqs t1, t0

else

error

end

returnq(ctx, t0)

throwException(DivisionByZero)

throwException(IntegerOverflow)

end)

wasmOp(i64_div_u, WasmI64DivU, macro (ctx)

mloadq(ctx, m_lhs, t0)

mloadq(ctx, m_rhs, t1)

btqz t1, .throwDivisionByZero

if X86_64

xorq t2, t2

udivq t1

elsif ARM64 or ARM64E

divq t1, t0

else

error