using ErrorType = String;

using PartialResult = Expected<void, ErrorType>;

using Address = MacroAssembler::Address;

using BaseIndex = MacroAssembler::BaseIndex;

using Imm32 = MacroAssembler::Imm32;

using Imm64 = MacroAssembler::Imm64;

using TrustedImm32 = MacroAssembler::TrustedImm32;

using TrustedImm64 = MacroAssembler::TrustedImm64;

using TrustedImmPtr = MacroAssembler::TrustedImmPtr;

using RelationalCondition = MacroAssembler::RelationalCondition;

using ResultCondition = MacroAssembler::ResultCondition;

using DoubleCondition = MacroAssembler::DoubleCondition;

using StatusCondition = MacroAssembler::StatusCondition;

using Jump = MacroAssembler::Jump;

using JumpList = MacroAssembler::JumpList;

using DataLabelPtr = MacroAssembler::DataLabelPtr;

// Functions can have up to 1000000 instructions, so 32 bits is a sensible maximum number of stack items or locals.

using LocalOrTempIndex = uint32_t;

static constexpr unsigned LocalIndexBits = 21;

static_assert(maxFunctionLocals < 1 << LocalIndexBits);

static constexpr GPRReg wasmScratchGPR = GPRInfo::nonPreservedNonArgumentGPR0; // Scratch registers to hold temporaries in operations.

#if USE(JSVALUE32_64)

static constexpr GPRReg wasmScratchGPR2 = GPRInfo::nonPreservedNonArgumentGPR1;

#else

static constexpr GPRReg wasmScratchGPR2 = InvalidGPRReg;

#endif

static constexpr FPRReg wasmScratchFPR = FPRInfo::nonPreservedNonArgumentFPR0;

#if CPU(X86) || CPU(X86_64)

static constexpr GPRReg shiftRCX = X86Registers::ecx;

#else

static constexpr GPRReg shiftRCX = InvalidGPRReg;

#endif

#if USE(JSVALUE64)

static constexpr GPRReg wasmBaseMemoryPointer = GPRInfo::wasmBaseMemoryPointer;

static constexpr GPRReg wasmBoundsCheckingSizeRegister = GPRInfo::wasmBoundsCheckingSizeRegister;

#else

static constexpr GPRReg wasmBaseMemoryPointer = InvalidGPRReg;

static constexpr GPRReg wasmBoundsCheckingSizeRegister = InvalidGPRReg;

#endif

struct Location {

enum Kind : uint8_t {

None = 0,

Stack = 1,

Gpr = 2,

Fpr = 3,

Global = 4,

StackArgument = 5,

Gpr2 = 6

};

Location()

: m_kind(None)

{ }

static Location none();

static Location fromStack(int32_t stackOffset);

static Location fromStackArgument(int32_t stackOffset);

static Location fromGPR(GPRReg gpr);

static Location fromGPR2(GPRReg hi, GPRReg lo);

static Location fromFPR(FPRReg fpr);

static Location fromGlobal(int32_t globalOffset);

static Location fromArgumentLocation(ArgumentLocation argLocation, TypeKind type);

bool isNone() const;

bool isGPR() const;

bool isGPR2() const;

bool isFPR() const;

bool isRegister() const;

bool isStack() const;

bool isStackArgument() const;

bool isGlobal() const;

bool isMemory() const;

int32_t asStackOffset() const;

Address asStackAddress() const;

int32_t asGlobalOffset() const;

Address asGlobalAddress() const;

int32_t asStackArgumentOffset() const;

Address asStackArgumentAddress() const;

Address asAddress() const;

GPRReg asGPR() const;

FPRReg asFPR() const;

GPRReg asGPRlo() const;

GPRReg asGPRhi() const;

void dump(PrintStream& out) const;

bool operator==(Location other) const;

Kind kind() const;

private:

union {

// It's useful to we be able to cram a location into a 4-byte space, so that

// we can store them efficiently in ControlData.

struct {

unsigned m_kind : 3;

int32_t m_offset : 29;

};

struct {

Kind m_padGpr;

GPRReg m_gpr;

};

struct {

Kind m_padFpr;

FPRReg m_fpr;

};

struct {

Kind m_padGpr2;

GPRReg m_gprhi, m_gprlo;

};

};

};

static_assert(sizeof(Location) == 4);

static bool isValidValueTypeKind(TypeKind kind);

static TypeKind pointerType();

static bool isFloatingPointType(TypeKind type);

static bool typeNeedsGPR2(TypeKind type);

static uint32_t sizeOfType(TypeKind type);

static TypeKind toValueKind(TypeKind kind);

class Value {

public:

// Represents the location in which this value is stored.

enum Kind : uint8_t {

None = 0,

Const = 1,

Temp = 2,

Local = 3,

Pinned = 4 // Used if we need to represent a Location as a Value, mostly in operation calls

};

ALWAYS_INLINE bool isNone() const

{

return m_kind == None;

}

ALWAYS_INLINE bool isTemp() const

{

return m_kind == Temp;

}

ALWAYS_INLINE bool isLocal() const

{

return m_kind == Local;

}

ALWAYS_INLINE bool isPinned() const

{

return m_kind == Pinned;

}

ALWAYS_INLINE Kind kind() const

{

return m_kind;

}

ALWAYS_INLINE int32_t asI32() const

{

ASSERT(m_kind == Const);

return m_i32;

}

ALWAYS_INLINE int64_t asI64() const

{

ASSERT(m_kind == Const);

return m_i64;

}

ALWAYS_INLINE float asF32() const

{

ASSERT(m_kind == Const);

return m_f32;

}

ALWAYS_INLINE double asF64() const

{

ASSERT(m_kind == Const);

return m_f64;

}

ALWAYS_INLINE EncodedJSValue asRef() const

{

ASSERT(m_kind == Const);

return m_ref;

}

ALWAYS_INLINE LocalOrTempIndex asTemp() const

{

ASSERT(m_kind == Temp);

return m_index;

}

ALWAYS_INLINE LocalOrTempIndex asLocal() const

{

ASSERT(m_kind == Local);

return m_index;

}

ALWAYS_INLINE bool isConst() const

{

return m_kind == Const;

}

ALWAYS_INLINE Location asPinned() const

{

ASSERT(m_kind == Pinned);

return m_pinned;

}

ALWAYS_INLINE static Value fromI32(int32_t immediate)

{

Value val;

val.m_kind = Const;

val.m_type = TypeKind::I32;

val.m_i32 = immediate;

return val;

}

ALWAYS_INLINE static Value fromI64(int64_t immediate)

{

Value val;

val.m_kind = Const;

val.m_type = TypeKind::I64;

val.m_i64 = immediate;

return val;

}

ALWAYS_INLINE static Value fromF32(float immediate)

{

Value val;

val.m_kind = Const;

val.m_type = TypeKind::F32;

val.m_f32 = immediate;

return val;

}

ALWAYS_INLINE static Value fromF64(double immediate)

{

Value val;

val.m_kind = Const;

val.m_type = TypeKind::F64;

val.m_f64 = immediate;

return val;

}

ALWAYS_INLINE static Value fromRef(TypeKind refType, EncodedJSValue ref)

{

Value val;

val.m_kind = Const;

val.m_type = toValueKind(refType);

val.m_ref = ref;

return val;

}

ALWAYS_INLINE static Value fromTemp(TypeKind type, LocalOrTempIndex temp)

{

Value val;

val.m_kind = Temp;

val.m_type = toValueKind(type);

val.m_index = temp;

return val;

}

ALWAYS_INLINE static Value fromLocal(TypeKind type, LocalOrTempIndex local)

{

Value val;

val.m_kind = Local;

val.m_type = toValueKind(type);

val.m_index = local;

return val;

}

ALWAYS_INLINE static Value pinned(TypeKind type, Location location)

{

Value val;

val.m_kind = Pinned;

val.m_type = toValueKind(type);

val.m_pinned = location;

return val;

}

ALWAYS_INLINE static Value none()

{

Value val;

val.m_kind = None;

return val;

}

ALWAYS_INLINE uint32_t size() const

{

return sizeOfType(m_type);

}

ALWAYS_INLINE bool isFloat() const

{

return isFloatingPointType(m_type);

}

ALWAYS_INLINE TypeKind type() const

{

ASSERT(isValidValueTypeKind(m_type));

return m_type;

}

ALWAYS_INLINE Value()

: m_kind(None)

{ }

int32_t asI64hi() const;

int32_t asI64lo() const;

void dump(PrintStream& out) const;

private:

union {

int32_t m_i32;

struct {

int32_t lo, hi;

} m_i32_pair;

int64_t m_i64;

float m_f32;

double m_f64;

LocalOrTempIndex m_index;

Location m_pinned;

EncodedJSValue m_ref;

};

Kind m_kind;

TypeKind m_type;

};

struct RegisterBinding {

enum Kind : uint8_t {

None = 0,

Local = 1,

Temp = 2,

Scratch = 3, // Denotes a register bound for use as a scratch, not as a local or temp's location.

};

union {

uint32_t m_uintValue;

struct {

TypeKind m_type;

unsigned m_kind : 3;

unsigned m_index : LocalIndexBits;

};

};

RegisterBinding()

: m_uintValue(0)

{ }

RegisterBinding(uint32_t uintValue)

: m_uintValue(uintValue)

{ }

static RegisterBinding fromValue(Value value);

static RegisterBinding none();

static RegisterBinding scratch();

Value toValue() const;

bool isNone() const;

bool isValid() const;

bool isScratch() const;

bool operator==(RegisterBinding other) const;

void dump(PrintStream& out) const;

unsigned hash() const;

uint32_t encode() const;

};

struct ControlData {

static bool isIf(const ControlData& control) { return control.blockType() == BlockType::If; }

static bool isTry(const ControlData& control) { return control.blockType() == BlockType::Try; }

static bool isAnyCatch(const ControlData& control) { return control.blockType() == BlockType::Catch; }

static bool isCatch(const ControlData& control) { return isAnyCatch(control) && control.catchKind() == CatchKind::Catch; }

static bool isTopLevel(const ControlData& control) { return control.blockType() == BlockType::TopLevel; }

static bool isLoop(const ControlData& control) { return control.blockType() == BlockType::Loop; }

static bool isBlock(const ControlData& control) { return control.blockType() == BlockType::Block; }

ControlData()

: m_enclosedHeight(0)

{ }

ControlData(BBQJIT& generator, BlockType blockType, BlockSignature signature, LocalOrTempIndex enclosedHeight, RegisterSet liveScratchGPRs);

// Re-use the argument layout of another block (eg. else will re-use the argument/result locations from if)

enum BranchCallingConventionReuseTag { UseBlockCallingConventionOfOtherBranch };

ControlData(BranchCallingConventionReuseTag, BlockType blockType, ControlData& otherBranch)

: m_signature(otherBranch.m_signature)

, m_blockType(blockType)

, m_argumentLocations(otherBranch.m_argumentLocations)

, m_resultLocations(otherBranch.m_resultLocations)

, m_enclosedHeight(otherBranch.m_enclosedHeight)

{

}

// This function is intentionally not using implicitSlots since arguments and results should not include implicit slot.

Location allocateArgumentOrResult(BBQJIT& generator, TypeKind type, unsigned i, RegisterSet& remainingGPRs, RegisterSet& remainingFPRs);

template<typename Stack>

void flushAtBlockBoundary(BBQJIT& generator, unsigned targetArity, Stack& expressionStack, bool endOfWasmBlock)

{

// First, we flush all locals that were allocated outside of their designated slots in this block.

for (unsigned i = 0; i < expressionStack.size(); ++i) {

if (expressionStack[i].value().isLocal())

m_touchedLocals.add(expressionStack[i].value().asLocal());

}

for (LocalOrTempIndex touchedLocal : m_touchedLocals) {

Value value = Value::fromLocal(generator.m_localTypes[touchedLocal], touchedLocal);

if (generator.locationOf(value).isRegister())

generator.flushValue(value);

}

// If we are a catch block, we need to flush the exception value, since it's not represented on the expression stack.

if (isAnyCatch(*this)) {

Value value = generator.exception(*this);

if (!endOfWasmBlock)

generator.flushValue(value);

else

generator.consume(value);

}

for (unsigned i = 0; i < expressionStack.size(); ++i) {

Value& value = expressionStack[i].value();

int resultIndex = static_cast<int>(i) - static_cast<int>(expressionStack.size() - targetArity);

// Next, we turn all constants into temporaries, so they can be given persistent slots on the stack.

// If this is the end of the enclosing wasm block, we know we won't need them again, so this can be skipped.

if (value.isConst() && (resultIndex < 0 || !endOfWasmBlock)) {

Value constant = value;

value = Value::fromTemp(value.type(), static_cast<LocalOrTempIndex>(enclosedHeight() + implicitSlots() + i));

Location slot = generator.locationOf(value);

generator.emitMoveConst(constant, slot);

}

// Next, we flush or consume all the temp values on the stack.

if (value.isTemp()) {

if (!endOfWasmBlock)

generator.flushValue(value);

else if (resultIndex < 0)

generator.consume(value);

}

}

}

template<typename Stack, size_t N>

bool addExit(BBQJIT& generator, const Vector<Location, N>& targetLocations, Stack& expressionStack)

{

unsigned targetArity = targetLocations.size();

if (!targetArity)

return false;

// We move all passed temporaries to the successor, in its argument slots.

unsigned offset = expressionStack.size() - targetArity;

Vector<Value, 8> resultValues;

Vector<Location, 8> resultLocations;

for (unsigned i = 0; i < targetArity; ++i) {

resultValues.append(expressionStack[i + offset].value());

resultLocations.append(targetLocations[i]);

}

generator.emitShuffle(resultValues, resultLocations);

return true;

}

template<typename Stack>

void finalizeBlock(BBQJIT& generator, unsigned targetArity, Stack& expressionStack, bool preserveArguments)

{

// Finally, as we are leaving the block, we convert any constants into temporaries on the stack, so we don't blindly assume they have

// the same constant values in the successor.

unsigned offset = expressionStack.size() - targetArity;

for (unsigned i = 0; i < targetArity; ++i) {

Value& value = expressionStack[i + offset].value();

if (value.isConst()) {

Value constant = value;

value = Value::fromTemp(value.type(), static_cast<LocalOrTempIndex>(enclosedHeight() + implicitSlots() + i + offset));

if (preserveArguments)

generator.emitMoveConst(constant, generator.canonicalSlot(value));

} else if (value.isTemp()) {

if (preserveArguments)

generator.flushValue(value);

else

generator.consume(value);

}

}

}

template<typename Stack>

void flushAndSingleExit(BBQJIT& generator, ControlData& target, Stack& expressionStack, bool isChildBlock, bool endOfWasmBlock, bool unreachable = false)

{

// Helper to simplify the common case where we don't need to handle multiple exits.

const auto& targetLocations = isChildBlock ? target.argumentLocations() : target.targetLocations();

flushAtBlockBoundary(generator, targetLocations.size(), expressionStack, endOfWasmBlock);

if (!unreachable)

addExit(generator, targetLocations, expressionStack);

finalizeBlock(generator, targetLocations.size(), expressionStack, false);

}

template<typename Stack>

void startBlock(BBQJIT& generator, Stack& expressionStack)

{

ASSERT(expressionStack.size() >= m_argumentLocations.size());

for (unsigned i = 0; i < m_argumentLocations.size(); ++i) {

ASSERT(!expressionStack[i + expressionStack.size() - m_argumentLocations.size()].value().isConst());

generator.bind(expressionStack[i].value(), m_argumentLocations[i]);

}

}

template<typename Stack>

void resumeBlock(BBQJIT& generator, const ControlData& predecessor, Stack& expressionStack)

{

ASSERT(expressionStack.size() >= predecessor.resultLocations().size());

for (unsigned i = 0; i < predecessor.resultLocations().size(); ++i) {

unsigned offset = expressionStack.size() - predecessor.resultLocations().size();

// Intentionally not using implicitSlots since results should not include implicit slot.

expressionStack[i + offset].value() = Value::fromTemp(expressionStack[i + offset].type().kind, predecessor.enclosedHeight() + i);

generator.bind(expressionStack[i + offset].value(), predecessor.resultLocations()[i]);

}

}

void convertIfToBlock();

void convertLoopToBlock();

void addBranch(Jump jump);

void addLabel(Box<CCallHelpers::Label>&& label);

void delegateJumpsTo(ControlData& delegateTarget);

void linkJumps(MacroAssembler::AbstractMacroAssemblerType* masm);

void linkJumpsTo(MacroAssembler::Label label, MacroAssembler::AbstractMacroAssemblerType* masm);

void linkIfBranch(MacroAssembler::AbstractMacroAssemblerType* masm);

void dump(PrintStream& out) const;

LocalOrTempIndex enclosedHeight() const;

unsigned implicitSlots() const;

const Vector<Location, 2>& targetLocations() const;

const Vector<Location, 2>& argumentLocations() const;

const Vector<Location, 2>& resultLocations() const;

BlockType blockType() const;

BlockSignature signature() const;

FunctionArgCount branchTargetArity() const;

Type branchTargetType(unsigned i) const;

Type argumentType(unsigned i) const;

CatchKind catchKind() const;

void setCatchKind(CatchKind catchKind);

unsigned tryStart() const;

unsigned tryEnd() const;

unsigned tryCatchDepth() const;

void setTryInfo(unsigned tryStart, unsigned tryEnd, unsigned tryCatchDepth);

void setIfBranch(MacroAssembler::Jump branch);

void setLoopLabel(MacroAssembler::Label label);

const MacroAssembler::Label& loopLabel() const;

void touch(LocalOrTempIndex local);

private:

friend class BBQJIT;

void fillLabels(CCallHelpers::Label label);

BlockSignature m_signature;

BlockType m_blockType;

CatchKind m_catchKind { CatchKind::Catch };

Vector<Location, 2> m_argumentLocations; // List of input locations to write values into when entering this block.

Vector<Location, 2> m_resultLocations; // List of result locations to write values into when exiting this block.

JumpList m_branchList; // List of branch control info for branches targeting the end of this block.

Vector<Box<CCallHelpers::Label>> m_labels; // List of labels filled.

MacroAssembler::Label m_loopLabel;

MacroAssembler::Jump m_ifBranch;

LocalOrTempIndex m_enclosedHeight; // Height of enclosed expression stack, used as the base for all temporary locations.

BitVector m_touchedLocals; // Number of locals allocated to registers in this block.

unsigned m_tryStart { 0 };

unsigned m_tryEnd { 0 };

unsigned m_tryCatchDepth { 0 };

};

friend struct ControlData;

using ExpressionType = Value;

using ControlType = ControlData;

using CallType = CallLinkInfo::CallType;

using ResultList = Vector<ExpressionType, 8>;

using ControlEntry = typename FunctionParserTypes<ControlType, ExpressionType, CallType>::ControlEntry;

using TypedExpression = typename FunctionParserTypes<ControlType, ExpressionType, CallType>::TypedExpression;

using Stack = FunctionParser<BBQJIT>::Stack;

using ControlStack = FunctionParser<BBQJIT>::ControlStack;

unsigned stackCheckSize() const { return alignedFrameSize(m_maxCalleeStackSize + m_frameSize); }

unsigned m_loggingIndent = 0;

template<typename... Args>

void logInstructionData(bool first, const Value& value, const Location& location, const Args&... args)

{

if (!first)

dataLog(", ");

dataLog(value);

if (location.kind() != Location::None)

dataLog(":", location);

logInstructionData(false, args...);

}

template<typename... Args>

void logInstructionData(bool first, const Value& value, const Args&... args)

{

if (!first)

dataLog(", ");

dataLog(value);

if (!value.isConst() && !value.isPinned())

dataLog(":", locationOf(value));

logInstructionData(false, args...);

}

template<size_t N, typename OverflowHandler, typename... Args>

void logInstructionData(bool first, const Vector<TypedExpression, N, OverflowHandler>& expressions, const Args&... args)

{

for (const TypedExpression& expression : expressions) {

if (!first)

dataLog(", ");

const Value& value = expression.value();

dataLog(value);

if (!value.isConst() && !value.isPinned())

dataLog(":", locationOf(value));

first = false;

}

logInstructionData(false, args...);

}

template<size_t N, typename OverflowHandler, typename... Args>

void logInstructionData(bool first, const Vector<Value, N, OverflowHandler>& values, const Args&... args)

{

for (const Value& value : values) {

if (!first)

dataLog(", ");

dataLog(value);

if (!value.isConst() && !value.isPinned())

dataLog(":", locationOf(value));

first = false;

}

logInstructionData(false, args...);

}

template<size_t N, typename OverflowHandler, typename... Args>

void logInstructionData(bool first, const Vector<Location, N, OverflowHandler>& values, const Args&... args)

{

for (const Location& value : values) {

if (!first)

dataLog(", ");

dataLog(value);

first = false;

}

logInstructionData(false, args...);

}

inline void logInstructionData(bool)

{

dataLogLn();

}

template<typename... Data>

ALWAYS_INLINE void logInstruction(const char* opcode, SIMDLaneOperation op, const Data&... data)

{

dataLog("BBQ\t");

for (unsigned i = 0; i < m_loggingIndent; i ++)

dataLog(" ");

dataLog(opcode, SIMDLaneOperationDump(op), " ");

logInstructionData(true, data...);

}

template<typename... Data>

ALWAYS_INLINE void logInstruction(const char* opcode, const Data&... data)

{

dataLog("BBQ\t");

for (unsigned i = 0; i < m_loggingIndent; i ++)

dataLog(" ");

dataLog(opcode, " ");

logInstructionData(true, data...);

}

template<typename T>

struct Result {

T value;

Result(const T& value_in)

: value(value_in)

{ }

};

template<typename... Args>

void logInstructionData(bool first, const char* const& literal, const Args&... args)

{

if (!first)

dataLog(" ");

dataLog(literal);

if (!std::strcmp(literal, "=> "))

logInstructionData(true, args...);

else

logInstructionData(false, args...);

}

template<typename T, typename... Args>

void logInstructionData(bool first, const Result<T>& result, const Args&... args)

{

if (!first)

dataLog(" ");

dataLog("=> ");

logInstructionData(true, result.value, args...);

}

template<typename T, typename... Args>

void logInstructionData(bool first, const T& t, const Args&... args)

{

if (!first)

dataLog(", ");

dataLog(t);

logInstructionData(false, args...);

}

#define RESULT(...) Result { __VA_ARGS__ }

#define LOG_INSTRUCTION(...) do { if (UNLIKELY(Options::verboseBBQJITInstructions())) { logInstruction(__VA_ARGS__); } } while (false)

#define LOG_INDENT() do { if (UNLIKELY(Options::verboseBBQJITInstructions())) { m_loggingIndent += 2; } } while (false);

#define LOG_DEDENT() do { if (UNLIKELY(Options::verboseBBQJITInstructions())) { m_loggingIndent -= 2; } } while (false);

static constexpr bool tierSupportsSIMD = true;

BBQJIT(CCallHelpers& jit, const TypeDefinition& signature, BBQCallee& callee, const FunctionData& function, uint32_t functionIndex, const ModuleInformation& info, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, InternalFunction* compilation, std::optional<bool> hasExceptionHandlers, unsigned loopIndexForOSREntry, TierUpCount* tierUp);

ALWAYS_INLINE static Value emptyExpression()

{

return Value::none();

}

void setParser(FunctionParser<BBQJIT>* parser);

bool addArguments(const TypeDefinition& signature);

Value addConstant(Type type, uint64_t value);

PartialResult addDrop(Value value);

PartialResult addLocal(Type type, uint32_t numberOfLocals);

Value instanceValue();

// Tables

PartialResult WARN_UNUSED_RETURN addTableGet(unsigned tableIndex, Value index, Value& result);

PartialResult WARN_UNUSED_RETURN addTableSet(unsigned tableIndex, Value index, Value value);

PartialResult WARN_UNUSED_RETURN addTableInit(unsigned elementIndex, unsigned tableIndex, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

PartialResult WARN_UNUSED_RETURN addElemDrop(unsigned elementIndex);

PartialResult WARN_UNUSED_RETURN addTableSize(unsigned tableIndex, Value& result);

PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned tableIndex, Value fill, Value delta, Value& result);

PartialResult WARN_UNUSED_RETURN addTableFill(unsigned tableIndex, Value offset, Value fill, Value count);

PartialResult WARN_UNUSED_RETURN addTableCopy(unsigned dstTableIndex, unsigned srcTableIndex, Value dstOffset, Value srcOffset, Value length);

// Locals

PartialResult WARN_UNUSED_RETURN getLocal(uint32_t localIndex, Value& result);

PartialResult WARN_UNUSED_RETURN setLocal(uint32_t localIndex, Value value);

// Globals

Value topValue(TypeKind type);

Value exception(const ControlData& control);

PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, Value& result);

void emitWriteBarrier(GPRReg cellGPR);

PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, Value value);

// Memory

inline Location emitCheckAndPreparePointer(Value pointer, uint32_t uoffset, uint32_t sizeOfOperation)

{

ScratchScope<1, 0> scratches(*this);

Location pointerLocation;

if (pointer.isConst()) {

pointerLocation = Location::fromGPR(scratches.gpr(0));

emitMoveConst(pointer, pointerLocation);

} else

pointerLocation = loadIfNecessary(pointer);

ASSERT(pointerLocation.isGPR());

#if USE(JSVALUE32_64)

ScratchScope<2, 0> globals(*this);

GPRReg wasmBaseMemoryPointer = globals.gpr(0);

GPRReg wasmBoundsCheckingSizeRegister = globals.gpr(1);

loadWebAssemblyGlobalState(wasmBaseMemoryPointer, wasmBoundsCheckingSizeRegister);

#endif

uint64_t boundary = static_cast<uint64_t>(sizeOfOperation) + uoffset - 1;

switch (m_mode) {

case MemoryMode::BoundsChecking: {

// We're not using signal handling only when the memory is not shared.

// Regardless of signaling, we must check that no memory access exceeds the current memory size.

m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR);

if (boundary)

m_jit.addPtr(TrustedImmPtr(boundary), wasmScratchGPR);

throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, wasmBoundsCheckingSizeRegister));

break;

}

case MemoryMode::Signaling: {

// We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],

// and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register

// memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above

// the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any

// register + large offset access. We don't think this will be generated frequently.

//

// We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the

// PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater

// than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that

// any access equal to or greater than 4GiB will trap, no need to add the redzone.

if (uoffset >= Memory::fastMappedRedzoneBytes()) {

uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();

m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR);

if (boundary)

m_jit.addPtr(TrustedImmPtr(boundary), wasmScratchGPR);

throwExceptionIf(ExceptionType::OutOfBoundsMemoryAccess, m_jit.branchPtr(RelationalCondition::AboveOrEqual, wasmScratchGPR, TrustedImmPtr(static_cast<int64_t>(maximum))));

}

break;

}

}

#if CPU(ARM64)

m_jit.addZeroExtend64(wasmBaseMemoryPointer, pointerLocation.asGPR(), wasmScratchGPR);

#else

m_jit.zeroExtend32ToWord(pointerLocation.asGPR(), wasmScratchGPR);

m_jit.addPtr(wasmBaseMemoryPointer, wasmScratchGPR);

#endif

consume(pointer);

return Location::fromGPR(wasmScratchGPR);

}

template<typename Functor>

auto emitCheckAndPrepareAndMaterializePointerApply(Value pointer, uint32_t uoffset, uint32_t sizeOfOperation, Functor&& functor) -> decltype(auto);

static inline uint32_t sizeOfLoadOp(LoadOpType op)

{

switch (op) {

case LoadOpType::I32Load8S:

case LoadOpType::I32Load8U:

case LoadOpType::I64Load8S:

case LoadOpType::I64Load8U:

return 1;

case LoadOpType::I32Load16S:

case LoadOpType::I64Load16S:

case LoadOpType::I32Load16U:

case LoadOpType::I64Load16U:

return 2;

case LoadOpType::I32Load:

case LoadOpType::I64Load32S:

case LoadOpType::I64Load32U:

case LoadOpType::F32Load:

return 4;

case LoadOpType::I64Load:

case LoadOpType::F64Load:

return 8;

}

RELEASE_ASSERT_NOT_REACHED();

}

static inline TypeKind typeOfLoadOp(LoadOpType op)

{

switch (op) {