{

JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);

proc->dump(WTF::dataFile());

static constexpr uintptr_t isMaterializedMask = 0x1;

OMGExpression() = default;

OMGExpression(Value* value)

: m_storage(reinterpret_cast<uintptr_t>(value))

{

ASSERT(!isEmpty() && !isMaterialized() && b3Value() == value);

}

bool isEmpty() const

{

return !m_storage;

}

bool isMaterialized() const

{

return m_storage & isMaterializedMask;

}

void setMaterialized(B3::Variable* var)

{

ASSERT(!isMaterialized());

m_storage = reinterpret_cast<uintptr_t>(var) | isMaterializedMask;

ASSERT(isMaterialized() && b3Variable() == var);

}

// Not named value() to disambiguate with TypedExpression::value()

B3::Value* b3Value() const

{

ASSERT(!isMaterialized());

// No need to mask in this common case since the isMaterialized bit will be 0

return reinterpret_cast<B3::Value*>(m_storage);

}

B3::Variable* b3Variable() const

{

ASSERT(isMaterialized());

return reinterpret_cast<B3::Variable*>(m_storage & ~isMaterializedMask);

}

B3::Type type() const

{

if (isMaterialized()) [[unlikely]]

return b3Variable()->type();

return b3Value()->type();

}

void dump(PrintStream& out) const

{

if (isMaterialized())

out.print(*b3Variable());

else

out.print(*b3Value());

}

uintptr_t m_storage;

WTF_MAKE_TZONE_ALLOCATED(OMGIRGenerator);

using ExpressionType = OMGExpression;

using ResultList = Vector<ExpressionType, 8>;

using CallType = CallLinkInfo::CallType;

using CallPatchpointData = std::tuple<B3::PatchpointValue*, RefPtr<PatchpointExceptionHandle>, RefPtr<B3::StackmapGenerator>>;

using WasmConstRefValue = Const64Value;

static constexpr bool shouldFuseBranchCompare = false;

static constexpr bool tierSupportsSIMD() { return true; }

static constexpr bool validateFunctionBodySize = true;

struct ControlData {

ControlData(Procedure& proc, Origin origin, BlockSignature&& signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)

: controlBlockType(type)

, m_signature(WTF::move(signature))

, continuation(continuation)

, special(special)

{

ASSERT(type != BlockType::Try && type != BlockType::Catch);

if (type == BlockType::Loop) {

for (unsigned i = 0; i < m_signature.argumentCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(m_signature.argumentType(i)), origin));

} else {

for (unsigned i = 0; i < m_signature.returnCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(m_signature.returnType(i)), origin));

}

}

ControlData(Procedure& proc, Origin origin, BlockSignature&& signature, BlockType type, BasicBlock* continuation, unsigned tryStart, unsigned tryDepth)

: controlBlockType(type)

, m_signature(WTF::move(signature))

, continuation(continuation)

, special(nullptr)

, m_tryStart(tryStart)

, m_tryCatchDepth(tryDepth)

{

ASSERT(type == BlockType::Try || type == BlockType::TryTable);

for (unsigned i = 0; i < m_signature.returnCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(m_signature.returnType(i)), origin));

}

ControlData()

{

}

static bool isIf(const ControlData& control) { return control.blockType() == BlockType::If; }

static bool isElse(const ControlData& control) { return control.blockType() == BlockType::Else; }

static bool isTry(const ControlData& control) { return control.blockType() == BlockType::Try; }

static bool isTryTable(const ControlData& control) { return control.blockType() == BlockType::TryTable; }

static bool isAnyCatch(const ControlData& control) { return control.blockType() == BlockType::Catch; }

static bool isTopLevel(const ControlData& control) { return control.blockType() == BlockType::TopLevel; }

static bool isLoop(const ControlData& control) { return control.blockType() == BlockType::Loop; }

static bool isBlock(const ControlData& control) { return control.blockType() == BlockType::Block; }

static bool isCatch(const ControlData& control)

{

if (control.blockType() != BlockType::Catch)

return false;

return control.catchKind() == CatchKind::Catch;

}

void dump(PrintStream& out) const

{

switch (blockType()) {

case BlockType::If:

out.print("If: ");

break;

case BlockType::Else:

out.print("Else: ");

break;

case BlockType::Block:

out.print("Block: ");

break;

case BlockType::Loop:

out.print("Loop: ");

break;

case BlockType::TopLevel:

out.print("TopLevel: ");

break;

case BlockType::Try:

out.print("Try: ");

break;

case BlockType::TryTable:

out.print("TryTable: ");

break;

case BlockType::Catch:

out.print("Catch: ");

break;

}

out.print("Continuation: ", *continuation, ", Special: ");

if (special)

out.print(*special);

else

out.print("None");

}

BlockType blockType() const { return controlBlockType; }

const BlockSignature& signature() const { return m_signature; }

bool hasNonVoidresult() const { return m_signature.returnCount() > 0; }

BasicBlock* targetBlockForBranch()

{

if (blockType() == BlockType::Loop)

return special;

return continuation;

}

void convertIfToBlock()

{

ASSERT(blockType() == BlockType::If);

controlBlockType = BlockType::Block;

special = nullptr;

}

void convertTryToCatch(unsigned tryEndCallSiteIndex, Variable* exception)

{

ASSERT(blockType() == BlockType::Try);

controlBlockType = BlockType::Catch;

m_catchKind = CatchKind::Catch;

m_tryEnd = tryEndCallSiteIndex;

m_exception = exception;

}

void convertTryToCatchAll(unsigned tryEndCallSiteIndex, Variable* exception)

{

ASSERT(blockType() == BlockType::Try);

controlBlockType = BlockType::Catch;

m_catchKind = CatchKind::CatchAll;

m_tryEnd = tryEndCallSiteIndex;

m_exception = exception;

}

struct TryTableTarget {

CatchKind type;

uint32_t tag;

const TypeDefinition* exceptionSignature;

ControlRef target;

};

using TargetList = Vector<TryTableTarget>;

void setTryTableTargets(TargetList&& targets)

{

m_tryTableTargets = WTF::move(targets);

}

void endTryTable(unsigned tryEndCallSiteIndex)

{

ASSERT(blockType() == BlockType::TryTable);

m_tryEnd = tryEndCallSiteIndex;

}

FunctionArgCount branchTargetArity() const

{

if (blockType() == BlockType::Loop)

return m_signature.argumentCount();

return m_signature.returnCount();

}

Type branchTargetType(unsigned i) const

{

ASSERT(i < branchTargetArity());

if (blockType() == BlockType::Loop)

return m_signature.argumentType(i);

return m_signature.returnType(i);

}

unsigned tryStart() const

{

ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::TryTable || controlBlockType == BlockType::Catch);

return m_tryStart;

}

unsigned tryEnd() const

{

ASSERT(controlBlockType == BlockType::Catch || controlBlockType == BlockType::TryTable);

return m_tryEnd;

}

unsigned tryDepth() const

{

ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::TryTable || controlBlockType == BlockType::Catch);

return m_tryCatchDepth;

}

CatchKind catchKind() const

{

ASSERT(controlBlockType == BlockType::Catch);

return m_catchKind;

}

Variable* exception() const

{

ASSERT(controlBlockType == BlockType::Catch || controlBlockType == BlockType::TryTable);

return m_exception;

}

private:

// FIXME: Compress OMGIRGenerator::ControlData fields using an union

// https://bugs.webkit.org/show_bug.cgi?id=231212

friend class OMGIRGenerator;

BlockType controlBlockType;

BlockSignature m_signature;

BasicBlock* continuation;

BasicBlock* special;

Vector<Value*> phis;

unsigned m_tryStart;

unsigned m_tryEnd;

unsigned m_tryCatchDepth;

CatchKind m_catchKind;

Variable* m_exception;

TargetList m_tryTableTargets;

};

using ControlType = ControlData;

using ExpressionList = Vector<ExpressionType, 1>;

using ControlEntry = FunctionParser<OMGIRGenerator>::ControlEntry;

using ControlStack = FunctionParser<OMGIRGenerator>::ControlStack;

using Stack = FunctionParser<OMGIRGenerator>::Stack;

using TypedExpression = FunctionParser<OMGIRGenerator>::TypedExpression;

using CatchHandler = FunctionParser<OMGIRGenerator>::CatchHandler;

using ArgumentList = FunctionParser<OMGIRGenerator>::ArgumentList;

static_assert(std::is_same_v<ResultList, FunctionParser<OMGIRGenerator>::ResultList>);

typedef String ErrorType;

typedef Unexpected<ErrorType> UnexpectedResult;

typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;

typedef Expected<void, ErrorType> PartialResult;

static ExpressionType emptyExpression() { return { }; };

enum class CastKind { Cast, Test };

template <typename ...Args>

[[nodiscard]] NEVER_INLINE UnexpectedResult fail(Args... args) const

{

using namespace FailureHelper; // See ADL comment in WasmParser.h.

return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));

}

#define WASM_COMPILE_FAIL_IF(condition, ...) do { \

if (condition) [[unlikely]] \

return fail(__VA_ARGS__); \

} while (0)

unsigned advanceCallSiteIndex()

{

if (m_inlineParent)

return m_inlineRoot->advanceCallSiteIndex();

return ++m_callSiteIndex;

}

unsigned callSiteIndex() const

{

if (m_inlineParent)

return m_inlineRoot->callSiteIndex();

return m_callSiteIndex;

}

OMGIRGenerator(AbstractHeapRepository&, CompilationContext&, Module&, CalleeGroup&, const ModuleInformation&, IPIntCallee&, InliningNode*, OptimizingJITCallee&, Procedure&, Vector<UnlinkedWasmToWasmCall>&, FixedBitVector& outgoingDirectCallees, unsigned& osrEntryScratchBufferSize, MemoryMode, CompilationMode, unsigned functionIndex, unsigned loopIndexForOSREntry);

OMGIRGenerator(AbstractHeapRepository&, CompilationContext&, OMGIRGenerator& inlineCaller, OMGIRGenerator& inlineRoot, Module&, CalleeGroup&, unsigned functionIndex, IPIntCallee&, InliningNode*, BasicBlock* returnContinuation, Vector<Value*> args);

void computeStackCheckSize(bool& needsOverflowCheck, int32_t& checkSize);

Value* wasmRefOfCell(Value* cell)

{

return cell;

}

Value* pointerOfWasmRef(Value* ref)

{

return ref;

}

Value* pointerOfInt32(Value* value)

{

return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);

}

Value* int32OfPointer(Value* value)

{

return m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);

}

// SIMD

bool usesSIMD() { return m_info.usesSIMD(m_functionIndex); }

void notifyFunctionUsesSIMD() { ASSERT(m_info.usesSIMD(m_functionIndex)); }

[[nodiscard]] PartialResult addSIMDLoad(ExpressionType pointer, uint32_t offset, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDStore(ExpressionType value, ExpressionType pointer, uint32_t offset);

[[nodiscard]] PartialResult addSIMDSplat(SIMDLane, ExpressionType scalar, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDShuffle(v128_t imm, ExpressionType a, ExpressionType b, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDShift(SIMDLaneOperation, SIMDInfo, ExpressionType v, ExpressionType shift, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDExtmul(SIMDLaneOperation, SIMDInfo, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDLoadSplat(SIMDLaneOperation, ExpressionType pointer, uint32_t offset, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDLoadLane(SIMDLaneOperation, ExpressionType pointer, ExpressionType vector, uint32_t offset, uint8_t laneIndex, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDStoreLane(SIMDLaneOperation, ExpressionType pointer, ExpressionType vector, uint32_t offset, uint8_t laneIndex);

[[nodiscard]] PartialResult addSIMDLoadExtend(SIMDLaneOperation, ExpressionType pointer, uint32_t offset, ExpressionType& result);

[[nodiscard]] PartialResult addSIMDLoadPad(SIMDLaneOperation, ExpressionType pointer, uint32_t offset, ExpressionType& result);

[[nodiscard]] ExpressionType addSIMDConstant(v128_t value)

{

return push(constant(B3::V128, value));

}

// SIMD generated

#define B3_OP_CASE(OP) \

else if (op == SIMDLaneOperation::OP) b3Op = B3::Vector##OP;

#define B3_OP_CASES() \

B3::Opcode b3Op = B3::Oops; \

if (false) { }

auto addSIMDExtractLane(SIMDInfo info, uint8_t lane, ExpressionType v, ExpressionType& result) -> PartialResult

{

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), B3::VectorExtractLane, toB3Type(simdScalarType(info.lane)), info,

lane,

get(v)));

return { };

}

auto addSIMDReplaceLane(SIMDInfo info, uint8_t lane, ExpressionType v, ExpressionType s, ExpressionType& result) -> PartialResult

{

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), B3::VectorReplaceLane, B3::V128, info,

lane,

get(v),

get(s)));

return { };

}

auto addSIMDI_V(SIMDLaneOperation op, SIMDInfo info, ExpressionType v, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(Bitmask)

B3_OP_CASE(AnyTrue)

B3_OP_CASE(AllTrue)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::Int32, info,

get(v)));

return { };

}

auto addSIMDV_V(SIMDLaneOperation op, SIMDInfo info, ExpressionType v, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(Demote)

B3_OP_CASE(Promote)

B3_OP_CASE(Abs)

B3_OP_CASE(Popcnt)

B3_OP_CASE(Ceil)

B3_OP_CASE(Floor)

B3_OP_CASE(Trunc)

B3_OP_CASE(Nearest)

B3_OP_CASE(Sqrt)

B3_OP_CASE(ExtaddPairwise)

B3_OP_CASE(Convert)

B3_OP_CASE(ConvertLow)

B3_OP_CASE(ExtendHigh)

B3_OP_CASE(ExtendLow)

B3_OP_CASE(TruncSat)

B3_OP_CASE(RelaxedTruncSat)

B3_OP_CASE(Not)

B3_OP_CASE(Neg)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(v)));

return { };

}

auto addSIMDBitwiseSelect(ExpressionType v1, ExpressionType v2, ExpressionType c, ExpressionType& result) -> PartialResult

{

auto b3Op = B3::VectorBitwiseSelect;

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, SIMDInfo { SIMDLane::v128, SIMDSignMode::None },

get(v1), get(v2), get(c)));

return { };

}

auto addSIMDRelOp(SIMDLaneOperation, SIMDInfo info, ExpressionType lhs, ExpressionType rhs, Air::Arg relOp, ExpressionType& result) -> PartialResult

{

B3::Opcode b3Op = Oops;

if (scalarTypeIsIntegral(info.lane)) {

switch (relOp.asRelationalCondition()) {

case MacroAssembler::Equal:

b3Op = VectorEqual;

break;

case MacroAssembler::NotEqual:

b3Op = VectorNotEqual;

break;

case MacroAssembler::LessThan:

b3Op = VectorLessThan;

break;

case MacroAssembler::LessThanOrEqual:

b3Op = VectorLessThanOrEqual;

break;

case MacroAssembler::Below:

b3Op = VectorBelow;

break;

case MacroAssembler::BelowOrEqual:

b3Op = VectorBelowOrEqual;

break;

case MacroAssembler::GreaterThan:

b3Op = VectorGreaterThan;

break;

case MacroAssembler::GreaterThanOrEqual:

b3Op = VectorGreaterThanOrEqual;

break;

case MacroAssembler::Above:

b3Op = VectorAbove;

break;

case MacroAssembler::AboveOrEqual:

b3Op = VectorAboveOrEqual;

break;

}

} else {

switch (relOp.asDoubleCondition()) {

case MacroAssembler::DoubleEqualAndOrdered:

b3Op = VectorEqual;

break;

case MacroAssembler::DoubleNotEqualOrUnordered:

b3Op = VectorNotEqual;

break;

case MacroAssembler::DoubleLessThanAndOrdered:

b3Op = VectorLessThan;

break;

case MacroAssembler::DoubleLessThanOrEqualAndOrdered:

b3Op = VectorLessThanOrEqual;

break;

case MacroAssembler::DoubleGreaterThanAndOrdered:

b3Op = VectorGreaterThan;

break;

case MacroAssembler::DoubleGreaterThanOrEqualAndOrdered:

b3Op = VectorGreaterThanOrEqual;

break;

default:

RELEASE_ASSERT_NOT_REACHED();

break;

}

}

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(lhs), get(rhs)));

return { };

}

Value* fixupOutOfBoundsIndicesForSwizzle(Value* input, Value* indexes)

{

// The intel version of the swizzle instruction does not handle OOB indices properly,

// so we need to fix them up.

ASSERT(isX86());

// Let each byte mask be 112 (0x70) then after VectorAddSat

// each index > 15 would set the saturated index's bit 7 to 1,

// whose corresponding byte will be zero cleared in VectorSwizzle.

// https://github.com/WebAssembly/simd/issues/93

v128_t mask;

mask.u64x2[0] = 0x7070707070707070;

mask.u64x2[1] = 0x7070707070707070;

auto saturatingMask = m_currentBlock->appendNew<Const128Value>(m_proc, origin(), mask);

auto saturatedIndexes = m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), VectorAddSat, B3::V128, SIMDLane::i8x16, SIMDSignMode::Unsigned, saturatingMask, indexes);

return m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), VectorSwizzle, B3::V128, SIMDLane::i8x16, SIMDSignMode::None, input, saturatedIndexes);

}

auto addSIMDV_VV(SIMDLaneOperation op, SIMDInfo info, ExpressionType a, ExpressionType b, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(And)

B3_OP_CASE(Andnot)

B3_OP_CASE(AvgRound)

B3_OP_CASE(DotProduct)

B3_OP_CASE(Add)

B3_OP_CASE(Mul)

B3_OP_CASE(MulSat)

B3_OP_CASE(Sub)

B3_OP_CASE(Div)

B3_OP_CASE(Pmax)

B3_OP_CASE(Pmin)

B3_OP_CASE(Or)

B3_OP_CASE(Swizzle)

B3_OP_CASE(RelaxedSwizzle)

B3_OP_CASE(Xor)

B3_OP_CASE(Narrow)

B3_OP_CASE(AddSat)

B3_OP_CASE(SubSat)

B3_OP_CASE(Max)

B3_OP_CASE(Min)

if (isX86() && b3Op == B3::VectorSwizzle) {

result = push(fixupOutOfBoundsIndicesForSwizzle(get(a), get(b)));

return { };

}

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(a), get(b)));

return { };

}

auto addSIMDRelaxedFMA(SIMDLaneOperation op, SIMDInfo info, ExpressionType m1, ExpressionType m2, ExpressionType add, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(RelaxedMAdd)

B3_OP_CASE(RelaxedNMAdd)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(m1), get(m2), get(add)));

return { };

}

[[nodiscard]] PartialResult addDrop(ExpressionType);

[[nodiscard]] PartialResult addInlinedArguments(const TypeDefinition&);

[[nodiscard]] PartialResult addArguments(const TypeDefinition&);

[[nodiscard]] PartialResult addLocal(Type, uint32_t);

ExpressionType addConstant(Type, uint64_t);

// References

[[nodiscard]] PartialResult addRefIsNull(ExpressionType value, ExpressionType& result);

[[nodiscard]] PartialResult addRefFunc(FunctionSpaceIndex index, ExpressionType& result);

[[nodiscard]] PartialResult addRefAsNonNull(TypedExpression, ExpressionType&);

[[nodiscard]] PartialResult addRefEq(ExpressionType, ExpressionType, ExpressionType&);

// Tables

[[nodiscard]] PartialResult addTableGet(unsigned, ExpressionType index, ExpressionType& result);

[[nodiscard]] PartialResult addTableSet(unsigned, ExpressionType index, ExpressionType value);

[[nodiscard]] PartialResult addTableInit(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

[[nodiscard]] PartialResult addElemDrop(unsigned);

[[nodiscard]] PartialResult addTableSize(unsigned, ExpressionType& result);

[[nodiscard]] PartialResult addTableGrow(unsigned, ExpressionType fill, ExpressionType delta, ExpressionType& result);

[[nodiscard]] PartialResult addTableFill(unsigned, ExpressionType offset, ExpressionType fill, ExpressionType count);

[[nodiscard]] PartialResult addTableCopy(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

// Locals

[[nodiscard]] PartialResult getLocal(uint32_t index, ExpressionType& result);

[[nodiscard]] PartialResult setLocal(uint32_t index, ExpressionType value);

[[nodiscard]] PartialResult teeLocal(uint32_t, ExpressionType, ExpressionType& result);

// Globals

[[nodiscard]] PartialResult getGlobal(uint32_t index, ExpressionType& result);

[[nodiscard]] PartialResult setGlobal(uint32_t index, ExpressionType value);

// Memory

[[nodiscard]] PartialResult load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset, uint8_t memoryIndex);

[[nodiscard]] PartialResult addGrowMemory(ExpressionType delta, ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] PartialResult addCurrentMemory(ExpressionType& result, uint8_t memoryIndex);

[[nodiscard]] PartialResult addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count, uint8_t memoryIndex);

[[nodiscard]] PartialResult addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count, uint8_t dstMemoryIndex, uint8_t srcMemoryIndex);

[[nodiscard]] PartialResult addMemoryInit(unsigned, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length, uint8_t memoryIndex);

[[nodiscard]] PartialResult addDataDrop(unsigned);

// Atomics

[[nodiscard]] PartialResult atomicLoad(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType& result, uint32_t offset);

[[nodiscard]] PartialResult atomicStore(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);

[[nodiscard]] PartialResult atomicBinaryRMW(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);

[[nodiscard]] PartialResult atomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint32_t offset);

[[nodiscard]] PartialResult atomicWait(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType timeout, ExpressionType& result, uint32_t offset);

[[nodiscard]] PartialResult atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);

[[nodiscard]] PartialResult atomicFence(ExtAtomicOpType, uint8_t flags);

// Saturated truncation.

[[nodiscard]] PartialResult truncSaturated(Ext1OpType, ExpressionType operand, ExpressionType& result, Type returnType, Type operandType);

// GC

[[nodiscard]] PartialResult addRefI31(ExpressionType value, ExpressionType& result);

[[nodiscard]] PartialResult addI31GetS(TypedExpression ref, ExpressionType& result);

[[nodiscard]] PartialResult addI31GetU(TypedExpression ref, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNew(uint32_t index, ExpressionType size, ExpressionType value, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNewDefault(uint32_t index, ExpressionType size, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNewFixed(uint32_t typeIndex, ArgumentList& args, ExpressionType& result);

[[nodiscard]] PartialResult addArrayGet(ExtGCOpType arrayGetKind, uint32_t typeIndex, TypedExpression arrayref, ExpressionType index, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNewData(uint32_t typeIndex, uint32_t dataIndex, ExpressionType size, ExpressionType offset, ExpressionType& result);

[[nodiscard]] PartialResult addArrayNewElem(uint32_t typeIndex, uint32_t elemSegmentIndex, ExpressionType size, ExpressionType offset, ExpressionType& result);

[[nodiscard]] PartialResult addArraySet(uint32_t typeIndex, TypedExpression arrayref, ExpressionType index, ExpressionType value);

[[nodiscard]] PartialResult addArrayLen(TypedExpression arrayref, ExpressionType& result);

[[nodiscard]] PartialResult addArrayFill(uint32_t, TypedExpression, ExpressionType, ExpressionType, ExpressionType);

[[nodiscard]] PartialResult addArrayCopy(uint32_t, TypedExpression, ExpressionType, uint32_t, TypedExpression, ExpressionType, ExpressionType);

[[nodiscard]] PartialResult addArrayInitElem(uint32_t, TypedExpression, ExpressionType, uint32_t, ExpressionType, ExpressionType);

[[nodiscard]] PartialResult addArrayInitData(uint32_t, TypedExpression, ExpressionType, uint32_t, ExpressionType, ExpressionType);

[[nodiscard]] PartialResult addStructNew(uint32_t typeIndex, ArgumentList& args, ExpressionType& result);

[[nodiscard]] PartialResult addStructNewDefault(uint32_t index, ExpressionType& result);

[[nodiscard]] PartialResult addStructGet(ExtGCOpType structGetKind, TypedExpression structReference, const StructType&, const RTT&, uint32_t fieldIndex, ExpressionType& result);

[[nodiscard]] PartialResult addStructSet(TypedExpression structReference, const StructType&, const RTT&, uint32_t fieldIndex, ExpressionType value);

[[nodiscard]] PartialResult addRefTest(TypedExpression reference, bool allowNull, int32_t heapType, bool shouldNegate, ExpressionType& result);

[[nodiscard]] PartialResult addRefCast(TypedExpression reference, bool allowNull, int32_t heapType, ExpressionType& result);

[[nodiscard]] PartialResult addAnyConvertExtern(ExpressionType reference, ExpressionType& result);

[[nodiscard]] PartialResult addExternConvertAny(ExpressionType reference, ExpressionType& result);

// Basic operators

#define X(name, opcode, short, idx, ...) \

[[nodiscard]] PartialResult add##name(ExpressionType arg, ExpressionType& result);

FOR_EACH_WASM_UNARY_OP(X)

#undef X

#define X(name, opcode, short, idx, ...) \

[[nodiscard]] PartialResult add##name(ExpressionType left, ExpressionType right, ExpressionType& result);

FOR_EACH_WASM_BINARY_OP(X)

#undef X

[[nodiscard]] PartialResult addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);

// Control flow

[[nodiscard]] ControlData addTopLevel(BlockSignature&&);

[[nodiscard]] PartialResult addBlock(BlockSignature&&, Stack& enclosingStack, ControlType& newBlock, Stack& newStack);

[[nodiscard]] PartialResult addLoop(BlockSignature&&, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex);

[[nodiscard]] PartialResult addIf(ExpressionType condition, BlockSignature&&, Stack& enclosingStack, ControlType& result, Stack& newStack);

[[nodiscard]] PartialResult addElse(ControlData&, const Stack&);

[[nodiscard]] PartialResult addElseToUnreachable(ControlData&);

[[nodiscard]] PartialResult addTry(BlockSignature&&, Stack& enclosingStack, ControlType& result, Stack& newStack);

[[nodiscard]] PartialResult addTryTable(BlockSignature&&, Stack& enclosingStack, const Vector<CatchHandler>& targets, ControlType& result, Stack& newStack);

[[nodiscard]] PartialResult addCatch(unsigned exceptionIndex, const TypeDefinition&, Stack&, ControlType&, ResultList&);

[[nodiscard]] PartialResult addCatchToUnreachable(unsigned exceptionIndex, const TypeDefinition&, ControlType&, ResultList&);

[[nodiscard]] PartialResult addCatchAll(Stack&, ControlType&);

[[nodiscard]] PartialResult addCatchAllToUnreachable(ControlType&);

[[nodiscard]] PartialResult addDelegate(ControlType&, ControlType&);

[[nodiscard]] PartialResult addDelegateToUnreachable(ControlType&, ControlType&);

[[nodiscard]] PartialResult addThrow(unsigned exceptionIndex, ArgumentList& args, Stack&);

[[nodiscard]] PartialResult addRethrow(unsigned, ControlType&);

[[nodiscard]] PartialResult addThrowRef(TypedExpression exception, Stack&);

[[nodiscard]] PartialResult addInlinedReturn(const auto& returnValues);

[[nodiscard]] PartialResult addReturn(const ControlData&, const Stack& returnValues);

[[nodiscard]] PartialResult addBranch(ControlData&, ExpressionType condition, const Stack& returnValues);

[[nodiscard]] PartialResult addBranchNull(ControlType&, ExpressionType, const Stack&, bool, ExpressionType&);

[[nodiscard]] PartialResult addBranchCast(ControlType&, TypedExpression, const Stack&, bool, int32_t, bool);

[[nodiscard]] PartialResult addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const Stack& expressionStack);

[[nodiscard]] PartialResult endBlock(ControlEntry&, Stack& expressionStack);

[[nodiscard]] PartialResult addEndToUnreachable(ControlEntry&, const Stack& = { });

[[nodiscard]] PartialResult endTopLevel(const Stack&) { return { }; }

// Fused comparison stubs (B3 will do this for us later).

[[nodiscard]] PartialResult addFusedBranchCompare(OpType, ControlType&, ExpressionType, const Stack&) { RELEASE_ASSERT_NOT_REACHED(); }

[[nodiscard]] PartialResult addFusedBranchCompare(OpType, ControlType&, ExpressionType, ExpressionType, const Stack&) { RELEASE_ASSERT_NOT_REACHED(); }

[[nodiscard]] PartialResult addFusedIfCompare(OpType, ExpressionType, BlockSignature&&, Stack&, ControlType&, Stack&) { RELEASE_ASSERT_NOT_REACHED(); }

[[nodiscard]] PartialResult addFusedIfCompare(OpType, ExpressionType, ExpressionType, BlockSignature&&, Stack&, ControlType&, Stack&) { RELEASE_ASSERT_NOT_REACHED(); }

// Calls

[[nodiscard]] PartialResult addCall(unsigned, FunctionSpaceIndex functionIndexSpace, const TypeDefinition&, ArgumentList& args, ResultList& results, CallType = CallType::Call);

[[nodiscard]] PartialResult addCallIndirect(unsigned, unsigned tableIndex, const TypeDefinition&, ArgumentList& args, ResultList& results, CallType = CallType::Call);

[[nodiscard]] PartialResult addCallRef(unsigned, const TypeDefinition&, ArgumentList& args, ResultList& results, CallType = CallType::Call);

[[nodiscard]] PartialResult addUnreachable();

[[nodiscard]] PartialResult addCrash();

using ValueResults = Vector<Value*, 16>;

void fillCallResults(Value* callResult, const TypeDefinition& signature, ValueResults&);

[[nodiscard]] PartialResult emitDirectCall(unsigned, FunctionSpaceIndex functionIndexSpace, const TypeDefinition&, const ArgumentList& args, ValueResults&, CallType = CallType::Call);

[[nodiscard]] PartialResult emitIndirectCall(Value* calleeInstance, Value* calleeCode, Value* boxedCalleeCallee, const TypeDefinition&, const ArgumentList& args, ValueResults&, CallType = CallType::Call);

Vector<ConstrainedValue> createCallConstrainedArgs(BasicBlock*, const CallInformation& wasmCalleeInfo, const ArgumentList&);

auto createCallPatchpoint(BasicBlock*, const TypeDefinition&, const CallInformation&, const ArgumentList& tmpArgs) -> CallPatchpointData;

auto createTailCallPatchpoint(BasicBlock*, const TypeDefinition&, const CallInformation& wasmCallerInfoAsCallee, const CallInformation& wasmCalleeInfoAsCallee, const ArgumentList& tmpArgSourceLocations, Vector<B3::ConstrainedValue> patchArgs) -> CallPatchpointData;

InliningNode* canInline(FunctionSpaceIndex functionIndexSpace, unsigned callProfileIndex) const;

[[nodiscard]] PartialResult emitInlineDirectCall(InliningNode*, FunctionCodeIndex calleeIndex, const TypeDefinition&, const ArgumentList& args, ValueResults&);

void dump(const ControlStack&, const Stack* expressionStack);

void setParser(FunctionParser<OMGIRGenerator>* parser) { m_parser = parser; };

ALWAYS_INLINE void willParseOpcode() { }

ALWAYS_INLINE void willParseExtendedOpcode() { }

ALWAYS_INLINE void didParseOpcode() { }

void didFinishParsingLocals() { }

void didPopValueFromStack(ExpressionType expr, ASCIILiteral message)

{

TRACE_VALUE(Wasm::Types::Void, get(expr), "pop at height: ", m_parser->expressionStack().size() + 1, " site: [", message, "], ", expr);

}

const Ref<TypeDefinition> getTypeDefinition(uint32_t typeIndex) { return m_info.typeSignatures[typeIndex]; }

const ArrayType* getArrayTypeDefinition(uint32_t);

void getArrayElementType(uint32_t, StorageType&);

void getArrayRefType(uint32_t, Type&);

Value* constant(B3::Type, uint64_t bits, std::optional<Origin> = std::nullopt);

Value* constant(B3::Type, v128_t bits, std::optional<Origin> = std::nullopt);

Value* framePointer();

void insertEntrySwitch();

void insertConstants();

B3::Type toB3ResultType(const TypeDefinition*);

void addStackMap(unsigned callSiteIndex, StackMap&& stackmap)

{

if (m_inlineParent) {

m_inlineRoot->addStackMap(callSiteIndex, WTF::move(stackmap));

return;

}

m_stackmaps.add(CallSiteIndex(callSiteIndex), WTF::move(stackmap));

}

StackMaps&& takeStackmaps()

{

RELEASE_ASSERT(m_inlineRoot == this);

return WTF::move(m_stackmaps);

}

Vector<UnlinkedHandlerInfo>&& takeExceptionHandlers()

{

RELEASE_ASSERT(m_inlineRoot == this);

return WTF::move(m_exceptionHandlers);

}

void emitPrepareWasmOperation(BasicBlock* block)

{

#if !USE(BUILTIN_FRAME_ADDRESS) || ASSERT_ENABLED

// Prepare wasm operation calls.

block->appendNew<B3::MemoryValue>(m_proc, B3::Store, origin(), framePointer(), instanceValue(), safeCast<int32_t>(JSWebAssemblyInstance::offsetOfTemporaryCallFrame()));

#else

UNUSED_PARAM(block);

#endif

}

template<typename OperationType, typename ...Args>

Value* callWasmOperation(BasicBlock* block, B3::Type resultType, OperationType operation, Args&&... args)

{

emitPrepareWasmOperation(block);

static_assert(FunctionTraits<OperationType>::cCallArity() == sizeof...(Args), "Sanity check");

Value* operationValue = block->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operation));

return block->appendNew<CCallValue>(m_proc, resultType, origin(), operationValue, std::forward<Args>(args)...);

}

void emitExceptionCheck(CCallHelpers&, Origin, ExceptionType);

void emitWriteBarrierForJSWrapper();

void emitWriteBarrier(Value* cell);

Value* emitCheckAndPreparePointer(Value* pointer, uint32_t offset, uint32_t sizeOfOp, uint8_t memoryIndex);

B3::Kind memoryKind(B3::Opcode memoryOp);

Value* emitLoadOp(LoadOpType, Value* pointer, uint32_t offset);

void emitStoreOp(StoreOpType, Value* pointer, Value*, uint32_t offset);

Value* sanitizeAtomicResult(ExtAtomicOpType, Type, Value* result);

Value* emitAtomicLoadOp(ExtAtomicOpType, Type, Value* pointer, uint32_t offset);

void emitAtomicStoreOp(ExtAtomicOpType, Type, Value* pointer, Value*, uint32_t offset);

Value* emitAtomicBinaryRMWOp(ExtAtomicOpType, Type, Value* pointer, Value*, uint32_t offset);

Value* emitAtomicCompareExchange(ExtAtomicOpType, Type, Value* pointer, Value* expected, Value*, uint32_t offset);

Value* encodeStructureID(Value* structure);

Value* allocatorForWasmGCHeapCellSize(Value* size, BasicBlock* slowPath);

Value* allocateWasmGCHeapCell(Value* allocator, BasicBlock* slowPath);

Value* allocateWasmGCObject(Value* allocator, Value* structureID, Value* typeInfo, BasicBlock* slowPath);

Value* allocateWasmGCArrayUninitialized(uint32_t typeIndex, Value* size);

void mutatorFence();

Value* emitGetArrayPayloadBase(Wasm::StorageType, Value*);

Value* emitGetArraySizeWithNullCheck(Type arrayType, Value*);

void emitNullCheck(Value*, ExceptionType);

bool emitNullCheckBeforeAccess(Value*, ptrdiff_t offset);

void emitArraySetUnchecked(uint32_t, Value*, Value*, Value*);

bool emitArraySetUncheckedWithoutWriteBarrier(uint32_t, Value*, Value*, Value*);

// Returns true if a writeBarrier/mutatorFence is needed.

[[nodiscard]] bool emitStructSet(bool canTrap, Value*, uint32_t, const StructType&, const RTT&, Value*);

[[nodiscard]] Value* allocateWasmGCArray(uint32_t typeIndex, Value* initValue, Value* size);

using ArraySegmentOperation = EncodedJSValue SYSV_ABI (&)(JSC::JSWebAssemblyInstance*, uint32_t, uint32_t, uint32_t, uint32_t);

[[nodiscard]] ExpressionType pushArrayNewFromSegment(ArraySegmentOperation, uint32_t typeIndex, uint32_t segmentIndex, ExpressionType arraySize, ExpressionType offset, ExceptionType);

void emitRefTestOrCast(CastKind, TypedExpression, bool, int32_t, bool, ExpressionType&);

const B3::AbstractHeap* structFieldHeap(const RTT& rtt, uint32_t fieldIndex)

{

return &m_heaps.JSWebAssemblyStruct_fields[rtt.fieldHeapKey(fieldIndex)];

}

void unify(Value* phi, const ExpressionType& source);

void unifyValuesWithBlock(const Stack& resultStack, const ControlData& block);

void emitChecksForModOrDiv(B3::Opcode, Value* left, Value* right);

[[nodiscard]] int32_t fixupPointerPlusOffset(Value*&, uint32_t);

[[nodiscard]] Value* fixupPointerPlusOffsetForAtomicOps(ExtAtomicOpType, Value*, uint32_t);

void restoreWasmContextInstance(BasicBlock*, Value*);

void restoreWebAssemblyGlobalState(const Vector<MemoryInformation>&, Value* instance, BasicBlock*);

void reloadMemoryRegistersFromInstance(const MemoryInformation&, Value* instance, BasicBlock*);

void materializeExpressionStackIntoVariables();

Value* loadFromScratchBuffer(unsigned& indexInBuffer, Value* pointer, B3::Type);

void connectValuesAtEntrypoint(unsigned& indexInBuffer, Value* pointer, Stack& expressionStack);

Value* emitCatchImpl(CatchKind, ControlType&, unsigned exceptionIndex = 0);

void emitCatchTableImpl(ControlData& entryData, const ControlData::TryTableTarget&);

RefPtr<PatchpointExceptionHandle> preparePatchpointForExceptions(BasicBlock*, PatchpointValue*);

void connectValuesForCatchEntrypoint(ControlData& catchData, Value* pointer);

Origin origin();

ExpressionType push(Value* value)

{

if constexpr (WasmOMGIRGeneratorInternal::traceExecution) {

String site;

#if ASSERT_ENABLED

if constexpr (WasmOMGIRGeneratorInternal::traceExecutionIncludesConstructionSite)

site = Value::generateCompilerConstructionSite();

#endif

TRACE_VALUE(Wasm::Types::Void, value, "push to stack height ", m_parser->expressionStack().size(), " site: [", site, "] ", value);

}

return value;

}

Value* get(BasicBlock* block, Variable* variable)

{

return block->appendNew<VariableValue>(m_proc, B3::Get, origin(), variable);

}

Value* get(Variable* variable)

{

return get(m_currentBlock, variable);

}

Value* get(BasicBlock* block, const OMGExpression& expr)

{