{

JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);

proc->dump(WTF::dataFile());

WTF_MAKE_TZONE_ALLOCATED(OMGIRGenerator);

using ExpressionType = Variable*;

using ResultList = Vector<ExpressionType, 8>;

using CallType = CallLinkInfo::CallType;

static constexpr bool tierSupportsSIMD = true;

struct ControlData {

ControlData(Procedure& proc, Origin origin, BlockSignature signature, BlockType type, unsigned stackSize, BasicBlock* continuation, BasicBlock* special = nullptr)

: controlBlockType(type)

, m_signature(signature)

, m_stackSize(stackSize)

, continuation(continuation)

, special(special)

{

ASSERT(type != BlockType::Try && type != BlockType::Catch);

if (type != BlockType::TopLevel)

m_stackSize -= signature->argumentCount();

if (type == BlockType::Loop) {

for (unsigned i = 0; i < signature->argumentCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(signature->argumentType(i)), origin));

} else {

for (unsigned i = 0; i < signature->returnCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(signature->returnType(i)), origin));

}

}

ControlData(Procedure& proc, Origin origin, BlockSignature signature, BlockType type, unsigned stackSize, BasicBlock* continuation, unsigned tryStart, unsigned tryDepth)

: controlBlockType(type)

, m_signature(signature)

, m_stackSize(stackSize)

, continuation(continuation)

, special(nullptr)

, m_tryStart(tryStart)

, m_tryCatchDepth(tryDepth)

{

ASSERT(type == BlockType::Try);

m_stackSize -= signature->argumentCount();

for (unsigned i = 0; i < signature->returnCount(); ++i)

phis.append(proc.add<Value>(Phi, toB3Type(signature->returnType(i)), origin));

}

ControlData()

{

}

static bool isIf(const ControlData& control) { return control.blockType() == BlockType::If; }

static bool isTry(const ControlData& control) { return control.blockType() == BlockType::Try; }

static bool isAnyCatch(const ControlData& control) { return control.blockType() == BlockType::Catch; }

static bool isTopLevel(const ControlData& control) { return control.blockType() == BlockType::TopLevel; }

static bool isLoop(const ControlData& control) { return control.blockType() == BlockType::Loop; }

static bool isBlock(const ControlData& control) { return control.blockType() == BlockType::Block; }

static bool isCatch(const ControlData& control)

{

if (control.blockType() != BlockType::Catch)

return false;

return control.catchKind() == CatchKind::Catch;

}

void dump(PrintStream& out) const

{

switch (blockType()) {

case BlockType::If:

out.print("If: ");

break;

case BlockType::Block:

out.print("Block: ");

break;

case BlockType::Loop:

out.print("Loop: ");

break;

case BlockType::TopLevel:

out.print("TopLevel: ");

break;

case BlockType::Try:

out.print("Try: ");

break;

case BlockType::Catch:

out.print("Catch: ");

break;

}

out.print("Continuation: ", *continuation, ", Special: ");

if (special)

out.print(*special);

else

out.print("None");

}

BlockType blockType() const { return controlBlockType; }

BlockSignature signature() const { return m_signature; }

bool hasNonVoidresult() const { return m_signature->returnsVoid(); }

BasicBlock* targetBlockForBranch()

{

if (blockType() == BlockType::Loop)

return special;

return continuation;

}

void convertIfToBlock()

{

ASSERT(blockType() == BlockType::If);

controlBlockType = BlockType::Block;

special = nullptr;

}

void convertTryToCatch(unsigned tryEndCallSiteIndex, Variable* exception)

{

ASSERT(blockType() == BlockType::Try);

controlBlockType = BlockType::Catch;

m_catchKind = CatchKind::Catch;

m_tryEnd = tryEndCallSiteIndex;

m_exception = exception;

}

void convertTryToCatchAll(unsigned tryEndCallSiteIndex, Variable* exception)

{

ASSERT(blockType() == BlockType::Try);

controlBlockType = BlockType::Catch;

m_catchKind = CatchKind::CatchAll;

m_tryEnd = tryEndCallSiteIndex;

m_exception = exception;

}

FunctionArgCount branchTargetArity() const

{

if (blockType() == BlockType::Loop)

return m_signature->argumentCount();

return m_signature->returnCount();

}

Type branchTargetType(unsigned i) const

{

ASSERT(i < branchTargetArity());

if (blockType() == BlockType::Loop)

return m_signature->argumentType(i);

return m_signature->returnType(i);

}

unsigned tryStart() const

{

ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::Catch);

return m_tryStart;

}

unsigned tryEnd() const

{

ASSERT(controlBlockType == BlockType::Catch);

return m_tryEnd;

}

unsigned tryDepth() const

{

ASSERT(controlBlockType == BlockType::Try || controlBlockType == BlockType::Catch);

return m_tryCatchDepth;

}

CatchKind catchKind() const

{

ASSERT(controlBlockType == BlockType::Catch);

return m_catchKind;

}

Variable* exception() const

{

ASSERT(controlBlockType == BlockType::Catch);

return m_exception;

}

unsigned stackSize() const { return m_stackSize; }

private:

// FIXME: Compress OMGIRGenerator::ControlData fields using an union

// https://bugs.webkit.org/show_bug.cgi?id=231212

friend class OMGIRGenerator;

BlockType controlBlockType;

BlockSignature m_signature;

unsigned m_stackSize;

BasicBlock* continuation;

BasicBlock* special;

Vector<Value*> phis;

unsigned m_tryStart;

unsigned m_tryEnd;

unsigned m_tryCatchDepth;

CatchKind m_catchKind;

Variable* m_exception;

};

using ControlType = ControlData;

using ExpressionList = Vector<ExpressionType, 1>;

using ControlEntry = FunctionParser<OMGIRGenerator>::ControlEntry;

using ControlStack = FunctionParser<OMGIRGenerator>::ControlStack;

using Stack = FunctionParser<OMGIRGenerator>::Stack;

using TypedExpression = FunctionParser<OMGIRGenerator>::TypedExpression;

static_assert(std::is_same_v<ResultList, FunctionParser<OMGIRGenerator>::ResultList>);

typedef String ErrorType;

typedef Unexpected<ErrorType> UnexpectedResult;

typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;

typedef Expected<void, ErrorType> PartialResult;

static ExpressionType emptyExpression() { return nullptr; };

enum class CastKind { Cast, Test };

template <typename ...Args>

NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const

{

using namespace FailureHelper; // See ADL comment in WasmParser.h.

return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));

}

#define WASM_COMPILE_FAIL_IF(condition, ...) do { \

if (UNLIKELY(condition)) \

return fail(__VA_ARGS__); \

} while (0)

unsigned advanceCallSiteIndex()

{

if (m_inlineParent)

return m_inlineRoot->advanceCallSiteIndex();

return ++m_callSiteIndex;

}

unsigned callSiteIndex() const

{

if (m_inlineParent)

return m_inlineRoot->callSiteIndex();

return m_callSiteIndex;

}

OMGIRGenerator(const ModuleInformation&, OptimizingJITCallee&, Procedure&, Vector<UnlinkedWasmToWasmCall>&, unsigned& osrEntryScratchBufferSize, MemoryMode, CompilationMode, unsigned functionIndex, std::optional<bool> hasExceptionHandlers, unsigned loopIndexForOSREntry, TierUpCount*);

OMGIRGenerator(OMGIRGenerator& inlineCaller, OMGIRGenerator& inlineRoot, unsigned functionIndex, BasicBlock* returnContinuation, Vector<Value*> args);

void computeStackCheckSize(bool& needsOverflowCheck, int32_t& checkSize);

// SIMD

void notifyFunctionUsesSIMD() { ASSERT(m_info.usesSIMD(m_functionIndex)); }

PartialResult WARN_UNUSED_RETURN addSIMDLoad(ExpressionType pointer, uint32_t offset, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSIMDStore(ExpressionType value, ExpressionType pointer, uint32_t offset);

PartialResult WARN_UNUSED_RETURN addSIMDSplat(SIMDLane, ExpressionType scalar, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSIMDShuffle(v128_t imm, ExpressionType a, ExpressionType b, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSIMDShift(SIMDLaneOperation, SIMDInfo, ExpressionType v, ExpressionType shift, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSIMDExtmul(SIMDLaneOperation, SIMDInfo, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSIMDLoadSplat(SIMDLaneOperation, ExpressionType pointer, uint32_t offset, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSIMDLoadLane(SIMDLaneOperation, ExpressionType pointer, ExpressionType vector, uint32_t offset, uint8_t laneIndex, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSIMDStoreLane(SIMDLaneOperation, ExpressionType pointer, ExpressionType vector, uint32_t offset, uint8_t laneIndex);

PartialResult WARN_UNUSED_RETURN addSIMDLoadExtend(SIMDLaneOperation, ExpressionType pointer, uint32_t offset, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addSIMDLoadPad(SIMDLaneOperation, ExpressionType pointer, uint32_t offset, ExpressionType& result);

ExpressionType WARN_UNUSED_RETURN addConstant(v128_t value)

{

return push(m_currentBlock->appendNew<Const128Value>(m_proc, origin(), value));

}

// SIMD generated

#define B3_OP_CASE(OP) \

else if (op == SIMDLaneOperation::OP) b3Op = B3::Vector##OP;

#define B3_OP_CASES() \

B3::Opcode b3Op = B3::Oops; \

if (false) { }

auto addExtractLane(SIMDInfo info, uint8_t lane, ExpressionType v, ExpressionType& result) -> PartialResult

{

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), B3::VectorExtractLane, toB3Type(simdScalarType(info.lane)), info,

lane,

get(v)));

return { };

}

auto addReplaceLane(SIMDInfo info, uint8_t lane, ExpressionType v, ExpressionType s, ExpressionType& result) -> PartialResult

{

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), B3::VectorReplaceLane, B3::V128, info,

lane,

get(v),

get(s)));

return { };

}

auto addSIMDI_V(SIMDLaneOperation op, SIMDInfo info, ExpressionType v, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(Bitmask)

B3_OP_CASE(AnyTrue)

B3_OP_CASE(AllTrue)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::Int32, info,

get(v)));

return { };

}

auto addSIMDV_V(SIMDLaneOperation op, SIMDInfo info, ExpressionType v, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(Demote)

B3_OP_CASE(Promote)

B3_OP_CASE(Abs)

B3_OP_CASE(Popcnt)

B3_OP_CASE(Ceil)

B3_OP_CASE(Floor)

B3_OP_CASE(Trunc)

B3_OP_CASE(Nearest)

B3_OP_CASE(Sqrt)

B3_OP_CASE(ExtaddPairwise)

B3_OP_CASE(Convert)

B3_OP_CASE(ConvertLow)

B3_OP_CASE(ExtendHigh)

B3_OP_CASE(ExtendLow)

B3_OP_CASE(TruncSat)

B3_OP_CASE(RelaxedTruncSat)

B3_OP_CASE(Not)

B3_OP_CASE(Neg)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(v)));

return { };

}

auto addSIMDBitwiseSelect(ExpressionType v1, ExpressionType v2, ExpressionType c, ExpressionType& result) -> PartialResult

{

auto b3Op = B3::VectorBitwiseSelect;

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, SIMDInfo { SIMDLane::v128, SIMDSignMode::None },

get(v1), get(v2), get(c)));

return { };

}

auto addSIMDRelOp(SIMDLaneOperation, SIMDInfo info, ExpressionType lhs, ExpressionType rhs, Air::Arg relOp, ExpressionType& result) -> PartialResult

{

B3::Opcode b3Op = Oops;

if (scalarTypeIsIntegral(info.lane)) {

switch (relOp.asRelationalCondition()) {

case MacroAssembler::Equal:

b3Op = VectorEqual;

break;

case MacroAssembler::NotEqual:

b3Op = VectorNotEqual;

break;

case MacroAssembler::LessThan:

b3Op = VectorLessThan;

break;

case MacroAssembler::LessThanOrEqual:

b3Op = VectorLessThanOrEqual;

break;

case MacroAssembler::Below:

b3Op = VectorBelow;

break;

case MacroAssembler::BelowOrEqual:

b3Op = VectorBelowOrEqual;

break;

case MacroAssembler::GreaterThan:

b3Op = VectorGreaterThan;

break;

case MacroAssembler::GreaterThanOrEqual:

b3Op = VectorGreaterThanOrEqual;

break;

case MacroAssembler::Above:

b3Op = VectorAbove;

break;

case MacroAssembler::AboveOrEqual:

b3Op = VectorAboveOrEqual;

break;

}

} else {

switch (relOp.asDoubleCondition()) {

case MacroAssembler::DoubleEqualAndOrdered:

b3Op = VectorEqual;

break;

case MacroAssembler::DoubleNotEqualOrUnordered:

b3Op = VectorNotEqual;

break;

case MacroAssembler::DoubleLessThanAndOrdered:

b3Op = VectorLessThan;

break;

case MacroAssembler::DoubleLessThanOrEqualAndOrdered:

b3Op = VectorLessThanOrEqual;

break;

case MacroAssembler::DoubleGreaterThanAndOrdered:

b3Op = VectorGreaterThan;

break;

case MacroAssembler::DoubleGreaterThanOrEqualAndOrdered:

b3Op = VectorGreaterThanOrEqual;

break;

default:

RELEASE_ASSERT_NOT_REACHED();

break;

}

}

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(lhs), get(rhs)));

return { };

}

Value* fixupOutOfBoundsIndicesForSwizzle(Value* input, Value* indexes)

{

// The intel version of the swizzle instruction does not handle OOB indices properly,

// so we need to fix them up.

ASSERT(isX86());

// Let each byte mask be 112 (0x70) then after VectorAddSat

// each index > 15 would set the saturated index's bit 7 to 1,

// whose corresponding byte will be zero cleared in VectorSwizzle.

// https://github.com/WebAssembly/simd/issues/93

v128_t mask;

mask.u64x2[0] = 0x7070707070707070;

mask.u64x2[1] = 0x7070707070707070;

auto saturatingMask = m_currentBlock->appendNew<Const128Value>(m_proc, origin(), mask);

auto saturatedIndexes = m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), VectorAddSat, B3::V128, SIMDLane::i8x16, SIMDSignMode::Unsigned, saturatingMask, indexes);

return m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), VectorSwizzle, B3::V128, SIMDLane::i8x16, SIMDSignMode::None, input, saturatedIndexes);

}

auto addSIMDV_VV(SIMDLaneOperation op, SIMDInfo info, ExpressionType a, ExpressionType b, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(And)

B3_OP_CASE(Andnot)

B3_OP_CASE(AvgRound)

B3_OP_CASE(DotProduct)

B3_OP_CASE(Add)

B3_OP_CASE(Mul)

B3_OP_CASE(MulSat)

B3_OP_CASE(Sub)

B3_OP_CASE(Div)

B3_OP_CASE(Pmax)

B3_OP_CASE(Pmin)

B3_OP_CASE(Or)

B3_OP_CASE(Swizzle)

B3_OP_CASE(RelaxedSwizzle)

B3_OP_CASE(Xor)

B3_OP_CASE(Narrow)

B3_OP_CASE(AddSat)

B3_OP_CASE(SubSat)

B3_OP_CASE(Max)

B3_OP_CASE(Min)

if (isX86() && b3Op == B3::VectorSwizzle) {

result = push(fixupOutOfBoundsIndicesForSwizzle(get(a), get(b)));

return { };

}

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(a), get(b)));

return { };

}

auto addSIMDRelaxedFMA(SIMDLaneOperation op, SIMDInfo info, ExpressionType m1, ExpressionType m2, ExpressionType add, ExpressionType& result) -> PartialResult

{

B3_OP_CASES()

B3_OP_CASE(RelaxedMAdd)

B3_OP_CASE(RelaxedNMAdd)

result = push(m_currentBlock->appendNew<SIMDValue>(m_proc, origin(), b3Op, B3::V128, info,

get(m1), get(m2), get(add)));

return { };

}

PartialResult WARN_UNUSED_RETURN addDrop(ExpressionType);

PartialResult WARN_UNUSED_RETURN addInlinedArguments(const TypeDefinition&);

PartialResult WARN_UNUSED_RETURN addArguments(const TypeDefinition&);

PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);

ExpressionType addConstant(Type, uint64_t);

// References

PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType value, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addRefAsNonNull(ExpressionType, ExpressionType&);

PartialResult WARN_UNUSED_RETURN addRefEq(ExpressionType, ExpressionType, ExpressionType&);

// Tables

PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType index, ExpressionType value);

PartialResult WARN_UNUSED_RETURN addTableInit(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

PartialResult WARN_UNUSED_RETURN addElemDrop(unsigned);

PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType fill, ExpressionType delta, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType offset, ExpressionType fill, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addTableCopy(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

// Locals

PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);

// Globals

PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);

// Memory

PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);

PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addMemoryInit(unsigned, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length);

PartialResult WARN_UNUSED_RETURN addDataDrop(unsigned);

// Atomics

PartialResult WARN_UNUSED_RETURN atomicLoad(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicStore(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicBinaryRMW(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicWait(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType timeout, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicFence(ExtAtomicOpType, uint8_t flags);

// Saturated truncation.

PartialResult WARN_UNUSED_RETURN truncSaturated(Ext1OpType, ExpressionType operand, ExpressionType& result, Type returnType, Type operandType);

// GC

PartialResult WARN_UNUSED_RETURN addRefI31(ExpressionType value, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addI31GetS(ExpressionType ref, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addI31GetU(ExpressionType ref, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNew(uint32_t index, ExpressionType size, ExpressionType value, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNewDefault(uint32_t index, ExpressionType size, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNewFixed(uint32_t typeIndex, Vector<ExpressionType>& args, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayGet(ExtGCOpType arrayGetKind, uint32_t typeIndex, ExpressionType arrayref, ExpressionType index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNewData(uint32_t typeIndex, uint32_t dataIndex, ExpressionType size, ExpressionType offset, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNewElem(uint32_t typeIndex, uint32_t elemSegmentIndex, ExpressionType size, ExpressionType offset, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArraySet(uint32_t typeIndex, ExpressionType arrayref, ExpressionType index, ExpressionType value);

PartialResult WARN_UNUSED_RETURN addArrayLen(ExpressionType arrayref, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayFill(uint32_t, ExpressionType, ExpressionType, ExpressionType, ExpressionType);

PartialResult WARN_UNUSED_RETURN addArrayCopy(uint32_t, ExpressionType, ExpressionType, uint32_t, ExpressionType, ExpressionType, ExpressionType);

PartialResult WARN_UNUSED_RETURN addArrayInitElem(uint32_t, ExpressionType, ExpressionType, uint32_t, ExpressionType, ExpressionType);

PartialResult WARN_UNUSED_RETURN addArrayInitData(uint32_t, ExpressionType, ExpressionType, uint32_t, ExpressionType, ExpressionType);

PartialResult WARN_UNUSED_RETURN addStructNew(uint32_t typeIndex, Vector<ExpressionType>& args, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addStructNewDefault(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addStructGet(ExtGCOpType structGetKind, ExpressionType structReference, const StructType&, uint32_t fieldIndex, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addStructSet(ExpressionType structReference, const StructType&, uint32_t fieldIndex, ExpressionType value);

PartialResult WARN_UNUSED_RETURN addRefTest(ExpressionType reference, bool allowNull, int32_t heapType, bool shouldNegate, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addRefCast(ExpressionType reference, bool allowNull, int32_t heapType, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addAnyConvertExtern(ExpressionType reference, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addExternConvertAny(ExpressionType reference, ExpressionType& result);

// Basic operators

#define X(name, opcode, short, idx, ...) \

PartialResult WARN_UNUSED_RETURN add##name(ExpressionType arg, ExpressionType& result);

FOR_EACH_WASM_UNARY_OP(X)

#undef X

#define X(name, opcode, short, idx, ...) \

PartialResult WARN_UNUSED_RETURN add##name(ExpressionType left, ExpressionType right, ExpressionType& result);

FOR_EACH_WASM_BINARY_OP(X)

#undef X

PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);

// Control flow

ControlData WARN_UNUSED_RETURN addTopLevel(BlockSignature);

PartialResult WARN_UNUSED_RETURN addBlock(BlockSignature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addLoop(BlockSignature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex);

PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const Stack&);

PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);

PartialResult WARN_UNUSED_RETURN addTry(BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addCatch(unsigned exceptionIndex, const TypeDefinition&, Stack&, ControlType&, ResultList&);

PartialResult WARN_UNUSED_RETURN addCatchToUnreachable(unsigned exceptionIndex, const TypeDefinition&, ControlType&, ResultList&);

PartialResult WARN_UNUSED_RETURN addCatchAll(Stack&, ControlType&);

PartialResult WARN_UNUSED_RETURN addCatchAllToUnreachable(ControlType&);

PartialResult WARN_UNUSED_RETURN addDelegate(ControlType&, ControlType&);

PartialResult WARN_UNUSED_RETURN addDelegateToUnreachable(ControlType&, ControlType&);

PartialResult WARN_UNUSED_RETURN addThrow(unsigned exceptionIndex, Vector<ExpressionType>& args, Stack&);

PartialResult WARN_UNUSED_RETURN addRethrow(unsigned, ControlType&);

PartialResult WARN_UNUSED_RETURN addInlinedReturn(const Stack& returnValues);

PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const Stack& returnValues);

PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const Stack& returnValues);

PartialResult WARN_UNUSED_RETURN addBranchNull(ControlType&, ExpressionType, const Stack&, bool, ExpressionType&);

PartialResult WARN_UNUSED_RETURN addBranchCast(ControlType&, ExpressionType, const Stack&, bool, int32_t, bool);

PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const Stack& expressionStack);

PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, Stack& expressionStack);

PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& = { });

PartialResult WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&) { return { }; }

// Calls

PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const TypeDefinition&, Vector<ExpressionType>& args, ResultList& results, CallType = CallType::Call);

PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const TypeDefinition&, Vector<ExpressionType>& args, ResultList& results, CallType = CallType::Call);

PartialResult WARN_UNUSED_RETURN addCallRef(const TypeDefinition&, Vector<ExpressionType>& args, ResultList& results);

PartialResult WARN_UNUSED_RETURN addUnreachable();

PartialResult WARN_UNUSED_RETURN addCrash();

PartialResult WARN_UNUSED_RETURN emitIndirectCall(Value* calleeInstance, Value* calleeCode, Value* boxedCalleeCallee, Value* jsCalleeAnchor, const TypeDefinition&, const Vector<ExpressionType>& args, ResultList&, CallType = CallType::Call);

B3::PatchpointValue* createCallPatchpoint(BasicBlock*, Value* jsCalleeAnchor, B3::Type, const CallInformation&, const Vector<ExpressionType>& tmpArgs, const ScopedLambda<void(PatchpointValue*, Box<PatchpointExceptionHandle>)>& patchpointFunctor);

B3::PatchpointValue* createTailCallPatchpoint(BasicBlock*, const Vector<ArgumentLocation>&, const Vector<ExpressionType>& tmpArgs, const Checked<int32_t>& tailCallStackOffsetFromFP, const ScopedLambda<void(PatchpointValue*, Box<PatchpointExceptionHandle>)>& patchpointFunctor);

PartialResult WARN_UNUSED_RETURN emitInlineDirectCall(uint32_t calleeIndex, const TypeDefinition&, Vector<ExpressionType>& args, ResultList& results);

void dump(const ControlStack&, const Stack* expressionStack);

void setParser(FunctionParser<OMGIRGenerator>* parser) { m_parser = parser; };

ALWAYS_INLINE void willParseOpcode() { }

ALWAYS_INLINE void didParseOpcode() { }

void didFinishParsingLocals() { }

void didPopValueFromStack(ExpressionType expr, String msg)

{

--m_stackSize;

TRACE_VALUE(Wasm::Types::Void, get(expr), "pop at height: ", m_stackSize.value() + 1, " site: [", msg, "], var ", *expr);

}

const Ref<TypeDefinition> getTypeDefinition(uint32_t typeIndex) { return m_info.typeSignatures[typeIndex]; }

const ArrayType* getArrayTypeDefinition(uint32_t);

void getArrayElementType(uint32_t, StorageType&);

void getArrayRefType(uint32_t, Type&);

bool canInline() const;

Value* constant(B3::Type, uint64_t bits, std::optional<Origin> = std::nullopt);

Value* constant(B3::Type, v128_t bits, std::optional<Origin> = std::nullopt);

Value* framePointer();

void insertEntrySwitch();

void insertConstants();

B3::Type toB3ResultType(const TypeDefinition*);

void addStackMap(unsigned callSiteIndex, StackMap&& stackmap)

{

if (m_inlineParent) {

m_inlineRoot->addStackMap(callSiteIndex, WTFMove(stackmap));

return;

}

m_stackmaps.add(CallSiteIndex(callSiteIndex), WTFMove(stackmap));

}

StackMaps&& takeStackmaps()

{

RELEASE_ASSERT(m_inlineRoot == this);

return WTFMove(m_stackmaps);

}

Vector<UnlinkedHandlerInfo>&& takeExceptionHandlers()

{

RELEASE_ASSERT(m_inlineRoot == this);

return WTFMove(m_exceptionHandlers);

}

void emitPrepareWasmOperation(BasicBlock* block)

{

#if !USE(BUILTIN_FRAME_ADDRESS) || ASSERT_ENABLED

// Prepare wasm operation calls.

block->appendNew<B3::MemoryValue>(m_proc, B3::Store, origin(), framePointer(), instanceValue(), Instance::offsetOfTemporaryCallFrame());

#else

UNUSED_PARAM(block);

#endif

}

template<typename OperationType, typename ...Args>

Value* callWasmOperation(BasicBlock* block, B3::Type resultType, OperationType operation, Args&&... args)

{

emitPrepareWasmOperation(block);

static_assert(FunctionTraits<OperationType>::cCallArity() == sizeof...(Args), "Sanity check");

Value* operationValue = block->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunction<OperationPtrTag>(operation));

return block->appendNew<CCallValue>(m_proc, resultType, origin(), operationValue, std::forward<Args>(args)...);

}

void emitExceptionCheck(CCallHelpers&, ExceptionType);

void emitEntryTierUpCheck();

void emitLoopTierUpCheck(uint32_t loopIndex, const Stack& enclosingStack, const Stack& newStack);

void emitWriteBarrierForJSWrapper();

void emitWriteBarrier(Value* cell, Value* instanceCell);

Value* emitCheckAndPreparePointer(Value* pointer, uint32_t offset, uint32_t sizeOfOp);

B3::Kind memoryKind(B3::Opcode memoryOp);

Value* emitLoadOp(LoadOpType, Value* pointer, uint32_t offset);

void emitStoreOp(StoreOpType, Value* pointer, Value*, uint32_t offset);

Value* sanitizeAtomicResult(ExtAtomicOpType, Type, Value* result);

Value* emitAtomicLoadOp(ExtAtomicOpType, Type, Value* pointer, uint32_t offset);

void emitAtomicStoreOp(ExtAtomicOpType, Type, Value* pointer, Value*, uint32_t offset);

Value* emitAtomicBinaryRMWOp(ExtAtomicOpType, Type, Value* pointer, Value*, uint32_t offset);

Value* emitAtomicCompareExchange(ExtAtomicOpType, Type, Value* pointer, Value* expected, Value*, uint32_t offset);

void emitArrayNullCheck(Value*, ExceptionType);

void emitArraySetUnchecked(uint32_t, Value*, Value*, Value*);

void emitStructSet(Value*, uint32_t, const StructType&, Value*);

ExpressionType WARN_UNUSED_RETURN pushArrayNew(uint32_t typeIndex, Value* initValue, ExpressionType size);

using arraySegmentOperation = EncodedJSValue (&)(JSC::Wasm::Instance*, uint32_t, uint32_t, uint32_t, uint32_t);

ExpressionType WARN_UNUSED_RETURN pushArrayNewFromSegment(arraySegmentOperation, uint32_t typeIndex, uint32_t segmentIndex, ExpressionType arraySize, ExpressionType offset, ExceptionType);

void emitRefTestOrCast(CastKind, ExpressionType, bool, int32_t, bool, ExpressionType&);

template <typename Generator>

void emitCheckOrBranchForCast(CastKind, Value*, const Generator&, BasicBlock*);

Value* emitLoadRTTFromFuncref(Value*);

Value* emitLoadRTTFromObject(Value*);

Value* emitNotRTTKind(Value*, RTTKind);

void unify(Value* phi, const ExpressionType source);

void unifyValuesWithBlock(const Stack& resultStack, const ControlData& block);

void emitChecksForModOrDiv(B3::Opcode, Value* left, Value* right);

int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(Value*&, uint32_t);

Value* WARN_UNUSED_RETURN fixupPointerPlusOffsetForAtomicOps(ExtAtomicOpType, Value*, uint32_t);

void restoreWasmContextInstance(BasicBlock*, Value*);

void restoreWebAssemblyGlobalState(const MemoryInformation&, Value* instance, BasicBlock*);

void reloadMemoryRegistersFromInstance(const MemoryInformation&, Value* instance, BasicBlock*);

Value* loadFromScratchBuffer(unsigned& indexInBuffer, Value* pointer, B3::Type);

void connectControlAtEntrypoint(unsigned& indexInBuffer, Value* pointer, ControlData&, Stack& expressionStack, ControlData& currentData, bool fillLoopPhis = false);

Value* emitCatchImpl(CatchKind, ControlType&, unsigned exceptionIndex = 0);

PatchpointExceptionHandle preparePatchpointForExceptions(BasicBlock*, PatchpointValue*);

Origin origin();

uint32_t outerLoopIndex() const

{

if (m_outerLoops.isEmpty())

return UINT32_MAX;

return m_outerLoops.last();

}

ExpressionType getPushVariable(B3::Type type)

{

++m_stackSize;

if (m_stackSize > m_maxStackSize) {

m_maxStackSize = m_stackSize;

Variable* var = m_proc.addVariable(type);

if constexpr (WasmOMGIRGeneratorInternal::traceStackValues)

set(var, constant(type, 0xBADBEEFEF));

m_stack.append(var);

return var;

}

if constexpr (WasmOMGIRGeneratorInternal::traceStackValues) {

// When we push, everything else *should* be dead

for (unsigned i = m_stackSize - 1; i < m_stack.size(); ++i)

set(m_stack[i], constant(m_stack[i]->type(), 0xBADBEEFEF));

}

Variable* var = m_stack[m_stackSize - 1];

if (var->type() == type)

return var;

var = m_proc.addVariable(type);

m_stack[m_stackSize - 1] = var;

return var;

}

ExpressionType push(Value* value)

{

Variable* var = getPushVariable(value->type());

set(var, value);

if constexpr (!WasmOMGIRGeneratorInternal::traceExecution)

return var;

String site;

#if ASSERT_ENABLED

if constexpr (WasmOMGIRGeneratorInternal::traceExecutionIncludesConstructionSite)

site = Value::generateCompilerConstructionSite();

#endif

TRACE_VALUE(Wasm::Types::Void, get(var), "push to stack height ", m_stackSize.value(), " site: [", site, "] var ", *var);

return var;

}

Value* get(BasicBlock* block, Variable* variable)

{

return block->appendNew<VariableValue>(m_proc, B3::Get, origin(), variable);

}

Value* get(Variable* variable)

{

return get(m_currentBlock, variable);

}

Value* set(BasicBlock* block, Variable* dst, Value* src)

{

return block->appendNew<VariableValue>(m_proc, B3::Set, origin(), dst, src);

}

Value* set(Variable* dst, Value* src)

{

return set(m_currentBlock, dst, src);

}

Value* set(Variable* dst, Variable* src)

{

return set(dst, get(src));

}

bool useSignalingMemory() const

{

return m_mode == MemoryMode::Signaling;

}

template<typename... Args>

void traceValue(Type, Value*, Args&&... info);

template<typename... Args>

void traceCF(Args&&... info);

FunctionParser<OMGIRGenerator>* m_parser { nullptr };

const ModuleInformation& m_info;

OptimizingJITCallee* m_callee;

const MemoryMode m_mode { MemoryMode::BoundsChecking };

const CompilationMode m_compilationMode;

const unsigned m_functionIndex { UINT_MAX };

const unsigned m_loopIndexForOSREntry { UINT_MAX };

TierUpCount* m_tierUp { nullptr };

Procedure& m_proc;

Vector<BasicBlock*> m_rootBlocks;

BasicBlock* m_topLevelBlock;

BasicBlock* m_currentBlock { nullptr };

// Only used when this is an inlined context

BasicBlock* m_returnContinuation { nullptr };

OMGIRGenerator* m_inlineRoot { nullptr };

OMGIRGenerator* m_inlineParent { nullptr };

Vector<Value*> m_inlinedArgs;

Vector<Variable*> m_inlinedResults;

unsigned m_inlineDepth { 0 };

Checked<uint32_t> m_inlinedBytes { 0 };

Vector<uint32_t> m_outerLoops;

Vector<Variable*> m_locals;

Vector<Variable*> m_stack;

Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.

unsigned* m_osrEntryScratchBufferSize;

HashMap<ValueKey, Value*> m_constantPool;

HashMap<const TypeDefinition*, B3::Type> m_tupleMap;

InsertionSet m_constantInsertionValues;

Value* m_framePointer { nullptr };

bool m_makesCalls { false };

bool m_makesTailCalls { false };

// This tracks the maximum stack offset for a tail call, to be used in the stack overflow check.

Checked<int32_t> m_tailCallStackOffsetFromFP { 0 };

std::optional<bool> m_hasExceptionHandlers;

Value* m_instanceValue { nullptr };

Value* m_baseMemoryValue { nullptr };

Value* m_boundsCheckingSizeValue { nullptr };

Value* instanceValue()

{

return m_instanceValue;

}

Value* baseMemoryValue()

{

return m_baseMemoryValue;

}

Value* boundsCheckingSizeValue()

{

return m_boundsCheckingSizeValue;

}

uint32_t m_maxNumJSCallArguments { 0 };

unsigned m_numImportFunctions;

Checked<unsigned> m_tryCatchDepth { 0 };

Checked<unsigned> m_callSiteIndex { 0 };

Checked<unsigned> m_stackSize { 0 };

Checked<unsigned> m_maxStackSize { 0 };

StackMaps m_stackmaps;

Vector<UnlinkedHandlerInfo> m_exceptionHandlers;

RefPtr<B3::Air::PrologueGenerator> m_prologueGenerator;

Vector<std::unique_ptr<OMGIRGenerator>> m_protectedInlineeGenerators;

Vector<std::unique_ptr<FunctionParser<OMGIRGenerator>>> m_protectedInlineeParsers;

{

if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {

ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));

return 0;

}

return offset;

{

// FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.

// This prevents us from using ArgumentReg to this (logically) immutable pinned register.

PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(m_proc, B3::Void, Origin());

Effects effects = Effects::none();