using ExpressionType = VirtualRegister;

using CallType = CallLinkInfo::CallType;

static constexpr bool tierSupportsSIMD = false;

struct ControlLoop {

Ref<Label> m_body;

};

struct ControlTopLevel {

};

struct ControlBlock {

};

struct ControlIf {

Ref<Label> m_alternate;

};

struct ControlTry {

Ref<Label> m_try;

unsigned m_tryDepth;

};

struct ControlCatch {

CatchKind m_kind;

Ref<Label> m_tryStart;

Ref<Label> m_tryEnd;

unsigned m_tryDepth;

VirtualRegister m_exception;

};

struct CatchRewriteInfo {

unsigned m_instructionOffset;

unsigned m_tryDepth;

};

struct ControlType : public std::variant<ControlLoop, ControlTopLevel, ControlBlock, ControlIf, ControlTry, ControlCatch> {

using Base = std::variant<ControlLoop, ControlTopLevel, ControlBlock, ControlIf, ControlTry, ControlCatch>;

ControlType()

: Base(ControlBlock { })

{

}

static ControlType topLevel(BlockSignature signature, unsigned stackSize, RefPtr<Label>&& continuation)

{

return ControlType(signature, stackSize, WTFMove(continuation), ControlTopLevel { });

}

static ControlType loop(BlockSignature signature, unsigned stackSize, Ref<Label>&& body, RefPtr<Label>&& continuation)

{

return ControlType(signature, stackSize - signature->argumentCount(), WTFMove(continuation), ControlLoop { WTFMove(body) });

}

static ControlType block(BlockSignature signature, unsigned stackSize, RefPtr<Label>&& continuation)

{

return ControlType(signature, stackSize - signature->argumentCount(), WTFMove(continuation), ControlBlock { });

}

static ControlType if_(BlockSignature signature, unsigned stackSize, Ref<Label>&& alternate, RefPtr<Label>&& continuation)

{

return ControlType(signature, stackSize - signature->argumentCount(), WTFMove(continuation), ControlIf { WTFMove(alternate) });

}

static ControlType createTry(BlockSignature signature, unsigned stackSize, Ref<Label>&& tryLabel, RefPtr<Label>&& continuation, unsigned tryDepth)

{

return ControlType(signature, stackSize - signature->argumentCount(), WTFMove(continuation), ControlTry { WTFMove(tryLabel), tryDepth });

}

static bool isLoop(const ControlType& control) { return std::holds_alternative<ControlLoop>(control); }

static bool isTopLevel(const ControlType& control) { return std::holds_alternative<ControlTopLevel>(control); }

static bool isBlock(const ControlType& control) { return std::holds_alternative<ControlBlock>(control); }

static bool isIf(const ControlType& control) { return std::holds_alternative<ControlIf>(control); }

static bool isTry(const ControlType& control) { return std::holds_alternative<ControlTry>(control); }

static bool isAnyCatch(const ControlType& control) { return std::holds_alternative<ControlCatch>(control); }

static bool isCatch(const ControlType& control)

{

if (!isAnyCatch(control))

return false;

ControlCatch catchData = std::get<ControlCatch>(control);

return catchData.m_kind == CatchKind::Catch;

}

static bool isCatchAll(const ControlType& control)

{

if (!isAnyCatch(control))

return false;

ControlCatch catchData = std::get<ControlCatch>(control);

return catchData.m_kind == CatchKind::CatchAll;

}

unsigned stackSize() const { return m_stackSize; }

BlockSignature signature() const { return m_signature; }

RefPtr<Label> targetLabelForBranch() const

{

if (isLoop(*this))

return std::get<ControlLoop>(*this).m_body.ptr();

return m_continuation;

}

FunctionArgCount branchTargetArity() const

{

if (isLoop(*this))

return m_signature->argumentCount();

return m_signature->returnCount();

}

Type branchTargetType(unsigned i) const

{

ASSERT(i < branchTargetArity());

if (isLoop(*this))

return m_signature->argumentType(i);

return m_signature->returnType(i);

}

void dump(PrintStream& out) const

{

if (isIf(*this))

out.print("If: ");

else if (isBlock(*this))

out.print("Block: ");

else if (isLoop(*this))

out.print("Loop: ");

else if (isTopLevel(*this))

out.print("TopLevel: ");

else if (isTry(*this))

out.print("Try: ");

else if (isCatch(*this))

out.print("Catch: ");

else if (isCatchAll(*this))

out.print("CatchAll: ");

out.print("stackSize:(", stackSize(), ") ");

}

void convertToCatch(Ref<Label> tryEnd, VirtualRegister exception)

{

ASSERT(isTry(*this));

auto& tryData = std::get<ControlTry>(*this);

auto tryStart = WTFMove(tryData.m_try);

auto tryDepth = WTFMove(tryData.m_tryDepth);

auto continuation = WTFMove(m_continuation);

*this = ControlType(m_signature, m_stackSize, WTFMove(continuation), ControlCatch { CatchKind::Catch, WTFMove(tryStart), WTFMove(tryEnd), WTFMove(tryDepth), exception });

}

BlockSignature m_signature;

unsigned m_stackSize;

RefPtr<Label> m_continuation;

private:

template<typename T>

ControlType(BlockSignature signature, unsigned stackSize, RefPtr<Label>&& continuation, T&& t)

: Base(std::forward<T>(t))

, m_signature(signature)

, m_stackSize(stackSize)

, m_continuation(WTFMove(continuation))

{

}

};

using ErrorType = String;

using PartialResult = Expected<void, ErrorType>;

using UnexpectedResult = Unexpected<ErrorType>;

using ControlEntry = FunctionParser<LLIntGenerator>::ControlEntry;

using ControlStack = FunctionParser<LLIntGenerator>::ControlStack;

using ResultList = FunctionParser<LLIntGenerator>::ResultList;

using Stack = FunctionParser<LLIntGenerator>::Stack;

using TypedExpression = FunctionParser<LLIntGenerator>::TypedExpression;

static ExpressionType emptyExpression() { return { }; };

template <typename ...Args>

NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const

{

using namespace FailureHelper; // See ADL comment in WasmParser.h.

return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));

}

LLIntGenerator(ModuleInformation&, unsigned functionIndex, const TypeDefinition&);

std::unique_ptr<FunctionCodeBlockGenerator> finalize();

template<typename ExpressionListA, typename ExpressionListB>

void unifyValuesWithBlock(const ExpressionListA& destinations, const ExpressionListB& values)

{

ASSERT(destinations.size() <= values.size());

auto offset = values.size() - destinations.size();

for (size_t i = 0; i < destinations.size(); ++i) {

auto& src = values[offset + i];

auto& dst = destinations[i];

if (static_cast<VirtualRegister>(src) != static_cast<VirtualRegister>(dst))

WasmMov::emit(this, dst, src);

}

}

enum NoConsistencyCheckTag { NoConsistencyCheck };

ExpressionType push(NoConsistencyCheckTag)

{

m_maxStackSize = std::max(m_maxStackSize, ++m_stackSize);

return virtualRegisterForLocal(m_stackSize - 1);

}

ExpressionType push()

{

checkConsistency();

return push(NoConsistencyCheck);

}

void didPopValueFromStack(ExpressionType, String) { --m_stackSize; }

void notifyFunctionUsesSIMD() { ASSERT(Options::useWebAssemblySIMD()); m_usesSIMD = true; }

PartialResult WARN_UNUSED_RETURN addDrop(ExpressionType);

PartialResult WARN_UNUSED_RETURN addArguments(const TypeDefinition&);

PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);

ExpressionType addConstant(Type, int64_t);

ExpressionType addConstantWithoutPush(Type, int64_t);

// References

PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType value, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addRefAsNonNull(ExpressionType, ExpressionType&);

PartialResult WARN_UNUSED_RETURN addRefEq(ExpressionType, ExpressionType, ExpressionType&);

// Tables

PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType index, ExpressionType value);

PartialResult WARN_UNUSED_RETURN addTableInit(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

PartialResult WARN_UNUSED_RETURN addElemDrop(unsigned);

PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType fill, ExpressionType delta, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType offset, ExpressionType fill, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addTableCopy(unsigned, unsigned, ExpressionType dstOffset, ExpressionType srcOffset, ExpressionType length);

// Locals

PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);

// Globals

PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);

// Memory

PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);

PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addMemoryFill(ExpressionType dstAddress, ExpressionType targetValue, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addMemoryCopy(ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType count);

PartialResult WARN_UNUSED_RETURN addMemoryInit(unsigned, ExpressionType dstAddress, ExpressionType srcAddress, ExpressionType length);

PartialResult WARN_UNUSED_RETURN addDataDrop(unsigned);

// Atomics

PartialResult WARN_UNUSED_RETURN atomicLoad(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicStore(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicBinaryRMW(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicCompareExchange(ExtAtomicOpType, Type, ExpressionType pointer, ExpressionType expected, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicWait(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType timeout, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicNotify(ExtAtomicOpType, ExpressionType pointer, ExpressionType value, ExpressionType& result, uint32_t offset);

PartialResult WARN_UNUSED_RETURN atomicFence(ExtAtomicOpType, uint8_t flags);

// Saturated truncation.

PartialResult WARN_UNUSED_RETURN truncSaturated(Ext1OpType, ExpressionType operand, ExpressionType& result, Type, Type);

// GC

PartialResult WARN_UNUSED_RETURN addRefI31(ExpressionType value, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addI31GetS(ExpressionType ref, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addI31GetU(ExpressionType ref, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNew(uint32_t index, ExpressionType size, ExpressionType value, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNewDefault(uint32_t index, ExpressionType size, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNewFixed(uint32_t index, Vector<ExpressionType>& args, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayGet(ExtGCOpType arrayGetKind, uint32_t typeIndex, ExpressionType arrayref, ExpressionType index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNewData(uint32_t typeIndex, uint32_t dataIndex, ExpressionType size, ExpressionType offset, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayNewElem(uint32_t typeIndex, uint32_t elemSegmentIndex, ExpressionType size, ExpressionType offset, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArraySet(uint32_t typeIndex, ExpressionType arrayref, ExpressionType index, ExpressionType value);

PartialResult WARN_UNUSED_RETURN addArrayLen(ExpressionType arrayref, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addArrayFill(uint32_t, ExpressionType, ExpressionType, ExpressionType, ExpressionType);

PartialResult WARN_UNUSED_RETURN addArrayCopy(uint32_t, ExpressionType, ExpressionType, uint32_t, ExpressionType, ExpressionType, ExpressionType);

PartialResult WARN_UNUSED_RETURN addArrayInitElem(uint32_t, ExpressionType, ExpressionType, uint32_t, ExpressionType, ExpressionType);

PartialResult WARN_UNUSED_RETURN addArrayInitData(uint32_t, ExpressionType, ExpressionType, uint32_t, ExpressionType, ExpressionType);

PartialResult WARN_UNUSED_RETURN addStructNew(uint32_t index, Vector<ExpressionType>& args, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addStructNewDefault(uint32_t index, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addStructGet(ExtGCOpType structGetKind, ExpressionType structReference, const StructType&, uint32_t fieldIndex, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addStructSet(ExpressionType structReference, const StructType&, uint32_t fieldIndex, ExpressionType value);

PartialResult WARN_UNUSED_RETURN addRefTest(ExpressionType reference, bool allowNull, int32_t heapType, bool shouldNegate, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addRefCast(ExpressionType reference, bool allowNull, int32_t heapType, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addAnyConvertExtern(ExpressionType reference, ExpressionType& result);

PartialResult WARN_UNUSED_RETURN addExternConvertAny(ExpressionType reference, ExpressionType& result);

// Basic operators

#define X(name, opcode, short, idx, ...) \

PartialResult WARN_UNUSED_RETURN add##name(ExpressionType arg, ExpressionType& result);

FOR_EACH_WASM_UNARY_OP(X)

#undef X

#define X(name, opcode, short, idx, ...) \

PartialResult WARN_UNUSED_RETURN add##name(ExpressionType left, ExpressionType right, ExpressionType& result);

FOR_EACH_WASM_BINARY_OP(X)

#undef X

PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);

// Control flow

ControlType WARN_UNUSED_RETURN addTopLevel(BlockSignature);

PartialResult WARN_UNUSED_RETURN addBlock(BlockSignature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addLoop(BlockSignature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex);

PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addElse(ControlType&, Stack&);

PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlType&);

PartialResult WARN_UNUSED_RETURN addTry(BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);

PartialResult WARN_UNUSED_RETURN addCatch(unsigned exceptionIndex, const TypeDefinition&, Stack&, ControlType&, ResultList&);

PartialResult WARN_UNUSED_RETURN addCatchToUnreachable(unsigned exceptionIndex, const TypeDefinition&, ControlType&, ResultList&);

PartialResult WARN_UNUSED_RETURN addCatchAll(Stack&, ControlType&);

PartialResult WARN_UNUSED_RETURN addCatchAllToUnreachable(ControlType&);

PartialResult WARN_UNUSED_RETURN addDelegate(ControlType&, ControlType&);

PartialResult WARN_UNUSED_RETURN addDelegateToUnreachable(ControlType&, ControlType&);

PartialResult WARN_UNUSED_RETURN addThrow(unsigned exceptionIndex, Vector<ExpressionType>& args, Stack&);

PartialResult WARN_UNUSED_RETURN addRethrow(unsigned, ControlType&);

PartialResult WARN_UNUSED_RETURN addReturn(const ControlType&, Stack& returnValues);

PartialResult WARN_UNUSED_RETURN addBranch(ControlType&, ExpressionType condition, Stack& returnValues);

PartialResult WARN_UNUSED_RETURN addBranchNull(ControlType&, ExpressionType, Stack&, bool, ExpressionType&);

PartialResult WARN_UNUSED_RETURN addBranchCast(ControlType&, ExpressionType, Stack&, bool, int32_t, bool);

PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlType*>& targets, ControlType& defaultTargets, Stack& expressionStack);

PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, Stack& expressionStack);

PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, Stack& expressionStack, bool unreachable = true);

PartialResult WARN_UNUSED_RETURN endTopLevel(BlockSignature, const Stack&);

// Calls

PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const TypeDefinition&, Vector<ExpressionType>& args, ResultList& results, CallType = CallType::Call);

PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const TypeDefinition&, Vector<ExpressionType>& args, ResultList& results, CallType = CallType::Call);

PartialResult WARN_UNUSED_RETURN addCallRef(const TypeDefinition&, Vector<ExpressionType>& args, ResultList& results);

PartialResult WARN_UNUSED_RETURN addUnreachable();

PartialResult WARN_UNUSED_RETURN addCrash();

ALWAYS_INLINE void willParseOpcode() { }

ALWAYS_INLINE void didParseOpcode() { }

void didFinishParsingLocals();

void setParser(FunctionParser<LLIntGenerator>* parser) { m_parser = parser; };

// We need this for autogenerated templates used by JS bytecodes.

void setUsesCheckpoints() const { UNREACHABLE_FOR_PLATFORM(); }

void dump(const ControlStack&, const Stack*);

friend GenericLabel<Wasm::GeneratorTraits>;

struct LLIntCallInformation {

unsigned stackOffset;

unsigned numberOfStackArguments;

ResultList arguments;

CompletionHandler<void(ResultList&)> commitResults;

};

LLIntCallInformation callInformationForCaller(const FunctionSignature&);

Vector<VirtualRegister, 2> callInformationForCallee(const FunctionSignature&);

void linkSwitchTargets(Label&, unsigned location);

VirtualRegister virtualRegisterForWasmLocal(uint32_t index)

{

if (index < m_codeBlock->m_numArguments)

return m_normalizedArguments[index];

const auto& callingConvention = wasmCallingConvention();

const uint32_t gprCount = callingConvention.jsrArgs.size();

const uint32_t fprCount = callingConvention.fprArgs.size();

return virtualRegisterForLocal(index - m_codeBlock->m_numArguments + gprCount + fprCount + numberOfLLIntCalleeSaveRegisters + numberOfLLIntInternalRegisters);

}

ExpressionType jsNullConstant()

{

if (UNLIKELY(!m_jsNullConstant.isValid())) {

m_jsNullConstant = VirtualRegister(FirstConstantRegisterIndex + m_codeBlock->m_constants.size());

m_codeBlock->m_constants.append(JSValue::encode(jsNull()));

if (UNLIKELY(Options::dumpGeneratedWasmBytecodes()))

m_codeBlock->m_constantTypes.append(Types::Externref);

}

return m_jsNullConstant;

}

ExpressionType zeroConstant()

{

if (UNLIKELY(!m_zeroConstant.isValid())) {

m_zeroConstant = VirtualRegister(FirstConstantRegisterIndex + m_codeBlock->m_constants.size());

m_codeBlock->m_constants.append(0);

if (UNLIKELY(Options::dumpGeneratedWasmBytecodes()))

m_codeBlock->m_constantTypes.append(Types::I32);

}

return m_zeroConstant;

}

void getDropKeepCount(const ControlType& target, unsigned& startOffset, unsigned& drop, unsigned& keep)

{

startOffset = target.stackSize() + 1;

keep = target.branchTargetArity();

drop = m_stackSize - target.stackSize() - target.branchTargetArity();

}

void dropKeep(Stack& values, const ControlType& target, bool dropValues)

{

unsigned startOffset;

unsigned keep;

unsigned drop;

getDropKeepCount(target, startOffset, drop, keep);

if (dropValues)

values.shrink(keep);

if (!drop)

return;

if (keep)

WasmDropKeep::emit(this, startOffset, drop, keep);

}

template<typename Stack, typename Functor>

void walkExpressionStack(Stack& expressionStack, unsigned stackSize, const Functor& functor)

{

for (unsigned i = expressionStack.size(); i > 0; --i) {

VirtualRegister slot = virtualRegisterForLocal(stackSize - i);

functor(expressionStack[expressionStack.size() - i], slot);

}

}

template<typename Stack, typename Functor>

void walkExpressionStack(Stack& expressionStack, const Functor& functor)

{

walkExpressionStack(expressionStack, m_stackSize, functor);

}

template<typename Functor>

void walkExpressionStack(ControlEntry& entry, const Functor& functor)

{

unsigned stackSize = entry.controlData.stackSize();

walkExpressionStack(entry.enclosedExpressionStack, stackSize, functor);

}

void checkConsistency()

{

#if ASSERT_ENABLED

// The rules for locals and constants in the stack are:

// 1) Locals have to be materialized whenever a control entry is pushed to the control stack (i.e. every time we splitStack)

// NOTE: This is a trade-off so that set_local does not have to walk up the control stack looking for delayed get_locals

// 2) If the control entry is a loop, we also need to materialize constants in the newStack, since those slots will be written

// to from loop back edges

// 3) Both locals and constants have to be materialized before branches, since multiple branches might share the same target,

// we can't make any assumptions about the stack state at that point, so we materialize the stack.

for (ControlEntry& controlEntry : m_parser->controlStack()) {

walkExpressionStack(controlEntry, [&](VirtualRegister expression, VirtualRegister slot) {

ASSERT(expression == slot || expression.isConstant());

});

}

walkExpressionStack(m_parser->expressionStack(), [&](VirtualRegister expression, VirtualRegister slot) {

ASSERT(expression == slot || expression.isConstant() || expression.isArgument() || static_cast<unsigned>(expression.toLocal()) < m_codeBlock->m_numVars);

});

#endif // ASSERT_ENABLED

}

void materializeConstantsAndLocals(Stack& expressionStack, NoConsistencyCheckTag)

{

walkExpressionStack(expressionStack, [&](auto& expression, VirtualRegister slot) {

ASSERT(expression.value() == slot || expression.value().isConstant() || expression.value().isArgument() || static_cast<unsigned>(expression.value().toLocal()) < m_codeBlock->m_numVars);

if (expression.value() == slot)

return;

WasmMov::emit(this, slot, expression);

expression = TypedExpression { expression.type(), slot };

});

}

void materializeConstantsAndLocals(Stack& expressionStack)

{

if (expressionStack.isEmpty())

return;

checkConsistency();

materializeConstantsAndLocals(expressionStack, NoConsistencyCheck);

checkConsistency();

}

void splitStack(BlockSignature signature, Stack& enclosingStack, Stack& newStack)

{

JSC::Wasm::splitStack(signature, enclosingStack, newStack);

m_stackSize -= newStack.size();

checkConsistency();

walkExpressionStack(enclosingStack, [&](TypedExpression& expression, VirtualRegister slot) {

ASSERT(expression.value() == slot || expression.value().isConstant() || expression.value().isArgument() || static_cast<unsigned>(expression.value().toLocal()) < m_codeBlock->m_numVars);

if (expression.value() == slot || expression.value().isConstant())

return;

WasmMov::emit(this, slot, expression);

expression = TypedExpression { expression.type(), slot };

});

checkConsistency();

m_stackSize += newStack.size();

}

void finalizePreviousBlockForCatch(ControlType&, Stack&);

void addCallBuiltin(LLIntBuiltin, const Vector<ExpressionType> args, ResultList& results);

struct SwitchEntry {

WasmInstructionStream::Offset offset;

int* jumpTarget;

};

struct ConstantMapHashTraits : HashTraits<EncodedJSValue> {

static constexpr bool emptyValueIsZero = true;

static void constructDeletedValue(EncodedJSValue& slot) { slot = JSValue::encode(jsNull()); }

static bool isDeletedValue(EncodedJSValue value) { return value == JSValue::encode(jsNull()); }

};

FunctionParser<LLIntGenerator>* m_parser { nullptr };

ModuleInformation& m_info;

const unsigned m_functionIndex { UINT_MAX };

Vector<VirtualRegister> m_normalizedArguments;

HashMap<Label*, Vector<SwitchEntry>> m_switches;

ExpressionType m_jsNullConstant;

ExpressionType m_zeroConstant;

ResultList m_uninitializedLocals;

HashMap<EncodedJSValue, VirtualRegister, WTF::IntHash<EncodedJSValue>, ConstantMapHashTraits> m_constantMap;

Vector<VirtualRegister, 2> m_results;

Checked<unsigned> m_stackSize { 0 };

Checked<unsigned> m_maxStackSize { 0 };

Checked<unsigned> m_tryDepth { 0 };

bool m_usesExceptions { false };

bool m_usesAtomics { false };

bool m_usesSIMD { false };

{

LLIntGenerator llintGenerator(info, functionIndex, signature);

FunctionParser<LLIntGenerator> parser(llintGenerator, function, signature, info);

WASM_FAIL_IF_HELPER_FAILS(parser.parse());

return llintGenerator.finalize();

: BytecodeGeneratorBase(makeUnique<FunctionCodeBlockGenerator>(functionIndex), 0)

, m_info(info)

, m_functionIndex(functionIndex)

{

{

auto& threadSpecific = threadSpecificBuffer();

Buffer buffer = WTFMove(*threadSpecific);

*threadSpecific = Buffer();

m_writer.setInstructionBuffer(WTFMove(buffer));

}

m_maxStackSize = m_stackSize = m_codeBlock->m_numVars = numberOfLLIntCalleeSaveRegisters + numberOfLLIntInternalRegisters;

WasmEnter::emit(this);

{

RELEASE_ASSERT(m_codeBlock);

size_t numCalleeLocals = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_maxStackSize);

m_codeBlock->m_numCalleeLocals = numCalleeLocals;

RELEASE_ASSERT(numCalleeLocals == m_codeBlock->m_numCalleeLocals);

auto& threadSpecific = threadSpecificBuffer();

Buffer usedBuffer;

m_codeBlock->setInstructions(m_writer.finalize(usedBuffer));

size_t oldCapacity = usedBuffer.capacity();

usedBuffer.shrink(0);

RELEASE_ASSERT(usedBuffer.capacity() == oldCapacity);

*threadSpecific = WTFMove(usedBuffer);

return WTFMove(m_codeBlock);

{

// This function sets up the stack layout for calls. The desired stack layout is:

// FPRn |

// ... v stack-growth towards lower memory

// FPR1

// FPR0

// ---

// GPRn

// ...

// GPR1

// GPR0

// ----

// stackN

// ...

// stack1

// stack0

// ---

// call frame header

// We need to allocate at least space for all GPRs and FPRs.

// Return value stack0 is at stackN - stackReturnValues

const auto initialStackSize = m_stackSize;

const auto& callingConvention = wasmCallingConvention();

const uint32_t gprCount = callingConvention.jsrArgs.size();

const uint32_t fprCount = callingConvention.fprArgs.size();

uint32_t stackResults = callingConvention.numberOfStackResults(signature);

uint32_t stackCountAligned = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), std::max(callingConvention.numberOfStackArguments(signature), stackResults));

uint32_t gprIndex = 0;

uint32_t fprIndex = 0;

uint32_t stackIndex = 0;

m_stackSize = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_stackSize.value());

// FIXME: we are allocating the extra space for the argument/return count in order to avoid interference, but we could do better

// NOTE: We increase arg count by 1 for the case of indirect calls

m_stackSize += std::max(signature.argumentCount() + 1, signature.returnCount()) + gprCount + fprCount + stackCountAligned + CallFrame::headerSizeInRegisters + 1;

m_stackSize = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), m_stackSize.value());

if (m_maxStackSize < m_stackSize)

m_maxStackSize = m_stackSize;

ResultList arguments(signature.argumentCount());

ResultList temporaryResults(signature.returnCount());

const unsigned stackOffset = m_stackSize;

const unsigned base = stackOffset - CallFrame::headerSizeInRegisters - 1;

const uint32_t gprLimit = base - stackCountAligned - gprCount;

const uint32_t fprLimit = gprLimit - fprCount;

stackIndex = base;

gprIndex = base - stackCountAligned;

fprIndex = gprIndex - gprCount;

for (uint32_t i = 0; i < signature.argumentCount(); i++) {

switch (signature.argumentType(i).kind) {

case TypeKind::I32:

case TypeKind::I64:

case TypeKind::Externref:

case TypeKind::Funcref:

case TypeKind::RefNull:

case TypeKind::Ref:

if (gprIndex > gprLimit)

arguments[i] = virtualRegisterForLocal(--gprIndex);

else

arguments[i] = virtualRegisterForLocal(--stackIndex);

break;

case TypeKind::F32:

case TypeKind::F64:

case TypeKind::V128:

if (fprIndex > fprLimit)

arguments[i] = virtualRegisterForLocal(--fprIndex);

else

arguments[i] = virtualRegisterForLocal(--stackIndex);

break;

case TypeKind::Void:

case TypeKind::Func:

case TypeKind::Struct:

case TypeKind::Structref:

case TypeKind::Array:

case TypeKind::Arrayref:

case TypeKind::Eqref:

case TypeKind::Anyref:

case TypeKind::Nullref:

case TypeKind::Nullfuncref:

case TypeKind::Nullexternref:

case TypeKind::I31ref:

case TypeKind::Rec:

case TypeKind::Sub:

case TypeKind::Subfinal:

RELEASE_ASSERT_NOT_REACHED();

}

}

gprIndex = base - stackCountAligned;

stackIndex = gprIndex + stackResults;

fprIndex = gprIndex - gprCount;

for (uint32_t i = 0; i < signature.returnCount(); i++) {

switch (signature.returnType(i).kind) {

case TypeKind::I32:

case TypeKind::I64:

case TypeKind::Externref:

case TypeKind::Funcref:

case TypeKind::RefNull:

case TypeKind::Ref:

if (gprIndex > gprLimit)

temporaryResults[i] = virtualRegisterForLocal(--gprIndex);

else

temporaryResults[i] = virtualRegisterForLocal(--stackIndex);

break;

case TypeKind::F32:

case TypeKind::F64:

case TypeKind::V128:

if (fprIndex > fprLimit)

temporaryResults[i] = virtualRegisterForLocal(--fprIndex);

else

temporaryResults[i] = virtualRegisterForLocal(--stackIndex);

break;

case TypeKind::Void:

case TypeKind::Func:

case TypeKind::Struct:

case TypeKind::Structref:

case TypeKind::Array:

case TypeKind::Arrayref:

case TypeKind::Eqref:

case TypeKind::Anyref:

case TypeKind::Nullref:

case TypeKind::Nullfuncref:

case TypeKind::Nullexternref:

case TypeKind::I31ref:

case TypeKind::Rec:

case TypeKind::Sub:

case TypeKind::Subfinal:

RELEASE_ASSERT_NOT_REACHED();

}

}

m_stackSize = initialStackSize;

auto commitResults = [this, temporaryResults = WTFMove(temporaryResults)](ResultList& results) {

checkConsistency();

for (auto temporaryResult : temporaryResults) {

ExpressionType result = push(NoConsistencyCheck);

WasmMov::emit(this, result, temporaryResult);

results.append(result);

}

};

return LLIntCallInformation { stackOffset, stackCountAligned, WTFMove(arguments), WTFMove(commitResults) };

{

if (m_results.size())

return m_results;

m_results.reserveInitialCapacity(signature.returnCount());

const auto& callingConvention = wasmCallingConvention();

const uint32_t gprCount = callingConvention.jsrArgs.size();

const uint32_t fprCount = callingConvention.fprArgs.size();

uint32_t gprIndex = 0;

uint32_t fprIndex = gprCount;

const uint32_t maxGPRIndex = gprCount;

const uint32_t maxFPRIndex = maxGPRIndex + fprCount;

uint32_t stackResults = callingConvention.numberOfStackResults(signature);

uint32_t stackCountAligned = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), std::max(callingConvention.numberOfStackArguments(signature), stackResults));

uint32_t stackIndex = 1 + stackCountAligned - stackResults;

for (uint32_t i = 0; i < signature.returnCount(); i++) {

switch (signature.returnType(i).kind) {

case TypeKind::I32:

case TypeKind::I64:

case TypeKind::Externref:

case TypeKind::Funcref:

case TypeKind::RefNull:

case TypeKind::Ref:

if (gprIndex < maxGPRIndex)

m_results.append(virtualRegisterForLocal(numberOfLLIntCalleeSaveRegisters + numberOfLLIntInternalRegisters + gprIndex++));

else

m_results.append(virtualRegisterForArgumentIncludingThis(stackIndex++));

break;

case TypeKind::F32:

case TypeKind::F64:

case TypeKind::V128:

if (fprIndex < maxFPRIndex)

m_results.append(virtualRegisterForLocal(numberOfLLIntCalleeSaveRegisters + numberOfLLIntInternalRegisters + fprIndex++));

else

m_results.append(virtualRegisterForArgumentIncludingThis(stackIndex++));

break;

case TypeKind::Void:

case TypeKind::Func:

case TypeKind::Struct:

case TypeKind::Structref:

case TypeKind::Array:

case TypeKind::Arrayref:

case TypeKind::Eqref:

case TypeKind::Anyref:

case TypeKind::Nullref:

case TypeKind::Nullfuncref:

case TypeKind::Nullexternref:

case TypeKind::I31ref:

case TypeKind::Rec:

case TypeKind::Sub:

case TypeKind::Subfinal:

RELEASE_ASSERT_NOT_REACHED();

}

}

return m_results;

{

checkConsistency();

m_codeBlock->m_numArguments = signature.as<FunctionSignature>()->argumentCount();

m_normalizedArguments.resize(m_codeBlock->m_numArguments);

const auto& callingConvention = wasmCallingConvention();

const uint32_t gprCount = callingConvention.jsrArgs.size();

const uint32_t fprCount = callingConvention.fprArgs.size();

const uint32_t maxGPRIndex = gprCount;

const uint32_t maxFPRIndex = gprCount + fprCount;

uint32_t gprIndex = 0;

uint32_t fprIndex = maxGPRIndex;

uint32_t stackIndex = 1;

Vector<VirtualRegister> registerArguments(gprCount + fprCount);

for (uint32_t i = 0; i < gprCount + fprCount; i++)

registerArguments[i] = push(NoConsistencyCheck);

const auto addArgument = [&](uint32_t index, uint32_t& count, uint32_t max) {

if (count < max)

m_normalizedArguments[index] = registerArguments[count++];

else

m_normalizedArguments[index] = virtualRegisterForArgumentIncludingThis(stackIndex++);

};

for (uint32_t i = 0; i < signature.as<FunctionSignature>()->argumentCount(); i++) {

switch (signature.as<FunctionSignature>()->argumentType(i).kind) {

case TypeKind::I32:

case TypeKind::I64:

case TypeKind::Externref:

case TypeKind::Funcref:

case TypeKind::RefNull:

case TypeKind::Ref:

addArgument(i, gprIndex, maxGPRIndex);

break;

case TypeKind::F32:

case TypeKind::F64:

case TypeKind::V128:

addArgument(i, fprIndex, maxFPRIndex);

break;

case TypeKind::Void:

case TypeKind::Func:

case TypeKind::Struct:

case TypeKind::Structref:

case TypeKind::Array:

case TypeKind::Arrayref:

case TypeKind::Eqref:

case TypeKind::Anyref:

case TypeKind::Nullref:

case TypeKind::Nullfuncref:

case TypeKind::Nullexternref:

case TypeKind::I31ref:

case TypeKind::Rec:

case TypeKind::Sub:

case TypeKind::Subfinal:

RELEASE_ASSERT_NOT_REACHED();

}

}

m_codeBlock->m_numVars += gprCount + fprCount;

return { };

{

checkConsistency();

m_codeBlock->m_numVars += count;

// All ref-typed locals (funcref, externref, GC types) have to be

// initialized to the JS null value (not 0)

if (isRefType(type)) {

while (count--)

m_uninitializedLocals.append(push(NoConsistencyCheck));

} else

m_stackSize += count;

if (m_maxStackSize < m_stackSize)

m_maxStackSize = m_stackSize;

return { };

{

if (m_uninitializedLocals.isEmpty())

return;

auto null = jsNullConstant();

for (auto local : m_uninitializedLocals)

WasmMov::emit(this, local, null);

m_uninitializedLocals.clear();

{

if (!value)

return zeroConstant();

if (value == JSValue::encode(jsNull()))

return jsNullConstant();

VirtualRegister source(FirstConstantRegisterIndex + m_codeBlock->m_constants.size());

auto result = m_constantMap.add(value, source);

if (!result.isNewEntry)

return result.iterator->value;

m_codeBlock->m_constants.append(value);

if (UNLIKELY(Options::dumpGeneratedWasmBytecodes()))

m_codeBlock->m_constantTypes.append(type);

return source;

{

// leave a hole if we need to materialize the constant

push();

return addConstantWithoutPush(type, value);

{

// leave a hole if we need to materialize the local

push();

result = virtualRegisterForWasmLocal(index);

return { };

{

VirtualRegister target = virtualRegisterForWasmLocal(index);

// If this local is currently on the stack we need to materialize it, otherwise it'll see the new value instead of the old one

walkExpressionStack(m_parser->expressionStack(), [&](TypedExpression& expression, VirtualRegister slot) {

if (expression.value() != target)

return;