WTF_MAKE_TZONE_ALLOCATED(CCallHelpers);

CCallHelpers(CodeBlock* codeBlock = nullptr)

: AssemblyHelpers(codeBlock)

{

}

// Wrapper to encode JSCell GPR into JSValue.

class CellValue {

public:

explicit CellValue(GPRReg gpr)

: m_gpr(gpr)

{

}

GPRReg gpr() const { return m_gpr; }

private:

GPRReg m_gpr;

};

// Base class for constant materializers.

// It offers DerivedClass::materialize and poke functions.

class ConstantMaterializer { };

// The most general helper for setting arguments that fit in a GPR, if you can compute each

// argument without using any argument registers. You usually want one of the setupArguments*()

// methods below instead of this. This thing is most useful if you have *a lot* of arguments.

template<typename Functor>

void setupArgument(unsigned argumentIndex, const Functor& functor)

{

unsigned numberOfRegs = GPRInfo::numberOfArgumentRegisters; // Disguise the constant from clang's tautological compare warning.

if (argumentIndex < numberOfRegs) {

functor(GPRInfo::toArgumentRegister(argumentIndex));

return;

}

functor(GPRInfo::nonArgGPR0);

poke(GPRInfo::nonArgGPR0, POKE_ARGUMENT_OFFSET + argumentIndex - GPRInfo::numberOfArgumentRegisters);

}

enum class ShuffleStatus : uint8_t {

ToMove,

BeingMoved,

Moved

};

template<typename RegType, size_t N, typename RegPair = std::pair<RegType, RegType>>

void emitShuffleMove(Vector<RegPair, N>& moves, Vector<ShuffleStatus, N>& status, unsigned index, RegType scratch)

{

status[index] = ShuffleStatus::BeingMoved;

for (unsigned i = 0; i < moves.size(); i ++) {

if (moves[i].first == moves[index].second) {

ASSERT(i != index);

switch (status[i]) {

case ShuffleStatus::ToMove:

emitShuffleMove(moves, status, i, scratch);

break;

case ShuffleStatus::BeingMoved: {

if constexpr (std::is_same_v<RegType, FPRegisterID>)

moveDouble(moves[i].first, scratch);

else

move(moves[i].first, scratch);

moves[i].first = scratch;

break;

}

case ShuffleStatus::Moved:

break;

}

}

}

if constexpr (std::is_same_v<RegType, FPRegisterID>)

moveDouble(moves[index].first, moves[index].second);

else

move(moves[index].first, moves[index].second);

status[index] = ShuffleStatus::Moved;

}

template<typename RegType, unsigned NumberOfRegisters>

ALWAYS_INLINE void shuffleRegisters(std::array<RegType, NumberOfRegisters> sources, std::array<RegType, NumberOfRegisters> destinations)

{

if (ASSERT_ENABLED) {

RegisterSet set;

for (RegType dest : destinations)

set.add(dest, IgnoreVectors);

ASSERT_WITH_MESSAGE(set.numberOfSetRegisters() == NumberOfRegisters, "Destinations should not be aliased.");

}

using RegPair = std::pair<RegType, RegType>;

Vector<RegPair, NumberOfRegisters> pairs;

// if constexpr avoids warnings when NumberOfRegisters is 0.

if constexpr (NumberOfRegisters > 0) {

for (unsigned i = 0; i < NumberOfRegisters; ++i) {

if (sources[i] != destinations[i])

pairs.append(std::make_pair(sources[i], destinations[i]));

}

} else {

// Silence some older compilers (GCC up to 9.X) about unused but set parameters.

UNUSED_PARAM(sources);

UNUSED_PARAM(destinations);

}

Vector<ShuffleStatus, NumberOfRegisters> status;

status.fill(ShuffleStatus::ToMove, pairs.size());

RELEASE_ASSERT(m_allowScratchRegister); // We absolutely need one for the recursive shuffle algorithm.

auto scratch = scratchRegister();

{

DisallowMacroScratchRegisterUsage disallowScratch(*this);

for (size_t i = 0; i < pairs.size(); i ++) {

if (status[i] == ShuffleStatus::ToMove) {

if constexpr (std::is_same_v<RegType, RegisterID>)

emitShuffleMove(pairs, status, i, scratch);

else

emitShuffleMove(pairs, status, i, fpTempRegister);

}

}

}

}

template<unsigned NumberOfJSRs>

ALWAYS_INLINE void shuffleJSRs(std::array<JSValueRegs, NumberOfJSRs> sources, std::array<JSValueRegs, NumberOfJSRs> destinations)

{

#if USE(JSVALUE64)

constexpr unsigned NumberOfRegisters = NumberOfJSRs;

#else

constexpr unsigned NumberOfRegisters = NumberOfJSRs * 2;

#endif

std::array<GPRReg, NumberOfRegisters> sourceRegs;

std::array<GPRReg, NumberOfRegisters> destinationRegs;

for (unsigned i = 0; i < NumberOfJSRs; ++i) {

sourceRegs[i] = sources[i].payloadGPR();

destinationRegs[i] = destinations[i].payloadGPR();

#if !USE(JSVALUE64)

sourceRegs[i + NumberOfJSRs] = sources[i].tagGPR();

destinationRegs[i + NumberOfJSRs] = destinations[i].tagGPR();

#endif

}

shuffleRegisters<GPRReg, NumberOfRegisters>(sourceRegs, destinationRegs);

}

template<typename RegType>

using InfoTypeForReg = decltype(toInfoFromReg(RegType(-1)));

// extraGPRArgs is used to track 64-bit argument types passed in register on 32-bit architectures.

// extraPoke is used to track 64-bit argument types passed on the stack.

template<unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke>

struct ArgCollection {

ArgCollection()

{

gprSources.fill(InvalidGPRReg);

gprDestinations.fill(InvalidGPRReg);

fprSources.fill(InvalidFPRReg);

fprDestinations.fill(InvalidFPRReg);

crossSources.fill(InvalidFPRReg);

crossDestinations.fill(InvalidGPRReg);

}

template<unsigned a, unsigned b, unsigned c, unsigned d, unsigned e, unsigned f, unsigned g, unsigned h>

ArgCollection(ArgCollection<a, b, c, d, e, f, g, h>& other)

{

gprSources = other.gprSources;

gprDestinations = other.gprDestinations;

fprSources = other.fprSources;

fprDestinations = other.fprDestinations;

crossSources = other.crossSources;

crossDestinations = other.crossDestinations;

}

ArgCollection<numGPRArgs + 1, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> pushRegArg(GPRReg argument, GPRReg destination)

{

ArgCollection<numGPRArgs + 1, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> result(*this);

result.gprSources[numGPRSources] = argument;

result.gprDestinations[numGPRSources] = destination;

return result;

}

ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources + 1, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> pushRegArg(FPRReg argument, FPRReg destination)

{

ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources + 1, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> result(*this);

result.fprSources[numFPRSources] = argument;

result.fprDestinations[numFPRSources] = destination;

return result;

}

ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources, numCrossSources + 1, extraGPRArgs, nonArgGPRs, extraPoke> pushRegArg(FPRReg argument, GPRReg destination)

{

ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources, numCrossSources + 1, extraGPRArgs, nonArgGPRs, extraPoke> result(*this);

result.crossSources[numCrossSources] = argument;

result.crossDestinations[numCrossSources] = destination;

return result;

}

ArgCollection<numGPRArgs, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs + 1, nonArgGPRs, extraPoke> pushExtraRegArg(GPRReg argument, GPRReg destination)

{

ArgCollection<numGPRArgs, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs + 1, nonArgGPRs, extraPoke> result(*this);

result.gprSources[numGPRSources] = argument;

result.gprDestinations[numGPRSources] = destination;

return result;

}

ArgCollection<numGPRArgs, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs + 1, extraPoke> pushNonArg(GPRReg argument, GPRReg destination)

{

ArgCollection<numGPRArgs, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs + 1, extraPoke> result(*this);

result.gprSources[numGPRSources] = argument;

result.gprDestinations[numGPRSources] = destination;

return result;

}

ArgCollection<numGPRArgs + 1, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> addGPRArg()

{

return ArgCollection<numGPRArgs + 1, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke>(*this);

}

ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs + 1, nonArgGPRs, extraPoke> addGPRExtraArg()

{

return ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs + 1, nonArgGPRs, extraPoke>(*this);

}

ArgCollection<numGPRArgs + 1, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> addStackArg(GPRReg)

{

return ArgCollection<numGPRArgs + 1, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke>(*this);

}

ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> addStackArg(FPRReg)

{

return ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke>(*this);

}

ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke + 1> addPoke()

{

return ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke + 1>(*this);

}

unsigned argCount(GPRReg) { return numGPRArgs + extraGPRArgs; }

unsigned argCount(FPRReg) { return numFPRArgs; }

// store GPR -> GPR assignments

std::array<GPRReg, GPRInfo::numberOfRegisters> gprSources;

std::array<GPRReg, GPRInfo::numberOfRegisters> gprDestinations;

// store FPR -> FPR assignments

std::array<FPRReg, FPRInfo::numberOfRegisters> fprSources;

std::array<FPRReg, FPRInfo::numberOfRegisters> fprDestinations;

// store FPR -> GPR assignments

std::array<FPRReg, GPRInfo::numberOfRegisters> crossSources;

std::array<GPRReg, GPRInfo::numberOfRegisters> crossDestinations;

};

template<unsigned TargetSize, typename RegType>

std::array<RegType, TargetSize> clampArrayToSize(std::array<RegType, InfoTypeForReg<RegType>::numberOfRegisters> sourceArray)

{

static_assert(TargetSize <= sourceArray.size(), "TargetSize is bigger than source.size()");

RELEASE_ASSERT(TargetSize <= InfoTypeForReg<RegType>::numberOfRegisters);

std::array<RegType, TargetSize> result { };

// if constexpr avoids warnings when TargetSize is 0.

if constexpr (TargetSize > 0) {

for (unsigned i = 0; i < TargetSize; i++) {

ASSERT(sourceArray[i] != static_cast<int32_t>(InfoTypeForReg<RegType>::InvalidIndex));

result[i] = sourceArray[i];

}

}

return result;

}

ALWAYS_INLINE unsigned calculatePokeOffset(unsigned currentGPRArgument, unsigned currentFPRArgument, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke)

{

// Clang claims that it cannot find the symbol for FPRReg/GPRReg::numberOfArgumentRegisters when they are passed directly to std::max... seems like a bug

unsigned numberOfFPArgumentRegisters = FPRInfo::numberOfArgumentRegisters;

unsigned numberOfGPArgumentRegisters = GPRInfo::numberOfArgumentRegisters;

UNUSED_PARAM(nonArgGPRs);

currentGPRArgument += extraGPRArgs;

currentFPRArgument -= numCrossSources;

IGNORE_WARNINGS_BEGIN("type-limits")

ASSERT(currentGPRArgument >= GPRInfo::numberOfArgumentRegisters || currentFPRArgument >= FPRInfo::numberOfArgumentRegisters);

IGNORE_WARNINGS_END

unsigned pokeOffset = POKE_ARGUMENT_OFFSET + extraPoke;

pokeOffset += std::max(currentGPRArgument, numberOfGPArgumentRegisters) - numberOfGPArgumentRegisters;

pokeOffset += std::max(currentFPRArgument, numberOfFPArgumentRegisters) - numberOfFPArgumentRegisters;

return pokeOffset;

}

template<typename ArgType>

ALWAYS_INLINE void pokeForArgument(ArgType arg, unsigned currentGPRArgument, unsigned currentFPRArgument, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke)

{

unsigned pokeOffset = calculatePokeOffset(currentGPRArgument, currentFPRArgument, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke);

if constexpr (std::derived_from<ArgType, ConstantMaterializer>)

arg.store(*this, addressForPoke(pokeOffset));

else

poke(arg, pokeOffset);

}

ALWAYS_INLINE bool stackAligned(unsigned currentGPRArgument, unsigned currentFPRArgument, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke)

{

unsigned pokeOffset = calculatePokeOffset(currentGPRArgument, currentFPRArgument, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke);

return !(pokeOffset & 1);

}

// In the auto-calling convention code below the order of operations is:

// 1) spill arguments to stack slots

// 2) shuffle incomming argument values in registers to argument registers

// 3) fill immediate values to argument registers

// To do this, we recurse forwards through our args collecting argument values in registers and spilling stack slots.

// when we run out of args we then run our shuffling code to relocate registers. Finally, as we unwind from our

// recursion we can fill immediates.

#define CURRENT_ARGUMENT_TYPE typename FunctionTraits<OperationType>::template ArgumentType<numGPRArgs + numFPRArgs>

#define RESULT_TYPE typename FunctionTraits<OperationType>::ResultType

#if USE(JSVALUE64)

// Avoid MSVC optimization time explosion associated with __forceinline in recursive templates.

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename RegType, typename... Args>

ALWAYS_INLINE void marshallArgumentRegister(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, RegType arg, Args... args)

{

using InfoType = InfoTypeForReg<RegType>;

unsigned numArgRegisters = InfoType::numberOfArgumentRegisters;

unsigned currentArgCount = argSourceRegs.argCount(arg);

if (currentArgCount < numArgRegisters) {

auto updatedArgSourceRegs = argSourceRegs.pushRegArg(arg, InfoType::toArgumentRegister(currentArgCount));

setupArgumentsImpl<OperationType>(updatedArgSourceRegs, args...);

return;

}

pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke);

setupArgumentsImpl<OperationType>(argSourceRegs.addStackArg(arg), args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, FPRReg arg, Args... args)

{

static_assert(std::is_same_v<CURRENT_ARGUMENT_TYPE, double>, "We should only be passing FPRRegs to a double. We use moveDouble / loadDouble / storeDouble exclusively");

marshallArgumentRegister<OperationType>(argSourceRegs, arg, args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, GPRReg arg, Args... args)

{

marshallArgumentRegister<OperationType>(argSourceRegs, arg, args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, JSValueRegs arg, Args... args)

{

marshallArgumentRegister<OperationType>(argSourceRegs, arg.gpr(), args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, CellValue arg, Args... args)

{

marshallArgumentRegister<OperationType>(argSourceRegs, arg.gpr(), args...);

}

#else // USE(JSVALUE64)

#if CPU(ARM_THUMB2)

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, FPRReg arg, Args... args)

{

static_assert(std::is_same_v<CURRENT_ARGUMENT_TYPE, double>, "We should only be passing FPRRegs to a double. We use moveDouble / loadDouble / storeDouble exclusively");

// ARM (hardfp, which we require) passes FP arguments in FP registers.

unsigned numberOfFPArgumentRegisters = FPRInfo::numberOfArgumentRegisters;

unsigned currentFPArgCount = argSourceRegs.argCount(arg);

if (currentFPArgCount < numberOfFPArgumentRegisters) {

auto updatedArgSourceRegs = argSourceRegs.pushRegArg(arg, FPRInfo::toArgumentRegister(currentFPArgCount));

setupArgumentsImpl<OperationType>(updatedArgSourceRegs, args...);

return;

}

// Otherwise pass FP argument on stack.

if (stackAligned(numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke)) {

pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke);

setupArgumentsImpl<OperationType>(argSourceRegs.addStackArg(arg).addPoke(), args...);

} else {

pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke + 1);

setupArgumentsImpl<OperationType>(argSourceRegs.addStackArg(arg).addPoke().addPoke(), args...);

}

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

requires (sizeof(CURRENT_ARGUMENT_TYPE) <= 4)

void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, GPRReg arg, Args... args)

{

unsigned numArgRegisters = GPRInfo::numberOfArgumentRegisters;

unsigned currentArgCount = argSourceRegs.argCount(arg);

if (currentArgCount < numArgRegisters) {

auto updatedArgSourceRegs = argSourceRegs.pushRegArg(arg, GPRInfo::toArgumentRegister(currentArgCount));

setupArgumentsImpl<OperationType>(updatedArgSourceRegs, args...);

return;

}

pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke);

setupArgumentsImpl<OperationType>(argSourceRegs.addStackArg(arg), args...);

}

template<typename OperationType, typename Arg1, typename Arg2, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

void pokeArgumentsAligned(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, Arg1 arg1, Arg2 arg2, Args... args)

{

unsigned numArgRegisters = GPRInfo::numberOfArgumentRegisters;

unsigned currentArgCount = argSourceRegs.argCount(GPRInfo::regT0);

if (currentArgCount + 1 == numArgRegisters) {

pokeForArgument(arg1, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs + 1, nonArgGPRs, extraPoke);

pokeForArgument(arg2, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs + 1, nonArgGPRs, extraPoke + 1);

setupArgumentsImpl<OperationType>(argSourceRegs.addGPRExtraArg().addGPRArg().addPoke(), args...);

} else if (stackAligned(numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke)) {

pokeForArgument(arg1, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke);

pokeForArgument(arg2, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke + 1);

setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg().addPoke(), args...);

} else {

pokeForArgument(arg1, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke + 1);

pokeForArgument(arg2, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke + 2);

setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg().addPoke().addPoke(), args...);

}

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

requires std::same_as<CURRENT_ARGUMENT_TYPE, EncodedJSValue>

void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, CellValue payload, Args... args)

{

unsigned numArgRegisters = GPRInfo::numberOfArgumentRegisters;

unsigned currentArgCount = argSourceRegs.argCount(payload.gpr());

unsigned alignedArgCount = roundUpToMultipleOf<2>(currentArgCount);

if (alignedArgCount + 1 < numArgRegisters) {

auto updatedArgSourceRegs = argSourceRegs.pushRegArg(payload.gpr(), GPRInfo::toArgumentRegister(alignedArgCount));

if (alignedArgCount > currentArgCount)

setupArgumentsImpl<OperationType>(updatedArgSourceRegs.addGPRExtraArg().addGPRExtraArg(), args...);

else

setupArgumentsImpl<OperationType>(updatedArgSourceRegs.addGPRExtraArg(), args...);

move(TrustedImm32(JSValue::CellTag), GPRInfo::toArgumentRegister(alignedArgCount + 1));

} else

pokeArgumentsAligned<OperationType>(argSourceRegs, payload.gpr(), TrustedImm32(JSValue::CellTag), args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

requires std::same_as<CURRENT_ARGUMENT_TYPE, EncodedJSValue>

void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, JSValueRegs arg, Args... args)

{

unsigned numArgRegisters = GPRInfo::numberOfArgumentRegisters;

unsigned currentArgCount = argSourceRegs.argCount(arg.tagGPR());

unsigned alignedArgCount = roundUpToMultipleOf<2>(currentArgCount);

if (alignedArgCount + 1 < numArgRegisters) {

// JSValueRegs is passed in two 32-bit registers on these architectures. Increase both numGPRArgs and extraGPRArgs by 1.

// We can't just add 2 to numGPRArgs, since it is used for CURRENT_ARGUMENT_TYPE. Adding 2 would lead to a skipped argument.

auto updatedArgSourceRegs1 = argSourceRegs.pushRegArg(arg.payloadGPR(), GPRInfo::toArgumentRegister(alignedArgCount));

auto updatedArgSourceRegs2 = updatedArgSourceRegs1.pushExtraRegArg(arg.tagGPR(), GPRInfo::toArgumentRegister(alignedArgCount + 1));

if (alignedArgCount > currentArgCount)

setupArgumentsImpl<OperationType>(updatedArgSourceRegs2.addGPRExtraArg(), args...);

else

setupArgumentsImpl<OperationType>(updatedArgSourceRegs2, args...);

} else

pokeArgumentsAligned<OperationType>(argSourceRegs, arg.payloadGPR(), arg.tagGPR(), args...);

}

#endif // CPU(ARM_THUMB2)

#endif // USE(JSVALUE64)

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename Arg, typename... Args>

requires DerivedFromOrConvertibleTo<Arg, TrustedImm> // DerivedFromOrConvertibleTo instead of derived_from since DFGSpeculativeJIT has its own implementation of TrustedImmPtr

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, Arg arg, Args... args)

{

// Right now this only supports non-floating point immediate arguments since we never call operations with non-register values.

// If we ever needed to support immediate floating point arguments we would need to duplicate this logic for both types, which sounds

// gross so it's probably better to do that marshalling before the call operation...

static_assert(!std::is_floating_point_v<CURRENT_ARGUMENT_TYPE>, "We don't support immediate floats/doubles in setupArguments");

auto numArgRegisters = GPRInfo::numberOfArgumentRegisters;

auto currentArgCount = numGPRArgs + extraGPRArgs;

if (currentArgCount < numArgRegisters) {

setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg(), args...);

move(arg, GPRInfo::toArgumentRegister(currentArgCount));

return;

}

pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke);

setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg(), args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename Arg, typename... Args>

requires (std::same_as<CURRENT_ARGUMENT_TYPE, Arg> && std::integral<CURRENT_ARGUMENT_TYPE> && sizeof(CURRENT_ARGUMENT_TYPE) <= 4)

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, Arg arg, Args... args)

{

setupArgumentsImpl<OperationType>(argSourceRegs, TrustedImm32(arg), args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename Arg, typename... Args>

requires (std::same_as<CURRENT_ARGUMENT_TYPE, Arg> && std::integral<CURRENT_ARGUMENT_TYPE> && sizeof(CURRENT_ARGUMENT_TYPE) == 8)

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, Arg arg, Args... args)

{

setupArgumentsImpl<OperationType>(argSourceRegs, TrustedImm64(arg), args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename Arg, typename... Args>

requires (std::is_pointer_v<CURRENT_ARGUMENT_TYPE> && std::same_as<Arg, std::nullptr_t>)

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, Arg arg, Args... args)

{

setupArgumentsImpl<OperationType>(argSourceRegs, TrustedImmPtr(arg), args...);

}

// Special case DFG::RegisteredStructure because it's really annoying to deal with otherwise...

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename Arg, typename... Args>

requires (std::same_as<CURRENT_ARGUMENT_TYPE, Structure*> && std::same_as<Arg, DFG::RegisteredStructure>)

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, Arg arg, Args... args)

{

setupArgumentsImpl<OperationType>(argSourceRegs, TrustedImmPtr(arg.get()), args...);

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename Arg, typename... Args>

requires std::derived_from<Arg, ConstantMaterializer>

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, Arg arg, Args... args)

{

static_assert(!std::is_floating_point_v<CURRENT_ARGUMENT_TYPE>, "We don't support immediate floats/doubles in setupArguments");

auto numArgRegisters = GPRInfo::numberOfArgumentRegisters;

auto currentArgCount = numGPRArgs + extraGPRArgs;

if (currentArgCount < numArgRegisters) {

setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg(), args...);

arg.materialize(*this, GPRInfo::toArgumentRegister(currentArgCount));

return;

}

pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke);

setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg(), args...);

}

template<typename OperationType, unsigned gprIndex>

constexpr void finalizeGPRArguments(std::index_sequence<>)

{

}

template<typename OperationType, unsigned gprIndex, size_t arityIndex, size_t... remainingArityIndices>

constexpr void finalizeGPRArguments(std::index_sequence<arityIndex, remainingArityIndices...>)

{

using NextIndexSequenceType = std::index_sequence<remainingArityIndices...>;

using ArgumentType = typename FunctionTraits<OperationType>::template ArgumentType<arityIndex>;

// Every non-double-typed argument should be passed in through GPRRegs, and inversely only double-typed

// arguments should be passed through FPRRegs. This is asserted in the invocation of the lastly-called

// setupArgumentsImpl(ArgCollection<>) overload, by matching the number of handled GPR and FPR arguments

// with the corresponding count of properly-typed arguments for this operation.

if constexpr (!std::is_same_v<ArgumentType, double>) {

// RV64 calling convention requires all 32-bit values to be sign-extended into the whole register.

// JSC JIT is tailored for other ISAs that pass these values in 32-bit-wide registers, which RISC-V

// doesn't support, so any 32-bit value passed in argument registers has to be manually sign-extended.

if (isRISCV64() && gprIndex < GPRInfo::numberOfArgumentRegisters

&& std::is_integral_v<ArgumentType> && sizeof(ArgumentType) == 4) {

GPRReg argReg = GPRInfo::toArgumentRegister(gprIndex);

signExtend32ToPtr(argReg, argReg);

}

finalizeGPRArguments<OperationType, gprIndex + 1>(NextIndexSequenceType());

} else

finalizeGPRArguments<OperationType, gprIndex>(NextIndexSequenceType());

}

template<typename ArgType> using GPRArgCountValue = std::conditional_t<std::is_same_v<ArgType, double>,

std::integral_constant<unsigned, 0>, std::integral_constant<unsigned, 1>>;

template<typename ArgType> using FPRArgCountValue = std::conditional_t<std::is_same_v<ArgType, double>,

std::integral_constant<unsigned, 1>, std::integral_constant<unsigned, 0>>;

template<typename OperationTraitsType, size_t... argIndices>

static constexpr unsigned gprArgsCount(std::index_sequence<argIndices...>)

{

return (0 + ... + (GPRArgCountValue<typename OperationTraitsType::template ArgumentType<argIndices>>::value));

}

template<typename OperationTraitsType, size_t... argIndices>

static constexpr unsigned fprArgsCount(std::index_sequence<argIndices...>)

{

return (0 + ... + (FPRArgCountValue<typename OperationTraitsType::template ArgumentType<argIndices>>::value));

}

// Base case; set up the argument registers.

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke>

ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs)

{

using TraitsType = FunctionTraits<OperationType>;

static_assert(TraitsType::arity == numGPRArgs + numFPRArgs, "One last sanity check");

#if USE(JSVALUE64)

static_assert(TraitsType::cCallArity() == numGPRArgs + numFPRArgs + extraPoke, "Check the CCall arity");

#endif

static_assert(gprArgsCount<TraitsType>(std::make_index_sequence<TraitsType::arity>()) == numGPRArgs);

static_assert(fprArgsCount<TraitsType>(std::make_index_sequence<TraitsType::arity>()) == numFPRArgs);

shuffleRegisters<GPRReg, numGPRSources>(clampArrayToSize<numGPRSources, GPRReg>(argSourceRegs.gprSources), clampArrayToSize<numGPRSources, GPRReg>(argSourceRegs.gprDestinations));

static_assert(!numCrossSources, "shouldn't be used on this architecture.");

shuffleRegisters<FPRReg, numFPRSources>(clampArrayToSize<numFPRSources, FPRReg>(argSourceRegs.fprSources), clampArrayToSize<numFPRSources, FPRReg>(argSourceRegs.fprDestinations));

}

template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned nonArgGPRs, unsigned extraPoke, typename... Args>

ALWAYS_INLINE void setupArgumentsEntryImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, nonArgGPRs, extraPoke> argSourceRegs, Args... args)

{

using FirstArgumentType = typename FunctionTraits<OperationType>::template ArgumentType<0>;

if constexpr (std::same_as<FirstArgumentType, CallFrame*>) {

#if USE(JSVALUE64)

// This only really works for 64-bit since jsvalue regs mess things up for 32-bit...

static_assert(FunctionTraits<OperationType>::cCallArity() == sizeof...(Args) + 1, "Basic sanity check; Did you explicitly pass callFrameRegister for the first argument?");

#endif

setupArgumentsImpl<OperationType>(argSourceRegs, GPRInfo::callFrameRegister, args...);

} else {

#if USE(JSVALUE64)

// This only really works for 64-bit since jsvalue regs mess things up for 32-bit...

static_assert(FunctionTraits<OperationType>::cCallArity() == sizeof...(Args), "Basic sanity check");

#endif

setupArgumentsImpl<OperationType>(argSourceRegs, args...);

}

finalizeGPRArguments<OperationType, 0>(std::make_index_sequence<FunctionTraits<OperationType>::arity>());

}

#undef CURRENT_ARGUMENT_TYPE

#undef RESULT_TYPE

template<typename OperationType, typename... Args>

ALWAYS_INLINE void setupArguments(Args... args)

{

setupArgumentsEntryImpl<OperationType>(ArgCollection<0, 0, 0, 0, 0, 0, 0, 0>(), args...);

}

template<typename OperationType, typename... Args>

ALWAYS_INLINE void setupArgumentsForIndirectCall(GPRReg functionGPR, Args... args)

{

setupArgumentsEntryImpl<OperationType>(ArgCollection<0, 0, 0, 0, 0, 0, 0, 0>().pushNonArg(functionGPR, GPRInfo::nonArgGPR0), args...);

}

template<typename OperationType, typename... Args>

ALWAYS_INLINE void setupArgumentsForIndirectCall(Address address, Args... args)

{

setupArgumentsEntryImpl<OperationType>(ArgCollection<0, 0, 0, 0, 0, 0, 0, 0>().pushNonArg(address.base, GPRInfo::nonArgGPR0), args...);

}

void setupResults(GPRReg destA, GPRReg destB = InvalidGPRReg)

{

GPRReg srcA = GPRInfo::returnValueGPR;

GPRReg srcB = GPRInfo::returnValueGPR2;

if (destA == InvalidGPRReg)

move(srcB, destB);

else if (destB == InvalidGPRReg)

move(srcA, destA);

else if (srcB != destA) {

// Handle the easy cases - two simple moves.

move(srcA, destA);

move(srcB, destB);

} else if (srcA != destB) {

// Handle the non-swap case - just put srcB in place first.

move(srcB, destB);

move(srcA, destA);

} else

swap(destA, destB);

}

void setupResults(JSValueRegs regs)

{

#if USE(JSVALUE64)

move(GPRInfo::returnValueGPR, regs.gpr());

#else

setupResults(regs.payloadGPR(), regs.tagGPR());

#endif

}

void setupResults(FPRReg destA)

{

if (destA != InvalidFPRReg)

moveDouble(FPRInfo::returnValueFPR, destA);

}

void jumpToExceptionHandler(VM& vm)

{

// genericUnwind() leaves the handler CallFrame* in vm->callFrameForCatch,

// and the address of the handler in vm->targetMachinePCForThrow.

loadPtr(&vm.targetMachinePCForThrow, GPRInfo::regT1);

farJump(GPRInfo::regT1, ExceptionHandlerPtrTag);

}

#if USE(JSVALUE64)

template<typename T>

requires (isExceptionOperationResult<T>)

static constexpr GPRReg operationExceptionRegister()

{

static_assert(assertNotOperationSignature<T>);

if constexpr (std::is_floating_point_v<typename T::ResultType> || std::is_same_v<typename T::ResultType, void>)

return GPRInfo::returnValueGPR;

else

return GPRInfo::returnValueGPR2;

}

#else

template<typename T>

requires (isExceptionOperationResult<T>)

static constexpr GPRReg operationExceptionRegister()

{

static_assert(assertNotOperationSignature<T>);

if constexpr (std::is_same_v<T, ExceptionOperationResult<void>>)

return GPRInfo::returnValueGPR;

else

return GPRInfo::returnValueGPR2;

}

#endif

template<typename T>

requires (!isExceptionOperationResult<T>)

static constexpr GPRReg operationExceptionRegister()

{

static_assert(assertNotOperationSignature<T>);

return InvalidGPRReg;

}

void prepareForTailCallSlow(RegisterSet preserved = { })

{

GPRReg temp1 = selectScratchGPR(preserved);

preserved.add(temp1, IgnoreVectors);

GPRReg temp2 = selectScratchGPR(preserved);

preserved.add(temp2, IgnoreVectors);

#if CPU(ARM64E)

GPRReg temp3 = selectScratchGPR(preserved);

preserved.add(temp3, IgnoreVectors);

#endif

ASSERT(!preserved.numberOfSetFPRs());

GPRReg newFramePointer = temp1;

GPRReg newFrameSizeGPR = temp2;

{

// The old frame size is its number of arguments (or number of

// parameters in case of arity fixup), plus the frame header size,

// aligned

GPRReg oldFrameSizeGPR = temp2;

{

GPRReg argCountGPR = oldFrameSizeGPR;

load32(Address(framePointerRegister, CallFrameSlot::argumentCountIncludingThis * static_cast<int>(sizeof(Register)) + PayloadOffset), argCountGPR);

{

GPRReg numParametersGPR = temp1;

{

GPRReg codeBlockGPR = numParametersGPR;

loadPtr(Address(framePointerRegister, CallFrameSlot::codeBlock * static_cast<int>(sizeof(Register))), codeBlockGPR);

load32(Address(codeBlockGPR, CodeBlock::offsetOfNumParameters()), numParametersGPR);

}

ASSERT(numParametersGPR != argCountGPR);

Jump argumentCountWasNotFixedUp = branch32(BelowOrEqual, numParametersGPR, argCountGPR);

move(numParametersGPR, argCountGPR);

argumentCountWasNotFixedUp.link(this);

}

add32(TrustedImm32(stackAlignmentRegisters() + CallFrame::headerSizeInRegisters - 1), argCountGPR, oldFrameSizeGPR);

and32(TrustedImm32(-stackAlignmentRegisters()), oldFrameSizeGPR);

// We assume < 2^28 arguments

mul32(TrustedImm32(sizeof(Register)), oldFrameSizeGPR, oldFrameSizeGPR);

}

// The new frame pointer is at framePointer + oldFrameSize - newFrameSize

ASSERT(newFramePointer != oldFrameSizeGPR);

addPtr(framePointerRegister, oldFrameSizeGPR, newFramePointer);

// The new frame size is just the number of arguments plus the

// frame header size, aligned

ASSERT(newFrameSizeGPR != newFramePointer);

load32(Address(stackPointerRegister, CallFrameSlot::argumentCountIncludingThis * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)),

newFrameSizeGPR);

add32(TrustedImm32(stackAlignmentRegisters() + CallFrame::headerSizeInRegisters - 1), newFrameSizeGPR);

and32(TrustedImm32(-stackAlignmentRegisters()), newFrameSizeGPR);

// We assume < 2^28 arguments

mul32(TrustedImm32(sizeof(Register)), newFrameSizeGPR, newFrameSizeGPR);

}

// We don't need the current frame beyond this point. Masquerade as our

// caller.

#if CPU(ARM_THUMB2) || CPU(ARM64) || CPU(RISCV64)

loadPtr(Address(framePointerRegister, CallFrame::returnPCOffset()), linkRegister);

subPtr(TrustedImm32(2 * sizeof(void*)), newFrameSizeGPR);

#if CPU(ARM64E)

GPRReg tempGPR = temp3;

ASSERT(tempGPR != newFramePointer && tempGPR != newFrameSizeGPR);

addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), MacroAssembler::framePointerRegister, tempGPR);

untagPtr(tempGPR, linkRegister);

validateUntaggedPtr(linkRegister, tempGPR);

#endif

#elif CPU(X86_64)

push(Address(framePointerRegister, sizeof(void*)));

subPtr(TrustedImm32(sizeof(void*)), newFrameSizeGPR);

#else

UNREACHABLE_FOR_PLATFORM();

#endif

subPtr(newFrameSizeGPR, newFramePointer);

loadPtr(Address(framePointerRegister), framePointerRegister);

// We need to move the newFrameSizeGPR slots above the stack pointer by

// newFramePointer registers. We use pointer-sized chunks.

MacroAssembler::Label copyLoop(label());

subPtr(TrustedImm32(sizeof(void*)), newFrameSizeGPR);

transferPtr(BaseIndex(stackPointerRegister, newFrameSizeGPR, TimesOne), BaseIndex(newFramePointer, newFrameSizeGPR, TimesOne));

branchTest32(MacroAssembler::NonZero, newFrameSizeGPR).linkTo(copyLoop, this);

// Ready for a jump!

move(newFramePointer, stackPointerRegister);

}

static Address addressOfCalleeCalleeFromCallerPerspective(int offset)

{

CCallHelpers::Address calleeFrame = CCallHelpers::Address(MacroAssembler::stackPointerRegister, 0);

return calleeFrame.withOffset(CallFrameSlot::callee * static_cast<int>(sizeof(Register)))

// calleeFrame is from the caller's perspective

.withOffset(-safeCast<int>(sizeof(CallerFrameAndPC)))

.withOffset(PayloadOffset)

.withOffset(offset);

}

// This function is used to store the wasm callee in case this function hasn't tiered up yet.

// The IPInt is going to expect this so that the common entrypoint can read bytecode/metadata.

void storeWasmCalleeToCalleeCallFrame(RegisterID value, int offset = 0)

{

JIT_COMMENT(*this, "< Store Callee's wasm callee");

auto addr = CCallHelpers::addressOfCalleeCalleeFromCallerPerspective(offset);

#if USE(JSVALUE64)

storePtr(value, addr);

#elif USE(JSVALUE32_64)

store32(value, addr.withOffset(PayloadOffset));

store32(TrustedImm32(JSValue::NativeCalleeTag), addr.withOffset(TagOffset));

#else

#error "Unsupported configuration"

#endif

}

void storeWasmCalleeToCalleeCallFrame(CalleeBits boxedCallee)

{

JIT_COMMENT(*this, "> ", RawPointer(boxedCallee.asNativeCallee()));

storeWasmCalleeToCalleeCallFrame(TrustedImmPtr(boxedCallee.rawPtr()));

}

void storeWasmCalleeToCalleeCallFrame(TrustedImmPtr imm, int offset = 0)

{

JIT_COMMENT(*this, "Store Callee's wasm callee");

auto addr = CCallHelpers::addressOfCalleeCalleeFromCallerPerspective(offset);

#if USE(JSVALUE64)

store64(imm, addr);

#elif USE(JSVALUE32_64)

move(imm, scratchRegister());

store32(scratchRegister(), addr.withOffset(PayloadOffset));

store32(TrustedImm32(JSValue::NativeCalleeTag), addr.withOffset(TagOffset));

#else

#error "Unsupported configuration"

#endif

}

// These operations clobber all volatile registers. They assume that there is room on the top of

// stack to marshall call arguments.

void logShadowChickenProloguePacket(GPRReg shadowPacket, GPRReg scratch1, GPRReg scope);

template <typename CodeBlockType>

void logShadowChickenTailPacketImpl(GPRReg shadowPacket, JSValueRegs thisRegs, GPRReg scope, CodeBlockType codeBlock, CallSiteIndex callSiteIndex);

void logShadowChickenTailPacket(GPRReg shadowPacket, JSValueRegs thisRegs, GPRReg scope, GPRReg codeBlock, CallSiteIndex callSiteIndex);

// Leaves behind a pointer to the Packet we should write to in shadowPacket.

void ensureShadowChickenPacket(VM&, GPRReg shadowPacket, GPRReg scratch1NonArgGPR, GPRReg scratch2);

void emitCTIThunkPrologue(bool returnAddressAlreadyTagged = false);

void emitCTIThunkEpilogue();