LowerToAir(Procedure& procedure)

: m_valueToTmp(procedure.values().size())

, m_phiToTmp(procedure.values().size())

, m_blockToBlock(procedure.size())

, m_useCounts(procedure)

, m_phiChildren(procedure)

, m_dominators(procedure.dominators())

, m_procedure(procedure)

, m_code(procedure.code())

{

}

void run()

{

for (B3::BasicBlock* block : m_procedure)

m_blockToBlock[block] = m_code.addBlock(block->frequency());

for (Value* value : m_procedure.values()) {

switch (value->opcode()) {

case Phi: {

m_phiToTmp[value] = m_code.newTmp(Arg::typeForB3Type(value->type()));

if (verbose)

dataLog("Phi tmp for ", *value, ": ", m_phiToTmp[value], "\n");

break;

}

default:

break;

}

}

for (B3::StackSlot* stack : m_procedure.stackSlots())

m_stackToStack.add(stack, m_code.addStackSlot(stack));

for (Variable* variable : m_procedure.variables())

m_variableToTmp.add(variable, m_code.newTmp(Arg::typeForB3Type(variable->type())));

// Figure out which blocks are not rare.

m_fastWorklist.push(m_procedure[0]);

while (B3::BasicBlock* block = m_fastWorklist.pop()) {

for (B3::FrequentedBlock& successor : block->successors()) {

if (!successor.isRare())

m_fastWorklist.push(successor.block());

}

}

m_procedure.resetValueOwners(); // Used by crossesInterference().

// Lower defs before uses on a global level. This is a good heuristic to lock down a

// hoisted address expression before we duplicate it back into the loop.

for (B3::BasicBlock* block : m_procedure.blocksInPreOrder()) {

m_block = block;

// Reset some state.

m_insts.resize(0);

m_isRare = !m_fastWorklist.saw(block);

if (verbose)

dataLog("Lowering Block ", *block, ":\n");

// Process blocks in reverse order so we see uses before defs. That's what allows us

// to match patterns effectively.

for (unsigned i = block->size(); i--;) {

m_index = i;

m_value = block->at(i);

if (m_locked.contains(m_value))

continue;

m_insts.append(Vector<Inst>());

if (verbose)

dataLog("Lowering ", deepDump(m_procedure, m_value), ":\n");

lower();

if (verbose) {

for (Inst& inst : m_insts.last())

dataLog(" ", inst, "\n");

}

}

// Now append the instructions. m_insts contains them in reverse order, so we process

// it in reverse.

for (unsigned i = m_insts.size(); i--;) {

for (Inst& inst : m_insts[i])

m_blockToBlock[block]->appendInst(WTFMove(inst));

}

// Make sure that the successors are set up correctly.

ASSERT(block->successors().size() <= 2);

for (B3::FrequentedBlock successor : block->successors()) {

m_blockToBlock[block]->successors().append(

Air::FrequentedBlock(m_blockToBlock[successor.block()], successor.frequency()));

}

}

Air::InsertionSet insertionSet(m_code);

for (Inst& inst : m_prologue)

insertionSet.insertInst(0, WTFMove(inst));

insertionSet.execute(m_code[0]);

}

bool shouldCopyPropagate(Value* value)

{

switch (value->opcode()) {

case Trunc:

case Identity:

return true;

default:

return false;

}

}

class ArgPromise {

public:

ArgPromise() { }

ArgPromise(const Arg& arg, Value* valueToLock = nullptr)

: m_arg(arg)

, m_value(valueToLock)

{

}

static ArgPromise tmp(Value* value)

{

ArgPromise result;

result.m_value = value;

return result;

}

explicit operator bool() const { return m_arg || m_value; }

Arg::Kind kind() const

{

if (!m_arg && m_value)

return Arg::Tmp;

return m_arg.kind();

}

const Arg& peek() const

{

return m_arg;

}

Arg consume(LowerToAir& lower) const

{

if (!m_arg && m_value)

return lower.tmp(m_value);

if (m_value)

lower.commitInternal(m_value);

return m_arg;

}

private:

// Three forms:

// Everything null: invalid.

// Arg non-null, value null: just use the arg, nothing special.

// Arg null, value non-null: it's a tmp, pin it when necessary.

// Arg non-null, value non-null: use the arg, lock the value.

Arg m_arg;

Value* m_value;

};

// Consider using tmpPromise() in cases where you aren't sure that you want to pin the value yet.

// Here are three canonical ways of using tmp() and tmpPromise():

//

// Idiom #1: You know that you want a tmp() and you know that it will be valid for the

// instruction you're emitting.

//

// append(Foo, tmp(bar));

//

// Idiom #2: You don't know if you want to use a tmp() because you haven't determined if the

// instruction will accept it, so you query first. Note that the call to tmp() happens only after

// you are sure that you will use it.

//

// if (isValidForm(Foo, Arg::Tmp))

// append(Foo, tmp(bar))

//

// Idiom #3: Same as Idiom #2, but using tmpPromise. Notice that this calls consume() only after

// it's sure it will use the tmp. That's deliberate.

//

// ArgPromise promise = tmpPromise(bar);

// if (isValidForm(Foo, promise.kind()))

// append(Foo, promise.consume(*this))

//

// In both idiom #2 and idiom #3, we don't pin the value to a temporary except when we actually

// emit the instruction. Both tmp() and tmpPromise().consume(*this) will pin it. Pinning means

// that we will henceforth require that the value of 'bar' is generated as a separate

// instruction. We don't want to pin the value to a temporary if we might change our minds, and

// pass an address operand representing 'bar' to Foo instead.

//

// Because tmp() pins, the following is not an idiom you should use:

//

// Tmp tmp = this->tmp(bar);

// if (isValidForm(Foo, tmp.kind()))

// append(Foo, tmp);

//

// That's because if isValidForm() returns false, you will have already pinned the 'bar' to a

// temporary. You might later want to try to do something like loadPromise(), and that will fail.

// This arises in operations that have both a Addr,Tmp and Tmp,Addr forms. The following code

// seems right, but will actually fail to ever match the Tmp,Addr form because by then, the right

// value is already pinned.

//

// auto tryThings = [this] (const Arg& left, const Arg& right) {

// if (isValidForm(Foo, left.kind(), right.kind()))

// return Inst(Foo, m_value, left, right);

// return Inst();

// };

// if (Inst result = tryThings(loadAddr(left), tmp(right)))

// return result;

// if (Inst result = tryThings(tmp(left), loadAddr(right))) // this never succeeds.

// return result;

// return Inst(Foo, m_value, tmp(left), tmp(right));

//

// If you imagine that loadAddr(value) is just loadPromise(value).consume(*this), then this code

// will run correctly - it will generate OK code - but the second form is never matched.

// loadAddr(right) will never succeed because it will observe that 'right' is already pinned.

// Of course, it's exactly because of the risky nature of such code that we don't have a

// loadAddr() helper and require you to balance ArgPromise's in code like this. Such code will

// work fine if written as:

//

// auto tryThings = [this] (const ArgPromise& left, const ArgPromise& right) {

// if (isValidForm(Foo, left.kind(), right.kind()))

// return Inst(Foo, m_value, left.consume(*this), right.consume(*this));

// return Inst();

// };

// if (Inst result = tryThings(loadPromise(left), tmpPromise(right)))

// return result;

// if (Inst result = tryThings(tmpPromise(left), loadPromise(right)))

// return result;

// return Inst(Foo, m_value, tmp(left), tmp(right));

//

// Notice that we did use tmp in the fall-back case at the end, because by then, we know for sure

// that we want a tmp. But using tmpPromise in the tryThings() calls ensures that doing so

// doesn't prevent us from trying loadPromise on the same value.

Tmp tmp(Value* value)

{

Tmp& tmp = m_valueToTmp[value];

if (!tmp) {

while (shouldCopyPropagate(value))

value = value->child(0);

if (value->opcode() == FramePointer)

return Tmp(GPRInfo::callFrameRegister);

Tmp& realTmp = m_valueToTmp[value];

if (!realTmp) {

realTmp = m_code.newTmp(Arg::typeForB3Type(value->type()));

if (m_procedure.isFastConstant(value->key()))

m_code.addFastTmp(realTmp);

if (verbose)

dataLog("Tmp for ", *value, ": ", realTmp, "\n");

}

tmp = realTmp;

}

return tmp;

}

ArgPromise tmpPromise(Value* value)

{

return ArgPromise::tmp(value);

}

bool canBeInternal(Value* value)

{

// If one of the internal things has already been computed, then we don't want to cause

// it to be recomputed again.

if (m_valueToTmp[value])

return false;

// We require internals to have only one use - us. It's not clear if this should be numUses() or

// numUsingInstructions(). Ideally, it would be numUsingInstructions(), except that it's not clear

// if we'd actually do the right thing when matching over such a DAG pattern. For now, it simply

// doesn't matter because we don't implement patterns that would trigger this.

if (m_useCounts.numUses(value) != 1)

return false;

return true;

}

// If you ask canBeInternal() and then construct something from that, and you commit to emitting

// that code, then you must commitInternal() on that value. This is tricky, and you only need to

// do it if you're pattern matching by hand rather than using the patterns language. Long story

// short, you should avoid this by using the pattern matcher to match patterns.

void commitInternal(Value* value)

{

m_locked.add(value);

}

bool crossesInterference(Value* value)

{

// If it's in a foreign block, then be conservative. We could handle this if we were

// willing to do heavier analysis. For example, if we had liveness, then we could label

// values as "crossing interference" if they interfere with anything that they are live

// across. But, it's not clear how useful this would be.

if (value->owner != m_value->owner)

return true;

Effects effects = value->effects();

for (unsigned i = m_index; i--;) {

Value* otherValue = m_block->at(i);

if (otherValue == value)

return false;

if (effects.interferes(otherValue->effects()))

return true;

}

ASSERT_NOT_REACHED();

return true;

}

// This turns the given operand into an address.

Arg effectiveAddr(Value* address, int32_t offset, Arg::Width width)

{

ASSERT(Arg::isValidAddrForm(offset, width));

auto fallback = [&] () -> Arg {

return Arg::addr(tmp(address), offset);

};

static const unsigned lotsOfUses = 10; // This is arbitrary and we should tune it eventually.

// Only match if the address value isn't used in some large number of places.

if (m_useCounts.numUses(address) > lotsOfUses)

return fallback();

switch (address->opcode()) {

case Add: {

Value* left = address->child(0);

Value* right = address->child(1);

auto tryIndex = [&] (Value* index, Value* base) -> Arg {

if (index->opcode() != Shl)

return Arg();

if (m_locked.contains(index->child(0)) || m_locked.contains(base))

return Arg();

if (!index->child(1)->hasInt32())

return Arg();

unsigned scale = 1 << (index->child(1)->asInt32() & 31);

if (!Arg::isValidIndexForm(scale, offset, width))

return Arg();

return Arg::index(tmp(base), tmp(index->child(0)), scale, offset);

};

if (Arg result = tryIndex(left, right))

return result;

if (Arg result = tryIndex(right, left))

return result;

if (m_locked.contains(left) || m_locked.contains(right)

|| !Arg::isValidIndexForm(1, offset, width))

return fallback();

return Arg::index(tmp(left), tmp(right), 1, offset);

}

case Shl: {

Value* left = address->child(0);

// We'll never see child(1)->isInt32(0), since that would have been reduced. If the shift

// amount is greater than 1, then there isn't really anything smart that we could do here.

// We avoid using baseless indexes because their encoding isn't particularly efficient.

if (m_locked.contains(left) || !address->child(1)->isInt32(1)

|| !Arg::isValidIndexForm(1, offset, width))

return fallback();

return Arg::index(tmp(left), tmp(left), 1, offset);

}

case FramePointer:

return Arg::addr(Tmp(GPRInfo::callFrameRegister), offset);

case SlotBase:

return Arg::stack(m_stackToStack.get(address->as<SlotBaseValue>()->slot()), offset);

default:

return fallback();

}

}

// This gives you the address of the given Load or Store. If it's not a Load or Store, then

// it returns Arg().

Arg addr(Value* memoryValue)

{

MemoryValue* value = memoryValue->as<MemoryValue>();

if (!value)

return Arg();

int32_t offset = value->offset();

Arg::Width width = Arg::widthForBytes(value->accessByteSize());

Arg result = effectiveAddr(value->lastChild(), offset, width);

ASSERT(result.isValidForm(width));

return result;

}

ArgPromise loadPromiseAnyOpcode(Value* loadValue)

{

if (!canBeInternal(loadValue))

return Arg();

if (crossesInterference(loadValue))

return Arg();

return ArgPromise(addr(loadValue), loadValue);

}

ArgPromise loadPromise(Value* loadValue, B3::Opcode loadOpcode)

{

if (loadValue->opcode() != loadOpcode)

return Arg();

return loadPromiseAnyOpcode(loadValue);

}

ArgPromise loadPromise(Value* loadValue)

{

return loadPromise(loadValue, Load);

}

Arg imm(Value* value)

{

if (value->hasInt()) {

int64_t intValue = value->asInt();

if (Arg::isValidImmForm(intValue))

return Arg::imm(intValue);

}

return Arg();

}

Arg bitImm(Value* value)

{

if (value->hasInt()) {

int64_t intValue = value->asInt();

if (Arg::isValidBitImmForm(intValue))

return Arg::bitImm(intValue);

}

return Arg();

}

Arg bitImm64(Value* value)

{

if (value->hasInt()) {

int64_t intValue = value->asInt();

if (Arg::isValidBitImm64Form(intValue))

return Arg::bitImm64(intValue);

}

return Arg();

}

Arg immOrTmp(Value* value)

{

if (Arg result = imm(value))

return result;

return tmp(value);

}

// By convention, we use Oops to mean "I don't know".

Air::Opcode tryOpcodeForType(

Air::Opcode opcode32, Air::Opcode opcode64, Air::Opcode opcodeDouble, Air::Opcode opcodeFloat, Type type)

{

Air::Opcode opcode;

switch (type) {

case Int32:

opcode = opcode32;

break;

case Int64:

opcode = opcode64;

break;

case Float:

opcode = opcodeFloat;

break;

case Double:

opcode = opcodeDouble;

break;

default:

opcode = Air::Oops;

break;

}

return opcode;

}

Air::Opcode tryOpcodeForType(Air::Opcode opcode32, Air::Opcode opcode64, Type type)

{

return tryOpcodeForType(opcode32, opcode64, Air::Oops, Air::Oops, type);

}

Air::Opcode opcodeForType(

Air::Opcode opcode32, Air::Opcode opcode64, Air::Opcode opcodeDouble, Air::Opcode opcodeFloat, Type type)

{

Air::Opcode opcode = tryOpcodeForType(opcode32, opcode64, opcodeDouble, opcodeFloat, type);

RELEASE_ASSERT(opcode != Air::Oops);

return opcode;

}

Air::Opcode opcodeForType(Air::Opcode opcode32, Air::Opcode opcode64, Type type)

{

return tryOpcodeForType(opcode32, opcode64, Air::Oops, Air::Oops, type);

}

template<Air::Opcode opcode32, Air::Opcode opcode64, Air::Opcode opcodeDouble = Air::Oops, Air::Opcode opcodeFloat = Air::Oops>

void appendUnOp(Value* value)

{

Air::Opcode opcode = opcodeForType(opcode32, opcode64, opcodeDouble, opcodeFloat, value->type());

Tmp result = tmp(m_value);

// Two operand forms like:

// Op a, b

// mean something like:

// b = Op a

ArgPromise addr = loadPromise(value);

if (isValidForm(opcode, addr.kind(), Arg::Tmp)) {

append(opcode, addr.consume(*this), result);

return;

}

if (isValidForm(opcode, Arg::Tmp, Arg::Tmp)) {

append(opcode, tmp(value), result);

return;

}

ASSERT(value->type() == m_value->type());

append(relaxedMoveForType(m_value->type()), tmp(value), result);

append(opcode, result);

}

// Call this method when doing two-operand lowering of a commutative operation. You have a choice of

// which incoming Value is moved into the result. This will select which one is likely to be most

// profitable to use as the result. Doing the right thing can have big performance consequences in tight

// kernels.

bool preferRightForResult(Value* left, Value* right)

{

// The default is to move left into result, because that's required for non-commutative instructions.

// The value that we want to move into result position is the one that dies here. So, if we're

// compiling a commutative operation and we know that actually right is the one that dies right here,

// then we can flip things around to help coalescing, which then kills the move instruction.

//

// But it's more complicated:

// - Used-once is a bad estimate of whether the variable dies here.

// - A child might be a candidate for coalescing with this value.

//

// Currently, we have machinery in place to recognize super obvious forms of the latter issue.

// We recognize when a child is a Phi that has this value as one of its children. We're very

// conservative about this; for example we don't even consider transitive Phi children.

bool leftIsPhiWithThis = m_phiChildren[left].transitivelyUses(m_value);

bool rightIsPhiWithThis = m_phiChildren[right].transitivelyUses(m_value);

if (leftIsPhiWithThis != rightIsPhiWithThis)

return rightIsPhiWithThis;

if (m_useCounts.numUsingInstructions(right) != 1)

return false;

if (m_useCounts.numUsingInstructions(left) != 1)

return true;

// The use count might be 1 if the variable is live around a loop. We can guarantee that we

// pick the the variable that is least likely to suffer this problem if we pick the one that

// is closest to us in an idom walk. By convention, we slightly bias this in favor of

// returning true.

// We cannot prefer right if right is further away in an idom walk.

if (m_dominators.strictlyDominates(right->owner, left->owner))

return false;

return true;

}

template<Air::Opcode opcode32, Air::Opcode opcode64, Air::Opcode opcodeDouble, Air::Opcode opcodeFloat, Commutativity commutativity = NotCommutative>

void appendBinOp(Value* left, Value* right)

{

Air::Opcode opcode = opcodeForType(opcode32, opcode64, opcodeDouble, opcodeFloat, left->type());

Tmp result = tmp(m_value);

// Three-operand forms like:

// Op a, b, c

// mean something like:

// c = a Op b

if (isValidForm(opcode, Arg::Imm, Arg::Tmp, Arg::Tmp)) {

if (commutativity == Commutative) {

if (imm(right)) {

append(opcode, imm(right), tmp(left), result);

return;

}

} else {

// A non-commutative operation could have an immediate in left.

if (imm(left)) {

append(opcode, imm(left), tmp(right), result);

return;

}

}

}

if (isValidForm(opcode, Arg::BitImm, Arg::Tmp, Arg::Tmp)) {

if (commutativity == Commutative) {

if (Arg rightArg = bitImm(right)) {

append(opcode, rightArg, tmp(left), result);

return;

}

} else {

// A non-commutative operation could have an immediate in left.

if (Arg leftArg = bitImm(left)) {

append(opcode, leftArg, tmp(right), result);

return;

}

}

}

if (isValidForm(opcode, Arg::BitImm64, Arg::Tmp, Arg::Tmp)) {

if (commutativity == Commutative) {

if (Arg rightArg = bitImm64(right)) {

append(opcode, rightArg, tmp(left), result);

return;

}

} else {

// A non-commutative operation could have an immediate in left.

if (Arg leftArg = bitImm64(left)) {

append(opcode, leftArg, tmp(right), result);

return;

}

}

}

if (imm(right) && isValidForm(opcode, Arg::Tmp, Arg::Imm, Arg::Tmp)) {

append(opcode, tmp(left), imm(right), result);

return;

}

// Note that no extant architecture has a three-operand form of binary operations that also

// load from memory. If such an abomination did exist, we would handle it somewhere around

// here.

// Two-operand forms like:

// Op a, b

// mean something like:

// b = b Op a

// At this point, we prefer versions of the operation that have a fused load or an immediate

// over three operand forms.

if (left != right) {

if (commutativity == Commutative) {

ArgPromise leftAddr = loadPromise(left);

if (isValidForm(opcode, leftAddr.kind(), Arg::Tmp)) {

append(relaxedMoveForType(m_value->type()), tmp(right), result);

append(opcode, leftAddr.consume(*this), result);

return;

}

}

ArgPromise rightAddr = loadPromise(right);

if (isValidForm(opcode, rightAddr.kind(), Arg::Tmp)) {

append(relaxedMoveForType(m_value->type()), tmp(left), result);

append(opcode, rightAddr.consume(*this), result);

return;

}

}

if (imm(right) && isValidForm(opcode, Arg::Imm, Arg::Tmp)) {

append(relaxedMoveForType(m_value->type()), tmp(left), result);

append(opcode, imm(right), result);

return;

}

if (isValidForm(opcode, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {

append(opcode, tmp(left), tmp(right), result);

return;

}

if (commutativity == Commutative && preferRightForResult(left, right)) {

append(relaxedMoveForType(m_value->type()), tmp(right), result);

append(opcode, tmp(left), result);

return;

}

append(relaxedMoveForType(m_value->type()), tmp(left), result);

append(opcode, tmp(right), result);

}

template<Air::Opcode opcode32, Air::Opcode opcode64, Commutativity commutativity = NotCommutative>

void appendBinOp(Value* left, Value* right)

{

appendBinOp<opcode32, opcode64, Air::Oops, Air::Oops, commutativity>(left, right);

}

template<Air::Opcode opcode32, Air::Opcode opcode64>

void appendShift(Value* value, Value* amount)

{

Air::Opcode opcode = opcodeForType(opcode32, opcode64, value->type());

if (imm(amount)) {

if (isValidForm(opcode, Arg::Tmp, Arg::Imm, Arg::Tmp)) {

append(opcode, tmp(value), imm(amount), tmp(m_value));

return;

}

if (isValidForm(opcode, Arg::Imm, Arg::Tmp)) {

append(Move, tmp(value), tmp(m_value));

append(opcode, imm(amount), tmp(m_value));

return;

}

}

if (isValidForm(opcode, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {

append(opcode, tmp(value), tmp(amount), tmp(m_value));

return;

}

#if CPU(X86) || CPU(X86_64)

append(Move, tmp(value), tmp(m_value));

append(Move, tmp(amount), Tmp(X86Registers::ecx));

append(opcode, Tmp(X86Registers::ecx), tmp(m_value));

#endif

}

template<Air::Opcode opcode32, Air::Opcode opcode64>

bool tryAppendStoreUnOp(Value* value)

{

Air::Opcode opcode = tryOpcodeForType(opcode32, opcode64, value->type());

if (opcode == Air::Oops)

return false;

Arg storeAddr = addr(m_value);

ASSERT(storeAddr);

ArgPromise loadPromise = this->loadPromise(value);

if (loadPromise.peek() != storeAddr)

return false;

if (!isValidForm(opcode, storeAddr.kind()))

return false;

loadPromise.consume(*this);

append(opcode, storeAddr);

return true;

}

template<

Air::Opcode opcode32, Air::Opcode opcode64, Commutativity commutativity = NotCommutative>

bool tryAppendStoreBinOp(Value* left, Value* right)

{

Air::Opcode opcode = tryOpcodeForType(opcode32, opcode64, left->type());

if (opcode == Air::Oops)

return false;

Arg storeAddr = addr(m_value);

ASSERT(storeAddr);

auto getLoadPromise = [&] (Value* load) -> ArgPromise {

switch (m_value->opcode()) {

case B3::Store:

if (load->opcode() != B3::Load)

return ArgPromise();

break;

case B3::Store8:

if (load->opcode() != B3::Load8Z && load->opcode() != B3::Load8S)

return ArgPromise();

break;

case B3::Store16:

if (load->opcode() != B3::Load16Z && load->opcode() != B3::Load16S)

return ArgPromise();

break;

default:

return ArgPromise();

}

return loadPromiseAnyOpcode(load);

};

ArgPromise loadPromise;

Value* otherValue = nullptr;

loadPromise = getLoadPromise(left);

if (loadPromise.peek() == storeAddr)

otherValue = right;

else if (commutativity == Commutative) {

loadPromise = getLoadPromise(right);

if (loadPromise.peek() == storeAddr)

otherValue = left;

}

if (!otherValue)

return false;

if (isValidForm(opcode, Arg::Imm, storeAddr.kind()) && imm(otherValue)) {

loadPromise.consume(*this);

append(opcode, imm(otherValue), storeAddr);

return true;

}

if (!isValidForm(opcode, Arg::Tmp, storeAddr.kind()))

return false;

loadPromise.consume(*this);

append(opcode, tmp(otherValue), storeAddr);

return true;

}

Inst createStore(Air::Opcode move, Value* value, const Arg& dest)

{

if (imm(value) && isValidForm(move, Arg::Imm, dest.kind()))

return Inst(move, m_value, imm(value), dest);

return Inst(move, m_value, tmp(value), dest);

}

Inst createStore(Value* value, const Arg& dest)

{

Air::Opcode moveOpcode = moveForType(value->type());

return createStore(moveOpcode, value, dest);

}

void appendStore(Value* value, const Arg& dest)

{

m_insts.last().append(createStore(value, dest));

}

Air::Opcode moveForType(Type type)

{

switch (type) {

case Int32:

return Move32;

case Int64:

RELEASE_ASSERT(is64Bit());

return Move;

case Float:

return MoveFloat;

case Double:

return MoveDouble;

case Void:

break;

}

RELEASE_ASSERT_NOT_REACHED();

return Air::Oops;

}

Air::Opcode relaxedMoveForType(Type type)

{

switch (type) {

case Int32:

case Int64:

// For Int32, we could return Move or Move32. It's a trade-off.

//

// Move32: Using Move32 guarantees that we use the narrower move, but in cases where the

// register allocator can't prove that the variables involved are 32-bit, this will

// disable coalescing.

//

// Move: Using Move guarantees that the register allocator can coalesce normally, but in

// cases where it can't prove that the variables are 32-bit and it doesn't coalesce,

// this will force us to use a full 64-bit Move instead of the slightly cheaper

// 32-bit Move32.

//

// Coalescing is a lot more profitable than turning Move into Move32. So, it's better to

// use Move here because in cases where the register allocator cannot prove that

// everything is 32-bit, we still get coalescing.

return Move;

case Float:

// MoveFloat is always coalescable and we never convert MoveDouble to MoveFloat, so we

// should use MoveFloat when we know that the temporaries involved are 32-bit.

return MoveFloat;

case Double:

return MoveDouble;

case Void:

break;

}

RELEASE_ASSERT_NOT_REACHED();

return Air::Oops;

}

template<typename... Arguments>

void append(Air::Opcode opcode, Arguments&&... arguments)

{

m_insts.last().append(Inst(opcode, m_value, std::forward<Arguments>(arguments)...));

}

template<typename T, typename... Arguments>

T* ensureSpecial(T*& field, Arguments&&... arguments)

{

if (!field) {

field = static_cast<T*>(

m_code.addSpecial(std::make_unique<T>(std::forward<Arguments>(arguments)...)));

}

return field;

}

template<typename... Arguments>

CheckSpecial* ensureCheckSpecial(Arguments&&... arguments)

{

CheckSpecial::Key key(std::forward<Arguments>(arguments)...);

auto result = m_checkSpecials.add(key, nullptr);

return ensureSpecial(result.iterator->value, key);

}

void fillStackmap(Inst& inst, StackmapValue* stackmap, unsigned numSkipped)

{

for (unsigned i = numSkipped; i < stackmap->numChildren(); ++i) {

ConstrainedValue value = stackmap->constrainedChild(i);

Arg arg;

switch (value.rep().kind()) {

case ValueRep::WarmAny:

case ValueRep::ColdAny:

case ValueRep::LateColdAny:

if (imm(value.value()))

arg = imm(value.value());

else if (value.value()->hasInt64())

arg = Arg::bigImm(value.value()->asInt64());

else if (value.value()->hasDouble() && canBeInternal(value.value())) {

commitInternal(value.value());

arg = Arg::bigImm(bitwise_cast<int64_t>(value.value()->asDouble()));

} else

arg = tmp(value.value());

break;

case ValueRep::SomeRegister:

arg = tmp(value.value());

break;

case ValueRep::Register:

stackmap->earlyClobbered().clear(value.rep().reg());

arg = Tmp(value.rep().reg());

append(relaxedMoveForType(value.value()->type()), immOrTmp(value.value()), arg);

break;

case ValueRep::StackArgument:

arg = Arg::callArg(value.rep().offsetFromSP());

appendStore(value.value(), arg);

break;

default: