CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)

{

dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);

if (nodeIndex != UINT_MAX)

dataLog(", node @", nodeIndex);

dataLog(".\n");

CRASH();

WTF_MAKE_NONCOPYABLE(LowerDFGToB3);

LowerDFGToB3(State& state)

: m_graph(state.graph)

, m_ftlState(state)

, m_out(state)

, m_proc(*state.proc)

, m_state(state.graph)

, m_interpreter(state.graph, m_state)

{

}

void lower()

{

State* state = &m_ftlState;

CString name;

if (verboseCompilationEnabled()) {

name = toCString(

"jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),

"_", codeBlock()->hash());

} else

name = "jsBody";

m_graph.ensureDominators();

if (verboseCompilationEnabled())

dataLog("Function ready, beginning lowering.\n");

m_out.initialize(m_heaps);

// We use prologue frequency for all of the initialization code.

m_out.setFrequency(1);

m_prologue = m_out.newBlock();

LBasicBlock stackOverflow = m_out.newBlock();

m_handleExceptions = m_out.newBlock();

LBasicBlock checkArguments = m_out.newBlock();

for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {

m_highBlock = m_graph.block(blockIndex);

if (!m_highBlock)

continue;

m_out.setFrequency(m_highBlock->executionCount);

m_blocks.add(m_highBlock, m_out.newBlock());

}

// Back to prologue frequency for any bocks that get sneakily created in the initialization code.

m_out.setFrequency(1);

m_out.appendTo(m_prologue, stackOverflow);

m_out.initializeConstants(m_proc, m_prologue);

createPhiVariables();

size_t sizeOfCaptured = sizeof(JSValue) * m_graph.m_nextMachineLocal;

B3::SlotBaseValue* capturedBase = m_out.lockedStackSlot(sizeOfCaptured);

m_captured = m_out.add(capturedBase, m_out.constIntPtr(sizeOfCaptured));

state->capturedValue = capturedBase->slot();

auto preOrder = m_graph.blocksInPreOrder();

// We should not create any alloca's after this point, since they will cease to

// be mem2reg candidates.

m_callFrame = m_out.framePointer();

m_tagTypeNumber = m_out.constInt64(TagTypeNumber);

m_tagMask = m_out.constInt64(TagMask);

// Make sure that B3 knows that we really care about the mask registers. This forces the

// constants to be materialized in registers.

m_proc.addFastConstant(m_tagTypeNumber->key());

m_proc.addFastConstant(m_tagMask->key());

m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));

m_out.branch(

didOverflowStack(), rarely(stackOverflow), usually(checkArguments));

m_out.appendTo(stackOverflow, m_handleExceptions);

m_out.call(m_out.voidType, m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));

m_out.patchpoint(Void)->setGenerator(

[=] (CCallHelpers& jit, const StackmapGenerationParams&) {

// We are terminal, so we can clobber everything. That's why we don't claim to

// clobber scratch.

AllowMacroScratchRegisterUsage allowScratch(jit);

jit.copyCalleeSavesToVMCalleeSavesBuffer();

jit.move(CCallHelpers::TrustedImmPtr(jit.vm()), GPRInfo::argumentGPR0);

jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);

CCallHelpers::Call call = jit.call();

jit.jumpToExceptionHandler();

jit.addLinkTask(

[=] (LinkBuffer& linkBuffer) {

linkBuffer.link(call, FunctionPtr(lookupExceptionHandlerFromCallerFrame));

});

});

m_out.unreachable();

m_out.appendTo(m_handleExceptions, checkArguments);

Box<CCallHelpers::Label> exceptionHandler = state->exceptionHandler;

m_out.patchpoint(Void)->setGenerator(

[=] (CCallHelpers& jit, const StackmapGenerationParams&) {

CCallHelpers::Jump jump = jit.jump();

jit.addLinkTask(

[=] (LinkBuffer& linkBuffer) {

linkBuffer.link(jump, linkBuffer.locationOf(*exceptionHandler));

});

});

m_out.unreachable();

m_out.appendTo(checkArguments, lowBlock(m_graph.block(0)));

availabilityMap().clear();

availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);

for (unsigned i = codeBlock()->numParameters(); i--;) {

availabilityMap().m_locals.argument(i) =

Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));

}

m_node = nullptr;

m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);

for (unsigned i = codeBlock()->numParameters(); i--;) {

Node* node = m_graph.m_arguments[i];

VirtualRegister operand = virtualRegisterForArgument(i);

LValue jsValue = m_out.load64(addressFor(operand));

if (node) {

DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal);

// This is a hack, but it's an effective one. It allows us to do CSE on the

// primordial load of arguments. This assumes that the GetLocal that got put in

// place of the original SetArgument doesn't have any effects before it. This

// should hold true.

m_loadedArgumentValues.add(node, jsValue);

}

switch (m_graph.m_argumentFormats[i]) {

case FlushedInt32:

speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue));

break;

case FlushedBoolean:

speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue));

break;

case FlushedCell:

speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue));

break;

case FlushedJSValue:

break;

default:

DFG_CRASH(m_graph, node, "Bad flush format for argument");

break;

}

}

m_out.jump(lowBlock(m_graph.block(0)));

for (DFG::BasicBlock* block : preOrder)

compileBlock(block);

// Make sure everything is decorated. This does a bunch of deferred decorating. This has

// to happen last because our abstract heaps are generated lazily. They have to be

// generated lazily because we have an infiniten number of numbered, indexed, and

// absolute heaps. We only become aware of the ones we actually mention while lowering.

m_heaps.computeRangesAndDecorateInstructions();

// We create all Phi's up front, but we may then decide not to compile the basic block

// that would have contained one of them. So this creates orphans, which triggers B3

// validation failures. Calling this fixes the issue.

//

// Note that you should avoid the temptation to make this call conditional upon

// validation being enabled. B3 makes no guarantees of any kind of correctness when

// dealing with IR that would have failed validation. For example, it would be valid to

// write a B3 phase that so aggressively assumes the lack of orphans that it would crash

// if any orphans were around. We might even have such phases already.

m_proc.deleteOrphans();

// We put the blocks into the B3 procedure in a super weird order. Now we reorder them.

m_out.applyBlockOrder();

}

void createPhiVariables()

{

for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {

DFG::BasicBlock* block = m_graph.block(blockIndex);

if (!block)

continue;

for (unsigned nodeIndex = block->size(); nodeIndex--;) {

Node* node = block->at(nodeIndex);

if (node->op() != DFG::Phi)

continue;

LType type;

switch (node->flags() & NodeResultMask) {

case NodeResultDouble:

type = m_out.doubleType;

break;

case NodeResultInt32:

type = m_out.int32;

break;

case NodeResultInt52:

type = m_out.int64;

break;

case NodeResultBoolean:

type = m_out.boolean;

break;

case NodeResultJS:

type = m_out.int64;

break;

default:

DFG_CRASH(m_graph, node, "Bad Phi node result type");

break;

}

m_phis.add(node, m_proc.add<Value>(B3::Phi, type, Origin(node)));

}

}

}

void compileBlock(DFG::BasicBlock* block)

{

if (!block)

return;

if (verboseCompilationEnabled())

dataLog("Compiling block ", *block, "\n");

m_highBlock = block;

// Make sure that any blocks created while lowering code in the high block have the frequency of

// the high block. This is appropriate because B3 doesn't need precise frequencies. It just needs

// something roughly approximate for things like register allocation.

m_out.setFrequency(m_highBlock->executionCount);

LBasicBlock lowBlock = m_blocks.get(m_highBlock);

m_nextHighBlock = 0;

for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {

m_nextHighBlock = m_graph.block(nextBlockIndex);

if (m_nextHighBlock)

break;

}

m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;

// All of this effort to find the next block gives us the ability to keep the

// generated IR in roughly program order. This ought not affect the performance

// of the generated code (since we expect B3 to reorder things) but it will

// make IR dumps easier to read.

m_out.appendTo(lowBlock, m_nextLowBlock);

if (Options::ftlCrashes())

m_out.trap();

if (!m_highBlock->cfaHasVisited) {

if (verboseCompilationEnabled())

dataLog("Bailing because CFA didn't reach.\n");

crash(m_highBlock->index, UINT_MAX);

return;

}

m_availabilityCalculator.beginBlock(m_highBlock);

m_state.reset();

m_state.beginBasicBlock(m_highBlock);

for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {

if (!compileNode(m_nodeIndex))

break;

}

}

void safelyInvalidateAfterTermination()

{

if (verboseCompilationEnabled())

dataLog("Bailing.\n");

crash();

// Invalidate dominated blocks. Under normal circumstances we would expect

// them to be invalidated already. But you can have the CFA become more

// precise over time because the structures of objects change on the main

// thread. Failing to do this would result in weird crashes due to a value

// being used but not defined. Race conditions FTW!

for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {

DFG::BasicBlock* target = m_graph.block(blockIndex);

if (!target)

continue;

if (m_graph.m_dominators->dominates(m_highBlock, target)) {

if (verboseCompilationEnabled())

dataLog("Block ", *target, " will bail also.\n");

target->cfaHasVisited = false;

}

}

}

bool compileNode(unsigned nodeIndex)

{

if (!m_state.isValid()) {

safelyInvalidateAfterTermination();

return false;

}

m_node = m_highBlock->at(nodeIndex);

m_origin = m_node->origin;

m_out.setOrigin(m_node);

if (verboseCompilationEnabled())

dataLog("Lowering ", m_node, "\n");

m_availableRecoveries.resize(0);

m_interpreter.startExecuting();

switch (m_node->op()) {

case DFG::Upsilon:

compileUpsilon();

break;

case DFG::Phi:

compilePhi();

break;

case JSConstant:

break;

case DoubleConstant:

compileDoubleConstant();

break;

case Int52Constant:

compileInt52Constant();

break;

case DoubleRep:

compileDoubleRep();

break;

case DoubleAsInt32:

compileDoubleAsInt32();

break;

case DFG::ValueRep:

compileValueRep();

break;

case Int52Rep:

compileInt52Rep();

break;

case ValueToInt32:

compileValueToInt32();

break;

case BooleanToNumber:

compileBooleanToNumber();

break;

case ExtractOSREntryLocal:

compileExtractOSREntryLocal();

break;

case GetStack:

compileGetStack();

break;

case PutStack:

compilePutStack();

break;

case DFG::Check:

compileNoOp();

break;

case ToThis:

compileToThis();

break;

case ValueAdd:

compileValueAdd();

break;

case StrCat:

compileStrCat();

break;

case ArithAdd:

case ArithSub:

compileArithAddOrSub();

break;

case ArithClz32:

compileArithClz32();

break;

case ArithMul:

compileArithMul();

break;

case ArithDiv:

compileArithDiv();

break;

case ArithMod:

compileArithMod();

break;

case ArithMin:

case ArithMax:

compileArithMinOrMax();

break;

case ArithAbs:

compileArithAbs();

break;

case ArithSin:

compileArithSin();

break;

case ArithCos:

compileArithCos();

break;

case ArithPow:

compileArithPow();

break;

case ArithRandom:

compileArithRandom();

break;

case ArithRound:

compileArithRound();

break;

case ArithFloor:

compileArithFloor();

break;

case ArithCeil:

compileArithCeil();

break;

case ArithSqrt:

compileArithSqrt();

break;

case ArithLog:

compileArithLog();

break;

case ArithFRound:

compileArithFRound();

break;

case ArithNegate:

compileArithNegate();

break;

case DFG::BitAnd:

compileBitAnd();

break;

case DFG::BitOr:

compileBitOr();

break;

case DFG::BitXor:

compileBitXor();

break;

case BitRShift:

compileBitRShift();

break;

case BitLShift:

compileBitLShift();

break;

case BitURShift:

compileBitURShift();

break;

case UInt32ToNumber:

compileUInt32ToNumber();

break;

case CheckStructure:

compileCheckStructure();

break;

case CheckCell:

compileCheckCell();

break;

case CheckNotEmpty:

compileCheckNotEmpty();

break;

case CheckBadCell:

compileCheckBadCell();

break;

case CheckIdent:

compileCheckIdent();

break;

case GetExecutable:

compileGetExecutable();

break;

case ArrayifyToStructure:

compileArrayifyToStructure();

break;

case PutStructure:

compilePutStructure();

break;

case GetById:

case GetByIdFlush:

compileGetById();

break;

case In:

compileIn();

break;

case PutById:

case PutByIdDirect:

case PutByIdFlush:

compilePutById();

break;

case PutGetterById:

case PutSetterById:

compilePutAccessorById();

break;

case PutGetterSetterById:

compilePutGetterSetterById();

break;

case PutGetterByVal:

case PutSetterByVal:

compilePutAccessorByVal();

break;

case GetButterfly:

compileGetButterfly();

break;

case GetButterflyReadOnly:

compileGetButterflyReadOnly();

break;

case ConstantStoragePointer:

compileConstantStoragePointer();

break;

case GetIndexedPropertyStorage:

compileGetIndexedPropertyStorage();

break;

case CheckArray:

compileCheckArray();

break;

case GetArrayLength:

compileGetArrayLength();

break;

case CheckInBounds:

compileCheckInBounds();

break;

case GetByVal:

compileGetByVal();

break;

case GetMyArgumentByVal:

compileGetMyArgumentByVal();

break;

case PutByVal:

case PutByValAlias:

case PutByValDirect:

compilePutByVal();

break;

case ArrayPush:

compileArrayPush();

break;

case ArrayPop:

compileArrayPop();

break;

case CreateActivation:

compileCreateActivation();

break;

case NewFunction:

case NewArrowFunction:

case NewGeneratorFunction:

compileNewFunction();

break;

case CreateDirectArguments:

compileCreateDirectArguments();

break;

case CreateScopedArguments:

compileCreateScopedArguments();

break;

case CreateClonedArguments:

compileCreateClonedArguments();

break;

case NewObject:

compileNewObject();

break;

case NewArray:

compileNewArray();

break;

case NewArrayBuffer:

compileNewArrayBuffer();

break;

case NewArrayWithSize:

compileNewArrayWithSize();

break;

case NewTypedArray:

compileNewTypedArray();

break;

case GetTypedArrayByteOffset:

compileGetTypedArrayByteOffset();

break;

case AllocatePropertyStorage:

compileAllocatePropertyStorage();

break;

case ReallocatePropertyStorage:

compileReallocatePropertyStorage();

break;

case ToString:

case CallStringConstructor:

compileToStringOrCallStringConstructor();

break;

case ToPrimitive:

compileToPrimitive();

break;

case MakeRope:

compileMakeRope();

break;

case StringCharAt:

compileStringCharAt();

break;

case StringCharCodeAt:

compileStringCharCodeAt();

break;

case StringFromCharCode:

compileStringFromCharCode();

break;

case GetByOffset:

case GetGetterSetterByOffset:

compileGetByOffset();

break;

case GetGetter:

compileGetGetter();

break;

case GetSetter:

compileGetSetter();

break;

case MultiGetByOffset:

compileMultiGetByOffset();

break;

case PutByOffset:

compilePutByOffset();

break;

case MultiPutByOffset:

compileMultiPutByOffset();

break;

case GetGlobalVar:

case GetGlobalLexicalVariable:

compileGetGlobalVariable();

break;

case PutGlobalVariable:

compilePutGlobalVariable();

break;

case NotifyWrite:

compileNotifyWrite();

break;

case GetCallee:

compileGetCallee();

break;

case GetArgumentCount:

compileGetArgumentCount();

break;

case GetScope:

compileGetScope();

break;

case SkipScope:

compileSkipScope();

break;

case GetClosureVar:

compileGetClosureVar();

break;

case PutClosureVar:

compilePutClosureVar();

break;

case GetFromArguments:

compileGetFromArguments();

break;

case PutToArguments:

compilePutToArguments();

break;

case CompareEq:

compileCompareEq();

break;

case CompareStrictEq:

compileCompareStrictEq();

break;

case CompareLess:

compileCompareLess();

break;

case CompareLessEq:

compileCompareLessEq();

break;

case CompareGreater:

compileCompareGreater();

break;

case CompareGreaterEq:

compileCompareGreaterEq();

break;

case LogicalNot:

compileLogicalNot();

break;

case Call:

case TailCallInlinedCaller:

case Construct:

compileCallOrConstruct();

break;

case TailCall:

compileTailCall();

break;

case CallVarargs:

case CallForwardVarargs:

case TailCallVarargs:

case TailCallVarargsInlinedCaller:

case TailCallForwardVarargs:

case TailCallForwardVarargsInlinedCaller:

case ConstructVarargs:

case ConstructForwardVarargs:

compileCallOrConstructVarargs();

break;

case LoadVarargs:

compileLoadVarargs();

break;

case ForwardVarargs:

compileForwardVarargs();

break;

case DFG::Jump:

compileJump();

break;

case DFG::Branch:

compileBranch();

break;

case DFG::Switch:

compileSwitch();

break;

case DFG::Return:

compileReturn();

break;

case ForceOSRExit:

compileForceOSRExit();

break;

case Throw:

case ThrowReferenceError:

compileThrow();

break;

case InvalidationPoint:

compileInvalidationPoint();

break;

case IsUndefined:

compileIsUndefined();

break;

case IsBoolean:

compileIsBoolean();

break;

case IsNumber:

compileIsNumber();

break;

case IsString:

compileIsString();

break;

case IsObject:

compileIsObject();

break;

case IsObjectOrNull:

compileIsObjectOrNull();

break;

case IsFunction:

compileIsFunction();

break;

case TypeOf:

compileTypeOf();

break;

case CheckTypeInfoFlags:

compileCheckTypeInfoFlags();

break;

case OverridesHasInstance:

compileOverridesHasInstance();

break;

case InstanceOf:

compileInstanceOf();

break;

case InstanceOfCustom:

compileInstanceOfCustom();

break;

case CountExecution:

compileCountExecution();

break;

case StoreBarrier:

compileStoreBarrier();

break;

case HasIndexedProperty:

compileHasIndexedProperty();

break;

case HasGenericProperty:

compileHasGenericProperty();

break;

case HasStructureProperty:

compileHasStructureProperty();

break;

case GetDirectPname:

compileGetDirectPname();

break;

case GetEnumerableLength:

compileGetEnumerableLength();

break;

case GetPropertyEnumerator:

compileGetPropertyEnumerator();

break;

case GetEnumeratorStructurePname:

compileGetEnumeratorStructurePname();

break;

case GetEnumeratorGenericPname:

compileGetEnumeratorGenericPname();

break;

case ToIndexString:

compileToIndexString();

break;

case CheckStructureImmediate:

compileCheckStructureImmediate();

break;

case MaterializeNewObject:

compileMaterializeNewObject();

break;

case MaterializeCreateActivation:

compileMaterializeCreateActivation();

break;

case CheckWatchdogTimer:

compileCheckWatchdogTimer();

break;

case CopyRest:

compileCopyRest();

break;

case GetRestLength:

compileGetRestLength();

break;

case RegExpExec:

compileRegExpExec();

break;

case RegExpTest:

compileRegExpTest();

break;

case NewRegexp:

compileNewRegexp();

break;

case StringReplace:

compileStringReplace();

break;

case PhantomLocal:

case LoopHint:

case MovHint:

case ZombieHint:

case ExitOK:

case PhantomNewObject:

case PhantomNewFunction:

case PhantomNewGeneratorFunction:

case PhantomCreateActivation:

case PhantomDirectArguments:

case PhantomClonedArguments:

case PutHint:

case BottomValue:

case KillStack:

break;

default:

DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");

break;

}

if (m_node->isTerminal())

return false;

if (!m_state.isValid()) {

safelyInvalidateAfterTermination();

return false;

}

m_availabilityCalculator.executeNode(m_node);

m_interpreter.executeEffects(nodeIndex);

return true;

}

void compileUpsilon()

{

LValue upsilonValue = nullptr;

switch (m_node->child1().useKind()) {

case DoubleRepUse:

upsilonValue = lowDouble(m_node->child1());

break;

case Int32Use:

case KnownInt32Use:

upsilonValue = lowInt32(m_node->child1());

break;

case Int52RepUse:

upsilonValue = lowInt52(m_node->child1());

break;

case BooleanUse:

case KnownBooleanUse:

upsilonValue = lowBoolean(m_node->child1());

break;

case CellUse:

case KnownCellUse:

upsilonValue = lowCell(m_node->child1());

break;

case UntypedUse:

upsilonValue = lowJSValue(m_node->child1());

break;

default:

DFG_CRASH(m_graph, m_node, "Bad use kind");

break;

}

ValueFromBlock upsilon = m_out.anchor(upsilonValue);

LValue phiNode = m_phis.get(m_node->phi());

m_out.addIncomingToPhi(phiNode, upsilon);

}

void compilePhi()

{

LValue phi = m_phis.get(m_node);