Skip to content

Commit ff283f7

Browse files
expatdannoCommit bot
expatdanno
authored and
Commit bot
committed
[turbofan] Better and more sane support for tail calls
* Limit triggering of tail calls to explicit use of a new inline runtime function %_TailCall. %_TailCall works just like %_Call except for using tail-calling mechanics (currently only in TF). * Remove hack that recognized some specific usages of %_Call and converted them into tail calls. * Support tail calls for all calls where the number of callee stack parameters is less than or equal to the number of caller stack parameters. * Use the gap resolver to swizzle parameters and registers to tail calls. BUG=v8:4076 LOG=n Review URL: https://codereview.chromium.org/1439613003 Cr-Commit-Position: refs/heads/master@{#31987}
1 parent c42f188 commit ff283f7

18 files changed

Lines changed: 252 additions & 119 deletions

‎src/compiler/arm/code-generator-arm.cc‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -354,12 +354,16 @@ Condition FlagsConditionToCondition(FlagsCondition condition) {
354354
} while (0)
355355

356356

357-
void CodeGenerator::AssembleDeconstructActivationRecord() {
357+
void CodeGenerator::AssembleDeconstructActivationRecord(int stack_param_delta) {
358358
CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
359359
int stack_slots = frame()->GetSpillSlotCount();
360360
if (descriptor->IsJSFunctionCall() || stack_slots > 0) {
361361
__ LeaveFrame(StackFrame::MANUAL);
362362
}
363+
if (stack_param_delta < 0) {
364+
int offset = -stack_param_delta * kPointerSize;
365+
__ add(sp, sp, Operand(offset));
366+
}
363367
}
364368

365369

@@ -385,7 +389,8 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
385389
break;
386390
}
387391
case kArchTailCallCodeObject: {
388-
AssembleDeconstructActivationRecord();
392+
int stack_param_delta = i.InputInt32(instr->InputCount() - 1);
393+
AssembleDeconstructActivationRecord(stack_param_delta);
389394
if (instr->InputAt(0)->IsImmediate()) {
390395
__ Jump(Handle<Code>::cast(i.InputHeapObject(0)),
391396
RelocInfo::CODE_TARGET);
@@ -420,7 +425,8 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
420425
__ cmp(cp, kScratchReg);
421426
__ Assert(eq, kWrongFunctionContext);
422427
}
423-
AssembleDeconstructActivationRecord();
428+
int stack_param_delta = i.InputInt32(instr->InputCount() - 1);
429+
AssembleDeconstructActivationRecord(stack_param_delta);
424430
__ ldr(ip, FieldMemOperand(func, JSFunction::kCodeEntryOffset));
425431
__ Jump(ip);
426432
DCHECK_EQ(LeaveCC, i.OutputSBit());

‎src/compiler/arm64/code-generator-arm64.cc‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -448,13 +448,17 @@ Condition FlagsConditionToCondition(FlagsCondition condition) {
448448
} while (0)
449449

450450

451-
void CodeGenerator::AssembleDeconstructActivationRecord() {
451+
void CodeGenerator::AssembleDeconstructActivationRecord(int stack_param_delta) {
452452
CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
453453
int stack_slots = frame()->GetSpillSlotCount();
454454
if (descriptor->IsJSFunctionCall() || stack_slots > 0) {
455455
__ Mov(jssp, fp);
456456
__ Pop(fp, lr);
457457
}
458+
if (stack_param_delta < 0) {
459+
int offset = -stack_param_delta * kPointerSize;
460+
__ Add(jssp, jssp, Operand(offset));
461+
}
458462
}
459463

460464

@@ -477,7 +481,8 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
477481
break;
478482
}
479483
case kArchTailCallCodeObject: {
480-
AssembleDeconstructActivationRecord();
484+
int stack_param_delta = i.InputInt32(instr->InputCount() - 1);
485+
AssembleDeconstructActivationRecord(stack_param_delta);
481486
if (instr->InputAt(0)->IsImmediate()) {
482487
__ Jump(Handle<Code>::cast(i.InputHeapObject(0)),
483488
RelocInfo::CODE_TARGET);
@@ -514,7 +519,8 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
514519
__ cmp(cp, temp);
515520
__ Assert(eq, kWrongFunctionContext);
516521
}
517-
AssembleDeconstructActivationRecord();
522+
int stack_param_delta = i.InputInt32(instr->InputCount() - 1);
523+
AssembleDeconstructActivationRecord(stack_param_delta);
518524
__ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset));
519525
__ Jump(x10);
520526
break;

‎src/compiler/code-generator.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ class CodeGenerator final : public GapResolver::Assembler {
9494
void AssembleReturn();
9595

9696
// Generates code to deconstruct a the caller's frame, including arguments.
97-
void AssembleDeconstructActivationRecord();
97+
void AssembleDeconstructActivationRecord(int stack_param_delta);
9898

9999
// ===========================================================================
100100
// ============== Architecture-specific gap resolver methods. ================

‎src/compiler/ia32/code-generator-ia32.cc‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -326,13 +326,20 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
326326
} while (false)
327327

328328

329-
void CodeGenerator::AssembleDeconstructActivationRecord() {
329+
void CodeGenerator::AssembleDeconstructActivationRecord(int stack_param_delta) {
330330
CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
331331
int stack_slots = frame()->GetSpillSlotCount();
332332
if (descriptor->IsJSFunctionCall() || stack_slots > 0) {
333333
__ mov(esp, ebp);
334334
__ pop(ebp);
335335
}
336+
if (stack_param_delta < 0) {
337+
int offset = -(stack_param_delta + 1) * kPointerSize;
338+
__ pop(Operand(esp, offset));
339+
if (offset != 0) {
340+
__ add(esp, Immediate(offset));
341+
}
342+
}
336343
}
337344

338345

@@ -355,7 +362,8 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
355362
break;
356363
}
357364
case kArchTailCallCodeObject: {
358-
AssembleDeconstructActivationRecord();
365+
int stack_param_delta = i.InputInt32(instr->InputCount() - 1);
366+
AssembleDeconstructActivationRecord(stack_param_delta);
359367
if (HasImmediateInput(instr, 0)) {
360368
Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
361369
__ jmp(code, RelocInfo::CODE_TARGET);
@@ -385,7 +393,8 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
385393
__ cmp(esi, FieldOperand(func, JSFunction::kContextOffset));
386394
__ Assert(equal, kWrongFunctionContext);
387395
}
388-
AssembleDeconstructActivationRecord();
396+
int stack_param_delta = i.InputInt32(instr->InputCount() - 1);
397+
AssembleDeconstructActivationRecord(stack_param_delta);
389398
__ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset));
390399
break;
391400
}

‎src/compiler/instruction-selector.cc‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,8 @@ struct CallBuffer {
374374
// TODO(bmeurer): Get rid of the CallBuffer business and make
375375
// InstructionSelector::VisitCall platform independent instead.
376376
void InstructionSelector::InitializeCallBuffer(Node* call, CallBuffer* buffer,
377-
bool call_code_immediate,
378-
bool call_address_immediate) {
377+
CallBufferFlags flags,
378+
int stack_param_delta) {
379379
OperandGenerator g(this);
380380
DCHECK_LE(call->op()->ValueOutputCount(),
381381
static_cast<int>(buffer->descriptor->ReturnCount()));
@@ -426,6 +426,8 @@ void InstructionSelector::InitializeCallBuffer(Node* call, CallBuffer* buffer,
426426

427427
// The first argument is always the callee code.
428428
Node* callee = call->InputAt(0);
429+
bool call_code_immediate = (flags & kCallCodeImmediate) != 0;
430+
bool call_address_immediate = (flags & kCallAddressImmediate) != 0;
429431
switch (buffer->descriptor->kind()) {
430432
case CallDescriptor::kCallCodeObject:
431433
buffer->instruction_args.push_back(
@@ -478,14 +480,20 @@ void InstructionSelector::InitializeCallBuffer(Node* call, CallBuffer* buffer,
478480
// as an InstructionOperand argument to the call.
479481
auto iter(call->inputs().begin());
480482
size_t pushed_count = 0;
483+
bool call_tail = (flags & kCallTail) != 0;
481484
for (size_t index = 0; index < input_count; ++iter, ++index) {
482485
DCHECK(iter != call->inputs().end());
483486
DCHECK((*iter)->op()->opcode() != IrOpcode::kFrameState);
484487
if (index == 0) continue; // The first argument (callee) is already done.
488+
489+
LinkageLocation location = buffer->descriptor->GetInputLocation(index);
490+
if (call_tail) {
491+
location = LinkageLocation::ConvertToTailCallerLocation(
492+
location, stack_param_delta);
493+
}
485494
InstructionOperand op =
486-
g.UseLocation(*iter, buffer->descriptor->GetInputLocation(index),
487-
buffer->descriptor->GetInputType(index));
488-
if (UnallocatedOperand::cast(op).HasFixedSlotPolicy()) {
495+
g.UseLocation(*iter, location, buffer->descriptor->GetInputType(index));
496+
if (UnallocatedOperand::cast(op).HasFixedSlotPolicy() && !call_tail) {
489497
int stack_index = -UnallocatedOperand::cast(op).fixed_slot_index() - 1;
490498
if (static_cast<size_t>(stack_index) >= buffer->pushed_nodes.size()) {
491499
buffer->pushed_nodes.resize(stack_index + 1, NULL);
@@ -1173,7 +1181,8 @@ void InstructionSelector::VisitCall(Node* node, BasicBlock* handler) {
11731181
// the code object in a register if there are multiple uses of it.
11741182
// Improve constant pool and the heuristics in the register allocator
11751183
// for where to emit constants.
1176-
InitializeCallBuffer(node, &buffer, true, true);
1184+
CallBufferFlags call_buffer_flags(kCallCodeImmediate | kCallAddressImmediate);
1185+
InitializeCallBuffer(node, &buffer, call_buffer_flags);
11771186

11781187
EmitPrepareArguments(&(buffer.pushed_nodes), descriptor, node);
11791188

@@ -1226,11 +1235,17 @@ void InstructionSelector::VisitTailCall(Node* node) {
12261235

12271236
// TODO(turbofan): Relax restriction for stack parameters.
12281237

1229-
if (linkage()->GetIncomingDescriptor()->CanTailCall(node)) {
1238+
int stack_param_delta = 0;
1239+
if (linkage()->GetIncomingDescriptor()->CanTailCall(node,
1240+
&stack_param_delta)) {
12301241
CallBuffer buffer(zone(), descriptor, nullptr);
12311242

12321243
// Compute InstructionOperands for inputs and outputs.
1233-
InitializeCallBuffer(node, &buffer, true, IsTailCallAddressImmediate());
1244+
CallBufferFlags flags(kCallCodeImmediate | kCallTail);
1245+
if (IsTailCallAddressImmediate()) {
1246+
flags |= kCallAddressImmediate;
1247+
}
1248+
InitializeCallBuffer(node, &buffer, flags, stack_param_delta);
12341249

12351250
// Select the appropriate opcode based on the call type.
12361251
InstructionCode opcode;
@@ -1247,6 +1262,8 @@ void InstructionSelector::VisitTailCall(Node* node) {
12471262
}
12481263
opcode |= MiscField::encode(descriptor->flags());
12491264

1265+
buffer.instruction_args.push_back(g.TempImmediate(stack_param_delta));
1266+
12501267
// Emit the tailcall instruction.
12511268
Emit(opcode, 0, nullptr, buffer.instruction_args.size(),
12521269
&buffer.instruction_args.front());
@@ -1260,7 +1277,11 @@ void InstructionSelector::VisitTailCall(Node* node) {
12601277
CallBuffer buffer(zone(), descriptor, frame_state_descriptor);
12611278

12621279
// Compute InstructionOperands for inputs and outputs.
1263-
InitializeCallBuffer(node, &buffer, true, IsTailCallAddressImmediate());
1280+
CallBufferFlags flags = kCallCodeImmediate;
1281+
if (IsTailCallAddressImmediate()) {
1282+
flags |= kCallAddressImmediate;
1283+
}
1284+
InitializeCallBuffer(node, &buffer, flags);
12641285

12651286
EmitPrepareArguments(&(buffer.pushed_nodes), descriptor, node);
12661287

‎src/compiler/instruction-selector.h‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,14 +164,20 @@ class InstructionSelector final {
164164
// operand {op}.
165165
void MarkAsRepresentation(MachineType rep, const InstructionOperand& op);
166166

167+
enum CallBufferFlag {
168+
kCallCodeImmediate = 1u << 0,
169+
kCallAddressImmediate = 1u << 1,
170+
kCallTail = 1u << 2
171+
};
172+
typedef base::Flags<CallBufferFlag> CallBufferFlags;
173+
167174
// Initialize the call buffer with the InstructionOperands, nodes, etc,
168175
// corresponding
169176
// to the inputs and outputs of the call.
170177
// {call_code_immediate} to generate immediate operands to calls of code.
171178
// {call_address_immediate} to generate immediate operands to address calls.
172179
void InitializeCallBuffer(Node* call, CallBuffer* buffer,
173-
bool call_code_immediate,
174-
bool call_address_immediate);
180+
CallBufferFlags flags, int stack_param_delta = 0);
175181
bool IsTailCallAddressImmediate();
176182

177183
FrameStateDescriptor* GetFrameStateDescriptor(Node* node);

‎src/compiler/js-intrinsic-lowering.cc‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ Reduction JSIntrinsicLowering::Reduce(Node* node) {
114114
return ReduceThrowNotDateError(node);
115115
case Runtime::kInlineCall:
116116
return ReduceCall(node);
117+
case Runtime::kInlineTailCall:
118+
return ReduceTailCall(node);
117119
default:
118120
break;
119121
}
@@ -648,6 +650,16 @@ Reduction JSIntrinsicLowering::ReduceToString(Node* node) {
648650

649651

650652
Reduction JSIntrinsicLowering::ReduceCall(Node* node) {
653+
size_t const arity = CallRuntimeParametersOf(node->op()).arity();
654+
NodeProperties::ChangeOp(
655+
node, javascript()->CallFunction(arity, STRICT, VectorSlotPair(),
656+
ConvertReceiverMode::kAny,
657+
TailCallMode::kDisallow));
658+
return Changed(node);
659+
}
660+
661+
662+
Reduction JSIntrinsicLowering::ReduceTailCall(Node* node) {
651663
size_t const arity = CallRuntimeParametersOf(node->op()).arity();
652664
NodeProperties::ChangeOp(
653665
node, javascript()->CallFunction(arity, STRICT, VectorSlotPair(),

‎src/compiler/js-intrinsic-lowering.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ class JSIntrinsicLowering final : public AdvancedReducer {
7070
Reduction ReduceToPrimitive(Node* node);
7171
Reduction ReduceToString(Node* node);
7272
Reduction ReduceCall(Node* node);
73+
Reduction ReduceTailCall(Node* node);
7374

7475
Reduction Change(Node* node, const Operator* op);
7576
Reduction Change(Node* node, const Operator* op, Node* a, Node* b);

‎src/compiler/linkage.cc‎

Lines changed: 20 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -89,79 +89,36 @@ bool CallDescriptor::HasSameReturnLocationsAs(
8989
}
9090

9191

92-
bool CallDescriptor::CanTailCall(const Node* node) const {
93-
// Determine the number of stack parameters passed in
94-
size_t stack_params = 0;
95-
for (size_t i = 0; i < InputCount(); ++i) {
96-
if (!GetInputLocation(i).IsRegister()) {
97-
++stack_params;
98-
}
99-
}
100-
// Ensure the input linkage contains the stack parameters in the right order
101-
size_t current_stack_param = 0;
102-
for (size_t i = 0; i < InputCount(); ++i) {
103-
if (!GetInputLocation(i).IsRegister()) {
104-
if (GetInputLocation(i) != LinkageLocation::ForCallerFrameSlot(
105-
static_cast<int>(current_stack_param) -
106-
static_cast<int>(stack_params))) {
107-
return false;
108-
}
109-
++current_stack_param;
110-
}
111-
}
112-
// Tail calling is currently allowed if return locations match and all
113-
// parameters are either in registers or on the stack but match exactly in
114-
// number and content.
92+
bool CallDescriptor::CanTailCall(const Node* node,
93+
int* stack_param_delta) const {
94+
// TODO(danno): TF only current supports tail calls where the number of stack
95+
// parameters of the callee is the same or fewer of the caller.
11596
CallDescriptor const* other = OpParameter<CallDescriptor const*>(node);
11697
if (!HasSameReturnLocationsAs(other)) return false;
11798
size_t current_input = 0;
11899
size_t other_input = 0;
119-
while (true) {
120-
if (other_input >= other->InputCount()) {
121-
while (current_input < InputCount()) {
122-
if (!GetInputLocation(current_input).IsRegister()) {
123-
return false;
124-
}
125-
++current_input;
100+
*stack_param_delta = 0;
101+
bool more_other = true;
102+
bool more_this = true;
103+
while (more_other || more_this) {
104+
if (other_input < other->InputCount()) {
105+
if (!other->GetInputLocation(other_input).IsRegister()) {
106+
(*stack_param_delta)++;
126107
}
127-
return true;
108+
} else {
109+
more_other = false;
128110
}
129-
if (current_input >= InputCount()) {
130-
while (other_input < other->InputCount()) {
131-
if (!other->GetInputLocation(other_input).IsRegister()) {
132-
return false;
133-
}
134-
++other_input;
111+
if (current_input < InputCount()) {
112+
if (!GetInputLocation(current_input).IsRegister()) {
113+
(*stack_param_delta)--;
135114
}
136-
return true;
137-
}
138-
if (GetInputLocation(current_input).IsRegister()) {
139-
++current_input;
140-
continue;
141-
}
142-
if (other->GetInputLocation(other_input).IsRegister()) {
143-
++other_input;
144-
continue;
145-
}
146-
if (GetInputLocation(current_input) !=
147-
other->GetInputLocation(other_input)) {
148-
return false;
149-
}
150-
Node* input = node->InputAt(static_cast<int>(other_input));
151-
if (input->opcode() != IrOpcode::kParameter) {
152-
return false;
153-
}
154-
// Make sure that the parameter input passed through to the tail call
155-
// corresponds to the correct stack slot.
156-
size_t param_index = ParameterIndexOf(input->op());
157-
if (param_index != current_input - 1) {
158-
return false;
115+
} else {
116+
more_this = false;
159117
}
160118
++current_input;
161119
++other_input;
162120
}
163-
UNREACHABLE();
164-
return false;
121+
return *stack_param_delta <= 0;
165122
}
166123

167124

@@ -258,6 +215,7 @@ int Linkage::FrameStateInputCount(Runtime::FunctionId function) {
258215
case Runtime::kInlineToString:
259216
return 1;
260217
case Runtime::kInlineCall:
218+
case Runtime::kInlineTailCall:
261219
case Runtime::kInlineDeoptimizeNow:
262220
case Runtime::kInlineThrowNotDateError:
263221
return 2;

0 commit comments

Comments
 (0)