Skip to content

Commit e9f295a

Browse files
dstogovdstogov
committed
Tracing JIT support for megamorphic calls
1 parent 4edce91 commit e9f295a

File tree

6 files changed

+122
-21
lines changed

6 files changed

+122
-21
lines changed

‎ext/opcache/jit/zend_jit.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ typedef struct _zend_jit_globals {
104104
zend_long max_loops_unroll; /* max number of unrolled loops */
105105
zend_long max_recursive_calls; /* max number of recursive inlined call unrolls */
106106
zend_long max_recursive_returns; /* max number of recursive inlined return unrolls */
107+
zend_long max_polymorphic_calls; /* max number of inlined polymorphic calls */
107108

108109
zend_sym_node *symbols; /* symbols for disassembler */
109110

‎ext/opcache/jit/zend_jit_internal.h‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ typedef enum _zend_jit_trace_stop {
207207
#define ZEND_JIT_EXIT_BLACKLISTED (1<<1)
208208
#define ZEND_JIT_EXIT_TO_VM (1<<2) /* exit to VM without attempt to create a side trace */
209209
#define ZEND_JIT_EXIT_RESTORE_CALL (1<<3) /* deoptimizer should restore EX(call) chain */
210+
#define ZEND_JIT_EXIT_POLYMORPHISM (1<<4) /* exit becaus of polymorphic call */
210211

211212
typedef union _zend_op_trace_info {
212213
zend_op dummy; /* the size of this structure must be the same as zend_op */
@@ -346,6 +347,7 @@ typedef struct _zend_jit_trace_info {
346347
uint32_t exit_counters; /* offset in exit counters array */
347348
uint32_t stack_map_size;
348349
uint32_t flags; /* See ZEND_JIT_TRACE_... defines above */
350+
uint32_t polymorphism; /* Counter of polymorphic calls */
349351
const zend_op *opline; /* first opline */
350352
const void *code_start; /* address of native code */
351353
zend_jit_trace_exit_info *exit_info; /* info about side exits */
@@ -433,7 +435,7 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HAN
433435

434436
int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline);
435437
int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
436-
zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start);
438+
zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphc);
437439

438440
static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, zend_bool *exit_if_true)
439441
{
@@ -456,4 +458,25 @@ static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit
456458
return NULL;
457459
}
458460

461+
static zend_always_inline zend_bool zend_jit_may_be_polymorphic_call(const zend_op *opline)
462+
{
463+
if (opline->opcode == ZEND_INIT_FCALL
464+
|| opline->opcode == ZEND_INIT_FCALL_BY_NAME
465+
|| opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
466+
return 0;
467+
} else if (opline->opcode == ZEND_INIT_METHOD_CALL
468+
|| opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
469+
return 1;
470+
} else if (opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
471+
return (opline->op1_type != IS_CONST || opline->op2_type != IS_CONST);
472+
} else if (opline->opcode == ZEND_INIT_USER_CALL) {
473+
return (opline->op2_type != IS_CONST);
474+
} else if (opline->opcode == ZEND_NEW) {
475+
return (opline->op1_type != IS_CONST);
476+
} else {
477+
ZEND_ASSERT(0);
478+
return 0;
479+
}
480+
}
481+
459482
#endif /* ZEND_JIT_INTERNAL_H */

‎ext/opcache/jit/zend_jit_trace.c‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4177,6 +4177,11 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
41774177
}
41784178
call_info = call_info->next_callee;
41794179
}
4180+
if (!skip_guard
4181+
&& !zend_jit_may_be_polymorphic_call(init_opline)) {
4182+
// TODO: recompilation may change target ???
4183+
skip_guard = 1;
4184+
}
41804185
}
41814186
if (!skip_guard && !zend_jit_init_fcall_guard(&dasm_state, NULL, p->func, trace_buffer[1].opline)) {
41824187
goto jit_failure;
@@ -4422,6 +4427,7 @@ static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace
44224427
t->child_count = 0;
44234428
t->stack_map_size = 0;
44244429
t->flags = 0;
4430+
t->polymorphism = 0;
44254431
t->opline = trace_buffer[1].opline;
44264432
t->exit_info = exit_info;
44274433
t->stack_map = NULL;
@@ -4770,6 +4776,9 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
47704776
if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
47714777
fprintf(stderr, "/CALL");
47724778
}
4779+
if (t->exit_info[i].flags & ZEND_JIT_EXIT_POLYMORPHISM) {
4780+
fprintf(stderr, "/POLY");
4781+
}
47734782
for (j = 0; j < stack_size; j++) {
47744783
zend_uchar type = STACK_TYPE(stack, j);
47754784
if (type != IS_UNKNOWN) {
@@ -4839,7 +4848,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const
48394848
}
48404849

48414850
stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
4842-
ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK);
4851+
ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
48434852

48444853
if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
48454854
zend_jit_dump_trace(trace_buffer, NULL);
@@ -4957,7 +4966,7 @@ static zend_bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_nu
49574966
return 0;
49584967
}
49594968

4960-
static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num)
4969+
static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
49614970
{
49624971
zend_jit_trace_stop ret;
49634972
const void *handler;
@@ -4987,6 +4996,7 @@ static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace
49874996
t->child_count = 0;
49884997
t->stack_map_size = 0;
49894998
t->flags = 0;
4999+
t->polymorphism = polymorphism;
49905000
t->opline = NULL;
49915001
t->exit_info = exit_info;
49925002
t->stack_map = NULL;
@@ -5078,6 +5088,8 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
50785088
int ret = 0;
50795089
uint32_t trace_num;
50805090
zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
5091+
zend_bool is_megamorphic = 0;
5092+
uint32_t polymorphism = 0;
50815093

50825094
trace_num = ZEND_JIT_TRACE_NUM;
50835095

@@ -5105,7 +5117,19 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
51055117
goto abort;
51065118
}
51075119

5108-
stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE);
5120+
if (EX(call)
5121+
&& JIT_G(max_polymorphic_calls) > 0
5122+
&& (zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)) {
5123+
if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
5124+
is_megamorphic = 1;
5125+
} else if (!zend_jit_traces[parent_num].polymorphism) {
5126+
polymorphism = 1;
5127+
} else if (exit_num == 0) {
5128+
polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
5129+
}
5130+
}
5131+
5132+
stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
51095133

51105134
if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
51115135
zend_jit_dump_trace(trace_buffer, NULL);
@@ -5129,7 +5153,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
51295153
}
51305154
}
51315155
if (EXPECTED(stop != ZEND_JIT_TRACE_STOP_LOOP)) {
5132-
stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num);
5156+
stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
51335157
} else {
51345158
const zend_op_array *op_array = trace_buffer[0].op_array;
51355159
zend_jit_op_array_trace_extension *jit_extension =

‎ext/opcache/jit/zend_jit_vm_helpers.c‎

Lines changed: 66 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ static int zend_jit_trace_bad_loop_exit(const zend_op *opline)
457457
return 0;
458458
}
459459

460-
static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx)
460+
static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level, uint32_t *call_level)
461461
{
462462
zend_jit_trace_stop stop ZEND_ATTRIBUTE_UNUSED = ZEND_JIT_TRACE_STOP_ERROR;
463463

@@ -466,7 +466,10 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji
466466
zend_jit_op_array_trace_extension *jit_extension;
467467

468468
if (call->prev_execute_data) {
469-
idx = zend_jit_trace_record_fake_init_call(call->prev_execute_data, trace_buffer, idx);
469+
idx = zend_jit_trace_record_fake_init_call_ex(call->prev_execute_data, trace_buffer, idx, is_megamorphic, megamorphic, level, call_level);
470+
if (idx < 0) {
471+
return idx;
472+
}
470473
}
471474

472475
func = call->func;
@@ -483,11 +486,39 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji
483486
}
484487
func = (zend_function*)jit_extension->op_array;
485488
}
489+
if (is_megamorphic
490+
/* TODO: use more accurate check ??? */
491+
&& ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC)
492+
|| func->common.scope)) {
493+
func = NULL;
494+
*megamorphic |= (1 << (level + *call_level));
495+
} else {
496+
*megamorphic &= ~(1 << (level + *call_level));
497+
}
498+
(*call_level)++;
486499
TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_FAKE_INIT_CALL, func);
487500
} while (0);
488501
return idx;
489502
}
490503

504+
static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_jit_trace_rec *trace_buffer, int idx, zend_bool is_megamorphic, uint32_t *megamorphic, uint32_t level)
505+
{
506+
uint32_t call_level = 0;
507+
508+
return zend_jit_trace_record_fake_init_call_ex(call, trace_buffer, idx, is_megamorphic, megamorphic, level, &call_level);
509+
}
510+
511+
static int zend_jit_trace_call_level(const zend_execute_data *call)
512+
{
513+
int call_level = 0;
514+
515+
while (call->prev_execute_data) {
516+
call_level++;
517+
call = call->prev_execute_data;
518+
}
519+
return call_level;
520+
}
521+
491522
/*
492523
* Trace Linking Rules
493524
* ===================
@@ -518,7 +549,8 @@ static int zend_jit_trace_record_fake_init_call(zend_execute_data *call, zend_ji
518549
*
519550
*/
520551

521-
zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start)
552+
zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, const zend_op *op, zend_jit_trace_rec *trace_buffer, uint8_t start, zend_bool is_megamorphic)
553+
522554
{
523555
#ifdef HAVE_GCC_GLOBAL_REGS
524556
zend_execute_data *save_execute_data = execute_data;
@@ -528,6 +560,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
528560
zend_jit_trace_stop stop = ZEND_JIT_TRACE_STOP_ERROR;
529561
int level = 0;
530562
int ret_level = 0;
563+
int call_level;
531564
zend_vm_opcode_handler_t handler;
532565
const zend_op_array *op_array;
533566
zend_jit_op_array_trace_extension *jit_extension;
@@ -539,6 +572,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
539572
int backtrack_ret_recursion = -1;
540573
int backtrack_ret_recursion_level = 0;
541574
int loop_unroll_limit = 0;
575+
uint32_t megamorphic = 0;
542576
const zend_op_array *unrolled_calls[ZEND_JIT_TRACE_MAX_CALL_DEPTH + ZEND_JIT_TRACE_MAX_RET_DEPTH];
543577
#ifdef HAVE_GCC_GLOBAL_REGS
544578
zend_execute_data *prev_execute_data = ex;
@@ -571,7 +605,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
571605
TRACE_START(ZEND_JIT_TRACE_START, start, op_array, opline);
572606

573607
if (prev_call) {
574-
int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx);
608+
int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, is_megamorphic, &megamorphic, ret_level + level);
575609
if (ret < 0) {
576610
return ZEND_JIT_TRACE_STOP_BAD_FUNC;
577611
}
@@ -665,17 +699,18 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
665699
TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2);
666700
}
667701

668-
switch (opline->opcode) {
669-
case ZEND_DO_FCALL:
670-
case ZEND_DO_ICALL:
671-
case ZEND_DO_UCALL:
672-
case ZEND_DO_FCALL_BY_NAME:
673-
if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
674-
TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func);
675-
}
676-
break;
677-
default:
702+
if (opline->opcode == ZEND_DO_FCALL
703+
|| opline->opcode == ZEND_DO_ICALL
704+
|| opline->opcode == ZEND_DO_UCALL
705+
|| opline->opcode == ZEND_DO_FCALL_BY_NAME) {
706+
call_level = zend_jit_trace_call_level(EX(call));
707+
if (megamorphic & (1 << (ret_level + level + call_level))) {
708+
stop = ZEND_JIT_TRACE_STOP_INTERPRETER;
678709
break;
710+
}
711+
if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) {
712+
TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func);
713+
}
679714
}
680715

681716
handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
@@ -779,7 +814,7 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
779814
ret_level++;
780815

781816
if (prev_call) {
782-
int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx);
817+
int ret = zend_jit_trace_record_fake_init_call(prev_call, trace_buffer, idx, 0, &megamorphic, ret_level + level);
783818
if (ret < 0) {
784819
stop = ZEND_JIT_TRACE_STOP_BAD_FUNC;
785820
break;
@@ -827,6 +862,22 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
827862
}
828863
func = (zend_function*)jit_extension->op_array;
829864
}
865+
866+
#ifndef HAVE_GCC_GLOBAL_REGS
867+
opline = EX(opline);
868+
#endif
869+
870+
if (JIT_G(max_polymorphic_calls) == 0
871+
&& zend_jit_may_be_polymorphic_call(opline - 1)) {
872+
func = NULL;
873+
}
874+
call_level = zend_jit_trace_call_level(EX(call));
875+
ZEND_ASSERT(ret_level + level + call_level < 32);
876+
if (func) {
877+
megamorphic &= ~(1 << (ret_level + level + call_level));
878+
} else {
879+
megamorphic |= (1 << (ret_level + level + call_level));
880+
}
830881
TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func);
831882
}
832883
prev_call = EX(call);

‎ext/opcache/jit/zend_jit_x86.dasc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8009,7 +8009,7 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, const zend_op *opline, co
80098009
return 0;
80108010
}
80118011

8012-
exit_point = zend_jit_trace_get_exit_point(opline, to_opline, NULL, 0);
8012+
exit_point = zend_jit_trace_get_exit_point(opline, to_opline, NULL, ZEND_JIT_EXIT_POLYMORPHISM);
80138013
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
80148014
if (!exit_addr) {
80158015
return 0;

‎ext/opcache/zend_accelerator_module.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,7 @@ ZEND_INI_BEGIN()
303303
STD_PHP_INI_ENTRY("opcache.jit_max_loops_unroll" , "8", PHP_INI_ALL, OnUpdateUnrollL, max_loops_unroll, zend_jit_globals, jit_globals)
304304
STD_PHP_INI_ENTRY("opcache.jit_max_recursive_calls" , "2", PHP_INI_ALL, OnUpdateUnrollC, max_recursive_calls, zend_jit_globals, jit_globals)
305305
STD_PHP_INI_ENTRY("opcache.jit_max_recursive_returns" , "2", PHP_INI_ALL, OnUpdateUnrollR, max_recursive_returns, zend_jit_globals, jit_globals)
306+
STD_PHP_INI_ENTRY("opcache.jit_max_polymorphic_calls" , "2", PHP_INI_ALL, OnUpdateLong, max_polymorphic_calls, zend_jit_globals, jit_globals)
306307
#endif
307308
ZEND_INI_END()
308309

@@ -786,6 +787,7 @@ ZEND_FUNCTION(opcache_get_configuration)
786787
add_assoc_long(&directives, "opcache.jit_hot_return", JIT_G(hot_return));
787788
add_assoc_long(&directives, "opcache.jit_hot_side_exit", JIT_G(hot_side_exit));
788789
add_assoc_long(&directives, "opcache.jit_max_loops_unroll", JIT_G(max_loops_unroll));
790+
add_assoc_long(&directives, "opcache.jit_max_polymorphic_calls", JIT_G(max_polymorphic_calls));
789791
add_assoc_long(&directives, "opcache.jit_max_recursive_calls", JIT_G(max_recursive_calls));
790792
add_assoc_long(&directives, "opcache.jit_max_recursive_returns", JIT_G(max_recursive_returns));
791793
add_assoc_long(&directives, "opcache.jit_max_root_traces", JIT_G(max_root_traces));

0 commit comments

Comments
 (0)