Skip to content

Commit cb77441

Browse files
dstogovdstogov
committed
Better trace coverage (JIT trampoline calls)
1 parent 57bfca9 commit cb77441

File tree

4 files changed

+70
-59
lines changed

4 files changed

+70
-59
lines changed

‎ext/opcache/jit/zend_jit_internal.h

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -248,8 +248,11 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
248248
_(RECURSIVE_CALL, "recursive call") \
249249
_(RECURSIVE_RET, "recursive return") \
250250
_(RETURN, "return") \
251-
_(INTERPRETER, "exit to VM interpreter") \
252251
_(LINK, "link to another trace") \
252+
_(INTERPRETER, "exit to VM interpreter") \
253+
_(TRAMPOLINE, "trampoline call") \
254+
_(PROP_HOOK_CALL, "property hook call") \
255+
_(BAD_FUNC, "bad function call") \
253256
/* compilation and linking successful */ \
254257
_(COMPILED, "compiled") \
255258
_(ALREADY_DONE, "already prcessed") \
@@ -267,9 +270,6 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
267270
_(BLACK_LIST, "trace blacklisted") \
268271
_(INNER_LOOP, "inner loop") /* trace it */ \
269272
_(COMPILED_LOOP, "compiled loop") \
270-
_(TRAMPOLINE, "trampoline call") \
271-
_(PROP_HOOK_CALL, "property hook call") \
272-
_(BAD_FUNC, "bad function call") \
273273
_(COMPILER_ERROR, "JIT compilation error") \
274274
/* no recoverable error (blacklist immediately) */ \
275275
_(NO_SHM, "insufficient shared memory") \
@@ -380,6 +380,12 @@ typedef enum _zend_jit_trace_op {
380380
#define ZEND_JIT_TRACE_FAKE_INFO(level) \
381381
(((level) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) | ZEND_JIT_TRACE_FAKE_INIT_CALL)
382382

383+
#define ZEND_JIT_TRACE_NUM_ARGS_INFO(count) \
384+
((count) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)
385+
386+
#define ZEND_JIT_TRACE_NUM_ARGS(info) \
387+
(((info) & ZEND_JIT_TRACE_FAKE_LEVEL_MASK) >> ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)
388+
383389
#define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \
384390
_info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \
385391
} while (0)

‎ext/opcache/jit/zend_jit_ir.c

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9064,6 +9064,14 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
90649064
jit->delayed_call_level = call_level;
90659065
}
90669066

9067+
if (trace
9068+
&& trace->op == ZEND_JIT_TRACE_END
9069+
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
9070+
if (!zend_jit_set_ip(jit, opline + 1)) {
9071+
return 0;
9072+
}
9073+
}
9074+
90679075
return 1;
90689076
}
90699077

@@ -9323,7 +9331,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit,
93239331

93249332
if (trace
93259333
&& trace->op == ZEND_JIT_TRACE_END
9326-
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
9334+
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
93279335
if (!zend_jit_set_ip(jit, opline + 1)) {
93289336
return 0;
93299337
}
@@ -9932,7 +9940,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
99329940

99339941
if (trace && !func) {
99349942
if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
9935-
ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
9943+
ZEND_ASSERT(!trace->func || trace->func->type == ZEND_INTERNAL_FUNCTION);
99369944
#ifndef ZEND_WIN32
99379945
// TODO: ASLR may cause different addresses in different workers ???
99389946
func = trace->func;
@@ -10114,7 +10122,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
1011410122

1011510123
if (call_num_args <= func->op_array.num_args) {
1011610124
if (!trace || (trace->op == ZEND_JIT_TRACE_END
10117-
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
10125+
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) {
1011810126
uint32_t num_args;
1011910127

1012010128
if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
@@ -10148,7 +10156,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
1014810156
}
1014910157
} else {
1015010158
if (!trace || (trace->op == ZEND_JIT_TRACE_END
10151-
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
10159+
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) {
1015210160
ir_ref ip;
1015310161

1015410162
if (zend_accel_in_shm(func->op_array.opcodes)) {
@@ -10274,7 +10282,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
1027410282
ir_ref observer_handler;
1027510283
ir_ref rx = jit_FP(jit);
1027610284
struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref);
10277-
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
10285+
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
1027810286
ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
1027910287
jit_SET_EX_OPLINE(jit, trace[1].opline);
1028010288
} else if (GCC_GLOBAL_REGS) {
@@ -10567,7 +10575,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
1056710575
jit_LOAD_IP_ADDR(jit, opline + 1);
1056810576
} else if (trace
1056910577
&& trace->op == ZEND_JIT_TRACE_END
10570-
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
10578+
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
1057110579
jit_LOAD_IP_ADDR(jit, opline + 1);
1057210580
}
1057310581
}
@@ -16906,7 +16914,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
1690616914
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
1690716915
if (trace->op != ZEND_JIT_TRACE_END ||
1690816916
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
16909-
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
16917+
trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
1691016918
/* this check may be handled by the following OPLINE guard or jmp [IP] */
1691116919
ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
1691216920
jit_STUB_ADDR(jit, jit_stub_trace_halt));
@@ -16924,7 +16932,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
1692416932
}
1692516933
if (trace->op != ZEND_JIT_TRACE_END ||
1692616934
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
16927-
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
16935+
trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
1692816936

1692916937
const zend_op *next_opline = trace->opline;
1693016938
const zend_op *exit_opline = NULL;

‎ext/opcache/jit/zend_jit_trace.c

Lines changed: 26 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -427,25 +427,25 @@ static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa
427427
#define CHECK_OP1_DATA_TRACE_TYPE() \
428428
CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
429429

430-
static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
430+
static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array, uint32_t num_args)
431431
{
432432
if (op_array && op_array->type == ZEND_USER_FUNCTION) {
433433
return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
434434
} else if (op_array) {
435435
return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
436436
} else {
437-
return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
437+
return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(num_args * sizeof(zend_jit_trace_stack)));
438438
}
439439
}
440440

441-
static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
441+
static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array, uint32_t num_args)
442442
{
443-
return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
443+
return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array, num_args));
444444
}
445445

446446
static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
447447
{
448-
return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
448+
return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array, 0));
449449
}
450450

451451
static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type)
@@ -1323,7 +1323,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
13231323
* Calculate size of abstract stack;
13241324
* Construct regular SSA for involved op_array */
13251325
op_array = trace_buffer->op_array;
1326-
stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1326+
stack_top = stack_size = zend_jit_trace_frame_size(op_array, 0);
13271327
stack_bottom = 0;
13281328
p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
13291329
ssa_ops_count = 0;
@@ -1363,7 +1363,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
13631363
ssa_ops_count += zend_jit_trace_op_len(p->opline);
13641364
} else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
13651365
call_level++;
1366-
stack_top += zend_jit_trace_frame_size(p->op_array);
1366+
stack_top += zend_jit_trace_frame_size(p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info));
13671367
if (stack_top > stack_size) {
13681368
stack_size = stack_top;
13691369
}
@@ -1377,7 +1377,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
13771377
ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
13781378
}
13791379
}
1380-
frame_size = zend_jit_trace_frame_size(p->op_array);
1380+
frame_size = zend_jit_trace_frame_size(p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info));
13811381
if (call_level == 0) {
13821382
if (stack_top + frame_size > stack_size) {
13831383
stack_size = stack_top + frame_size;
@@ -1389,7 +1389,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
13891389
} else if (p->op == ZEND_JIT_TRACE_ENTER) {
13901390
op_array = p->op_array;
13911391
if (call_level == 0) {
1392-
stack_top += zend_jit_trace_frame_size(op_array);
1392+
stack_top += zend_jit_trace_frame_size(op_array, 0);
13931393
if (stack_top > stack_size) {
13941394
stack_size = stack_top;
13951395
}
@@ -1414,7 +1414,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
14141414
}
14151415
} else if (p->op == ZEND_JIT_TRACE_BACK) {
14161416
if (level == 0) {
1417-
stack_bottom += zend_jit_trace_frame_size(p->op_array);
1417+
stack_bottom += zend_jit_trace_frame_size(p->op_array, 0);
14181418
jit_extension =
14191419
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
14201420
ssa = &jit_extension->func_info.ssa;
@@ -1431,7 +1431,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
14311431
ssa = zend_jit_trace_build_ssa(op_array, script);
14321432
}
14331433
} else {
1434-
stack_top -= zend_jit_trace_frame_size(op_array);
1434+
stack_top -= zend_jit_trace_frame_size(op_array, 0);
14351435
level--;
14361436
}
14371437
op_array = p->op_array;
@@ -1534,7 +1534,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
15341534
len--;
15351535
}
15361536
} else if (p->op == ZEND_JIT_TRACE_ENTER) {
1537-
frame = zend_jit_trace_call_frame(frame, op_array);
1537+
frame = zend_jit_trace_call_frame(frame, op_array, 0);
15381538
stack = frame->stack;
15391539
op_array = p->op_array;
15401540
level++;
@@ -1754,7 +1754,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
17541754
}
17551755

17561756
frame = JIT_G(current_frame);
1757-
top = zend_jit_trace_call_frame(frame, op_array);
1757+
top = zend_jit_trace_call_frame(frame, op_array, 0);
17581758
TRACE_FRAME_INIT(frame, op_array, 0, 0);
17591759
TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
17601760
frame->used_stack = 0;
@@ -2448,7 +2448,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
24482448
call = top;
24492449
TRACE_FRAME_INIT(call, op_array, 0, 0);
24502450
call->used_stack = 0;
2451-
top = zend_jit_trace_call_frame(top, op_array);
2451+
top = zend_jit_trace_call_frame(top, op_array, 0);
24522452
} else {
24532453
ZEND_ASSERT(&call->func->op_array == op_array);
24542454
}
@@ -2583,7 +2583,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
25832583
call->prev = frame->call;
25842584
call->used_stack = 0;
25852585
frame->call = call;
2586-
top = zend_jit_trace_call_frame(top, p->op_array);
2586+
top = zend_jit_trace_call_frame(top, p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info));
25872587
if (p->func && p->func->type == ZEND_USER_FUNCTION) {
25882588
for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
25892589
SET_STACK_INFO(call->stack, i, -1);
@@ -2626,6 +2626,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
26262626

26272627
if (idx > 0
26282628
&& ssa_ops[idx-1].result_def >= 0
2629+
&& p->func
26292630
&& (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
26302631
&& !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
26312632
ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
@@ -3156,7 +3157,7 @@ static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *t
31563157
}
31573158
}
31583159

3159-
frame = zend_jit_trace_call_frame(frame, op_array);
3160+
frame = zend_jit_trace_call_frame(frame, op_array, 0);
31603161
frame->prev = prev_frame;
31613162
frame->func = (const zend_function*)p->op_array;
31623163
stack = frame->stack;
@@ -3306,8 +3307,7 @@ static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *t
33063307
}
33073308
phi = phi->next;
33083309
}
3309-
} else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
3310-
|| p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
3310+
} else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) {
33113311
for (i = 0; i < op_array->last_var + op_array->T; i++) {
33123312
int var = STACK_VAR(stack, i);
33133313
if (var >= 0 && RA_HAS_REG(var)
@@ -4123,7 +4123,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
41234123
ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
41244124
op_array = p->op_array;
41254125
frame = JIT_G(current_frame);
4126-
top = zend_jit_trace_call_frame(frame, op_array);
4126+
top = zend_jit_trace_call_frame(frame, op_array, 0);
41274127
TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
41284128
frame->used_stack = checked_stack = peek_checked_stack = 0;
41294129
stack = frame->stack;
@@ -7006,7 +7006,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
70067006
}
70077007
}
70087008
frame->call = call;
7009-
top = zend_jit_trace_call_frame(top, p->op_array);
7009+
top = zend_jit_trace_call_frame(top, p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info));
70107010
if (p->func) {
70117011
if (p->func->type == ZEND_USER_FUNCTION) {
70127012
if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
@@ -7192,8 +7192,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
71927192

71937193
zend_jit_trace_end_loop(&ctx, jit->trace_loop_ref, timeout_exit_addr); /* jump back to start of the trace loop */
71947194
}
7195-
} else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7196-
|| p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7195+
} else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) {
71977196
if (ra
71987197
&& (p-1)->op != ZEND_JIT_TRACE_ENTER
71997198
&& (p-1)->op != ZEND_JIT_TRACE_BACK
@@ -7303,8 +7302,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
73037302
break;
73047303
}
73057304
}
7306-
} else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7307-
|| p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7305+
} else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) {
73087306
if (opline
73097307
&& (opline->opcode == ZEND_DO_UCALL
73107308
|| opline->opcode == ZEND_DO_FCALL
@@ -7929,7 +7927,7 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa
79297927
level, ' ',
79307928
(p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
79317929
(p->func && p->func->common.scope) ? "::" : "",
7932-
p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7930+
(p->func && p->func->common.function_name) ? ZSTR_VAL(p->func->common.function_name) : "???");
79337931
} else {
79347932
fprintf(stderr, " %*c>skip\n",
79357933
level, ' ');
@@ -7938,9 +7936,9 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa
79387936
if (p->func != (zend_function*)&zend_pass_function) {
79397937
fprintf(stderr, " %*c>call %s%s%s\n",
79407938
level, ' ',
7941-
p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7942-
p->func->common.scope ? "::" : "",
7943-
ZSTR_VAL(p->func->common.function_name));
7939+
(p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7940+
(p->func && p->func->common.scope) ? "::" : "",
7941+
(p->func && p->func->common.function_name) ? ZSTR_VAL(p->func->common.function_name) : "???");
79447942
} else {
79457943
fprintf(stderr, " %*c>skip\n",
79467944
level, ' ');

0 commit comments

Comments
 (0)