Skip to content

Commit 1de16c7

Browse files
arnaud-lbarnaud-lb
committed
Merge branch 'PHP-8.4'
* PHP-8.4: Snapshotted poly_func / poly_this may be spilled
2 parents 4122daa + 18276a8 commit 1de16c7

File tree

3 files changed

+139
-69
lines changed

3 files changed

+139
-69
lines changed

‎ext/opcache/jit/zend_jit_internal.h

+15-9
Original file line numberDiff line numberDiff line change
@@ -437,16 +437,22 @@ struct _zend_jit_trace_rec {
437437

438438
#define ZEND_JIT_TRACE_START_REC_SIZE 2
439439

440+
typedef struct _zend_jit_ref_snapshot {
441+
union {
442+
int32_t ref; /* While generating code: The ir_ref to snapshot */
443+
int32_t offset; /* After compilation / during deopt: C stack offset if 'reg' is spilled */
444+
};
445+
int8_t reg; /* Set after compilation by zend_jit_snapshot_handler() */
446+
} zend_jit_ref_snapshot;
447+
440448
typedef struct _zend_jit_trace_exit_info {
441-
const zend_op *opline; /* opline where VM should continue execution */
442-
const zend_op_array *op_array;
443-
uint32_t flags; /* set of ZEND_JIT_EXIT_... */
444-
uint32_t stack_size;
445-
uint32_t stack_offset;
446-
int32_t poly_func_ref;
447-
int32_t poly_this_ref;
448-
int8_t poly_func_reg;
449-
int8_t poly_this_reg;
449+
const zend_op *opline; /* opline where VM should continue execution */
450+
const zend_op_array *op_array;
451+
uint32_t flags; /* set of ZEND_JIT_EXIT_... */
452+
uint32_t stack_size;
453+
uint32_t stack_offset;
454+
zend_jit_ref_snapshot poly_func;
455+
zend_jit_ref_snapshot poly_this;
450456
} zend_jit_trace_exit_info;
451457

452458
typedef struct _zend_jit_trace_stack {

‎ext/opcache/jit/zend_jit_ir.c

+73-30
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,8 @@ typedef struct _zend_jit_ctx {
277277
ir_ref tls;
278278
#endif
279279
ir_ref fp;
280+
ir_ref poly_func_ref; /* restored from parent trace snapshot */
281+
ir_ref poly_this_ref; /* restored from parent trace snapshot */
280282
ir_ref trace_loop_ref;
281283
ir_ref return_inputs;
282284
const zend_op_array *op_array;
@@ -624,12 +626,12 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
624626
uint32_t exit_point = 0, n = 0;
625627

626628
if (addr < 0) {
627-
if (t->exit_count > 0
628-
&& jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) {
629-
exit_point = t->exit_count - 1;
630-
if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
631-
n = 2;
632-
}
629+
/* addr is not always the address of the *last* exit point,
630+
* so we can not optimize this to 'exit_point = t->exit_count-1' */
631+
exit_point = zend_jit_exit_point_by_addr(ptr);
632+
ZEND_ASSERT(exit_point != -1);
633+
if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
634+
n = 2;
633635
}
634636
}
635637

@@ -660,8 +662,8 @@ static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
660662
ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);
661663
}
662664
if (n) {
663-
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref);
664-
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref);
665+
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func.ref);
666+
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this.ref);
665667
}
666668
}
667669
}
@@ -710,6 +712,31 @@ uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint
710712
return new_exit_point;
711713
}
712714

715+
static void zend_jit_resolve_ref_snapshot(zend_jit_ref_snapshot *dest, ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, int op)
716+
{
717+
int8_t *reg_ops = ctx->regs[snapshot_ref];
718+
ZEND_ASSERT(reg_ops[op] != ZREG_NONE);
719+
720+
int8_t reg = reg_ops[op];
721+
int32_t offset;
722+
723+
if (IR_REG_SPILLED(reg)) {
724+
reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD;
725+
offset = ir_get_spill_slot_offset(ctx, ir_insn_op(snapshot, op));
726+
} else {
727+
offset = 0;
728+
}
729+
730+
dest->reg = reg;
731+
dest->offset = offset;
732+
}
733+
734+
static bool zend_jit_ref_snapshot_equals(const zend_jit_ref_snapshot *a, const zend_jit_ref_snapshot *b)
735+
{
736+
return a->reg == b->reg
737+
&& (!IR_REG_SPILLED(a->reg) || (a->offset == b->offset));
738+
}
739+
713740
void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)
714741
{
715742
zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
@@ -722,18 +749,19 @@ void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snaps
722749
exit_flags = t->exit_info[exit_point].flags;
723750

724751
if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {
725-
int8_t *reg_ops = ctx->regs[snapshot_ref];
752+
zend_jit_ref_snapshot func, this;
753+
zend_jit_resolve_ref_snapshot(&func, ctx, snapshot_ref, snapshot, n - 1);
754+
zend_jit_resolve_ref_snapshot(&this, ctx, snapshot_ref, snapshot, n);
726755

727-
ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1);
728756
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
729-
&& (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1]
730-
|| t->exit_info[exit_point].poly_this_reg != reg_ops[n])) {
757+
&& (!zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_func, &func)
758+
|| !zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_this, &this))) {
731759
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
732760
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
733761
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
734762
}
735-
t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1];
736-
t->exit_info[exit_point].poly_this_reg = reg_ops[n];
763+
t->exit_info[exit_point].poly_func = func;
764+
t->exit_info[exit_point].poly_this = this;
737765
n -= 2;
738766
}
739767

@@ -2709,6 +2737,8 @@ static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
27092737
jit->tls = IR_UNUSED;
27102738
#endif
27112739
jit->fp = IR_UNUSED;
2740+
jit->poly_func_ref = IR_UNUSED;
2741+
jit->poly_this_ref = IR_UNUSED;
27122742
jit->trace_loop_ref = IR_UNUSED;
27132743
jit->return_inputs = IR_UNUSED;
27142744
jit->bb_start_ref = NULL;
@@ -4386,6 +4416,18 @@ static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg)
43864416
return ir_RLOAD(type, reg);
43874417
}
43884418

4419+
/* Same as zend_jit_deopt_rload(), but 'reg' may be spilled on C stack */
4420+
static ir_ref zend_jit_deopt_rload_spilled(zend_jit_ctx *jit, ir_type type, int8_t reg, int32_t offset)
4421+
{
4422+
ZEND_ASSERT(reg >= 0);
4423+
4424+
if (IR_REG_SPILLED(reg)) {
4425+
return ir_LOAD(type, ir_ADD_OFFSET(zend_jit_deopt_rload(jit, type, IR_REG_NUM(reg)), offset));
4426+
} else {
4427+
return zend_jit_deopt_rload(jit, type, reg);
4428+
}
4429+
}
4430+
43894431
static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val)
43904432
{
43914433
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
@@ -8440,10 +8482,9 @@ static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32
84408482
return 1;
84418483
}
84428484

8443-
static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg)
8485+
static int zend_jit_free_trampoline(zend_jit_ctx *jit, ir_ref func)
84448486
{
84458487
// JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
8446-
ir_ref func = ir_RLOAD_A(func_reg);
84478488
ir_ref if_trampoline = ir_IF(ir_AND_U32(
84488489
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))),
84498490
ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE)));
@@ -8537,8 +8578,8 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co
85378578
}
85388579

85398580
if (may_be_trampoline) {
8540-
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
8541-
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
8581+
jit->trace->exit_info[exit_point].poly_func.ref = func_ref;
8582+
jit->trace->exit_info[exit_point].poly_this.ref = this_ref;
85428583
}
85438584

85448585
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
@@ -8936,15 +8977,15 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
89368977
zend_class_entry *trace_ce,
89378978
zend_jit_trace_rec *trace,
89388979
int checked_stack,
8939-
int8_t func_reg,
8940-
int8_t this_reg,
8980+
ir_ref func_ref,
8981+
ir_ref this_ref,
89418982
bool polymorphic_side_trace)
89428983
{
89438984
zend_func_info *info = ZEND_FUNC_INFO(op_array);
89448985
zend_call_info *call_info = NULL;
89458986
zend_function *func = NULL;
89468987
zval *function_name;
8947-
ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL;
8988+
ir_ref if_static = IR_UNUSED, cold_path;
89488989

89498990
ZEND_ASSERT(opline->op2_type == IS_CONST);
89508991
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
@@ -8962,10 +9003,8 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
89629003
}
89639004

89649005
if (polymorphic_side_trace) {
8965-
/* function is passed in r0 from parent_trace */
8966-
ZEND_ASSERT(func_reg >= 0 && this_reg >= 0);
8967-
func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
8968-
this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
9006+
/* function is passed from parent snapshot */
9007+
ZEND_ASSERT(func_ref != IR_UNUSED && this_ref != IR_UNUSED);
89699008
} else {
89709009
ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2;
89719010

@@ -9111,8 +9150,8 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
91119150
return 0;
91129151
}
91139152

9114-
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
9115-
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
9153+
jit->trace->exit_info[exit_point].poly_func.ref = func_ref;
9154+
jit->trace->exit_info[exit_point].poly_this.ref = this_ref;
91169155

91179156
func = (zend_function*)trace->func;
91189157

@@ -17319,9 +17358,13 @@ static int zend_jit_trace_start(zend_jit_ctx *jit,
1731917358
}
1732017359

1732117360
if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
17322-
ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 && parent->exit_info[exit_num].poly_this_reg >= 0);
17323-
ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg);
17324-
ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg);
17361+
ZEND_ASSERT(parent->exit_info[exit_num].poly_func.reg >= 0 && parent->exit_info[exit_num].poly_this.reg >= 0);
17362+
if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_func.reg)) {
17363+
ir_RLOAD_A(parent->exit_info[exit_num].poly_func.reg);
17364+
}
17365+
if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_this.reg)) {
17366+
ir_RLOAD_A(parent->exit_info[exit_num].poly_this.reg);
17367+
}
1732517368
}
1732617369

1732717370
ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num));

‎ext/opcache/jit/zend_jit_trace.c

+51-30
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,8 @@ static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t
199199
t->exit_info[exit_point].flags = flags;
200200
t->exit_info[exit_point].stack_size = stack_size;
201201
t->exit_info[exit_point].stack_offset = stack_offset;
202-
t->exit_info[exit_point].poly_func_ref = 0;
203-
t->exit_info[exit_point].poly_this_ref = 0;
204-
t->exit_info[exit_point].poly_func_reg = ZREG_NONE;
205-
t->exit_info[exit_point].poly_this_reg = ZREG_NONE;
202+
t->exit_info[exit_point].poly_func = (zend_jit_ref_snapshot){.reg = ZREG_NONE};
203+
t->exit_info[exit_point].poly_this = (zend_jit_ref_snapshot){.reg = ZREG_NONE};
206204
}
207205

208206
return exit_point;
@@ -3514,17 +3512,18 @@ static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t
35143512
}
35153513

35163514
static int zend_jit_trace_deoptimization(
3517-
zend_jit_ctx *jit,
3518-
uint32_t flags,
3519-
const zend_op *opline,
3520-
zend_jit_trace_stack *parent_stack,
3521-
int parent_vars_count,
3522-
zend_ssa *ssa,
3523-
zend_jit_trace_stack *stack,
3524-
zend_jit_exit_const *constants,
3525-
int8_t func_reg,
3526-
bool polymorphic_side_trace)
3515+
zend_jit_ctx *jit,
3516+
const zend_jit_trace_exit_info *exit_info,
3517+
zend_jit_trace_stack *parent_stack,
3518+
int parent_vars_count,
3519+
zend_ssa *ssa,
3520+
zend_jit_trace_stack *stack,
3521+
zend_jit_exit_const *constants,
3522+
bool polymorphic_side_trace)
35273523
{
3524+
uint32_t flags = exit_info->flags;
3525+
const zend_op *opline = exit_info->opline;
3526+
35283527
int i;
35293528
int check2 = -1;
35303529

@@ -3668,9 +3667,16 @@ static int zend_jit_trace_deoptimization(
36683667
zend_jit_check_exception(jit);
36693668
}
36703669

3671-
if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3672-
if (!zend_jit_free_trampoline(jit, func_reg)) {
3673-
return 0;
3670+
if (flags & ZEND_JIT_EXIT_METHOD_CALL) {
3671+
jit->poly_func_ref = zend_jit_deopt_rload_spilled(jit, IR_ADDR,
3672+
exit_info->poly_func.reg, exit_info->poly_func.offset);
3673+
jit->poly_this_ref = zend_jit_deopt_rload_spilled(jit, IR_ADDR,
3674+
exit_info->poly_this.reg, exit_info->poly_this.offset);
3675+
3676+
if (!polymorphic_side_trace) {
3677+
if (!zend_jit_free_trampoline(jit, jit->poly_func_ref)) {
3678+
return 0;
3679+
}
36743680
}
36753681
}
36763682

@@ -4265,11 +4271,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
42654271
if (parent_trace) {
42664272
/* Deoptimization */
42674273
if (!zend_jit_trace_deoptimization(&ctx,
4268-
zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4269-
zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4274+
&zend_jit_traces[parent_trace].exit_info[exit_num],
42704275
parent_stack, parent_vars_count, ssa, stack,
42714276
zend_jit_traces[parent_trace].constants,
4272-
zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg,
42734277
polymorphic_side_trace)) {
42744278
goto jit_failure;
42754279
}
@@ -6381,8 +6385,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
63816385
op_array, ssa, ssa_op, frame->call_level,
63826386
op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
63836387
p + 1, peek_checked_stack - checked_stack,
6384-
polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg : -1,
6385-
polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg : -1,
6388+
polymorphic_side_trace ? jit->poly_func_ref : -1,
6389+
polymorphic_side_trace ? jit->poly_this_ref : -1,
63866390
polymorphic_side_trace)) {
63876391
goto jit_failure;
63886392
}
@@ -7419,11 +7423,9 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n
74197423
NULL;
74207424

74217425
if (!zend_jit_trace_deoptimization(&ctx,
7422-
zend_jit_traces[trace_num].exit_info[exit_num].flags,
7423-
zend_jit_traces[trace_num].exit_info[exit_num].opline,
7426+
&zend_jit_traces[trace_num].exit_info[exit_num],
74247427
stack, stack_size, NULL, NULL,
74257428
zend_jit_traces[trace_num].constants,
7426-
zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg,
74277429
0)) {
74287430
goto jit_failure;
74297431
}
@@ -7973,6 +7975,17 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa
79737975
}
79747976
}
79757977

7978+
static void zend_jit_dump_ref_snapshot(zend_jit_ref_snapshot *rs)
7979+
{
7980+
if (rs->reg == ZREG_NONE) {
7981+
fprintf(stderr, "?");
7982+
} else if (!IR_REG_SPILLED(rs->reg)) {
7983+
fprintf(stderr, "%s", zend_reg_name(rs->reg));
7984+
} else {
7985+
fprintf(stderr, "0x%x(%s)", rs->offset, zend_reg_name(IR_REG_NUM(rs->reg)));
7986+
}
7987+
}
7988+
79767989
static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
79777990
{
79787991
int i, j;
@@ -8003,9 +8016,11 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
80038016
if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
80048017
fprintf(stderr, "/POLY");
80058018
if (t->exit_info[i].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8006-
fprintf(stderr, "(%s, %s)",
8007-
t->exit_info[i].poly_func_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_func_reg) : "?",
8008-
t->exit_info[i].poly_this_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_this_reg) : "?");
8019+
fprintf(stderr, "(");
8020+
zend_jit_dump_ref_snapshot(&t->exit_info[i].poly_func);
8021+
fprintf(stderr, ", ");
8022+
zend_jit_dump_ref_snapshot(&t->exit_info[i].poly_this);
8023+
fprintf(stderr, ")");
80098024
}
80108025
}
80118026
if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
@@ -8737,9 +8752,15 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf
87378752
}
87388753
}
87398754
if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8740-
ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0);
8741-
zend_function *func = (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg];
8755+
zend_jit_ref_snapshot *func_snapshot = &t->exit_info[exit_num].poly_func;
8756+
ZEND_ASSERT(func_snapshot->reg >= 0);
87428757

8758+
zend_function *func;
8759+
if (IR_REG_SPILLED(func_snapshot->reg)) {
8760+
func = *(zend_function**)(regs->gpr[IR_REG_NUM(func_snapshot->reg)] + func_snapshot->offset);
8761+
} else {
8762+
func = (zend_function*)regs->gpr[func_snapshot->reg];
8763+
}
87438764
if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
87448765
zend_string_release_ex(func->common.function_name, 0);
87458766
zend_free_trampoline(func);

0 commit comments

Comments
 (0)