archive/php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2025-08-16 05:58:45 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

JIT refactoring to allow run-time changes of JIT options (triggers, optimization_level, debug flags, etc)

Browse source
This commit is contained in:
Dmitry Stogov 2020-05-18 10:23:06 +03:00
parent 1179686f62
commit 0695048e20
14 changed files with 532 additions and 472 deletions
Download patch file Download diff file Expand all files Collapse all files

26
ext/opcache/ZendAccelerator.c
View file

@ -2854,6 +2854,10 @@ static int accel_startup(zend_extension *extension)
accel_globals_ctor(&accel_globals); accel_globals_ctor(&accel_globals);
#endif #endif
#ifdef HAVE_JIT
zend_jit_init();
#endif
#ifdef ZEND_WIN32 #ifdef ZEND_WIN32
# if !defined(__has_feature) || !__has_feature(address_sanitizer) # if !defined(__has_feature) || !__has_feature(address_sanitizer)
_setmaxstdio(2048); /* The default configuration is limited to 512 stdio files */ _setmaxstdio(2048); /* The default configuration is limited to 512 stdio files */
@ -2937,8 +2941,7 @@ static int accel_post_startup(void)
size_t jit_size = 0; size_t jit_size = 0;
zend_bool reattached = 0; zend_bool reattached = 0;
if (ZCG(accel_directives).jit && if (JIT_G(enabled) && JIT_G(buffer_size)) {
ZCG(accel_directives).jit_buffer_size) {
size_t page_size; size_t page_size;
# ifdef _WIN32 # ifdef _WIN32
@ -2952,12 +2955,9 @@ static int accel_post_startup(void)
zend_accel_error(ACCEL_LOG_FATAL, "Failure to initialize shared memory structures - can't get page size."); zend_accel_error(ACCEL_LOG_FATAL, "Failure to initialize shared memory structures - can't get page size.");
abort(); abort();
} }
jit_size = ZCG(accel_directives).jit_buffer_size; jit_size = JIT_G(buffer_size);
jit_size = ZEND_MM_ALIGNED_SIZE_EX(jit_size, page_size); jit_size = ZEND_MM_ALIGNED_SIZE_EX(jit_size, page_size);
shm_size += jit_size; shm_size += jit_size;
} else {
ZCG(accel_directives).jit = 0;
ZCG(accel_directives).jit_buffer_size = 0;
} }
switch (zend_shared_alloc_startup(shm_size, jit_size)) { switch (zend_shared_alloc_startup(shm_size, jit_size)) {
@ -3010,13 +3010,13 @@ static int accel_post_startup(void)
zend_shared_alloc_lock(); zend_shared_alloc_lock();
#ifdef HAVE_JIT #ifdef HAVE_JIT
if (ZCG(accel_directives).jit && if (JIT_G(enabled)) {
ZCG(accel_directives).jit_buffer_size && if (JIT_G(buffer_size) == 0
ZSMMG(reserved) && || !ZSMMG(reserved)
zend_jit_startup(ZCG(accel_directives).jit, ZSMMG(reserved), jit_size, reattached) == SUCCESS) { || zend_jit_startup(ZSMMG(reserved), jit_size, reattached) != SUCCESS) {
ZCG(jit_enabled) = 1; JIT_G(enabled) = 0;
} else { JIT_G(on) = 0;
ZCG(jit_enabled) = 0; }
} }
#endif #endif
zend_shared_alloc_save_state(); zend_shared_alloc_save_state();

9
ext/opcache/ZendAccelerator.h
View file

@ -191,12 +191,6 @@ typedef struct _zend_accel_directives {
#ifdef ZEND_WIN32 #ifdef ZEND_WIN32
char *cache_id; char *cache_id;
#endif #endif
#ifdef HAVE_JIT
zend_long jit;
zend_long jit_buffer_size;
zend_long jit_debug;
zend_long jit_bisect_limit;
#endif
} zend_accel_directives; } zend_accel_directives;
typedef struct _zend_accel_globals { typedef struct _zend_accel_globals {
@ -227,9 +221,6 @@ typedef struct _zend_accel_globals {
void *arena_mem; void *arena_mem;
zend_persistent_script *current_persistent_script; zend_persistent_script *current_persistent_script;
zend_bool is_immutable_class; zend_bool is_immutable_class;
#ifdef HAVE_JIT
zend_bool jit_enabled;
#endif
/* cache to save hash lookup on the same INCLUDE opcode */ /* cache to save hash lookup on the same INCLUDE opcode */
const zend_op *cache_opline; const zend_op *cache_opline;
zend_persistent_script *cache_persistent_script; zend_persistent_script *cache_persistent_script;

479
ext/opcache/jit/zend_jit.c
View file

@ -22,6 +22,7 @@
#include "Zend/zend_vm.h" #include "Zend/zend_vm.h"
#include "Zend/zend_exceptions.h" #include "Zend/zend_exceptions.h"
#include "Zend/zend_constants.h" #include "Zend/zend_constants.h"
#include "Zend/zend_ini.h"
#include "zend_smart_str.h" #include "zend_smart_str.h"
#include "jit/zend_jit.h" #include "jit/zend_jit.h"
@ -37,7 +38,7 @@
#include "jit/zend_jit_internal.h" #include "jit/zend_jit_internal.h"
#ifdef ZTS #ifdef ZTS
int zend_jit_globals_id; int jit_globals_id;
#else #else
zend_jit_globals jit_globals; zend_jit_globals jit_globals;
#endif #endif
@ -82,11 +83,6 @@ typedef struct _zend_jit_stub {
#define JIT_STUB(name) \ #define JIT_STUB(name) \
{JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub} {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub}
static zend_uchar zend_jit_level = 0;
static zend_uchar zend_jit_trigger = 0;
static zend_uchar zend_jit_reg_alloc = 0;
static zend_uchar zend_jit_cpu_flags = 0;
zend_ulong zend_jit_profile_counter = 0; zend_ulong zend_jit_profile_counter = 0;
int zend_jit_profile_counter_rid = -1; int zend_jit_profile_counter_rid = -1;
@ -105,9 +101,11 @@ static zend_long jit_bisect_pos = 0;
static const void *zend_jit_runtime_jit_handler = NULL; static const void *zend_jit_runtime_jit_handler = NULL;
static const void *zend_jit_profile_jit_handler = NULL; static const void *zend_jit_profile_jit_handler = NULL;
static const void *zend_jit_func_counter_handler = NULL; static const void *zend_jit_func_hot_counter_handler = NULL;
static const void *zend_jit_ret_counter_handler = NULL; static const void *zend_jit_loop_hot_counter_handler = NULL;
static const void *zend_jit_loop_counter_handler = NULL; static const void *zend_jit_func_trace_counter_handler = NULL;
static const void *zend_jit_ret_trace_counter_handler = NULL;
static const void *zend_jit_loop_trace_counter_handler = NULL;
static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa); static int zend_may_overflow(const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa);
static void ZEND_FASTCALL zend_runtime_jit(void); static void ZEND_FASTCALL zend_runtime_jit(void);
@ -199,9 +197,11 @@ ZEND_EXT_API void zend_jit_status(zval *ret)
{ {
zval stats; zval stats;
array_init(&stats); array_init(&stats);
add_assoc_long(&stats, "level", zend_jit_level); add_assoc_bool(&stats, "enabled", JIT_G(enabled));
add_assoc_long(&stats, "trigger", zend_jit_trigger); add_assoc_bool(&stats, "on", JIT_G(on));
add_assoc_long(&stats, "reg-alloc", zend_jit_reg_alloc); add_assoc_long(&stats, "kind", JIT_G(trigger));
add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
if (dasm_buf) { if (dasm_buf) {
add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf); add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr); add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
@ -349,14 +349,14 @@ static void *dasm_link_and_encode(dasm_State **dasm_state,
#if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE) #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
if (!name) { if (!name) {
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) {
str = zend_jit_func_name(op_array); str = zend_jit_func_name(op_array);
if (str) { if (str) {
name = ZSTR_VAL(str); name = ZSTR_VAL(str);
} }
} }
#ifdef HAVE_DISASM #ifdef HAVE_DISASM
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
zend_jit_disasm( zend_jit_disasm(
name, name,
@ -367,9 +367,9 @@ static void *dasm_link_and_encode(dasm_State **dasm_state,
size); size);
} }
} else { } else {
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size); zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
if (trace_num || (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM_STUBS) != 0) { if (trace_num || (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_STUBS) != 0) {
zend_jit_disasm( zend_jit_disasm(
name, name,
(op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL, (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
@ -384,7 +384,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state,
#endif #endif
#ifdef HAVE_GDB #ifdef HAVE_GDB
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
if (name) { if (name) {
zend_jit_gdb_register( zend_jit_gdb_register(
name, name,
@ -396,7 +396,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state,
#endif #endif
#ifdef HAVE_OPROFILE #ifdef HAVE_OPROFILE
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
zend_jit_oprofile_register( zend_jit_oprofile_register(
name, name,
entry, entry,
@ -405,13 +405,13 @@ static void *dasm_link_and_encode(dasm_State **dasm_state,
#endif #endif
#ifdef HAVE_PERFTOOLS #ifdef HAVE_PERFTOOLS
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
if (name) { if (name) {
zend_jit_perf_map_register( zend_jit_perf_map_register(
name, name,
entry, entry,
size); size);
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
zend_jit_perf_jitdump_register( zend_jit_perf_jitdump_register(
name, name,
entry, entry,
@ -422,7 +422,7 @@ static void *dasm_link_and_encode(dasm_State **dasm_state,
#endif #endif
#ifdef HAVE_VTUNE #ifdef HAVE_VTUNE
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_VTUNE) { if (JIT_G(debug) & ZEND_JIT_DEBUG_VTUNE) {
if (name) { if (name) {
zend_jit_vtune_register( zend_jit_vtune_register(
name, name,
@ -656,7 +656,7 @@ static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script
} }
#endif #endif
if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
&& ssa->cfg.blocks && ssa->cfg.blocks
&& op_array->last_try_catch == 0 && op_array->last_try_catch == 0
&& !(op_array->fn_flags & ZEND_ACC_GENERATOR) && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
@ -683,7 +683,7 @@ static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script
static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa) static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
{ {
if ((zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
&& ssa->cfg.blocks && ssa->cfg.blocks
&& op_array->last_try_catch == 0 && op_array->last_try_catch == 0
&& !(op_array->fn_flags & ZEND_ACC_GENERATOR) && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
@ -1221,7 +1221,7 @@ static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ss
} }
if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) { if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
/* Register hinting (a cheap way for register coalescing) */ /* Register hinting (a cheap way for register coalescing) */
for (i = 0; i < ssa->vars_count; i++) { for (i = 0; i < ssa->vars_count; i++) {
if (intervals[i]) { if (intervals[i]) {
@ -1787,7 +1787,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array
} }
if (list) { if (list) {
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
ival = list; ival = list;
while (ival) { while (ival) {
@ -1815,7 +1815,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array
ival = next; ival = next;
} }
if (zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) { if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
/* Naive SSA resolution */ /* Naive SSA resolution */
for (i = 0; i < ssa->vars_count; i++) { for (i = 0; i < ssa->vars_count; i++) {
if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) { if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
@ -1929,7 +1929,7 @@ static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array
} }
} }
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]"); fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
for (i = 0; i < ssa->vars_count; i++) { for (i = 0; i < ssa->vars_count; i++) {
ival = intervals[i]; ival = intervals[i];
@ -1970,10 +1970,10 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
zend_class_entry *ce; zend_class_entry *ce;
if (ZCG(accel_directives).jit_bisect_limit) { if (JIT_G(bisect_limit)) {
jit_bisect_pos++; jit_bisect_pos++;
if (jit_bisect_pos >= ZCG(accel_directives).jit_bisect_limit) { if (jit_bisect_pos >= JIT_G(bisect_limit)) {
if (jit_bisect_pos == ZCG(accel_directives).jit_bisect_limit) { if (jit_bisect_pos == JIT_G(bisect_limit)) {
fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n", fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
op_array->scope ? "::" : "", op_array->scope ? "::" : "",
@ -1984,7 +1984,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
} }
} }
if (zend_jit_reg_alloc) { if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
checkpoint = zend_arena_checkpoint(CG(arena)); checkpoint = zend_arena_checkpoint(CG(arena));
ra = zend_jit_allocate_registers(op_array, ssa); ra = zend_jit_allocate_registers(op_array, ssa);
} }
@ -2017,7 +2017,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) { if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) {
if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) { if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
/* pass */ /* pass */
} else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE && } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
ssa->cfg.blocks[b].len == 1 && ssa->cfg.blocks[b].len == 1 &&
(ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) && (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) &&
op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) { op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) {
@ -2079,13 +2079,13 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
} else { } else {
if (recv_emitted) { if (recv_emitted) {
zend_jit_jmp(&dasm_state, b); zend_jit_jmp(&dasm_state, b);
} else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE && } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
ssa->cfg.blocks[b].len == 1 && ssa->cfg.blocks[b].len == 1 &&
(ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) { (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
/* don't generate code for BB with single opcode */ /* don't generate code for BB with single opcode */
dasm_free(&dasm_state); dasm_free(&dasm_state);
if (zend_jit_reg_alloc) { if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
zend_arena_release(&CG(arena), checkpoint); zend_arena_release(&CG(arena), checkpoint);
} }
return SUCCESS; return SUCCESS;
@ -2094,13 +2094,13 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
zend_jit_prologue(&dasm_state); zend_jit_prologue(&dasm_state);
recv_emitted = 1; recv_emitted = 1;
} }
} else if (zend_jit_level < ZEND_JIT_LEVEL_INLINE && } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
ssa->cfg.blocks[b].len == 1 && ssa->cfg.blocks[b].len == 1 &&
(ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) { (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
/* don't generate code for BB with single opcode */ /* don't generate code for BB with single opcode */
dasm_free(&dasm_state); dasm_free(&dasm_state);
if (zend_jit_reg_alloc) { if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
zend_arena_release(&CG(arena), checkpoint); zend_arena_release(&CG(arena), checkpoint);
} }
return SUCCESS; return SUCCESS;
@ -2113,7 +2113,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
is_terminated = 0; is_terminated = 0;
zend_jit_label(&dasm_state, b); zend_jit_label(&dasm_state, b);
if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) { if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
&& ssa->cfg.blocks[b].start != 0 && ssa->cfg.blocks[b].start != 0
&& (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
@ -2145,7 +2145,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
if (!ssa->cfg.blocks[b].len) { if (!ssa->cfg.blocks[b].len) {
continue; continue;
} }
if ((zend_jit_reg_alloc >= ZEND_JIT_REG_ALLOC_GLOBAL) && ra) { if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
zend_ssa_phi *phi = ssa->blocks[b].phis; zend_ssa_phi *phi = ssa->blocks[b].phis;
while (phi) { while (phi) {
@ -2185,7 +2185,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
call_level++; call_level++;
} }
if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
switch (opline->opcode) { switch (opline->opcode) {
case ZEND_PRE_INC: case ZEND_PRE_INC:
case ZEND_PRE_DEC: case ZEND_PRE_DEC:
@ -2892,7 +2892,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
case ZEND_SWITCH_STRING: case ZEND_SWITCH_STRING:
break; break;
case ZEND_JMP: case ZEND_JMP:
if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) { if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
const zend_op *target = OP_JMP_ADDR(opline, opline->op1); const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
if (!zend_jit_set_ip(&dasm_state, target)) { if (!zend_jit_set_ip(&dasm_state, target)) {
@ -2972,7 +2972,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
zend_class_entry *ce = NULL; zend_class_entry *ce = NULL;
if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
if (ssa->ops && ssa->var_info) { if (ssa->ops && ssa->var_info) {
zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def]; zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
if (res_ssa->ce && !res_ssa->is_instanceof) { if (res_ssa->ce && !res_ssa->is_instanceof) {
@ -2995,7 +2995,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
const zend_op *next_opline = opline + 1; const zend_op *next_opline = opline + 1;
zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]); zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]);
if (zend_jit_level < ZEND_JIT_LEVEL_INLINE) { if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
zend_jit_call(&dasm_state, next_opline, b + 1); zend_jit_call(&dasm_state, next_opline, b + 1);
is_terminated = 1; is_terminated = 1;
} else { } else {
@ -3027,7 +3027,7 @@ done:
} }
dasm_free(&dasm_state); dasm_free(&dasm_state);
if (zend_jit_reg_alloc) { if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
zend_arena_release(&CG(arena), checkpoint); zend_arena_release(&CG(arena), checkpoint);
} }
return SUCCESS; return SUCCESS;
@ -3036,7 +3036,7 @@ jit_failure:
if (dasm_state) { if (dasm_state) {
dasm_free(&dasm_state); dasm_free(&dasm_state);
} }
if (zend_jit_reg_alloc) { if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
zend_arena_release(&CG(arena), checkpoint); zend_arena_release(&CG(arena), checkpoint);
} }
return FAILURE; return FAILURE;
@ -3072,7 +3072,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons
goto jit_failure; goto jit_failure;
} }
if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
if (zend_jit_collect_calls(op_array, script) != SUCCESS) { if (zend_jit_collect_calls(op_array, script) != SUCCESS) {
ZEND_SET_FUNC_INFO(op_array, NULL); ZEND_SET_FUNC_INFO(op_array, NULL);
goto jit_failure; goto jit_failure;
@ -3088,7 +3088,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons
goto jit_failure; goto jit_failure;
} }
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa); zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
} }
@ -3096,7 +3096,7 @@ static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, cons
goto jit_failure; goto jit_failure;
} }
if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
ZEND_SET_FUNC_INFO(op_array, NULL); ZEND_SET_FUNC_INFO(op_array, NULL);
} }
@ -3212,8 +3212,8 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array)
zend_cfg cfg; zend_cfg cfg;
uint32_t i; uint32_t i;
ZEND_ASSERT(zend_jit_func_counter_handler != NULL); ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL);
ZEND_ASSERT(zend_jit_loop_counter_handler != NULL); ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL);
if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
return FAILURE; return FAILURE;
@ -3240,13 +3240,13 @@ static int zend_jit_setup_hot_counters(zend_op_array *op_array)
} }
} }
opline->handler = (const void*)zend_jit_func_counter_handler; opline->handler = (const void*)zend_jit_func_hot_counter_handler;
for (i = 0; i < cfg.blocks_count; i++) { for (i = 0; i < cfg.blocks_count; i++) {
if ((cfg.blocks[i].flags & ZEND_BB_REACHABLE) && if ((cfg.blocks[i].flags & ZEND_BB_REACHABLE) &&
(cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER)) { (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
op_array->opcodes[cfg.blocks[i].start].handler = op_array->opcodes[cfg.blocks[i].start].handler =
(const void*)zend_jit_loop_counter_handler; (const void*)zend_jit_loop_hot_counter_handler;
} }
} }
@ -3279,7 +3279,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
return FAILURE; return FAILURE;
} }
if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
zend_op *opline = op_array->opcodes; zend_op *opline = op_array->opcodes;
/* Set run-time JIT handler */ /* Set run-time JIT handler */
@ -3293,7 +3293,7 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
opline->handler = (const void*)zend_jit_runtime_jit_handler; opline->handler = (const void*)zend_jit_runtime_jit_handler;
return SUCCESS; return SUCCESS;
} else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
zend_op *opline = op_array->opcodes; zend_op *opline = op_array->opcodes;
ZEND_ASSERT(zend_jit_profile_jit_handler != NULL); ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
@ -3308,13 +3308,13 @@ ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
} }
return SUCCESS; return SUCCESS;
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
return zend_jit_setup_hot_counters(op_array); return zend_jit_setup_hot_counters(op_array);
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
return zend_jit_setup_hot_trace_counters(op_array); return zend_jit_setup_hot_trace_counters(op_array);
} else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD) { } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
return zend_real_jit_func(op_array, script, NULL); return zend_real_jit_func(op_array, script, NULL);
} else if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { } else if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT) {
if (zend_needs_manual_jit(op_array)) { if (zend_needs_manual_jit(op_array)) {
return zend_real_jit_func(op_array, script, NULL); return zend_real_jit_func(op_array, script, NULL);
} else { } else {
@ -3345,20 +3345,20 @@ ZEND_EXT_API int zend_jit_script(zend_script *script)
zend_analyze_call_graph(&CG(arena), script, &call_graph); zend_analyze_call_graph(&CG(arena), script, &call_graph);
if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC || if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
for (i = 0; i < call_graph.op_arrays_count; i++) { for (i = 0; i < call_graph.op_arrays_count; i++) {
ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL); ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) { if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
goto jit_failure; goto jit_failure;
} }
} }
} else if (zend_jit_trigger == ZEND_JIT_ON_SCRIPT_LOAD || } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD ||
zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT) {
if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT) { if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT) {
int do_jit = 0; int do_jit = 0;
for (i = 0; i < call_graph.op_arrays_count; i++) { for (i = 0; i < call_graph.op_arrays_count; i++) {
if (zend_needs_manual_jit(call_graph.op_arrays[i])) { if (zend_needs_manual_jit(call_graph.op_arrays[i])) {
@ -3391,7 +3391,7 @@ ZEND_EXT_API int zend_jit_script(zend_script *script)
} }
for (i = 0; i < call_graph.op_arrays_count; i++) { for (i = 0; i < call_graph.op_arrays_count; i++) {
if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT &&
!zend_needs_manual_jit(call_graph.op_arrays[i])) { !zend_needs_manual_jit(call_graph.op_arrays[i])) {
continue; continue;
} }
@ -3404,9 +3404,9 @@ ZEND_EXT_API int zend_jit_script(zend_script *script)
} }
} }
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
for (i = 0; i < call_graph.op_arrays_count; i++) { for (i = 0; i < call_graph.op_arrays_count; i++) {
if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT &&
!zend_needs_manual_jit(call_graph.op_arrays[i])) { !zend_needs_manual_jit(call_graph.op_arrays[i])) {
continue; continue;
} }
@ -3418,7 +3418,7 @@ ZEND_EXT_API int zend_jit_script(zend_script *script)
} }
for (i = 0; i < call_graph.op_arrays_count; i++) { for (i = 0; i < call_graph.op_arrays_count; i++) {
if (zend_jit_trigger == ZEND_JIT_ON_DOC_COMMENT && if (JIT_G(trigger) == ZEND_JIT_ON_DOC_COMMENT &&
!zend_needs_manual_jit(call_graph.op_arrays[i])) { !zend_needs_manual_jit(call_graph.op_arrays[i])) {
continue; continue;
} }
@ -3451,13 +3451,13 @@ jit_failure:
ZEND_EXT_API void zend_jit_unprotect(void) ZEND_EXT_API void zend_jit_unprotect(void)
{ {
#ifdef HAVE_MPROTECT #ifdef HAVE_MPROTECT
if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE) != 0) { if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE) != 0) {
fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
} }
} }
#elif _WIN32 #elif _WIN32
if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
DWORD old; DWORD old;
if (!VirtualProtect(dasm_buf, dasm_size, PAGE_READWRITE, &old)) { if (!VirtualProtect(dasm_buf, dasm_size, PAGE_READWRITE, &old)) {
@ -3470,13 +3470,13 @@ ZEND_EXT_API void zend_jit_unprotect(void)
ZEND_EXT_API void zend_jit_protect(void) ZEND_EXT_API void zend_jit_protect(void)
{ {
#ifdef HAVE_MPROTECT #ifdef HAVE_MPROTECT
if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) { if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
} }
} }
#elif _WIN32 #elif _WIN32
if (!(ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) { if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
DWORD old; DWORD old;
if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) { if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
@ -3505,83 +3505,21 @@ static int zend_jit_make_stubs(void)
} }
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { zend_jit_runtime_jit_handler = dasm_labels[zend_lbhybrid_runtime_jit];
dasm_setup(&dasm_state, dasm_actions); zend_jit_profile_jit_handler = dasm_labels[zend_lbhybrid_profile_jit];
if (!zend_jit_hybrid_runtime_jit_stub(&dasm_state)) { zend_jit_func_hot_counter_handler = dasm_labels[zend_lbhybrid_func_hot_counter];
return 0; zend_jit_loop_hot_counter_handler = dasm_labels[zend_lbhybrid_loop_hot_counter];
} zend_jit_func_trace_counter_handler = dasm_labels[zend_lbhybrid_func_trace_counter];
zend_jit_runtime_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_runtime_jit", 0); zend_jit_ret_trace_counter_handler = dasm_labels[zend_lbhybrid_ret_trace_counter];
if (!zend_jit_runtime_jit_handler) { zend_jit_loop_trace_counter_handler = dasm_labels[zend_lbhybrid_loop_trace_counter];
return 0;
}
} else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) {
dasm_setup(&dasm_state, dasm_actions);
if (!zend_jit_hybrid_profile_jit_stub(&dasm_state)) {
return 0;
}
zend_jit_profile_jit_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_profile_jit", 0);
if (!zend_jit_profile_jit_handler) {
return 0;
}
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) {
dasm_setup(&dasm_state, dasm_actions);
if (!zend_jit_hybrid_func_counter_stub(&dasm_state)) {
return 0;
}
zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter", 0);
if (!zend_jit_func_counter_handler) {
return 0;
}
dasm_setup(&dasm_state, dasm_actions);
if (!zend_jit_hybrid_loop_counter_stub(&dasm_state)) {
return 0;
}
zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter", 0);
if (!zend_jit_loop_counter_handler) {
return 0;
}
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
dasm_setup(&dasm_state, dasm_actions);
if (!zend_jit_hybrid_func_trace_counter_stub(&dasm_state)) {
return 0;
}
zend_jit_func_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_func_counter", 0);
if (!zend_jit_func_counter_handler) {
return 0;
}
dasm_setup(&dasm_state, dasm_actions);
if (!zend_jit_hybrid_ret_trace_counter_stub(&dasm_state)) {
return 0;
}
zend_jit_ret_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_ret_counter", 0);
if (!zend_jit_ret_counter_handler) {
return 0;
}
dasm_setup(&dasm_state, dasm_actions);
if (!zend_jit_hybrid_loop_trace_counter_stub(&dasm_state)) {
return 0;
}
zend_jit_loop_counter_handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, "JIT$$hybrid_loop_counter", 0);
if (!zend_jit_loop_counter_handler) {
return 0;
}
}
} else { } else {
if (zend_jit_trigger == ZEND_JIT_ON_FIRST_EXEC) { zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit; zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
} else if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper; zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
zend_jit_func_counter_handler = (const void*)zend_jit_func_counter_helper; zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
zend_jit_loop_counter_handler = (const void*)zend_jit_loop_counter_helper; zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
zend_jit_func_counter_handler = (const void*)zend_jit_func_trace_helper;
zend_jit_ret_counter_handler = (const void*)zend_jit_ret_trace_helper;
zend_jit_loop_counter_handler = (const void*)zend_jit_loop_trace_helper;
}
} }
dasm_free(&dasm_state); dasm_free(&dasm_state);
@ -3594,20 +3532,119 @@ static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
zend_jit_trace_init_caches(); zend_jit_trace_init_caches();
} }
ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bool reattached) static int zend_jit_parse_config_num(zend_long jit)
{ {
int ret; if (jit == 0) {
JIT_G(on) = 0;
return SUCCESS;
}
if (jit < 0) return FAILURE;
if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
JIT_G(opt_level) = jit % 10;
jit /= 10;
if (jit % 10 > 5) return FAILURE;
JIT_G(trigger) = jit % 10;
jit /= 10;
if (jit % 10 > 2) return FAILURE;
JIT_G(opt_flags) = jit % 10;
jit /= 10;
if (jit % 10 > 1) return FAILURE;
JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
if (jit / 10 != 0) return FAILURE;
JIT_G(on) = 1;
return SUCCESS;
}
ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
{
zend_ulong num;
if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
if (stage == ZEND_INI_STAGE_RUNTIME) {
zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
}
return FAILURE;
}
if (ZSTR_LEN(jit) == 0
|| zend_string_equals_literal_ci(jit, "disable")) {
JIT_G(enabled) = 0;
JIT_G(on) = 0;
return SUCCESS;
} else if (zend_string_equals_literal_ci(jit, "0")
|| zend_string_equals_literal_ci(jit, "off")
|| zend_string_equals_literal_ci(jit, "no")
|| zend_string_equals_literal_ci(jit, "false")) {
JIT_G(enabled) = 1;
JIT_G(on) = 0;
return SUCCESS;
} else if (zend_string_equals_literal_ci(jit, "1")
|| zend_string_equals_literal_ci(jit, "on")
|| zend_string_equals_literal_ci(jit, "yes")
|| zend_string_equals_literal_ci(jit, "true")) {
JIT_G(enabled) = 1;
JIT_G(on) = 1;
JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
return SUCCESS;
} else if (ZEND_HANDLE_NUMERIC(jit, num)) {
if (zend_jit_parse_config_num((zend_long)num) != SUCCESS) {
goto failure;
}
JIT_G(enabled) = 1;
return SUCCESS;
}
failure:
zend_error(E_WARNING, "Invalid opcache.jit setting. Should be \"disable\", \"on\", \"off\" or 4-digit number");
JIT_G(enabled) = 0;
JIT_G(on) = 0;
return FAILURE;
}
ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
{
if (stage != ZEND_INI_STAGE_STARTUP) {
if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
if (stage == ZEND_INI_STAGE_RUNTIME) {
zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
}
return FAILURE;
}
#ifdef HAVE_DISASM
if (new_val & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
if (JIT_G(enabled) && !JIT_G(symbols) && !zend_jit_disasm_init()) {
// TODO: error reporting and cleanup ???
return FAILURE;
}
// TODO: symbols for JIT-ed code compiled before are missing ???
}
#endif
}
return SUCCESS;
}
ZEND_EXT_API void zend_jit_init(void)
{
#ifdef ZTS #ifdef ZTS
zend_jit_globals_id = ts_allocate_id(&zend_jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL); jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL);
#else #else
zend_jit_globals_ctor(&jit_globals); zend_jit_globals_ctor(&jit_globals);
#endif #endif
}
zend_jit_level = ZEND_JIT_LEVEL(jit); ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, zend_bool reattached)
zend_jit_trigger = ZEND_JIT_TRIGGER(jit); {
zend_jit_reg_alloc = ZEND_JIT_REG_ALLOC(jit); int ret;
zend_jit_cpu_flags = ZEND_JIT_CPU_FLAGS(jit);
zend_jit_vm_kind = zend_vm_kind(); zend_jit_vm_kind = zend_vm_kind();
if (zend_jit_vm_kind != ZEND_VM_KIND_CALL && if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
@ -3623,16 +3660,14 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo
return FAILURE; return FAILURE;
} }
if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { zend_jit_profile_counter_rid = zend_get_op_array_extension_handle();
zend_jit_profile_counter_rid = zend_get_op_array_extension_handle();
}
#ifdef HAVE_GDB #ifdef HAVE_GDB
zend_jit_gdb_init(); zend_jit_gdb_init();
#endif #endif
#ifdef HAVE_OPROFILE #ifdef HAVE_OPROFILE
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
if (!zend_jit_oprofile_startup()) { if (!zend_jit_oprofile_startup()) {
// TODO: error reporting and cleanup ??? // TODO: error reporting and cleanup ???
return FAILURE; return FAILURE;
@ -3644,7 +3679,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo
dasm_size = size; dasm_size = size;
#ifdef HAVE_MPROTECT #ifdef HAVE_MPROTECT
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) { if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno)); fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
} }
@ -3654,7 +3689,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo
} }
} }
#elif _WIN32 #elif _WIN32
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
DWORD old; DWORD old;
if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) { if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
@ -3681,7 +3716,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo
} }
#ifdef HAVE_DISASM #ifdef HAVE_DISASM
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
if (!zend_jit_disasm_init()) { if (!zend_jit_disasm_init()) {
// TODO: error reporting and cleanup ??? // TODO: error reporting and cleanup ???
return FAILURE; return FAILURE;
@ -3690,7 +3725,7 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo
#endif #endif
#ifdef HAVE_PERFTOOLS #ifdef HAVE_PERFTOOLS
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
zend_jit_perf_jitdump_open(); zend_jit_perf_jitdump_open();
} }
#endif #endif
@ -3714,10 +3749,8 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo
#endif #endif
} }
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (zend_jit_trace_startup() != SUCCESS) {
if (zend_jit_trace_startup() != SUCCESS) { return FAILURE;
return FAILURE;
}
} }
return SUCCESS; return SUCCESS;
@ -3726,113 +3759,73 @@ ZEND_EXT_API int zend_jit_startup(zend_long jit, void *buf, size_t size, zend_bo
ZEND_EXT_API void zend_jit_shutdown(void) ZEND_EXT_API void zend_jit_shutdown(void)
{ {
#ifdef HAVE_OPROFILE #ifdef HAVE_OPROFILE
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_OPROFILE) { if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
zend_jit_oprofile_shutdown(); zend_jit_oprofile_shutdown();
} }
#endif #endif
#ifdef HAVE_GDB #ifdef HAVE_GDB
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_GDB) { if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
zend_jit_gdb_unregister(); zend_jit_gdb_unregister();
} }
#endif #endif
#ifdef HAVE_DISASM #ifdef HAVE_DISASM
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { zend_jit_disasm_shutdown();
zend_jit_disasm_shutdown();
}
#endif #endif
#ifdef HAVE_PERFTOOLS #ifdef HAVE_PERFTOOLS
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_PERF_DUMP) { if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
zend_jit_perf_jitdump_close(); zend_jit_perf_jitdump_close();
} }
#endif #endif
} }
static void zend_jit_reset_counters(void)
{
int i;
for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
}
}
ZEND_EXT_API void zend_jit_activate(void) ZEND_EXT_API void zend_jit_activate(void)
{ {
if (zend_jit_trigger == ZEND_JIT_ON_HOT_COUNTERS) { zend_jit_profile_counter = 0;
int i; if (JIT_G(on)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) { zend_jit_reset_counters();
zend_jit_hot_counters[i] = ZEND_JIT_HOT_COUNTER_INIT; } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
zend_jit_reset_counters();
zend_jit_trace_reset_caches();
} }
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) {
int i;
for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
zend_jit_hot_counters[i] = ZEND_JIT_TRACE_COUNTER_INIT;
}
zend_jit_trace_reset_caches();
} }
} }
ZEND_EXT_API void zend_jit_deactivate(void) ZEND_EXT_API void zend_jit_deactivate(void)
{ {
if (zend_jit_trigger == ZEND_JIT_ON_PROF_REQUEST) { if (zend_jit_profile_counter) {
if (!zend_jit_profile_counter) { zend_class_entry *ce;
return;
} else {
zend_class_entry *ce;
zend_shared_alloc_lock(); zend_shared_alloc_lock();
SHM_UNPROTECT(); SHM_UNPROTECT();
zend_jit_unprotect(); zend_jit_unprotect();
zend_jit_check_funcs(EG(function_table), 0); zend_jit_check_funcs(EG(function_table), 0);
ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) { ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) {
if (ce->type == ZEND_INTERNAL_CLASS) { if (ce->type == ZEND_INTERNAL_CLASS) {
break; break;
} }
zend_jit_check_funcs(&ce->function_table, 1); zend_jit_check_funcs(&ce->function_table, 1);
} ZEND_HASH_FOREACH_END(); } ZEND_HASH_FOREACH_END();
zend_jit_protect(); zend_jit_protect();
SHM_PROTECT(); SHM_PROTECT();
zend_shared_alloc_unlock(); zend_shared_alloc_unlock();
zend_jit_profile_counter = 0; zend_jit_profile_counter = 0;
}
} }
} }
#else /* HAVE_JIT */
ZEND_EXT_API int zend_jit_op_array(const zend_op_array *op_array, zend_script *script)
{
return FAILURE;
}
ZEND_EXT_API int zend_jit_script(zend_script *script)
{
return FAILURE;
}
ZEND_EXT_API void zend_jit_unprotect(void)
{
}
ZEND_EXT_API void zend_jit_protect(void)
{
}
ZEND_EXT_API int zend_jit_startup(zend_long jit, size_t size)
{
return FAILURE;
}
ZEND_EXT_API void zend_jit_shutdown(void)
{
}
ZEND_EXT_API void zend_jit_activate(void)
{
}
ZEND_EXT_API void zend_jit_deactivate(void)
{
}
#endif /* HAVE_JIT */ #endif /* HAVE_JIT */

96
ext/opcache/jit/zend_jit.h
View file

@ -26,8 +26,6 @@
#define ZEND_JIT_LEVEL_OPT_FUNCS 4 /* optimized JIT based on Type-Inference and call-tree */ #define ZEND_JIT_LEVEL_OPT_FUNCS 4 /* optimized JIT based on Type-Inference and call-tree */
#define ZEND_JIT_LEVEL_OPT_SCRIPT 5 /* optimized JIT based on Type-Inference and inner-procedure analysis */ #define ZEND_JIT_LEVEL_OPT_SCRIPT 5 /* optimized JIT based on Type-Inference and inner-procedure analysis */
#define ZEND_JIT_LEVEL(n) ((n) % 10)
#define ZEND_JIT_ON_SCRIPT_LOAD 0 #define ZEND_JIT_ON_SCRIPT_LOAD 0
#define ZEND_JIT_ON_FIRST_EXEC 1 #define ZEND_JIT_ON_FIRST_EXEC 1
#define ZEND_JIT_ON_PROF_REQUEST 2 /* compile the most frequently caled on first request functions */ #define ZEND_JIT_ON_PROF_REQUEST 2 /* compile the most frequently caled on first request functions */
@ -35,20 +33,17 @@
#define ZEND_JIT_ON_DOC_COMMENT 4 /* compile functions with "@jit" tag in doc-comments */ #define ZEND_JIT_ON_DOC_COMMENT 4 /* compile functions with "@jit" tag in doc-comments */
#define ZEND_JIT_ON_HOT_TRACE 5 /* trace functions after N calls or loop iterations */ #define ZEND_JIT_ON_HOT_TRACE 5 /* trace functions after N calls or loop iterations */
#define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10) #define ZEND_JIT_REG_ALLOC_LOCAL (1<<0) /* local linear scan register allocation */
#define ZEND_JIT_REG_ALLOC_GLOBAL (1<<1) /* global linear scan register allocation */
#define ZEND_JIT_CPU_AVX (1<<2) /* use AVX instructions, if available */
#define ZEND_JIT_REG_ALLOC_NONE 0 /* no register allocation */ //#define ZEND_JIT_LEVEL(n) ((n) % 10)
#define ZEND_JIT_REG_ALLOC_ENABLED 1 /* local linear scan register allocation */ //#define ZEND_JIT_TRIGGER(n) (((n) / 10) % 10)
#define ZEND_JIT_REG_ALLOC_GLOBAL 2 /* global linear scan register allocation */ //#define ZEND_JIT_REG_ALLOC(n) (((n) / 100) % 10)
//#define ZEND_JIT_CPU_FLAGS(n) (((n) / 1000) % 10)
#define ZEND_JIT_REG_ALLOC(n) (((n) / 100) % 10) #define ZEND_JIT_DEFAULT_OPTIONS "1205"
#define ZEND_JIT_DEFAULT_BUFFER_SIZE "0"
#define ZEND_JIT_CPU_AVX 1 /* use AVX instructions, if available */
#define ZEND_JIT_CPU_FLAGS(n) (((n) / 1000) % 10)
#define ZEND_JIT_DEFAULT "1205"
/* Makes profile based JIT (opcache.jit=2*) to generate code only for most /* Makes profile based JIT (opcache.jit=2*) to generate code only for most
@ -57,12 +52,13 @@
*/ */
#define ZEND_JIT_PROF_THRESHOLD 0.005 #define ZEND_JIT_PROF_THRESHOLD 0.005
/* Hot Counters based JIT parameters. /* Hot/Trace Counters based JIT parameters.
* TODO: this setting should be configurable * TODO: this setting should be configurable
*/ */
#define ZEND_JIT_HOT_FUNC_COST 1 #define ZEND_JIT_COUNTER_FUNC_COST 1
#define ZEND_JIT_HOT_LOOP_COST 2 #define ZEND_JIT_COUNTER_RET_COST 15
#define ZEND_JIT_HOT_COUNTER_INIT 127 #define ZEND_JIT_COUNTER_LOOP_COST 2
#define ZEND_JIT_COUNTER_INIT 127
#define ZEND_JIT_DEBUG_ASM (1<<0) #define ZEND_JIT_DEBUG_ASM (1<<0)
#define ZEND_JIT_DEBUG_SSA (1<<1) #define ZEND_JIT_DEBUG_SSA (1<<1)
@ -86,11 +82,73 @@
#define ZEND_JIT_DEBUG_TRACE_TSSA (1<<19) #define ZEND_JIT_DEBUG_TRACE_TSSA (1<<19)
#define ZEND_JIT_DEBUG_TRACE_EXIT_INFO (1<<20) #define ZEND_JIT_DEBUG_TRACE_EXIT_INFO (1<<20)
#define ZEND_JIT_DEBUG_PERSISTENT 0x1f0 /* profile and debbuger flags can't be changed at run-time */
#define ZEND_JIT_TRACE_MAX_TRACES 1024 /* max number of traces */
#define ZEND_JIT_TRACE_MAX_LENGTH 1024 /* max length of single trace */
#define ZEND_JIT_TRACE_MAX_EXITS 512 /* max number of side exits per trace */
#define ZEND_JIT_TRACE_MAX_SIDE_TRACES 128 /* max number of side traces of a root trace */
#define ZEND_JIT_TRACE_MAX_EXIT_COUNTERS 8192 /* max number of side exits for all trace */
#define ZEND_JIT_TRACE_MAX_FUNCS 30 /* max number of different functions in a single trace */
#define ZEND_JIT_TRACE_MAX_CALL_DEPTH 10 /* max depth of inlined calls */
#define ZEND_JIT_TRACE_MAX_RET_DEPTH 4 /* max depth of inlined returns */
#define ZEND_JIT_TRACE_MAX_RECURSION 2 /* max number of recursive inlined calls */
#define ZEND_JIT_TRACE_MAX_UNROLL_LOOPS 8 /* max number of unrolled loops */
#define ZEND_JIT_TRACE_HOT_SIDE_COUNT 8 /* number of exits before taking side trace */
#define ZEND_JIT_TRACE_HOT_RETURN_COUNT 8 /* number of returns before taking continuation trace */
#define ZEND_JIT_TRACE_MAX_ROOT_FAILURES 16 /* number of attempts to record/compile a root trace */
#define ZEND_JIT_TRACE_MAX_SIDE_FAILURES 4 /* number of attempts to record/compile a side trace */
#define ZEND_JIT_TRACE_BAD_ROOT_SLOTS 64 /* number of slots in bad root trace cache */
typedef struct _zend_jit_trace_rec zend_jit_trace_rec;
typedef struct _zend_jit_trace_stack_frame zend_jit_trace_stack_frame;
typedef struct _sym_node zend_sym_node;
typedef struct _zend_jit_globals {
zend_bool enabled;
zend_bool on;
uint8_t trigger;
uint8_t opt_level;
uint32_t opt_flags;
const char *options;
zend_long buffer_size;
zend_long debug;
zend_long bisect_limit;
zend_sym_node *symbols; /* symbols for disassembler */
zend_jit_trace_rec *current_trace;
zend_jit_trace_stack_frame *current_frame;
const zend_op *bad_root_cache_opline[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
uint8_t bad_root_cache_count[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
uint8_t bad_root_cache_stop[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
uint32_t bad_root_slot;
uint8_t exit_counters[ZEND_JIT_TRACE_MAX_EXIT_COUNTERS];
} zend_jit_globals;
#ifdef ZTS
# define JIT_G(v) ZEND_TSRMG(jit_globals_id, zend_jit_globals *, v)
extern int jit_globals_id;
#else
# define JIT_G(v) (jit_globals.v)
extern zend_jit_globals jit_globals;
#endif
ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script); ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script);
ZEND_EXT_API int zend_jit_script(zend_script *script); ZEND_EXT_API int zend_jit_script(zend_script *script);
ZEND_EXT_API void zend_jit_unprotect(void); ZEND_EXT_API void zend_jit_unprotect(void);
ZEND_EXT_API void zend_jit_protect(void); ZEND_EXT_API void zend_jit_protect(void);
ZEND_EXT_API int zend_jit_startup(zend_long jit, void *jit_buffer, size_t size, zend_bool reattached); ZEND_EXT_API void zend_jit_init(void);
ZEND_EXT_API int zend_jit_config(zend_string *jit_options, int stage);
ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage);
ZEND_EXT_API int zend_jit_startup(void *jit_buffer, size_t size, zend_bool reattached);
ZEND_EXT_API void zend_jit_shutdown(void); ZEND_EXT_API void zend_jit_shutdown(void);
ZEND_EXT_API void zend_jit_activate(void); ZEND_EXT_API void zend_jit_activate(void);
ZEND_EXT_API void zend_jit_deactivate(void); ZEND_EXT_API void zend_jit_deactivate(void);

27
ext/opcache/jit/zend_jit_disasm_x86.c
View file

@ -50,16 +50,14 @@ static void zend_jit_disasm_add_symbol(const char *name,
static struct ud ud; static struct ud ud;
typedef struct _sym_node { struct _sym_node {
uint64_t addr; uint64_t addr;
uint64_t end; uint64_t end;
struct _sym_node *parent; struct _sym_node *parent;
struct _sym_node *child[2]; struct _sym_node *child[2];
unsigned char info; unsigned char info;
char name[1]; char name[1];
} zend_sym_node; };
static zend_sym_node *symbols = NULL;
static void zend_syms_rotateleft(zend_sym_node *p) { static void zend_syms_rotateleft(zend_sym_node *p) {
zend_sym_node *r = p->child[1]; zend_sym_node *r = p->child[1];
@ -69,7 +67,7 @@ static void zend_syms_rotateleft(zend_sym_node *p) {
} }
r->parent = p->parent; r->parent = p->parent;
if (p->parent == NULL) { if (p->parent == NULL) {
symbols = r; JIT_G(symbols) = r;
} else if (p->parent->child[0] == p) { } else if (p->parent->child[0] == p) {
p->parent->child[0] = r; p->parent->child[0] = r;
} else { } else {
@ -87,7 +85,7 @@ static void zend_syms_rotateright(zend_sym_node *p) {
} }
l->parent = p->parent; l->parent = p->parent;
if (p->parent == NULL) { if (p->parent == NULL) {
symbols = l; JIT_G(symbols) = l;
} else if (p->parent->child[1] == p) { } else if (p->parent->child[1] == p) {
p->parent->child[1] = l; p->parent->child[1] = l;
} else { } else {
@ -113,8 +111,8 @@ static void zend_jit_disasm_add_symbol(const char *name,
memcpy((char*)&sym->name, name, len + 1); memcpy((char*)&sym->name, name, len + 1);
sym->parent = sym->child[0] = sym->child[1] = NULL; sym->parent = sym->child[0] = sym->child[1] = NULL;
sym->info = 1; sym->info = 1;
if (symbols) { if (JIT_G(symbols)) {
zend_sym_node *node = symbols; zend_sym_node *node = JIT_G(symbols);
/* insert it into rbtree */ /* insert it into rbtree */
do { do {
@ -147,7 +145,7 @@ static void zend_jit_disasm_add_symbol(const char *name,
} while (1); } while (1);
/* fix rbtree after instering */ /* fix rbtree after instering */
while (sym && sym != symbols && sym->parent->info == 1) { while (sym && sym != JIT_G(symbols) && sym->parent->info == 1) {
if (sym->parent == sym->parent->parent->child[0]) { if (sym->parent == sym->parent->parent->child[0]) {
node = sym->parent->parent->child[1]; node = sym->parent->parent->child[1];
if (node && node->info == 1) { if (node && node->info == 1) {
@ -183,9 +181,9 @@ static void zend_jit_disasm_add_symbol(const char *name,
} }
} }
} else { } else {
symbols = sym; JIT_G(symbols) = sym;
} }
symbols->info = 0; JIT_G(symbols)->info = 0;
} }
static void zend_jit_disasm_destroy_symbols(zend_sym_node *n) { static void zend_jit_disasm_destroy_symbols(zend_sym_node *n) {
@ -201,7 +199,7 @@ static void zend_jit_disasm_destroy_symbols(zend_sym_node *n) {
static const char* zend_jit_disasm_find_symbol(uint64_t addr, static const char* zend_jit_disasm_find_symbol(uint64_t addr,
int64_t *offset) { int64_t *offset) {
zend_sym_node *node = symbols; zend_sym_node *node = JIT_G(symbols);
while (node) { while (node) {
if (addr < node->addr) { if (addr < node->addr) {
node = node->child[0]; node = node->child[0];
@ -524,5 +522,8 @@ static int zend_jit_disasm_init(void)
static void zend_jit_disasm_shutdown(void) static void zend_jit_disasm_shutdown(void)
{ {
zend_jit_disasm_destroy_symbols(symbols); if (JIT_G(symbols)) {
zend_jit_disasm_destroy_symbols(JIT_G(symbols));
JIT_G(symbols) = NULL;
}
} }

2
ext/opcache/jit/zend_jit_gdb.c
View file

@ -485,7 +485,7 @@ static void zend_jit_gdb_init(void)
/* This might enable registration of all JIT-ed code, but unfortunately, /* This might enable registration of all JIT-ed code, but unfortunately,
* in case of many functions, this takes enormous time. */ * in case of many functions, this takes enormous time. */
if (zend_gdb_present()) { if (zend_gdb_present()) {
ZCG(accel_directives).jit_debug |= ZEND_JIT_DEBUG_GDB; JIT_G(debug) |= ZEND_JIT_DEBUG_GDB;
} }
#endif #endif
} }

51
ext/opcache/jit/zend_jit_internal.h
View file

@ -132,31 +132,6 @@ int ZEND_FASTCALL zend_jit_check_constant(const zval *key);
#define zend_jit_opline_hash(opline) \ #define zend_jit_opline_hash(opline) \
zend_jit_hash(opline) zend_jit_hash(opline)
#define ZEND_JIT_TRACE_FUNC_COST (1*250)
#define ZEND_JIT_TRACE_RET_COST (15*250)
#define ZEND_JIT_TRACE_LOOP_COST (2*250)
#define ZEND_JIT_TRACE_COUNTER_INIT (127*250)
#define ZEND_JIT_TRACE_MAX_TRACES 1024 /* max number of traces */
#define ZEND_JIT_TRACE_MAX_LENGTH 1024 /* max length of single trace */
#define ZEND_JIT_TRACE_MAX_EXITS 512 /* max number of side exits per trace */
#define ZEND_JIT_TRACE_MAX_SIDE_TRACES 128 /* max number of side traces of a root trace */
#define ZEND_JIT_TRACE_MAX_EXIT_COUNTERS 8192 /* max number of side exits for all trace */
#define ZEND_JIT_TRACE_MAX_FUNCS 30 /* max number of different functions in a single trace */
#define ZEND_JIT_TRACE_MAX_CALL_DEPTH 10 /* max depth of inlined calls */
#define ZEND_JIT_TRACE_MAX_RET_DEPTH 4 /* max depth of inlined returns */
#define ZEND_JIT_TRACE_MAX_RECURSION 2 /* max number of recursive inlined calls */
#define ZEND_JIT_TRACE_MAX_UNROLL_LOOPS 8 /* max number of unrolled loops */
#define ZEND_JIT_TRACE_HOT_SIDE_COUNT 8 /* number of exits before taking side trace */
#define ZEND_JIT_TRACE_HOT_RETURN_COUNT 8 /* number of returns before taking continuation trace */
#define ZEND_JIT_TRACE_MAX_ROOT_FAILURES 16 /* number of attemts to record/compile a root trace */
#define ZEND_JIT_TRACE_MAX_SIDE_FAILURES 4 /* number of attemts to record/compile a side trace */
#define ZEND_JIT_TRACE_BAD_ROOT_SLOTS 64 /* number of slots in bad root trace cache */
#define ZEND_JIT_TRACE_STOP(_) \ #define ZEND_JIT_TRACE_STOP(_) \
_(LOOP, "loop") \ _(LOOP, "loop") \
_(RECURSIVE_CALL, "recursive call") \ _(RECURSIVE_CALL, "recursive call") \
@ -275,7 +250,7 @@ typedef enum _zend_jit_trace_op {
#define ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(_info) \ #define ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(_info) \
(_info >> ZEND_JIT_TRACE_SSA_VAR_SHIFT) (_info >> ZEND_JIT_TRACE_SSA_VAR_SHIFT)
typedef struct _zend_jit_trace_rec { struct _zend_jit_trace_rec {
union { union {
struct { ZEND_ENDIAN_LOHI( struct { ZEND_ENDIAN_LOHI(
uint8_t op, /* zend_jit_trace_op */ uint8_t op, /* zend_jit_trace_op */
@ -304,7 +279,7 @@ typedef struct _zend_jit_trace_rec {
const zend_op *opline; const zend_op *opline;
const zend_class_entry *ce; const zend_class_entry *ce;
}; };
} zend_jit_trace_rec; };
#define ZEND_JIT_TRACE_START_REC_SIZE 2 #define ZEND_JIT_TRACE_START_REC_SIZE 2
@ -370,8 +345,6 @@ typedef struct _zend_jit_trace_info {
//uint32_t loop_offset; //uint32_t loop_offset;
} zend_jit_trace_info; } zend_jit_trace_info;
typedef struct _zend_jit_trace_stack_frame zend_jit_trace_stack_frame;
struct _zend_jit_trace_stack_frame { struct _zend_jit_trace_stack_frame {
zend_jit_trace_stack_frame *call; zend_jit_trace_stack_frame *call;
zend_jit_trace_stack_frame *prev; zend_jit_trace_stack_frame *prev;
@ -446,26 +419,6 @@ struct _zend_jit_trace_stack_frame {
(frame)->_info |= TRACE_FRAME_MASK_THIS_CHECKED; \ (frame)->_info |= TRACE_FRAME_MASK_THIS_CHECKED; \
} while (0) } while (0)
typedef struct _zend_jit_globals {
zend_jit_trace_rec *current_trace;
zend_jit_trace_stack_frame *current_frame;
const zend_op *bad_root_cache_opline[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
uint8_t bad_root_cache_count[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
uint8_t bad_root_cache_stop[ZEND_JIT_TRACE_BAD_ROOT_SLOTS];
uint32_t bad_root_slot;
uint8_t exit_counters[ZEND_JIT_TRACE_MAX_EXIT_COUNTERS];
} zend_jit_globals;
#ifdef ZTS
# define JIT_G(v) ZEND_TSRMG(zend_jit_globals_id, zend_jit_globals *, v)
extern int zend_jit_globals_id;
#else
# define JIT_G(v) (jit_globals.v)
extern zend_jit_globals jit_globals;
#endif
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS); ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS);

71
ext/opcache/jit/zend_jit_trace.c
View file

@ -83,7 +83,7 @@ static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
dasm_free(&dasm_state); dasm_free(&dasm_state);
#ifdef HAVE_DISASM #ifdef HAVE_DISASM
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_ASM) { if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
uint32_t i; uint32_t i;
for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
@ -443,13 +443,13 @@ static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_sc
jit_extension->func_info.return_value_used = -1; jit_extension->func_info.return_value_used = -1;
ssa = &jit_extension->func_info.ssa; ssa = &jit_extension->func_info.ssa;
if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNC) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
do { do {
if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) { if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
break; break;
} }
if (zend_jit_level >= ZEND_JIT_LEVEL_OPT_FUNCS) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
if (zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info) != SUCCESS) { if (zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info) != SUCCESS) {
break; break;
} }
@ -463,7 +463,7 @@ static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_sc
break; break;
} }
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_SSA) { if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa); zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
} }
return ssa; return ssa;
@ -1722,7 +1722,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin
} }
} }
if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_TSSA)) { if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
if (parent_trace) { if (parent_trace) {
fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s() %s:%d\n", fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s() %s:%d\n",
ZEND_JIT_TRACE_NUM, ZEND_JIT_TRACE_NUM,
@ -2281,7 +2281,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
} }
if (list) { if (list) {
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM); fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM);
ival = list; ival = list;
while (ival) { while (ival) {
@ -2381,7 +2381,7 @@ static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace
return NULL; return NULL;
} }
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_REG_ALLOC) { if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM); fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM);
for (i = 0; i < ssa->vars_count; i++) { for (i = 0; i < ssa->vars_count; i++) {
ival = intervals[i]; ival = intervals[i];
@ -2443,7 +2443,7 @@ static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offse
ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
} }
ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN; ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
next_opline->handler = (const void*)zend_jit_ret_counter_handler; next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
} }
} }
@ -2487,7 +2487,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
} }
/* Register allocation */ /* Register allocation */
if (zend_jit_reg_alloc && zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
&& JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num); ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
} }
@ -2723,7 +2724,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
frame->call_level++; frame->call_level++;
} }
if (zend_jit_level >= ZEND_JIT_LEVEL_INLINE) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
switch (opline->opcode) { switch (opline->opcode) {
case ZEND_PRE_INC: case ZEND_PRE_INC:
case ZEND_PRE_DEC: case ZEND_PRE_DEC:
@ -4276,7 +4277,7 @@ exit:
zend_shared_alloc_unlock(); zend_shared_alloc_unlock();
if ((ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
&& ret == ZEND_JIT_TRACE_STOP_COMPILED && ret == ZEND_JIT_TRACE_STOP_COMPILED
&& t->exit_count > 0) { && t->exit_count > 0) {
zend_jit_dump_exit_info(t); zend_jit_dump_exit_info(t);
@ -4602,7 +4603,7 @@ repeat:
return 0; return 0;
} }
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_START) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
fprintf(stderr, "---- TRACE %d start (%s) %s() %s:%d\n", fprintf(stderr, "---- TRACE %d start (%s) %s() %s:%d\n",
trace_num, trace_num,
zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
@ -4622,7 +4623,7 @@ repeat:
if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) { if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) {
/* op_array may be already deallocated */ /* op_array may be already deallocated */
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
fprintf(stderr, "---- TRACE %d abort (%s)\n", fprintf(stderr, "---- TRACE %d abort (%s)\n",
trace_num, trace_num,
zend_jit_trace_stop_description[stop]); zend_jit_trace_stop_description[stop]);
@ -4630,12 +4631,12 @@ repeat:
goto blacklist; goto blacklist;
} }
if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
zend_jit_dump_trace(trace_buffer, NULL); zend_jit_dump_trace(trace_buffer, NULL);
} }
if (ZEND_JIT_TRACE_STOP_OK(stop)) { if (ZEND_JIT_TRACE_STOP_OK(stop)) {
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
if (stop == ZEND_JIT_TRACE_STOP_LINK) { if (stop == ZEND_JIT_TRACE_STOP_LINK) {
uint32_t idx = trace_buffer[1].last; uint32_t idx = trace_buffer[1].last;
uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler); uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
@ -4650,7 +4651,7 @@ repeat:
} }
stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset); stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) { if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_COMPILED) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
fprintf(stderr, "---- TRACE %d %s\n", fprintf(stderr, "---- TRACE %d %s\n",
trace_num, trace_num,
zend_jit_trace_stop_description[stop]); zend_jit_trace_stop_description[stop]);
@ -4660,7 +4661,7 @@ repeat:
} }
} else { } else {
abort: abort:
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
fprintf(stderr, "---- TRACE %d abort (%s)\n", fprintf(stderr, "---- TRACE %d abort (%s)\n",
trace_num, trace_num,
zend_jit_trace_stop_description[stop]); zend_jit_trace_stop_description[stop]);
@ -4668,7 +4669,7 @@ abort:
blacklist: blacklist:
if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
|| zend_jit_trace_is_bad_root(orig_opline, stop, offset)) { || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
fprintf(stderr, "---- TRACE %d blacklisted\n", fprintf(stderr, "---- TRACE %d blacklisted\n",
trace_num); trace_num);
} }
@ -4681,7 +4682,7 @@ blacklist:
} }
} }
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
fprintf(stderr, "\n"); fprintf(stderr, "\n");
} }
@ -4847,7 +4848,7 @@ exit:
zend_shared_alloc_unlock(); zend_shared_alloc_unlock();
if ((ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
&& ret == ZEND_JIT_TRACE_STOP_COMPILED && ret == ZEND_JIT_TRACE_STOP_COMPILED
&& t->exit_count > 0) { && t->exit_count > 0) {
zend_jit_dump_exit_info(t); zend_jit_dump_exit_info(t);
@ -4869,7 +4870,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
return 0; return 0;
} }
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_START) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s() %s:%d\n", fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s() %s:%d\n",
trace_num, parent_num, exit_num, trace_num, parent_num, exit_num,
EX(func)->op_array.function_name ? EX(func)->op_array.function_name ?
@ -4892,7 +4893,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) { if (stop == ZEND_JIT_TRACE_STOP_TOPLEVEL) {
/* op_array may be already deallocated */ /* op_array may be already deallocated */
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
fprintf(stderr, "---- TRACE %d abort (%s)\n", fprintf(stderr, "---- TRACE %d abort (%s)\n",
trace_num, trace_num,
zend_jit_trace_stop_description[stop]); zend_jit_trace_stop_description[stop]);
@ -4900,12 +4901,12 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
goto blacklist; goto blacklist;
} }
if (UNEXPECTED(ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
zend_jit_dump_trace(trace_buffer, NULL); zend_jit_dump_trace(trace_buffer, NULL);
} }
if (ZEND_JIT_TRACE_STOP_OK(stop)) { if (ZEND_JIT_TRACE_STOP_OK(stop)) {
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_STOP) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
if (stop == ZEND_JIT_TRACE_STOP_LINK) { if (stop == ZEND_JIT_TRACE_STOP_LINK) {
uint32_t idx = trace_buffer[1].last; uint32_t idx = trace_buffer[1].last;
uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);; uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
@ -4929,7 +4930,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset); stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
} }
if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) { if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_COMPILED) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
fprintf(stderr, "---- TRACE %d %s\n", fprintf(stderr, "---- TRACE %d %s\n",
trace_num, trace_num,
zend_jit_trace_stop_description[stop]); zend_jit_trace_stop_description[stop]);
@ -4939,7 +4940,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
} }
} else { } else {
abort: abort:
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_ABORT) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
fprintf(stderr, "---- TRACE %d abort (%s)\n", fprintf(stderr, "---- TRACE %d abort (%s)\n",
trace_num, trace_num,
zend_jit_trace_stop_description[stop]); zend_jit_trace_stop_description[stop]);
@ -4948,14 +4949,14 @@ blacklist:
if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
|| zend_jit_trace_exit_is_bad(parent_num, exit_num)) { || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
zend_jit_blacklist_trace_exit(parent_num, exit_num); zend_jit_blacklist_trace_exit(parent_num, exit_num);
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
fprintf(stderr, "---- EXIT %d/%d blacklisted\n", fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
parent_num, exit_num); parent_num, exit_num);
} }
} }
} }
if (ZCG(accel_directives).jit_debug & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
fprintf(stderr, "\n"); fprintf(stderr, "\n");
} }
@ -5010,7 +5011,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf
ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes && ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last); EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_EXIT) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
fprintf(stderr, " TRACE %d exit %d %s() %s:%d\n", fprintf(stderr, " TRACE %d exit %d %s() %s:%d\n",
trace_num, trace_num,
exit_num, exit_num,
@ -5023,7 +5024,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf
if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) { if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) { if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
zend_jit_blacklist_trace_exit(trace_num, exit_num); zend_jit_blacklist_trace_exit(trace_num, exit_num);
if (ZCG(accel_directives).jit_debug & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
fprintf(stderr, "---- EXIT %d/%d blacklisted\n", fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
trace_num, exit_num); trace_num, exit_num);
} }
@ -5063,9 +5064,9 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
zend_cfg cfg; zend_cfg cfg;
uint32_t i; uint32_t i;
ZEND_ASSERT(zend_jit_func_counter_handler != NULL); ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
ZEND_ASSERT(zend_jit_ret_counter_handler != NULL); ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
ZEND_ASSERT(zend_jit_loop_counter_handler != NULL); ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op)); ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
@ -5100,7 +5101,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
/* function entry */ /* function entry */
opline->handler = (const void*)zend_jit_func_counter_handler; opline->handler = (const void*)zend_jit_func_trace_counter_handler;
ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
&zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
@ -5114,7 +5115,7 @@ static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
/* loop header */ /* loop header */
opline = op_array->opcodes + cfg.blocks[i].start; opline = op_array->opcodes + cfg.blocks[i].start;
if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
opline->handler = (const void*)zend_jit_loop_counter_handler; opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) { if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
&zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];

16
ext/opcache/jit/zend_jit_vm_helpers.c
View file

@ -198,10 +198,10 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_H
const zend_op *opline = EX(opline); const zend_op *opline = EX(opline);
#endif #endif
*(jit_extension->counter) -= ZEND_JIT_HOT_FUNC_COST; *(jit_extension->counter) -= ZEND_JIT_COUNTER_FUNC_COST;
if (UNEXPECTED(*(jit_extension->counter) <= 0)) { if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
*(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT; *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
zend_jit_hot_func(execute_data, opline); zend_jit_hot_func(execute_data, opline);
ZEND_OPCODE_RETURN(); ZEND_OPCODE_RETURN();
} else { } else {
@ -218,10 +218,10 @@ ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_H
const zend_op *opline = EX(opline); const zend_op *opline = EX(opline);
#endif #endif
*(jit_extension->counter) -= ZEND_JIT_HOT_LOOP_COST; *(jit_extension->counter) -= ZEND_JIT_COUNTER_LOOP_COST;
if (UNEXPECTED(*(jit_extension->counter) <= 0)) { if (UNEXPECTED(*(jit_extension->counter) <= 0)) {
*(jit_extension->counter) = ZEND_JIT_HOT_COUNTER_INIT; *(jit_extension->counter) = ZEND_JIT_COUNTER_INIT;
zend_jit_hot_func(execute_data, opline); zend_jit_hot_func(execute_data, opline);
ZEND_OPCODE_RETURN(); ZEND_OPCODE_RETURN();
} else { } else {
@ -291,7 +291,7 @@ static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_c
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost; *(ZEND_OP_TRACE_INFO(opline, offset)->counter) -= cost;
if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) { if (UNEXPECTED(*(ZEND_OP_TRACE_INFO(opline, offset)->counter) <= 0)) {
*(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_TRACE_COUNTER_INIT; *(ZEND_OP_TRACE_INFO(opline, offset)->counter) = ZEND_JIT_COUNTER_INIT;
if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) { if (UNEXPECTED(zend_jit_trace_hot_root(execute_data, opline) < 0)) {
#ifndef HAVE_GCC_GLOBAL_REGS #ifndef HAVE_GCC_GLOBAL_REGS
return -1; return -1;
@ -312,17 +312,17 @@ static zend_always_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_trace_c
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS) ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{ {
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_FUNC_COST); ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_COUNTER_FUNC_COST);
} }
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS) ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{ {
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_RET_COST); ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_COUNTER_RET_COST);
} }
ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS) ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS)
{ {
ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_TRACE_LOOP_COST); ZEND_OPCODE_TAIL_CALL_EX(zend_jit_trace_counter_helper, ZEND_JIT_COUNTER_LOOP_COST);
} }
#define TRACE_RECORD(_op, _info, _ptr) \ #define TRACE_RECORD(_op, _info, _ptr) \

165
ext/opcache/jit/zend_jit_x86.dasc
View file

@ -130,8 +130,6 @@ const char* zend_reg_name[] = {
# define GCC_GLOBAL_REGS 0 # define GCC_GLOBAL_REGS 0
#endif #endif
static uint32_t zend_jit_x86_flags = 0;
#if ZTS #if ZTS
static size_t tsrm_ls_cache_tcb_offset = 0; static size_t tsrm_ls_cache_tcb_offset = 0;
static size_t tsrm_tls_index; static size_t tsrm_tls_index;
@ -527,7 +525,7 @@ static void* dasm_labels[zend_lb_MAX];
|.endmacro |.endmacro
|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 |.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2
|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { || if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| avx_ins op1, op2 | avx_ins op1, op2
|| } else { || } else {
| sse_ins op1, op2 | sse_ins op1, op2
@ -569,7 +567,7 @@ static void* dasm_labels[zend_lb_MAX];
|.macro SSE_GET_LONG, reg, lval |.macro SSE_GET_LONG, reg, lval
|| if (lval == 0) { || if (lval == 0) {
|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { || if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) | vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
|| } else { || } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) | xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
@ -584,7 +582,7 @@ static void* dasm_labels[zend_lb_MAX];
|.else |.else
| mov r0, lval | mov r0, lval
|.endif |.endif
|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { || if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), r0 | vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), r0
|| } else { || } else {
| cvtsi2sd, xmm(reg-ZREG_XMM0), r0 | cvtsi2sd, xmm(reg-ZREG_XMM0), r0
@ -596,13 +594,13 @@ static void* dasm_labels[zend_lb_MAX];
|| if (Z_MODE(addr) == IS_CONST_ZVAL) { || if (Z_MODE(addr) == IS_CONST_ZVAL) {
| SSE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)) | SSE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr))
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { || } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { || if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] | vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else { || } else {
| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] | cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } || }
|| } else if (Z_MODE(addr) == IS_REG) { || } else if (Z_MODE(addr) == IS_REG) {
|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { || if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) | vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr))
|| } else { || } else {
| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) | cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr))
@ -870,7 +868,7 @@ static void* dasm_labels[zend_lb_MAX];
|| if (Z_TYPE_P(zv) == IS_DOUBLE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0;
|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { || if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) | vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| } else { || } else {
| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) | xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
@ -924,7 +922,7 @@ static void* dasm_labels[zend_lb_MAX];
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ?
|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); || Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0);
|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { || if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) | vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| } else { || } else {
| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) | xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
@ -2128,6 +2126,10 @@ static int zend_jit_double_one_stub(dasm_State **Dst)
static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst)
{ {
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_runtime_jit: |->hybrid_runtime_jit:
| EXT_CALL zend_runtime_jit, r0 | EXT_CALL zend_runtime_jit, r0
| JMP_IP | JMP_IP
@ -2136,6 +2138,10 @@ static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst)
static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst)
{ {
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_profile_jit: |->hybrid_profile_jit:
| // ++zend_jit_profile_counter; | // ++zend_jit_profile_counter;
| .if X64 | .if X64
@ -2191,13 +2197,17 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst)
* } * }
* *
*/ */
static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst)
{ {
|->hybrid_func_counter: if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_func_hot_counter:
| mov r0, EX->func | mov r0, EX->func
| mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)]
| sub word [r2], ZEND_JIT_HOT_FUNC_COST | sub word [r2], ZEND_JIT_COUNTER_FUNC_COST
| jle >1 | jle >1
| GET_IP r2 | GET_IP r2
| sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)]
@ -2216,7 +2226,7 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst)
| jmp aword [r1+r2*4+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] | jmp aword [r1+r2*4+offsetof(zend_jit_op_array_hot_extension, orig_handlers)]
| .endif | .endif
|1: |1:
| mov word [r2], ZEND_JIT_HOT_COUNTER_INIT | mov word [r2], ZEND_JIT_COUNTER_INIT
| mov FCARG1a, FP | mov FCARG1a, FP
| GET_IP FCARG2a | GET_IP FCARG2a
| EXT_CALL zend_jit_hot_func, r0 | EXT_CALL zend_jit_hot_func, r0
@ -2224,13 +2234,17 @@ static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst)
return 1; return 1;
} }
static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst)
{ {
|->hybrid_loop_counter: if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_loop_hot_counter:
| mov r0, EX->func | mov r0, EX->func
| mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)]
| sub word [r2], ZEND_JIT_HOT_LOOP_COST | sub word [r2], ZEND_JIT_COUNTER_LOOP_COST
| jle >1 | jle >1
| GET_IP r2 | GET_IP r2
| sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)]
@ -2249,7 +2263,7 @@ static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst)
| jmp aword [r1+r2*4+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] | jmp aword [r1+r2*4+offsetof(zend_jit_op_array_hot_extension, orig_handlers)]
| .endif | .endif
|1: |1:
| mov word [r2], ZEND_JIT_HOT_COUNTER_INIT | mov word [r2], ZEND_JIT_COUNTER_INIT
| mov FCARG1a, FP | mov FCARG1a, FP
| GET_IP FCARG2a | GET_IP FCARG2a
| EXT_CALL zend_jit_hot_func, r0 | EXT_CALL zend_jit_hot_func, r0
@ -2267,7 +2281,7 @@ static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost)
| jle >1 | jle >1
| jmp aword [IP + r1] | jmp aword [IP + r1]
|1: |1:
| mov word [r2], ZEND_JIT_TRACE_COUNTER_INIT | mov word [r2], ZEND_JIT_COUNTER_INIT
| mov FCARG1a, FP | mov FCARG1a, FP
| GET_IP FCARG2a | GET_IP FCARG2a
| EXT_CALL zend_jit_trace_hot_root, r0 | EXT_CALL zend_jit_trace_hot_root, r0
@ -2283,27 +2297,39 @@ static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost)
static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst)
{ {
|->hybrid_func_counter: |->hybrid_func_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_FUNC_COST); return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_COUNTER_FUNC_COST);
} }
static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst)
{ {
|->hybrid_ret_counter: if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_RET_COST); |->hybrid_ret_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_COUNTER_RET_COST);
} }
static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst)
{ {
|->hybrid_loop_counter: if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_TRACE_LOOP_COST); |->hybrid_loop_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst, ZEND_JIT_COUNTER_LOOP_COST);
} }
static int zend_jit_trace_halt_stub(dasm_State **Dst) static int zend_jit_trace_halt_stub(dasm_State **Dst)
{ {
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->trace_halt: |->trace_halt:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| add r4, HYBRID_SPAD | add r4, HYBRID_SPAD
@ -2535,6 +2561,13 @@ static const zend_jit_stub zend_jit_stubs[] = {
JIT_STUB(trace_halt), JIT_STUB(trace_halt),
JIT_STUB(trace_exit), JIT_STUB(trace_exit),
JIT_STUB(trace_escape), JIT_STUB(trace_escape),
JIT_STUB(hybrid_runtime_jit),
JIT_STUB(hybrid_profile_jit),
JIT_STUB(hybrid_func_hot_counter),
JIT_STUB(hybrid_loop_hot_counter),
JIT_STUB(hybrid_func_trace_counter),
JIT_STUB(hybrid_ret_trace_counter),
JIT_STUB(hybrid_loop_trace_counter),
JIT_STUB(double_one), JIT_STUB(double_one),
#ifdef CONTEXT_THREADED_JIT #ifdef CONTEXT_THREADED_JIT
JIT_STUB(context_threaded_call), JIT_STUB(context_threaded_call),
@ -2553,9 +2586,9 @@ static int zend_jit_setup(void)
zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); zend_error(E_CORE_ERROR, "CPU doesn't support SSE2");
return FAILURE; return FAILURE;
} }
if (zend_jit_cpu_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags)) {
if (zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) { if (!zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) {
zend_jit_x86_flags |= ZEND_JIT_CPU_AVX; JIT_G(opt_flags) &= ~ZEND_JIT_CPU_AVX;
} }
} }
@ -3634,13 +3667,13 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, const zend_
} }
| SSE_GET_ZVAL_DVAL tmp_reg, op1_addr | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one]
} else { } else {
| addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one]
} }
} else { } else {
if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one]
} else { } else {
| subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one]
@ -3769,7 +3802,7 @@ static int zend_jit_math_long_long(dasm_State **Dst,
| SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr
| SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr
if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2
} else { } else {
| SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2
@ -3799,7 +3832,7 @@ static int zend_jit_math_long_double(dasm_State **Dst,
(Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0;
| SSE_GET_ZVAL_LVAL result_reg, op1_addr | SSE_GET_ZVAL_LVAL result_reg, op1_addr
if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| AVX_MATH opcode, result_reg, result_reg, op2_addr | AVX_MATH opcode, result_reg, result_reg, op2_addr
} else { } else {
| SSE_MATH opcode, result_reg, op2_addr | SSE_MATH opcode, result_reg, op2_addr
@ -3831,7 +3864,7 @@ static int zend_jit_math_double_long(dasm_State **Dst,
result_reg = ZREG_XMM0; result_reg = ZREG_XMM0;
} }
| SSE_GET_ZVAL_LVAL result_reg, op2_addr | SSE_GET_ZVAL_LVAL result_reg, op2_addr
if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| AVX_MATH opcode, result_reg, result_reg, op1_addr | AVX_MATH opcode, result_reg, result_reg, op1_addr
} else { } else {
| SSE_MATH opcode, result_reg, op1_addr | SSE_MATH opcode, result_reg, op1_addr
@ -3849,7 +3882,7 @@ static int zend_jit_math_double_long(dasm_State **Dst,
result_reg = ZREG_XMM0; result_reg = ZREG_XMM0;
tmp_reg = ZREG_XMM1; tmp_reg = ZREG_XMM1;
} }
if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
zend_reg op1_reg; zend_reg op1_reg;
if (Z_MODE(op1_addr) == IS_REG) { if (Z_MODE(op1_addr) == IS_REG) {
@ -3897,7 +3930,7 @@ static int zend_jit_math_double_double(dasm_State **Dst,
result_reg = ZREG_XMM0; result_reg = ZREG_XMM0;
} }
if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
zend_reg op1_reg; zend_reg op1_reg;
zend_jit_addr val_addr; zend_jit_addr val_addr;
@ -4610,7 +4643,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
const void *exit_addr = NULL; const void *exit_addr = NULL;
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point); exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) { if (!exit_addr) {
@ -4645,7 +4678,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
|.endif |.endif
if (type == BP_JIT_IS) { if (type == BP_JIT_IS) {
| jbe >9 // NOT_FOUND | jbe >9 // NOT_FOUND
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) {
| jbe &exit_addr | jbe &exit_addr
} else { } else {
| jbe >2 // NOT_FOUND | jbe >2 // NOT_FOUND
@ -4674,7 +4707,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
|.endif |.endif
if (type == BP_JIT_IS) { if (type == BP_JIT_IS) {
| jbe >9 // NOT_FOUND | jbe >9 // NOT_FOUND
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) { } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (type == BP_VAR_R || type == BP_VAR_RW)) {
| jbe &exit_addr | jbe &exit_addr
} else { } else {
| jbe >2 // NOT_FOUND | jbe >2 // NOT_FOUND
@ -4713,13 +4746,13 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long val = Z_LVAL_P(Z_ZV(op2_addr)); zend_long val = Z_LVAL_P(Z_ZV(op2_addr));
if (val >= 0 && val < HT_MAX_SIZE) { if (val >= 0 && val < HT_MAX_SIZE) {
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jmp &exit_addr | jmp &exit_addr
} else { } else {
| jmp >2 // NOT_FOUND | jmp >2 // NOT_FOUND
} }
} }
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jmp &exit_addr | jmp &exit_addr
} else { } else {
| jmp >2 // NOT_FOUND | jmp >2 // NOT_FOUND
@ -4728,7 +4761,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
} }
| EXT_CALL zend_hash_index_find, r0 | EXT_CALL zend_hash_index_find, r0
| test r0, r0 | test r0, r0
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jz &exit_addr | jz &exit_addr
} else { } else {
| jz >2 // NOT_FOUND | jz >2 // NOT_FOUND
@ -4737,7 +4770,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
|2: |2:
switch (type) { switch (type) {
case BP_VAR_R: case BP_VAR_R:
if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
| // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval);
| // retval = &EG(uninitialized_zval); | // retval = &EG(uninitialized_zval);
| UNDEFINED_OFFSET opline | UNDEFINED_OFFSET opline
@ -4757,14 +4790,14 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
break; break;
case BP_VAR_RW: case BP_VAR_RW:
|2: |2:
if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
| SAVE_VALID_OPLINE opline, r0 | SAVE_VALID_OPLINE opline, r0
| // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval);
| //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval));
| EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0
} }
if (op1_info & MAY_BE_ARRAY_KEY_LONG) { if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
| jmp >8 | jmp >8
} }
|4: |4:
@ -4848,7 +4881,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
| EXT_CALL _zend_hash_find_known_hash, r0 | EXT_CALL _zend_hash_find_known_hash, r0
} }
| test r0, r0 | test r0, r0
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jz &exit_addr | jz &exit_addr
} else { } else {
| jz >2 // NOT_FOUND | jz >2 // NOT_FOUND
@ -4863,7 +4896,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o
|2: |2:
switch (type) { switch (type) {
case BP_VAR_R: case BP_VAR_R:
if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
// zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key));
| UNDEFINED_INDEX opline | UNDEFINED_INDEX opline
| jmp >9 | jmp >9
@ -7351,7 +7384,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, const z
} }
if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { if (JIT_G(opt_flags) & ZEND_JIT_CPU_AVX) {
| vxorps xmm0, xmm0, xmm0 | vxorps xmm0, xmm0, xmm0
} else { } else {
| xorps xmm0, xmm0 | xorps xmm0, xmm0
@ -7615,7 +7648,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con
| cmp r2, FCARG1a | cmp r2, FCARG1a
} }
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
@ -7997,7 +8030,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t
| // CACHE_PTR(opline->result.num, fbc); | // CACHE_PTR(opline->result.num, fbc);
| mov r1, EX->run_time_cache | mov r1, EX->run_time_cache
| mov aword [r1 + opline->result.num], r0 | mov aword [r1 + opline->result.num], r0
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, 0); int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
@ -8588,7 +8621,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend
|2: |2:
} }
if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) || !JIT_G(current_frame) ||
!JIT_G(current_frame)->call || !JIT_G(current_frame)->call ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call)) { !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call)) {
@ -8682,7 +8715,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend
ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame) && JIT_G(current_frame)
&& JIT_G(current_frame)->call && JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) { && JIT_G(current_frame)->call->func) {
@ -8690,7 +8723,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, const zend
/* Don't generate code that always throws exception */ /* Don't generate code that always throws exception */
return 0; return 0;
} }
} else if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) { if (!exit_addr) {
@ -8828,7 +8861,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
} }
if (opline->opcode == ZEND_SEND_VAR_EX) { if (opline->opcode == ZEND_SEND_VAR_EX) {
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame) && JIT_G(current_frame)
&& JIT_G(current_frame)->call && JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) { && JIT_G(current_frame)->call->func) {
@ -8853,7 +8886,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
|.code |.code
} }
} else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame) && JIT_G(current_frame)
&& JIT_G(current_frame)->call && JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) { && JIT_G(current_frame)->call->func) {
@ -8895,7 +8928,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
} }
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >7 | jnz >7
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) { if (!exit_addr) {
@ -8915,7 +8948,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
|.code |.code
} }
} else if (opline->opcode == ZEND_SEND_FUNC_ARG) { } else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame) && JIT_G(current_frame)
&& JIT_G(current_frame)->call && JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) { && JIT_G(current_frame)->call->func) {
@ -8962,7 +8995,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend
| cmp cl, IS_REFERENCE | cmp cl, IS_REFERENCE
| je >7 | je >7
} }
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) { if (!exit_addr) {
@ -9032,7 +9065,7 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline, cons
{ {
uint32_t arg_num = opline->op2.num; uint32_t arg_num = opline->op2.num;
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame) && JIT_G(current_frame)
&& JIT_G(current_frame)->call && JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) { && JIT_G(current_frame)->call->func) {
@ -9545,7 +9578,7 @@ static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, const ze
} }
| EXT_CALL zend_jit_leave_func_helper, r0 | EXT_CALL zend_jit_leave_func_helper, r0
if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) || !JIT_G(current_frame) ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame))) { !TRACE_FRAME_IS_NESTED(JIT_G(current_frame))) {
// TODO: try to avoid this check ??? // TODO: try to avoid this check ???
@ -9693,7 +9726,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o
ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
return_value_used = 1; return_value_used = 1;
} else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
@ -10334,7 +10367,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen
zval *zv = RT_CONSTANT(opline, opline->op2); zval *zv = RT_CONSTANT(opline, opline->op2);
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE || if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
| cmp dword EX->This.u2.num_args, arg_num | cmp dword EX->This.u2.num_args, arg_num
| jae >5 | jae >5
@ -10559,7 +10592,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, const zend_op *opline, cons
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
} }
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM); int32_t exit_point = zend_jit_trace_get_exit_point(opline, opline, NULL, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
@ -10647,7 +10680,7 @@ static int zend_jit_fetch_obj_read(dasm_State **Dst, const zend_op *opline, cons
| jmp >9 | jmp >9
} }
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && zend_jit_trigger != ZEND_JIT_ON_HOT_TRACE) { if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
|7: |7:
if (opline->opcode == ZEND_FETCH_OBJ_R) { if (opline->opcode == ZEND_FETCH_OBJ_R) {
| SAVE_VALID_OPLINE opline, r1 | SAVE_VALID_OPLINE opline, r1
@ -10830,7 +10863,7 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze
{ {
zend_jit_addr res_addr = RES_ADDR(); zend_jit_addr res_addr = RES_ADDR();
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) || if (!JIT_G(current_frame) ||
!TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
@ -11200,7 +11233,7 @@ static zend_bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
return 0; return 0;
} }
if (zend_jit_reg_alloc < ZEND_JIT_REG_ALLOC_GLOBAL) { if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
/* Disable global register allocation, /* Disable global register allocation,
* register allocation for SSA variables connected through Phi functions * register allocation for SSA variables connected through Phi functions
*/ */
@ -11535,7 +11568,7 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend
break; break;
} }
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (ssa_op == ssa->ops if (ssa_op == ssa->ops
&& JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL
&& (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
@ -11546,7 +11579,7 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend
#if ZTS #if ZTS
/* %r0 is used to check EG(vm_interrupt) */ /* %r0 is used to check EG(vm_interrupt) */
if (zend_jit_trigger == ZEND_JIT_ON_HOT_TRACE) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (ssa_op == ssa->ops if (ssa_op == ssa->ops
&& (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP ||
JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) {

1
ext/opcache/tests/bug78185.phpt
View file

@ -5,6 +5,7 @@ opcache.enable_cli=1
opcache.optimization_level=-1 opcache.optimization_level=-1
opcache.file_cache={PWD} opcache.file_cache={PWD}
opcache.file_cache_only=1 opcache.file_cache_only=1
opcache.jit=0
--SKIPIF-- --SKIPIF--
<?php require_once('skipif.inc'); ?> <?php require_once('skipif.inc'); ?>
--FILE-- --FILE--

46
ext/opcache/zend_accelerator_module.c
View file

@ -194,6 +194,28 @@ static ZEND_INI_MH(OnUpdateFileCache)
return SUCCESS; return SUCCESS;
} }
#ifdef HAVE_JIT
static ZEND_INI_MH(OnUpdateJit)
{
if (zend_jit_config(new_value, stage) == SUCCESS) {
return OnUpdateString(entry, new_value, mh_arg1, mh_arg2, mh_arg3, stage);
}
return FAILURE;
}
static ZEND_INI_MH(OnUpdateJitDebug)
{
zend_long *p = (zend_long *) ZEND_INI_GET_ADDR();
zend_long val = zend_atol(ZSTR_VAL(new_value), ZSTR_LEN(new_value));
if (zend_jit_debug_config(*p, val, stage) == SUCCESS) {
*p = val;
return SUCCESS;
}
return FAILURE;
}
#endif
ZEND_INI_BEGIN() ZEND_INI_BEGIN()
STD_PHP_INI_BOOLEAN("opcache.enable" , "1", PHP_INI_ALL, OnEnable, enabled , zend_accel_globals, accel_globals) STD_PHP_INI_BOOLEAN("opcache.enable" , "1", PHP_INI_ALL, OnEnable, enabled , zend_accel_globals, accel_globals)
STD_PHP_INI_BOOLEAN("opcache.use_cwd" , "1", PHP_INI_SYSTEM, OnUpdateBool, accel_directives.use_cwd , zend_accel_globals, accel_globals) STD_PHP_INI_BOOLEAN("opcache.use_cwd" , "1", PHP_INI_SYSTEM, OnUpdateBool, accel_directives.use_cwd , zend_accel_globals, accel_globals)
@ -251,10 +273,10 @@ ZEND_INI_BEGIN()
STD_PHP_INI_ENTRY("opcache.cache_id" , "" , PHP_INI_SYSTEM, OnUpdateString, accel_directives.cache_id, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.cache_id" , "" , PHP_INI_SYSTEM, OnUpdateString, accel_directives.cache_id, zend_accel_globals, accel_globals)
#endif #endif
#ifdef HAVE_JIT #ifdef HAVE_JIT
STD_PHP_INI_ENTRY("opcache.jit" , ZEND_JIT_DEFAULT, PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.jit" , ZEND_JIT_DEFAULT_OPTIONS, PHP_INI_ALL, OnUpdateJit, options, zend_jit_globals, jit_globals)
STD_PHP_INI_ENTRY("opcache.jit_buffer_size" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_buffer_size, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.jit_buffer_size" , ZEND_JIT_DEFAULT_BUFFER_SIZE, PHP_INI_SYSTEM, OnUpdateLong, buffer_size, zend_jit_globals, jit_globals)
STD_PHP_INI_ENTRY("opcache.jit_debug" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_debug, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.jit_debug" , "0", PHP_INI_ALL, OnUpdateJitDebug, debug, zend_jit_globals, jit_globals)
STD_PHP_INI_ENTRY("opcache.jit_bisect_limit" , "0" , PHP_INI_SYSTEM, OnUpdateLong, accel_directives.jit_bisect_limit, zend_accel_globals, accel_globals) STD_PHP_INI_ENTRY("opcache.jit_bisect_limit" , "0", PHP_INI_ALL, OnUpdateLong, bisect_limit, zend_jit_globals, jit_globals)
#endif #endif
ZEND_INI_END() ZEND_INI_END()
@ -387,8 +409,12 @@ void zend_accel_info(ZEND_MODULE_INFO_FUNC_ARGS)
php_info_print_table_row(2, "File Cache", "Disabled"); php_info_print_table_row(2, "File Cache", "Disabled");
} }
#if HAVE_JIT #if HAVE_JIT
if (ZCG(jit_enabled)) { if (JIT_G(enabled)) {
php_info_print_table_row(2, "JIT", "Enabled"); if (JIT_G(on)) {
php_info_print_table_row(2, "JIT", "On");
} else {
php_info_print_table_row(2, "JIT", "Off");
}
} else { } else {
php_info_print_table_row(2, "JIT", "Disabled"); php_info_print_table_row(2, "JIT", "Disabled");
} }
@ -722,10 +748,10 @@ ZEND_FUNCTION(opcache_get_configuration)
add_assoc_string(&directives, "opcache.cache_id", STRING_NOT_NULL(ZCG(accel_directives).cache_id)); add_assoc_string(&directives, "opcache.cache_id", STRING_NOT_NULL(ZCG(accel_directives).cache_id));
#endif #endif
#ifdef HAVE_JIT #ifdef HAVE_JIT
add_assoc_long(&directives, "opcache.jit", ZCG(accel_directives).jit); add_assoc_string(&directives, "opcache.jit", JIT_G(options));
add_assoc_long(&directives, "opcache.jit_buffer_size", ZCG(accel_directives).jit_buffer_size); add_assoc_long(&directives, "opcache.jit_buffer_size", JIT_G(buffer_size));
add_assoc_long(&directives, "opcache.jit_debug", ZCG(accel_directives).jit_debug); add_assoc_long(&directives, "opcache.jit_debug", JIT_G(debug));
add_assoc_long(&directives, "opcache.jit_bisect_limit", ZCG(accel_directives).jit_bisect_limit); add_assoc_long(&directives, "opcache.jit_bisect_limit", JIT_G(bisect_limit));
#endif #endif
add_assoc_zval(return_value, "directives", &directives); add_assoc_zval(return_value, "directives", &directives);

6
ext/opcache/zend_file_cache.c
View file

@ -33,6 +33,10 @@
#include "zend_accelerator_util_funcs.h" #include "zend_accelerator_util_funcs.h"
#include "zend_accelerator_hash.h" #include "zend_accelerator_hash.h"
#if HAVE_JIT
#include "jit/zend_jit.h"
#endif
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <fcntl.h> #include <fcntl.h>
@ -892,7 +896,7 @@ int zend_file_cache_script_store(zend_persistent_script *script, int in_shm)
#ifdef HAVE_JIT #ifdef HAVE_JIT
/* FIXME: dump jited codes out to file cache? */ /* FIXME: dump jited codes out to file cache? */
if (ZCG(jit_enabled)) { if (JIT_G(on)) {
return FAILURE; return FAILURE;
} }
#endif #endif

9
ext/opcache/zend_persist.c
View file

@ -566,8 +566,7 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc
ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem)))); ZCG(mem) = (void*)((char*)ZCG(mem) + ZEND_ALIGNED_SIZE(zend_extensions_op_array_persist(op_array, ZCG(mem))));
#ifdef HAVE_JIT #ifdef HAVE_JIT
if (ZCG(jit_enabled) && if (JIT_G(on) && JIT_G(opt_level) <= ZEND_JIT_LEVEL_OPT_FUNCS &&
ZEND_JIT_LEVEL(ZCG(accel_directives).jit) <= ZEND_JIT_LEVEL_OPT_FUNCS &&
!ZCG(current_persistent_script)->corrupted) { !ZCG(current_persistent_script)->corrupted) {
zend_jit_op_array(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL); zend_jit_op_array(op_array, ZCG(current_persistent_script) ? &ZCG(current_persistent_script)->script : NULL);
} }
@ -1122,7 +1121,7 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script
ZCG(mem) = (void*)((char*)ZCG(mem) + script->arena_size); ZCG(mem) = (void*)((char*)ZCG(mem) + script->arena_size);
#ifdef HAVE_JIT #ifdef HAVE_JIT
if (ZCG(jit_enabled) && for_shm) { if (JIT_G(on) && for_shm) {
zend_jit_unprotect(); zend_jit_unprotect();
} }
#endif #endif
@ -1143,8 +1142,8 @@ zend_persistent_script *zend_accel_script_persist(zend_persistent_script *script
} }
#ifdef HAVE_JIT #ifdef HAVE_JIT
if (ZCG(jit_enabled) && for_shm) { if (JIT_G(on) && for_shm) {
if (ZEND_JIT_LEVEL(ZCG(accel_directives).jit) >= ZEND_JIT_LEVEL_OPT_SCRIPT) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_SCRIPT) {
zend_jit_script(&script->script); zend_jit_script(&script->script);
} }
zend_jit_protect(); zend_jit_protect();