archive/php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2025-08-15 13:38:49 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 3
php-src/ext/opcache/jit/zend_jit_ir.c
Bob Weinand 6a2c5318f9
Optimize observers (#13649) 
Inline the lookup whether a function is observed at all.
This strategy is also used for FRAMELESS calls. If the frameless call is observed, we instead allocate a call frame and push the arguments, to call the the function afterwards.
Doing so is still a performance benefit as opposed to executing individual INIT_FCALL+SEND_VAL ops. Thus, even if the frameless call turns out to be observed, the call overhead is slightly lower than before.
If the internal function is not observed at all, the unavoidable overhead is fetching the FLF zend_function pointer and the run-time cache needs to be inspected.

As part of this work, it turned out to be most viable to put the result operand on the ZEND_OP_DATA instead of ZEND_FRAMELESS_ICALL_3, allowing seamless interoperability with the DO_ICALL opcode.
This is a bit unusual in comparison to all other ZEND_OP_DATA usages, but seems to not pose problems overall.

There is also a small issue resolved: trampolines would always use the ZEND_CALL_TRAMPOLINE_SPEC_OBSERVER function due to zend_observer_fcall_op_array_extension being set to -1 too late.
2024-06-15 14:42:27 +02:00

17308 lines
544 KiB
C
Raw Blame History

/*
* +----------------------------------------------------------------------+
* | Zend JIT |
* +----------------------------------------------------------------------+
* | Copyright (c) The PHP Group |
* +----------------------------------------------------------------------+
* | This source file is subject to version 3.01 of the PHP license, |
* | that is bundled with this package in the file LICENSE, and is |
* | available through the world-wide-web at the following url: |
* | https://www.php.net/license/3_01.txt |
* | If you did not receive a copy of the PHP license and are unable to |
* | obtain it through the world-wide-web, please send a note to |
* | license@php.net so we can mail you a copy immediately. |
* +----------------------------------------------------------------------+
* | Authors: Dmitry Stogov <dmitry@php.net> |
* +----------------------------------------------------------------------+
*/
#include "jit/ir/ir.h"
#include "jit/ir/ir_builder.h"
#if defined(IR_TARGET_X86)
# define IR_REG_SP 4 /* IR_REG_RSP */
# define IR_REG_FP 5 /* IR_REG_RBP */
# define ZREG_FP 6 /* IR_REG_RSI */
# define ZREG_IP 7 /* IR_REG_RDI */
# define ZREG_FIRST_FPR 8
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */
#elif defined(IR_TARGET_X64)
# define IR_REG_SP 4 /* IR_REG_RSP */
# define IR_REG_FP 5 /* IR_REG_RBP */
# define ZREG_FP 14 /* IR_REG_R14 */
# define ZREG_IP 15 /* IR_REG_R15 */
# define ZREG_FIRST_FPR 16
# if defined(_WIN64)
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15))
/*
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15) | \
(1<<(16+6)) | (1<<(16+7)) | (1<<(16+8)) | (1<<(16+9)) | (1<<(16+10)) | \
(1<<(16+11)) | (1<<(16+12)) | (1<<(16+13)) | (1<<(16+14)) | (1<<(16+15)))
*/
# else
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<12) | (1<<13) | (1<<14) | (1<<15)) /* all preserved registers */
# endif
#elif defined(IR_TARGET_AARCH64)
# define IR_REG_SP 31 /* IR_REG_RSP */
# define IR_REG_FP 29 /* IR_REG_X29 */
# define ZREG_FP 27 /* IR_REG_X27 */
# define ZREG_IP 28 /* IR_REG_X28 */
# define ZREG_FIRST_FPR 32
# define IR_REGSET_PRESERVED ((1<<19) | (1<<20) | (1<<21) | (1<<22) | (1<<23) | \
(1<<24) | (1<<25) | (1<<26) | (1<<27) | (1<<28)) /* all preserved registers */
#else
# error "Unknown IR target"
#endif
#define ZREG_RX ZREG_IP
#define OPTIMIZE_FOR_SIZE 0
/* IR builder defines */
#undef _ir_CTX
#define _ir_CTX (&jit->ctx)
#undef ir_CONST_ADDR
#define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr))
#define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)
#define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)
#define ir_CAST_FC_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \
ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_I32))
#define ir_CONST_FUNC_PROTO(_addr, _proto) \
jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto))
#undef ir_ADD_OFFSET
#define ir_ADD_OFFSET(_addr, _offset) \
jit_ADD_OFFSET(jit, _addr, _offset)
#ifdef ZEND_ENABLE_ZVAL_LONG64
# define IR_LONG IR_I64
# define ir_CONST_LONG ir_CONST_I64
# define ir_UNARY_OP_L ir_UNARY_OP_I64
# define ir_BINARY_OP_L ir_BINARY_OP_I64
# define ir_ADD_L ir_ADD_I64
# define ir_SUB_L ir_SUB_I64
# define ir_MUL_L ir_MUL_I64
# define ir_DIV_L ir_DIV_I64
# define ir_MOD_L ir_MOD_I64
# define ir_NEG_L ir_NEG_I64
# define ir_ABS_L ir_ABS_I64
# define ir_SEXT_L ir_SEXT_I64
# define ir_ZEXT_L ir_ZEXT_I64
# define ir_TRUNC_L ir_TRUNC_I64
# define ir_BITCAST_L ir_BITCAST_I64
# define ir_FP2L ir_FP2I64
# define ir_ADD_OV_L ir_ADD_OV_I64
# define ir_SUB_OV_L ir_SUB_OV_I64
# define ir_MUL_OV_L ir_MUL_OV_I64
# define ir_NOT_L ir_NOT_I64
# define ir_OR_L ir_OR_I64
# define ir_AND_L ir_AND_I64
# define ir_XOR_L ir_XOR_I64
# define ir_SHL_L ir_SHL_I64
# define ir_SHR_L ir_SHR_I64
# define ir_SAR_L ir_SAR_I64
# define ir_ROL_L ir_ROL_I64
# define ir_ROR_L ir_ROR_I64
# define ir_MIN_L ir_MIN_I64
# define ir_MAX_L ir_MAX_I64
# define ir_LOAD_L ir_LOAD_I64
#else
# define IR_LONG IR_I32
# define ir_CONST_LONG ir_CONST_I32
# define ir_UNARY_OP_L ir_UNARY_OP_I32
# define ir_BINARY_OP_L ir_BINARY_OP_I32
# define ir_ADD_L ir_ADD_I32
# define ir_SUB_L ir_SUB_I32
# define ir_MUL_L ir_MUL_I32
# define ir_DIV_L ir_DIV_I32
# define ir_MOD_L ir_MOD_I32
# define ir_NEG_L ir_NEG_I32
# define ir_ABS_L ir_ABS_I32
# define ir_SEXT_L ir_SEXT_I32
# define ir_ZEXT_L ir_ZEXT_I32
# define ir_TRUNC_L ir_TRUNC_I32
# define ir_BITCAST_L ir_BITCAST_I32
# define ir_FP2L ir_FP2I32
# define ir_ADD_OV_L ir_ADD_OV_I32
# define ir_SUB_OV_L ir_SUB_OV_I32
# define ir_MUL_OV_L ir_MUL_OV_I32
# define ir_NOT_L ir_NOT_I32
# define ir_OR_L ir_OR_I32
# define ir_AND_L ir_AND_I32
# define ir_XOR_L ir_XOR_I32
# define ir_SHL_L ir_SHL_I32
# define ir_SHR_L ir_SHR_I32
# define ir_SAR_L ir_SAR_I32
# define ir_ROL_L ir_ROL_I32
# define ir_ROR_L ir_ROR_I32
# define ir_MIN_L ir_MIN_I32
# define ir_MAX_L ir_MAX_I32
# define ir_LOAD_L ir_LOAD_I32
#endif
/* A helper structure to collect IT rers for the following use in (MERGE/PHI)_N */
typedef struct _ir_refs {
uint32_t count;
uint32_t limit;
ir_ref refs[] ZEND_ELEMENT_COUNT(count);
} ir_refs;
#define ir_refs_size(_n) (offsetof(ir_refs, refs) + sizeof(ir_ref) * (_n))
#define ir_refs_init(_name, _n) _name = alloca(ir_refs_size(_n)); \
do {_name->count = 0; _name->limit = (_n);} while (0)
static void ir_refs_add(ir_refs *refs, ir_ref ref)
{
ir_ref *ptr;
ZEND_ASSERT(refs->count < refs->limit);
ptr = refs->refs;
ptr[refs->count++] = ref;
}
static size_t zend_jit_trace_prologue_size = (size_t)-1;
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
static uint32_t allowed_opt_flags = 0;
static uint32_t default_mflags = 0;
#endif
static bool delayed_call_chain = 0; // TODO: remove this var (use jit->delayed_call_level) ???
#ifdef ZTS
# ifdef _WIN32
extern uint32_t _tls_index;
extern char *_tls_start;
extern char *_tls_end;
# endif
static size_t tsrm_ls_cache_tcb_offset = 0;
static size_t tsrm_tls_index = 0;
static size_t tsrm_tls_offset = 0;
# define EG_TLS_OFFSET(field) \
(executor_globals_offset + offsetof(zend_executor_globals, field))
# define CG_TLS_OFFSET(field) \
(compiler_globals_offset + offsetof(zend_compiler_globals, field))
# define jit_EG(_field) \
ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))
# define jit_CG(_field) \
ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))
#else
# define jit_EG(_field) \
ir_CONST_ADDR(&EG(_field))
# define jit_CG(_field) \
ir_CONST_ADDR(&CG(_field))
#endif
#define jit_CALL(_call, _field) \
ir_ADD_OFFSET(_call, offsetof(zend_execute_data, _field))
#define jit_EX(_field) \
jit_CALL(jit_FP(jit), _field)
#define jit_RX(_field) \
jit_CALL(jit_IP(jit), _field)
#define JIT_STUBS(_) \
_(exception_handler, IR_SKIP_PROLOGUE) \
_(exception_handler_undef, IR_SKIP_PROLOGUE) \
_(exception_handler_free_op2, IR_SKIP_PROLOGUE) \
_(exception_handler_free_op1_op2, IR_SKIP_PROLOGUE) \
_(interrupt_handler, IR_SKIP_PROLOGUE) \
_(leave_function_handler, IR_SKIP_PROLOGUE) \
_(negative_shift, IR_SKIP_PROLOGUE) \
_(mod_by_zero, IR_SKIP_PROLOGUE) \
_(invalid_this, IR_SKIP_PROLOGUE) \
_(undefined_function, IR_SKIP_PROLOGUE) \
_(throw_cannot_pass_by_ref, IR_SKIP_PROLOGUE) \
_(icall_throw, IR_SKIP_PROLOGUE) \
_(leave_throw, IR_SKIP_PROLOGUE) \
_(hybrid_runtime_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_profile_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_func_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_loop_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_func_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_ret_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_loop_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(trace_halt, IR_SKIP_PROLOGUE) \
_(trace_escape, IR_SKIP_PROLOGUE) \
_(trace_exit, IR_SKIP_PROLOGUE) \
_(undefined_offset, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(undefined_key, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(cannot_add_element, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_const, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_tmp, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_var, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_cv_noref, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_cv, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(new_array, IR_FUNCTION | IR_FASTCALL_FUNC) \
#define JIT_STUB_ID(name, flags) \
jit_stub_ ## name,
#define JIT_STUB_FORWARD(name, flags) \
static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit);
#define JIT_STUB(name, flags) \
{JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, flags},
typedef enum _jit_stub_id {
JIT_STUBS(JIT_STUB_ID)
jit_last_stub
} jit_stub_id;
typedef struct _zend_jit_reg_var {
ir_ref ref;
uint32_t flags;
} zend_jit_reg_var;
typedef struct _zend_jit_ctx {
ir_ctx ctx;
const zend_op *last_valid_opline;
bool use_last_valid_opline;
bool track_last_valid_opline;
bool reuse_ip;
uint32_t delayed_call_level;
int b; /* current basic block number or -1 */
#ifdef ZTS
ir_ref tls;
#endif
ir_ref fp;
ir_ref trace_loop_ref;
ir_ref return_inputs;
const zend_op_array *op_array;
const zend_op_array *current_op_array;
zend_ssa *ssa;
zend_string *name;
ir_ref *bb_start_ref; /* PHP BB -> IR ref mapping */
ir_ref *bb_predecessors; /* PHP BB -> index in bb_edges -> IR refs of predessors */
ir_ref *bb_edges;
zend_jit_trace_info *trace;
zend_jit_reg_var *ra;
int delay_var;
ir_refs *delay_refs;
ir_ref eg_exception_addr;
HashTable addr_hash;
ir_ref stub_addr[jit_last_stub];
} zend_jit_ctx;
typedef int8_t zend_reg;
typedef struct _zend_jit_registers_buf {
#if defined(IR_TARGET_X64)
uint64_t gpr[16]; /* general purpose integer register */
double fpr[16]; /* floating point registers */
#elif defined(IR_TARGET_X86)
uint32_t gpr[8]; /* general purpose integer register */
double fpr[8]; /* floating point registers */
#elif defined (IR_TARGET_AARCH64)
uint64_t gpr[32]; /* general purpose integer register */
double fpr[32]; /* floating point registers */
#else
# error "Unknown IR target"
#endif
} zend_jit_registers_buf;
/* Keep 32 exit points in a single code block */
#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes
#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
static uint32_t zend_jit_exit_point_by_addr(const void *addr);
int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr var_use_addr,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
zend_jit_addr ref_addr,
bool check_exception);
typedef struct _zend_jit_stub {
const char *name;
int (*stub)(zend_jit_ctx *jit);
uint32_t flags;
} zend_jit_stub;
JIT_STUBS(JIT_STUB_FORWARD)
static const zend_jit_stub zend_jit_stubs[] = {
JIT_STUBS(JIT_STUB)
};
#if defined(_WIN32) || defined(IR_TARGET_AARCH64)
/* We keep addresses in SHM to share them between sepaeate processes (on Windows) or to support veneers (on AArch64) */
static void** zend_jit_stub_handlers = NULL;
#else
static void* zend_jit_stub_handlers[sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])];
#endif
#if defined(IR_TARGET_AARCH64)
# ifdef __FreeBSD__
/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */
typedef struct TLSDescriptor {
void* thunk;
int index;
size_t offset;
} TLSDescriptor;
# endif
#define IR_HAS_VENEERS (1U<<31) /* IR_RESERVED_FLAG_1 */
static const void *zend_jit_get_veneer(ir_ctx *ctx, const void *addr)
{
int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
for (i = 0; i < count; i++) {
if (zend_jit_stub_handlers[i] == addr) {
return zend_jit_stub_handlers[count + i];
}
}
if (((zend_jit_ctx*)ctx)->trace
&& (void*)addr >= dasm_buf && (void*)addr < dasm_end) {
uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
if (exit_point != (uint32_t)-1) {
zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
ZEND_ASSERT(exit_point < t->exit_count);
return (const void*)((char*)ctx->deoptimization_exits_base + (exit_point * 4));
}
}
return NULL;
}
static bool zend_jit_set_veneer(ir_ctx *ctx, const void *addr, const void *veneer)
{
int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
if (exit_point != (uint32_t)-1) {
return 1;
}
for (i = 0; i < count; i++) {
if (zend_jit_stub_handlers[i] == addr) {
const void **ptr = (const void**)&zend_jit_stub_handlers[count + i];
*ptr = veneer;
ctx->flags2 |= IR_HAS_VENEERS;
#ifdef HAVE_CAPSTONE
int64_t offset;
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
const char *name = ir_disasm_find_symbol((uint64_t)(uintptr_t)addr, &offset);
if (name && !offset) {
if (strstr(name, "@veneer") == NULL) {
char *new_name;
zend_spprintf(&new_name, 0, "%s@veneer", name);
ir_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);
efree(new_name);
} else {
ir_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);
}
}
}
#endif
return 1;
}
}
return 0;
}
static void zend_jit_commit_veneers(void)
{
int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
for (i = 0; i < count; i++) {
if (zend_jit_stub_handlers[count + i]) {
zend_jit_stub_handlers[i] = zend_jit_stub_handlers[count + i];
zend_jit_stub_handlers[count + i] = NULL;
}
}
}
#endif
static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr)
{
#if defined(IR_TARGET_X86)
return 0; /* always use immediate value */
#elif defined(IR_TARGET_X64)
return addr > 0xffffffff; /* prefer loading long constant from memery */
#elif defined(IR_TARGET_AARCH64)
return addr > 0xffff;
#else
# error "Unknown IR target"
#endif
}
static const char* zend_reg_name(int8_t reg)
{
return ir_reg_name(reg, ir_reg_is_int(reg) ? IR_LONG : IR_DOUBLE);
}
/* IR helpers */
#ifdef ZTS
static ir_ref jit_TLS(zend_jit_ctx *jit)
{
ZEND_ASSERT(jit->ctx.control);
if (jit->tls) {
/* Emit "TLS" once for basic block */
ir_insn *insn;
ir_ref ref = jit->ctx.control;
while (1) {
if (ref == jit->tls) {
return jit->tls;
}
insn = &jit->ctx.ir_base[ref];
if (insn->op >= IR_START || insn->op == IR_CALL) {
break;
}
ref = insn->op1;
}
}
jit->tls = ir_TLS(
tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index,
tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset);
return jit->tls;
}
#endif
static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr)
{
ir_ref ref;
zval *zv;
if (addr == 0) {
return IR_NULL;
}
zv = zend_hash_index_lookup(&jit->addr_hash, addr);
if (Z_TYPE_P(zv) == IS_LONG) {
ref = Z_LVAL_P(zv);
ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_ADDR, IR_ADDR));
} else {
ref = ir_unique_const_addr(&jit->ctx, addr);
ZVAL_LONG(zv, ref);
}
return ref;
}
static ir_ref jit_CONST_FUNC_PROTO(zend_jit_ctx *jit, uintptr_t addr, ir_ref proto)
{
ir_ref ref;
ir_insn *insn;
zval *zv;
ZEND_ASSERT(addr != 0);
zv = zend_hash_index_lookup(&jit->addr_hash, addr);
if (Z_TYPE_P(zv) == IS_LONG) {
ref = Z_LVAL_P(zv);
ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_FUNC_ADDR, IR_ADDR) && jit->ctx.ir_base[ref].proto == proto);
} else {
ref = ir_unique_const_addr(&jit->ctx, addr);
insn = &jit->ctx.ir_base[ref];
insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
insn->proto = proto;
ZVAL_LONG(zv, ref);
}
return ref;
}
static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags)
{
#if defined(IR_TARGET_X86)
/* TODO: dummy prototype (only flags matter) ??? */
ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
#else
ir_ref proto = 0;
#endif
return jit_CONST_FUNC_PROTO(jit, addr, proto);
}
static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset)
{
if (offset) {
addr = ir_ADD_A(addr, ir_CONST_ADDR(offset));
}
return addr;
}
static ir_ref jit_EG_exception(zend_jit_ctx *jit)
{
#ifdef ZTS
return jit_EG(exception);
#else
ir_ref ref = jit->eg_exception_addr;
if (UNEXPECTED(!ref)) {
ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)&EG(exception));
jit->eg_exception_addr = ref;
}
return ref;
#endif
}
static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id)
{
ir_ref ref = jit->stub_addr[id];
if (UNEXPECTED(!ref)) {
ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
jit->stub_addr[id] = ref;
}
return ref;
}
static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags)
{
ir_ref ref = jit->stub_addr[id];
ir_insn *insn;
if (UNEXPECTED(!ref)) {
ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
insn = &jit->ctx.ir_base[ref];
insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
#if defined(IR_TARGET_X86)
/* TODO: dummy prototype (only flags matter) ??? */
insn->proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
#else
insn->proto = 0;
#endif
jit->stub_addr[id] = ref;
}
return ref;
}
static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
{
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr;
const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array;
uint32_t stack_size = op_array->last_var + op_array->T;
if (ptr == zend_jit_stub_handlers[jit_stub_exception_handler]
|| ptr == zend_jit_stub_handlers[jit_stub_exception_handler_undef]
|| ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op1_op2]
|| ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op2]
|| ptr == zend_jit_stub_handlers[jit_stub_interrupt_handler]
|| ptr == zend_jit_stub_handlers[jit_stub_leave_function_handler]
|| ptr == zend_jit_stub_handlers[jit_stub_negative_shift]
|| ptr == zend_jit_stub_handlers[jit_stub_mod_by_zero]
|| ptr == zend_jit_stub_handlers[jit_stub_invalid_this]
|| ptr == zend_jit_stub_handlers[jit_stub_undefined_function]
|| ptr == zend_jit_stub_handlers[jit_stub_throw_cannot_pass_by_ref]
|| ptr == zend_jit_stub_handlers[jit_stub_icall_throw]
|| ptr == zend_jit_stub_handlers[jit_stub_leave_throw]
|| ptr == zend_jit_stub_handlers[jit_stub_trace_halt]
|| ptr == zend_jit_stub_handlers[jit_stub_trace_escape]) {
/* This is a GUARD that trigger exit through a stub code (without deoptimization) */
return;
}
/* Check if we need snapshot entries for polymorphic method call */
zend_jit_trace_info *t = jit->trace;
uint32_t exit_point = 0, n = 0;
if (addr < 0) {
if (t->exit_count > 0
&& jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) {
exit_point = t->exit_count - 1;
if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
n = 2;
}
}
}
if (stack_size || n) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t snapshot_size, i;
snapshot_size = stack_size;
while (snapshot_size > 0) {
ir_ref ref = STACK_REF(stack, snapshot_size - 1);
if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, snapshot_size - 1) & (/*ZREG_LOAD|*/ZREG_STORE))) {
snapshot_size--;
} else {
break;
}
}
if (snapshot_size || n) {
ir_ref snapshot;
snapshot = ir_SNAPSHOT(snapshot_size + n);
for (i = 0; i < snapshot_size; i++) {
ir_ref ref = STACK_REF(stack, i);
if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, i) & (/*ZREG_LOAD|*/ZREG_STORE))) {
ref = IR_UNUSED;
}
ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);
}
if (n) {
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref);
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref);
}
}
}
}
}
static int32_t _add_trace_const(zend_jit_trace_info *t, int64_t val)
{
int32_t i;
for (i = 0; i < t->consts_count; i++) {
if (t->constants[i].i == val) {
return i;
}
}
ZEND_ASSERT(i < 0x7fffffff);
t->consts_count = i + 1;
t->constants = erealloc(t->constants, (i + 1) * sizeof(zend_jit_exit_const));
t->constants[i].i = val;
return i;
}
uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint32_t exit_point, ir_ref snapshot_ref)
{
uint32_t stack_size, stack_offset;
uint32_t new_exit_point = t->exit_count;
if (new_exit_point >= ZEND_JIT_TRACE_MAX_EXITS) {
ctx->status = -ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
return exit_point;
}
t->exit_count++;
memcpy(&t->exit_info[new_exit_point], &t->exit_info[exit_point], sizeof(zend_jit_trace_exit_info));
stack_size = t->exit_info[new_exit_point].stack_size;
if (stack_size != 0) {
stack_offset = t->stack_map_size;
t->stack_map_size += stack_size;
// TODO: reduce number of reallocations ???
t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
memcpy(t->stack_map + stack_offset, t->stack_map + t->exit_info[new_exit_point].stack_offset, stack_size * sizeof(zend_jit_trace_stack));
t->exit_info[new_exit_point].stack_offset = stack_offset;
}
t->exit_info[new_exit_point].flags &= ~ZEND_JIT_EXIT_FIXED;
return new_exit_point;
}
void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)
{
zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
uint32_t exit_point, exit_flags;
ir_ref n = snapshot->inputs_count;
ir_ref i;
exit_point = zend_jit_exit_point_by_addr(addr);
ZEND_ASSERT(exit_point < t->exit_count);
exit_flags = t->exit_info[exit_point].flags;
if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {
int8_t *reg_ops = ctx->regs[snapshot_ref];
ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1);
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1]
|| t->exit_info[exit_point].poly_this_reg != reg_ops[n])) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1];
t->exit_info[exit_point].poly_this_reg = reg_ops[n];
n -= 2;
}
for (i = 2; i <= n; i++) {
ir_ref ref = ir_insn_op(snapshot, i);
if (ref) {
int8_t *reg_ops = ctx->regs[snapshot_ref];
int8_t reg = reg_ops[i];
ir_ref var = i - 2;
ZEND_ASSERT(var < t->exit_info[exit_point].stack_size);
if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags == ZREG_ZVAL_COPY) {
ZEND_ASSERT(reg != ZREG_NONE);
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
} else if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_CONST) {
ZEND_ASSERT(t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_LONG ||
t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_DOUBLE);
if (ref > 0) {
if (reg != ZREG_NONE) {
if (reg & IR_REG_SPILL_LOAD) {
ZEND_ASSERT(!(reg & IR_REG_SPILL_SPECIAL));
/* spill slot on a CPU stack */
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& (t->stack_map[t->exit_info[exit_point].stack_offset + var].ref != ref
|| t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
|| !(t->stack_map[t->exit_info[exit_point].stack_offset + var].flags & ZREG_SPILL_SLOT))) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = ref;
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_SPILL_SLOT;
} else if (reg & IR_REG_SPILL_SPECIAL) {
/* spill slot on a VM stack */
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
|| t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
} else {
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
}
} else {
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
|| t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
}
} else if (!(exit_flags & ZEND_JIT_EXIT_FIXED)) {
int32_t idx = _add_trace_const(t, ctx->ir_base[ref].val.i64);
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_CONST;
t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = idx;
}
}
}
}
t->exit_info[exit_point].flags |= ZEND_JIT_EXIT_FIXED;
return addr;
}
static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr)
{
jit_SNAPSHOT(jit, addr);
ir_IJMP(addr);
}
/* PHP JIT helpers */
static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend_op *opline)
{
#if ZEND_DEBUG
return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_emalloc),
ir_CONST_ADDR(size),
op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
ir_CONST_U32(opline ? opline->lineno : 0),
IR_NULL,
ir_CONST_U32(0));
#elif defined(HAVE_BUILTIN_CONSTANT_P)
if (size > 24 && size <= 32) {
return ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_emalloc_32));
} else {
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
}
#else
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
#endif
}
static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, const zend_op *opline)
{
#if ZEND_DEBUG
return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_efree),
ptr,
op_array && op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
ir_CONST_U32(opline ? opline->lineno : 0),
IR_NULL,
ir_CONST_U32(0));
#elif defined(HAVE_BUILTIN_CONSTANT_P)
if (size > 24 && size <= 32) {
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree_32), ptr);
} else {
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
}
#else
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
#endif
}
static ir_ref jit_FP(zend_jit_ctx *jit)
{
ZEND_ASSERT(jit->ctx.control);
if (jit->fp == IR_UNUSED) {
/* Emit "RLOAD FP" once for basic block */
jit->fp = ir_RLOAD_A(ZREG_FP);
} else {
ir_insn *insn;
ir_ref ref = jit->ctx.control;
while (1) {
if (ref == jit->fp) {
break;
}
insn = &jit->ctx.ir_base[ref];
if (insn->op >= IR_START || insn->op == IR_CALL) {
jit->fp = ir_RLOAD_A(ZREG_FP);
break;
}
ref = insn->op1;
}
}
return jit->fp;
}
static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref)
{
ir_RSTORE(ZREG_FP, ref);
jit->fp = IR_UNUSED;
}
static ir_ref jit_IP(zend_jit_ctx *jit)
{
return ir_RLOAD_A(ZREG_IP);
}
static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref)
{
ir_RSTORE(ZREG_IP, ref);
}
static ir_ref jit_IP32(zend_jit_ctx *jit)
{
return ir_RLOAD_U32(ZREG_IP);
}
static void jit_LOAD_IP(zend_jit_ctx *jit, ir_ref ref)
{
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ref);
} else {
ir_STORE(jit_EX(opline), ref);
}
}
static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target)
{
jit_LOAD_IP(jit, ir_CONST_ADDR(target));
}
static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit)
{
jit->use_last_valid_opline = 0;
jit->track_last_valid_opline = 1;
}
static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit)
{
if (jit->track_last_valid_opline) {
jit->use_last_valid_opline = 1;
jit->track_last_valid_opline = 0;
}
}
static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit)
{
return jit->use_last_valid_opline;
}
static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline)
{
if (!jit->reuse_ip) {
jit->track_last_valid_opline = 1;
jit->last_valid_opline = opline;
}
}
static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit)
{
jit->track_last_valid_opline = 0;
jit->last_valid_opline = NULL;
}
static void zend_jit_start_reuse_ip(zend_jit_ctx *jit)
{
zend_jit_reset_last_valid_opline(jit);
jit->reuse_ip = 1;
}
static int zend_jit_reuse_ip(zend_jit_ctx *jit)
{
if (!jit->reuse_ip) {
zend_jit_start_reuse_ip(jit);
// RX = EX(call);
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
}
return 1;
}
static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit)
{
jit->reuse_ip = 0;
}
static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level)
{
ir_ref rx, call;
if (call_level == 1) {
// JIT: call = NULL;
call = IR_NULL;
} else {
// JIT: call = EX(call);
call = ir_LOAD_A(jit_EX(call));
}
rx = jit_IP(jit);
// JIT: call->prev_execute_data = call;
ir_STORE(jit_CALL(rx, prev_execute_data), call);
// JIT: EX(call) = call;
ir_STORE(jit_EX(call), rx);
jit->delayed_call_level = 0;
delayed_call_chain = 0;
return 1;
}
static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target)
{
ir_ref ref;
ir_ref addr = IR_UNUSED;
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
if (jit->last_valid_opline) {
zend_jit_use_last_valid_opline(jit);
if (jit->last_valid_opline != target) {
if (GCC_GLOBAL_REGS) {
ref = jit_IP(jit);
} else {
addr = jit_EX(opline);
ref = ir_LOAD_A(addr);
}
if (target > jit->last_valid_opline) {
ref = ir_ADD_OFFSET(ref, (uintptr_t)target - (uintptr_t)jit->last_valid_opline);
} else {
ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target));
}
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ref);
} else {
ir_STORE(addr, ref);
}
}
} else {
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ir_CONST_ADDR(target));
} else {
ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
}
}
jit->reuse_ip = 0;
zend_jit_set_last_valid_opline(jit, target);
return 1;
}
static int zend_jit_set_ip_ex(zend_jit_ctx *jit, const zend_op *target, bool set_ip_reg)
{
if (!GCC_GLOBAL_REGS && set_ip_reg && !jit->last_valid_opline) {
/* Optimization to avoid duplicate constant load */
ir_STORE(jit_EX(opline), ir_HARD_COPY_A(ir_CONST_ADDR(target)));
return 1;
}
return zend_jit_set_ip(jit, target);
}
static void jit_SET_EX_OPLINE(zend_jit_ctx *jit, const zend_op *target)
{
if (jit->last_valid_opline == target) {
zend_jit_use_last_valid_opline(jit);
if (GCC_GLOBAL_REGS) {
// EX(opline) = opline
ir_STORE(jit_EX(opline), jit_IP(jit));
}
} else {
ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
if (!GCC_GLOBAL_REGS) {
zend_jit_reset_last_valid_opline(jit);
}
}
}
static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
return ir_ADD_OFFSET(reg, Z_OFFSET(addr));
} else if (Z_MODE(addr) == IS_REF_ZVAL) {
return Z_IR_REF(addr);
} else {
ZEND_ASSERT(Z_MODE(addr) == IS_CONST_ZVAL);
return ir_CONST_ADDR(Z_ZV(addr));
}
}
static ir_ref jit_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type)));
}
static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_U8(Z_TYPE_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type)));
} else {
return jit_Z_TYPE_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static ir_ref jit_Z_TYPE_FLAGS_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type_flags)));
}
static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_U8(Z_TYPE_FLAGS_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type_flags)));
} else {
return jit_Z_TYPE_FLAGS_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static ir_ref jit_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)));
}
static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_U32(Z_TYPE_INFO_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
return ir_LOAD_U32(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)));
} else {
return jit_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static void jit_set_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type_info)
{
ir_STORE(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)), type_info);
}
static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref type_info)
{
if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
ir_STORE(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)), type_info);
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr), type_info);
}
}
static void jit_set_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr, uint32_t type_info)
{
if (type_info < IS_STRING
&& Z_MODE(addr) == IS_MEM_ZVAL
&& Z_REG(addr) == ZREG_FP
&& JIT_G(current_frame)
&& STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(addr))) == type_info) {
/* type is already set */
return;
}
jit_set_Z_TYPE_INFO_ex(jit, addr, ir_CONST_U32(type_info));
}
static ir_ref jit_if_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type)
{
return ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), type));
}
static ir_ref jit_if_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
{
ZEND_ASSERT(type != IS_UNDEF);
return ir_IF(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)));
}
static ir_ref jit_if_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
{
ir_ref ref = jit_Z_TYPE(jit, addr);
if (type != IS_UNDEF) {
ref = ir_NE(ref, ir_CONST_U8(type));
}
return ir_IF(ref);
}
static void jit_guard_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
{
ir_ref ref = jit_Z_TYPE(jit, addr);
if (type != IS_UNDEF) {
ir_GUARD(ir_EQ(ref, ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
}
static void jit_guard_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
{
ir_ref ref = jit_Z_TYPE(jit, addr);
if (type != IS_UNDEF) {
ref = ir_NE(ref, ir_CONST_U8(type));
}
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
}
static ir_ref jit_if_REFCOUNTED(zend_jit_ctx *jit, zend_jit_addr addr)
{
return ir_IF(jit_Z_TYPE_FLAGS(jit, addr));
}
static ir_ref jit_if_COLLECTABLE_ref(zend_jit_ctx *jit, ir_ref addr_ref)
{
return ir_IF(ir_AND_U8(jit_Z_TYPE_FLAGS_ref(jit, addr_ref), ir_CONST_U8(IS_TYPE_COLLECTABLE)));
}
static ir_ref jit_Z_LVAL_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_L(ref);
}
static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_D(ref);
}
static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val)
{
if (jit->ctx.ir_base[val].op == IR_RLOAD) {
/* Deoptimization */
return 0;
}
// if (jit->ctx.ir_base[val].op == IR_LOAD
// && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP
// && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2)
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr == (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)) {
// /* LOAD from the same location (the LOAD is pinned) */
// // TODO: should be anti-dependent with the following stores ???
// return 0;
// }
if (jit->ssa->vars[var].var < jit->current_op_array->last_var) {
/* IS_CV */
return 0;
}
return 1;
}
static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val)
{
int var;
ZEND_ASSERT(Z_MODE(addr) == IS_REG);
var = Z_SSA_VAR(addr);
if (var == jit->delay_var) {
ir_refs_add(jit->delay_refs, val);
return;
}
ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL);
/* Negative "var" has special meaning for IR */
if (val > 0 && !zend_jit_spilling_may_cause_conflict(jit, var, val)) {
val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val);
}
jit->ra[var].ref = val;
if (jit->ra[var].flags & ZREG_FORWARD) {
zend_ssa_phi *phi = jit->ssa->vars[var].phi_use_chain;
zend_basic_block *bb;
int n, j, *p;
ir_ref *q;
jit->ra[var].flags &= ~ZREG_FORWARD;
while (phi != NULL) {
zend_ssa_phi *dst_phi = phi;
int src_var = var;
if (dst_phi->pi >= 0) {
jit->ra[src_var].ref = val;
src_var = dst_phi->ssa_var;
if (!(jit->ra[src_var].flags & ZREG_FORWARD)) {
phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
continue;
}
dst_phi = jit->ssa->vars[src_var].phi_use_chain;
ZEND_ASSERT(dst_phi != NULL && "reg forwarding");
ZEND_ASSERT(!zend_ssa_next_use_phi(jit->ssa, src_var, dst_phi) && "reg forwarding");
jit->ra[src_var].flags &= ~ZREG_FORWARD;
}
if (jit->ra[dst_phi->ssa_var].ref > 0) {
ir_insn *phi_insn = &jit->ctx.ir_base[jit->ra[dst_phi->ssa_var].ref];
if (phi_insn->op == IR_PHI) {
// ZEND_ASSERT(ir_operands_count(ctx, phi_insn) == n + 1);
bb = &jit->ssa->cfg.blocks[dst_phi->block];
n = bb->predecessors_count;
for (j = 0, p = &dst_phi->sources[0], q = phi_insn->ops + 2; j < n; j++, p++, q++) {
if (*p == src_var) {
*q = val;
}
}
}
}
phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
}
}
}
static ir_ref zend_jit_use_reg(zend_jit_ctx *jit, zend_jit_addr addr)
{
int var = Z_SSA_VAR(addr);
ZEND_ASSERT(Z_MODE(addr) == IS_REG);
ZEND_ASSERT(jit->ra && jit->ra[var].ref);
if (jit->ra[var].ref == IR_NULL) {
zend_jit_addr mem_addr;
ir_ref ref;
ZEND_ASSERT(jit->ra[var].flags & ZREG_LOAD);
mem_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(jit->ssa->vars[var].var));
if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG) {
ref = jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
} else if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
ref = jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
} else {
ZEND_UNREACHABLE();
}
zend_jit_def_reg(jit, addr, ref);
return ref;
}
return jit->ra[Z_SSA_VAR(addr)].ref;
}
static void zend_jit_gen_pi(zend_jit_ctx *jit, zend_ssa_phi *phi)
{
int src_var = phi->sources[0];
int dst_var = phi->ssa_var;
ZEND_ASSERT(phi->pi >= 0);
ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
ZEND_ASSERT(jit->ra[src_var].ref);
if (jit->ra[src_var].ref == IR_NULL) {
/* Not defined yet */
if (jit->ssa->vars[dst_var].use_chain < 0
&& jit->ssa->vars[dst_var].phi_use_chain) {
zend_ssa_phi *phi = jit->ssa->vars[dst_var].phi_use_chain;
if (!zend_ssa_next_use_phi(jit->ssa, dst_var, phi)) {
/* This is a Pi forwarded to Phi */
jit->ra[src_var].flags |= ZREG_FORWARD;
return;
}
}
ZEND_ASSERT(0 && "Not defined Pi source");
}
/* Reuse register */
zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var),
zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)));
}
static void zend_jit_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
{
int dst_var = phi->ssa_var;
zend_basic_block *bb = &jit->ssa->cfg.blocks[phi->block];
int n = bb->predecessors_count;
int i;
ir_type type = (jit->ssa->var_info[phi->ssa_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE;
ir_ref merge = jit->bb_start_ref[phi->block];
ir_ref ref;
ir_ref old_insns_count = jit->ctx.insns_count;
ir_ref same_src_ref = IR_UNUSED;
bool phi_inputs_are_the_same = 1;
ZEND_ASSERT(phi->pi < 0);
ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
ZEND_ASSERT(merge);
ZEND_ASSERT(jit->ctx.ir_base[merge].op == IR_MERGE || jit->ctx.ir_base[merge].op == IR_LOOP_BEGIN);
ZEND_ASSERT(n == jit->ctx.ir_base[merge].inputs_count);
ref = ir_emit_N(&jit->ctx, IR_OPT(IR_PHI, type), n + 1);
ir_set_op(&jit->ctx, ref, 1, merge);
for (i = 0; i < n; i++) {
int src_var = phi->sources[i];
ZEND_ASSERT(jit->ra[src_var].ref);
if (jit->ra[src_var].ref == IR_NULL) {
jit->ra[src_var].flags |= ZREG_FORWARD;
phi_inputs_are_the_same = 0;
} else {
ir_ref src_ref = zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var));
if (i == 0) {
same_src_ref = src_ref;
} else if (same_src_ref != src_ref) {
phi_inputs_are_the_same = 0;
}
ir_set_op(&jit->ctx, ref, i + 2, src_ref);
}
}
if (phi_inputs_are_the_same) {
ref = same_src_ref;
jit->ctx.insns_count = old_insns_count;
}
zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
}
static ir_ref jit_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_LONG(Z_LVAL_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_REG) {
return zend_jit_use_reg(jit, addr);
} else {
return jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static void jit_set_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
{
if (Z_MODE(addr) == IS_REG) {
zend_jit_def_reg(jit, addr, lval);
} else {
ir_STORE(jit_ZVAL_ADDR(jit, addr), lval);
}
}
#if SIZEOF_ZEND_LONG == 4
static ir_ref jit_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_U32((Z_ZV(addr))->value.ww.w2);
} else {
return ir_LOAD_L(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)));
}
}
static void jit_set_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
{
ir_STORE(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)), lval);
}
#endif
static ir_ref jit_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_DOUBLE(Z_DVAL_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_REG) {
return zend_jit_use_reg(jit, addr);
} else {
return jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static void jit_set_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref dval)
{
if (Z_MODE(addr) == IS_REG) {
zend_jit_def_reg(jit, addr, dval);
} else {
ir_STORE(jit_ZVAL_ADDR(jit, addr), dval);
}
}
static ir_ref jit_Z_PTR_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_A(ref);
}
static ir_ref jit_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_ADDR(Z_PTR_P(Z_ZV(addr)));
} else {
return jit_Z_PTR_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static void jit_set_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ptr)
{
ir_STORE(jit_ZVAL_ADDR(jit, addr), ptr);
}
static ir_ref jit_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_U32(ref);
}
static void jit_set_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref, uint32_t refcount)
{
ir_STORE(ref, ir_CONST_U32(refcount));
}
static void jit_GC_ADDREF(zend_jit_ctx *jit, ir_ref ref)
{
ir_STORE(ref, ir_ADD_U32(ir_LOAD_U32(ref), ir_CONST_U32(1)));
}
static void jit_GC_ADDREF2(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref counter = ir_LOAD_U32(ref);
ir_STORE(ref, ir_ADD_U32(counter, ir_CONST_U32(2)));
}
static ir_ref jit_GC_DELREF(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref counter = ir_LOAD_U32(ref);
counter = ir_SUB_U32(counter, ir_CONST_U32(1));
ir_STORE(ref, counter);
return counter;
}
static ir_ref jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx *jit, ir_ref ref)
{
return ir_IF(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_refcounted, gc.u.type_info))),
ir_CONST_U32(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT))));
}
static void jit_ZVAL_COPY_CONST(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, uint32_t dst_def_info, zval *zv, bool addref)
{
ir_ref ref = IR_UNUSED;
if (Z_TYPE_P(zv) > IS_TRUE) {
if (Z_TYPE_P(zv) == IS_DOUBLE) {
jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE(Z_DVAL_P(zv)));
} else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE((double)Z_LVAL_P(zv)));
} else if (Z_TYPE_P(zv) == IS_LONG) {
jit_set_Z_LVAL(jit, dst, ir_CONST_LONG(Z_LVAL_P(zv)));
} else {
ref = ir_CONST_ADDR(Z_PTR_P(zv));
jit_set_Z_PTR(jit, dst, ref);
if (addref && Z_REFCOUNTED_P(zv)) {
jit_GC_ADDREF(jit, ref);
}
}
}
if (Z_MODE(dst) != IS_REG) {
if (dst_def_info == MAY_BE_DOUBLE) {
if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
} else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
jit_set_Z_TYPE_INFO(jit, dst, Z_TYPE_INFO_P(zv));
}
}
}
static ir_ref jit_if_TYPED_REF(zend_jit_ctx *jit, ir_ref ref)
{
return ir_IF(ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr))));
}
static void jit_ZVAL_COPY(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, bool addref)
{
ir_ref ref = IR_UNUSED;
if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
jit_set_Z_LVAL(jit, dst, jit_Z_LVAL(jit, src));
} else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, jit_Z_DVAL(jit, src));
} else {
#if SIZEOF_ZEND_LONG == 4
if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
jit_set_Z_W2(jit, dst, jit_Z_W2(jit, src));
}
#endif
ref = jit_Z_PTR(jit, src);
jit_set_Z_PTR(jit, dst, ref);
}
}
if (has_concrete_type(src_info & MAY_BE_ANY)
&& (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
&& !(src_info & MAY_BE_GUARD)) {
if (Z_MODE(dst) != IS_REG
&& (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
uint8_t type = concrete_type(src_info);
jit_set_Z_TYPE_INFO(jit, dst, type);
}
} else {
ir_ref type = jit_Z_TYPE_INFO(jit, src);
jit_set_Z_TYPE_INFO_ex(jit, dst, type);
if (addref) {
if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
ir_ref if_refcounted = IR_UNUSED;
if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
ir_IF_TRUE(if_refcounted);
}
jit_GC_ADDREF(jit, ref);
if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
}
}
}
}
}
static void jit_ZVAL_COPY_2(zend_jit_ctx *jit, zend_jit_addr dst2, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, int addref)
{
ir_ref ref = IR_UNUSED;
if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
ref = jit_Z_LVAL(jit, src);
jit_set_Z_LVAL(jit, dst, ref);
jit_set_Z_LVAL(jit, dst2, ref);
} else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
ref = jit_Z_DVAL(jit, src);
jit_set_Z_DVAL(jit, dst, ref);
jit_set_Z_DVAL(jit, dst2, ref);
} else {
#if SIZEOF_ZEND_LONG == 4
if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
ref = jit_Z_W2(jit, src);
jit_set_Z_W2(jit, dst, ref);
jit_set_Z_W2(jit, dst2, ref);
}
#endif
ref = jit_Z_PTR(jit, src);
jit_set_Z_PTR(jit, dst, ref);
jit_set_Z_PTR(jit, dst2, ref);
}
}
if (has_concrete_type(src_info & MAY_BE_ANY)
&& (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
&& !(src_info & MAY_BE_GUARD)) {
uint8_t type = concrete_type(src_info);
ir_ref type_ref = ir_CONST_U32(type);
if (Z_MODE(dst) != IS_REG
&& (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
jit_set_Z_TYPE_INFO_ex(jit, dst, type_ref);
}
if (Z_MODE(dst2) != IS_REG) {
jit_set_Z_TYPE_INFO_ex(jit, dst2, type_ref);
}
} else {
ir_ref type = jit_Z_TYPE_INFO(jit, src);
jit_set_Z_TYPE_INFO_ex(jit, dst, type);
jit_set_Z_TYPE_INFO_ex(jit, dst2, type);
if (addref) {
if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
ir_ref if_refcounted = IR_UNUSED;
if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
ir_IF_TRUE(if_refcounted);
}
if (addref == 2) {
jit_GC_ADDREF2(jit, ref);
} else {
jit_GC_ADDREF(jit, ref);
}
if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
}
}
}
}
}
static void jit_ZVAL_DTOR(zend_jit_ctx *jit, ir_ref ref, uint32_t op_info, const zend_op *opline)
{
if (!((op_info) & MAY_BE_GUARD)
&& has_concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
uint8_t type = concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
if (type == IS_STRING && !ZEND_DEBUG) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(_efree), ref);
return;
} else if (type == IS_ARRAY) {
if ((op_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
if (opline && ((op_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_array_destroy), ref);
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_array_free), ref);
}
return;
} else if (type == IS_OBJECT) {
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
return;
}
}
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(rc_dtor_func), ref);
}
static void jit_ZVAL_PTR_DTOR(zend_jit_ctx *jit,
zend_jit_addr addr,
uint32_t op_info,
bool gc,
const zend_op *opline)
{
ir_ref ref, ref2;
ir_ref if_refcounted = IR_UNUSED;
ir_ref if_not_zero = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if_refcounted = jit_if_REFCOUNTED(jit, addr);
ir_IF_FALSE(if_refcounted);
ir_END_list(end_inputs);
ir_IF_TRUE(if_refcounted);
}
ref = jit_Z_PTR(jit, addr);
ref2 = jit_GC_DELREF(jit, ref);
if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
if_not_zero = ir_IF(ref2);
ir_IF_FALSE(if_not_zero);
}
// zval_dtor_func(r);
jit_ZVAL_DTOR(jit, ref, op_info, opline);
if (if_not_zero) {
ir_END_list(end_inputs);
ir_IF_TRUE(if_not_zero);
}
}
if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
ir_ref if_may_not_leak;
if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
ir_ref if_ref, if_collectable;
if_ref = jit_if_Z_TYPE(jit, addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
if_collectable = jit_if_COLLECTABLE_ref(jit, ref2);
ir_IF_FALSE(if_collectable);
ir_END_list(end_inputs);
ir_IF_TRUE(if_collectable);
ref2 = jit_Z_PTR_ref(jit, ref2);
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
}
if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
ir_IF_TRUE(if_may_not_leak);
ir_END_list(end_inputs);
ir_IF_FALSE(if_may_not_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
}
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
}
}
static void jit_FREE_OP(zend_jit_ctx *jit,
uint8_t op_type,
znode_op op,
uint32_t op_info,
const zend_op *opline)
{
if (op_type & (IS_VAR|IS_TMP_VAR)) {
jit_ZVAL_PTR_DTOR(jit,
ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var),
op_info, 0, opline);
}
}
static void jit_OBJ_RELEASE(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref end_inputs = IR_UNUSED;
ir_ref if_not_zero, if_may_not_leak;
// JIT: if (GC_DELREF(obj) == 0) {
if_not_zero = ir_IF(jit_GC_DELREF(jit, ref));
ir_IF_FALSE(if_not_zero);
// JIT: zend_objects_store_del(obj)
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
ir_END_list(end_inputs);
ir_IF_TRUE(if_not_zero);
if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
ir_IF_TRUE(if_may_not_leak);
ir_END_list(end_inputs);
ir_IF_FALSE(if_may_not_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
static int zend_jit_check_timeout(zend_jit_ctx *jit, const zend_op *opline, const void *exit_addr)
{
ir_ref ref = ir_LOAD_U8(jit_EG(vm_interrupt));
if (exit_addr) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
} else if (!opline || jit->last_valid_opline == opline) {
ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
} else {
ir_ref if_timeout = ir_IF(ref);
ir_IF_TRUE_cold(if_timeout);
jit_LOAD_IP_ADDR(jit, opline);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
ir_IF_FALSE(if_timeout);
}
return 1;
}
/* stubs */
static int zend_jit_exception_handler_stub(zend_jit_ctx *jit)
{
const void *handler;
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
handler = zend_get_opcode_handler_func(EG(exception_op));
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
handler = EG(exception_op)->handler;
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
} else {
ir_ref ref, if_negative;
ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), jit_FP(jit));
if_negative = ir_IF(ir_LT(ref, ir_CONST_U32(0)));
ir_IF_TRUE(if_negative);
ir_MERGE_WITH_EMPTY_FALSE(if_negative);
ref = ir_PHI_2(IR_I32, ref, ir_CONST_I32(1));
ir_RETURN(ref);
}
}
return 1;
}
static int zend_jit_exception_handler_undef_stub(zend_jit_ctx *jit)
{
ir_ref ref, result_type, if_result_used;
ref = jit_EG(opline_before_exception);
result_type = ir_LOAD_U8(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result_type)));
if_result_used = ir_IF(ir_AND_U8(result_type, ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
ir_IF_TRUE(if_result_used);
ref = ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
ir_STORE(ir_ADD_OFFSET(ir_ADD_A(jit_FP(jit), ref), offsetof(zval, u1.type_info)), ir_CONST_U32(IS_UNDEF));
ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
return 1;
}
static int zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx *jit)
{
ir_ref ref, if_dtor;
zend_jit_addr var_addr;
ref = ir_LOAD_A(jit_EG(opline_before_exception));
if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op1_type))),
ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
ir_IF_TRUE(if_dtor);
ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op1.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
ref = ir_ADD_A(jit_FP(jit), ref);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
return 1;
}
static int zend_jit_exception_handler_free_op2_stub(zend_jit_ctx *jit)
{
ir_ref ref, if_dtor;
zend_jit_addr var_addr;
ref = ir_LOAD_A(jit_EG(opline_before_exception));
if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op2_type))),
ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
ir_IF_TRUE(if_dtor);
ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
ref = ir_ADD_A(jit_FP(jit), ref);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
return 1;
}
static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit)
{
ir_ref if_timeout, if_exception;
if (GCC_GLOBAL_REGS) {
// EX(opline) = opline
ir_STORE(jit_EX(opline), jit_IP(jit));
}
ir_STORE(jit_EG(vm_interrupt), ir_CONST_U8(0));
if_timeout = ir_IF(ir_EQ(ir_LOAD_U8(jit_EG(timed_out)), ir_CONST_U8(0)));
ir_IF_FALSE(if_timeout);
ir_CALL(IR_VOID, ir_CONST_FUNC(zend_timeout));
ir_MERGE_WITH_EMPTY_TRUE(if_timeout);
if (zend_interrupt_function) {
ir_CALL_1(IR_VOID, ir_CONST_FUNC(zend_interrupt_function), jit_FP(jit));
if_exception = ir_IF(ir_LOAD_A(jit_EG(exception)));
ir_IF_TRUE(if_exception);
ir_CALL(IR_VOID, ir_CONST_FUNC(zend_jit_exception_in_interrupt_handler_helper));
ir_MERGE_WITH_EMPTY_FALSE(if_exception);
jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
}
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(1));
}
return 1;
}
static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit)
{
ir_ref call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
ir_ref if_top = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_TOP)));
ir_IF_FALSE(if_top);
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
jit_STORE_IP(jit,
ir_LOAD_A(jit_EX(opline)));
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else if (GCC_GLOBAL_REGS) {
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
} else {
ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info, jit_FP(jit));
}
ir_IF_TRUE(if_top);
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else if (GCC_GLOBAL_REGS) {
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
} else {
ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info, jit_FP(jit));
}
return 1;
}
static int zend_jit_negative_shift_stub(zend_jit_ctx *jit)
{
ir_CALL_2(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
ir_CONST_ADDR(zend_ce_arithmetic_error),
ir_CONST_ADDR("Bit shift by negative number"));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
return 1;
}
static int zend_jit_mod_by_zero_stub(zend_jit_ctx *jit)
{
ir_CALL_2(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
ir_CONST_ADDR(zend_ce_division_by_zero_error),
ir_CONST_ADDR("Modulo by zero"));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
return 1;
}
static int zend_jit_invalid_this_stub(zend_jit_ctx *jit)
{
ir_CALL_2(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
IR_NULL,
ir_CONST_ADDR("Using $this when not in object context"));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
return 1;
}
static int zend_jit_undefined_function_stub(zend_jit_ctx *jit)
{
// JIT: load EX(opline)
ir_ref ref = ir_LOAD_A(jit_FP(jit));
ir_ref arg3 = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.constant)));
if (sizeof(void*) == 8) {
arg3 = ir_LOAD_A(ir_ADD_A(ref, ir_SEXT_A(arg3)));
} else {
arg3 = ir_LOAD_A(arg3);
}
arg3 = ir_ADD_OFFSET(arg3, offsetof(zend_string, val));
ir_CALL_3(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
IR_NULL,
ir_CONST_ADDR("Call to undefined function %s()"),
arg3);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
return 1;
}
static int zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx *jit)
{
ir_ref opline, ref, rx, if_eq, if_tmp;
// JIT: opline = EX(opline)
opline = ir_LOAD_A(jit_FP(jit));
// JIT: ZVAL_UNDEF(ZEND_CALL_VAR(RX, opline->result.var))
ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
rx = jit_IP(jit);
jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(rx, ref), ir_CONST_U32(IS_UNDEF));
// last EX(call) frame may be delayed
// JIT: if (EX(call) == RX)
ref = ir_LOAD_A(jit_EX(call));
if_eq = ir_IF(ir_EQ(rx, ref));
ir_IF_FALSE(if_eq);
// JIT: RX->prev_execute_data == EX(call)
ir_STORE(jit_CALL(rx, prev_execute_data), ref);
// JIT: EX(call) = RX
ir_STORE(jit_EX(call), rx);
ir_MERGE_WITH_EMPTY_TRUE(if_eq);
// JIT: IP = opline
jit_STORE_IP(jit, opline);
// JIT: zend_cannot_pass_by_reference(opline->op2.num)
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_cannot_pass_by_reference),
ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, op2.num))));
// JIT: if (IP->op1_type == IS_TMP_VAR)
ref = ir_LOAD_U8(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1_type)));
if_tmp = ir_IF(ir_EQ(ref, ir_CONST_U8(IS_TMP_VAR)));
ir_IF_TRUE(if_tmp);
// JIT: zval_ptr_dtor(EX_VAR(IP->op1.var))
ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
ref = ir_ADD_A(jit_FP(jit), ref);
jit_ZVAL_PTR_DTOR(jit,
ZEND_ADDR_REF_ZVAL(ref),
MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
ir_MERGE_WITH_EMPTY_FALSE(if_tmp);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
return 1;
}
static int zend_jit_icall_throw_stub(zend_jit_ctx *jit)
{
ir_ref ip, if_set;
// JIT: zend_rethrow_exception(zend_execute_data *execute_data)
// JIT: if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
ip = jit_IP(jit);
if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
ir_IF_FALSE(if_set);
// JIT: EG(opline_before_exception) = opline;
ir_STORE(jit_EG(opline_before_exception), ip);
ir_MERGE_WITH_EMPTY_TRUE(if_set);
// JIT: opline = EG(exception_op);
jit_STORE_IP(jit, jit_EG(exception_op));
if (GCC_GLOBAL_REGS) {
ir_STORE(jit_EX(opline), jit_IP(jit));
}
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
return 1;
}
static int zend_jit_leave_throw_stub(zend_jit_ctx *jit)
{
ir_ref ip, if_set;
// JIT: if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
ip = jit_IP(jit);
if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
ir_IF_FALSE(if_set);
// JIT: EG(opline_before_exception) = opline;
ir_STORE(jit_EG(opline_before_exception), ip);
ir_MERGE_WITH_EMPTY_TRUE(if_set);
// JIT: opline = EG(exception_op);
jit_LOAD_IP(jit, jit_EG(exception_op));
if (GCC_GLOBAL_REGS) {
ir_STORE(jit_EX(opline), jit_IP(jit));
// JIT: HANDLE_EXCEPTION()
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
} else {
ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
}
return 1;
}
static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 0;
}
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_runtime_jit));
ir_IJMP(ir_LOAD_A(jit_IP(jit)));
return 1;
}
static int zend_jit_hybrid_profile_jit_stub(zend_jit_ctx *jit)
{
ir_ref addr, func, run_time_cache, jit_extension;
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 0;
}
addr = ir_CONST_ADDR(&zend_jit_profile_counter),
ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
func = ir_LOAD_A(jit_EX(func));
run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
if (zend_jit_profile_counter_rid) {
addr = ir_ADD_OFFSET(run_time_cache, zend_jit_profile_counter_rid * sizeof(void*));
} else {
addr = run_time_cache;
}
ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
addr = ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_extension, orig_handler));
ir_IJMP(ir_LOAD_A(addr));
return 1;
}
static int _zend_jit_hybrid_hot_counter_stub(zend_jit_ctx *jit, uint32_t cost)
{
ir_ref func, jit_extension, addr, ref, if_overflow;
func = ir_LOAD_A(jit_EX(func));
jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
addr = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, counter)));
ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
ir_STORE(addr, ref);
if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
ir_IF_TRUE_cold(if_overflow);
ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_hot_func),
jit_FP(jit),
jit_IP(jit));
ir_IJMP(ir_LOAD_A(jit_IP(jit)));
ir_IF_FALSE(if_overflow);
ref = ir_SUB_A(jit_IP(jit),
ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, opcodes))));
ref = ir_DIV_A(ref, ir_CONST_ADDR(sizeof(zend_op) / sizeof(void*)));
addr = ir_ADD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, orig_handlers)),
ref);
ir_IJMP(ir_LOAD_A(addr));
return 1;
}
static int zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 0;
}
return _zend_jit_hybrid_hot_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 0;
}
return _zend_jit_hybrid_hot_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static ir_ref _zend_jit_orig_opline_handler(zend_jit_ctx *jit, ir_ref offset)
{
ir_ref addr;
if (GCC_GLOBAL_REGS) {
addr = ir_ADD_A(offset, jit_IP(jit));
} else {
addr = ir_ADD_A(offset, ir_LOAD_A(jit_EX(opline)));
}
return ir_LOAD_A(addr);
}
static ir_ref zend_jit_orig_opline_handler(zend_jit_ctx *jit)
{
ir_ref func, jit_extension, offset;
func = ir_LOAD_A(jit_EX(func));
jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
return _zend_jit_orig_opline_handler(jit, offset);
}
static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost)
{
ir_ref func, jit_extension, offset, addr, ref, if_overflow, ret, if_halt;
func = ir_LOAD_A(jit_EX(func));
jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
addr = ir_LOAD_A(ir_ADD_OFFSET(ir_ADD_A(offset, jit_IP(jit)), offsetof(zend_op_trace_info, counter)));
ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
ir_STORE(addr, ref);
if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
ir_IF_TRUE_cold(if_overflow);
ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
ret = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_trace_hot_root),
jit_FP(jit),
jit_IP(jit));
if_halt = ir_IF(ir_LT(ret, ir_CONST_I32(0)));
ir_IF_FALSE(if_halt);
ref = jit_EG(current_execute_data);
jit_STORE_FP(jit, ir_LOAD_A(ref));
ref = ir_LOAD_A(jit_EX(opline));
jit_STORE_IP(jit, ref);
ir_IJMP(ir_LOAD_A(jit_IP(jit)));
ir_IF_FALSE(if_overflow);
ir_IJMP(_zend_jit_orig_opline_handler(jit, offset));
ir_IF_TRUE(if_halt);
ir_IJMP(ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
return 1;
}
static int zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 0;
}
return _zend_jit_hybrid_trace_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
return 0;
}
return _zend_jit_hybrid_trace_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
}
static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 0;
}
return _zend_jit_hybrid_trace_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static int zend_jit_trace_halt_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
} else if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, IR_NULL);
ir_RETURN(IR_VOID);
} else {
ir_RETURN(ir_CONST_I32(-1)); // ZEND_VM_RETURN
}
return 1;
}
static int zend_jit_trace_escape_stub(zend_jit_ctx *jit)
{
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
}
return 1;
}
static int zend_jit_trace_exit_stub(zend_jit_ctx *jit)
{
ir_ref ref, ret, if_zero, addr;
if (GCC_GLOBAL_REGS) {
// EX(opline) = opline
ir_STORE(jit_EX(opline), jit_IP(jit));
}
ret = ir_EXITCALL(ir_CONST_FC_FUNC(zend_jit_trace_exit));
if_zero = ir_IF(ir_EQ(ret, ir_CONST_I32(0)));
ir_IF_TRUE(if_zero);
if (GCC_GLOBAL_REGS) {
ref = jit_EG(current_execute_data);
jit_STORE_FP(jit, ir_LOAD_A(ref));
ref = ir_LOAD_A(jit_EX(opline));
jit_STORE_IP(jit, ref);
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
}
ir_IF_FALSE(if_zero);
ir_GUARD(ir_GE(ret, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
ref = jit_EG(current_execute_data);
jit_STORE_FP(jit, ir_LOAD_A(ref));
if (GCC_GLOBAL_REGS) {
ref = ir_LOAD_A(jit_EX(opline));
jit_STORE_IP(jit, ref);
}
// check for interrupt (try to avoid this ???)
if (!zend_jit_check_timeout(jit, NULL, NULL)) {
return 0;
}
addr = zend_jit_orig_opline_handler(jit);
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, addr);
} else {
#if defined(IR_TARGET_X86)
addr = ir_CAST_FC_FUNC(addr);
#endif
ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
}
return 1;
}
static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit)
{
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key));
} else {
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit));
}
return 1;
}
static int zend_jit_undefined_key_stub(zend_jit_ctx *jit)
{
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key));
} else {
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit));
}
return 1;
}
static int zend_jit_cannot_add_element_stub(zend_jit_ctx *jit)
{
ir_ref opline = ir_LOAD_A(jit_EX(opline));
ir_ref ref, if_result_used;
if_result_used = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(opline, offsetof(zend_op, result_type))),
ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
ir_IF_TRUE(if_result_used);
ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(jit_FP(jit), ref), ir_CONST_U32(IS_UNDEF));
ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
ir_CALL_2(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
IR_NULL,
ir_CONST_ADDR("Cannot add element to the array as the next element is already occupied"));
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_assign_const_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_CONST, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_assign_tmp_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_TMP_VAR, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_assign_var_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_VAR, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_assign_cv_noref_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_new_array_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
ir_ref ref = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_zend_new_array_0));
jit_set_Z_PTR(jit, var_addr, ref);
jit_set_Z_TYPE_INFO(jit, var_addr, IS_ARRAY_EX);
ir_RETURN(ref);
return 1;
}
static int zend_jit_assign_cv_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
{
#if defined (__CET__) && (__CET__ & 1) != 0
flags |= IR_GEN_ENDBR;
#endif
flags |= IR_OPT_FOLDING | IR_OPT_CFG | IR_OPT_CODEGEN;
ir_init(&jit->ctx, flags, 256, 1024);
jit->ctx.ret_type = -1;
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
jit->ctx.mflags |= default_mflags;
if (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) {
jit->ctx.mflags |= IR_X86_AVX;
}
#elif defined(IR_TARGET_AARCH64)
jit->ctx.get_veneer = zend_jit_get_veneer;
jit->ctx.set_veneer = zend_jit_set_veneer;
#endif
jit->ctx.fixed_regset = (1<<ZREG_FP) | (1<<ZREG_IP);
if (!(flags & IR_FUNCTION)) {
jit->ctx.flags |= IR_NO_STACK_COMBINE;
if (zend_jit_vm_kind == ZEND_VM_KIND_CALL) {
jit->ctx.flags |= IR_FUNCTION;
/* Stack must be 16 byte aligned */
/* TODO: select stack size ??? */
#if defined(IR_TARGET_AARCH64)
jit->ctx.flags |= IR_USE_FRAME_POINTER;
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 bytes) */
#elif defined(_WIN64)
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 8 saved registers and 3 spill slots (8 bytes) */
#elif defined(IR_TARGET_X86_64)
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 9; /* 6 saved registers and 3 spill slots (8 bytes) */
#else /* IR_TARGET_x86 */
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 4 saved registers and 7 spill slots (4 bytes) */
#endif
if (GCC_GLOBAL_REGS) {
jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<<ZREG_FP) | (1<<ZREG_IP));
} else {
jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED;
//#ifdef _WIN64
// jit->ctx.fixed_save_regset &= 0xffff; // TODO: don't save FP registers ???
//#endif
}
jit->ctx.fixed_call_stack_size = 16;
} else {
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
jit->ctx.fixed_stack_red_zone = ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE;
if (jit->ctx.fixed_stack_red_zone > 16) {
jit->ctx.fixed_stack_frame_size = jit->ctx.fixed_stack_red_zone - 16;
jit->ctx.fixed_call_stack_size = 16;
}
jit->ctx.flags |= IR_MERGE_EMPTY_ENTRIES;
#else
jit->ctx.fixed_stack_red_zone = 0;
jit->ctx.fixed_stack_frame_size = 32; /* 4 spill slots (8 bytes) or 8 spill slots (4 bytes) */
jit->ctx.fixed_call_stack_size = 16;
#endif
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
jit->ctx.fixed_regset |= (1<<IR_REG_FP); /* prevent %rbp (%r5) usage */
#endif
}
}
jit->ctx.snapshot_create = (ir_snapshot_create_t)jit_SNAPSHOT;
jit->op_array = NULL;
jit->current_op_array = NULL;
jit->ssa = NULL;
jit->name = NULL;
jit->last_valid_opline = NULL;
jit->use_last_valid_opline = 0;
jit->track_last_valid_opline = 0;
jit->reuse_ip = 0;
jit->delayed_call_level = 0;
delayed_call_chain = 0;
jit->b = -1;
#ifdef ZTS
jit->tls = IR_UNUSED;
#endif
jit->fp = IR_UNUSED;
jit->trace_loop_ref = IR_UNUSED;
jit->return_inputs = IR_UNUSED;
jit->bb_start_ref = NULL;
jit->bb_predecessors = NULL;
jit->bb_edges = NULL;
jit->trace = NULL;
jit->ra = NULL;
jit->delay_var = -1;
jit->delay_refs = NULL;
jit->eg_exception_addr = 0;
zend_hash_init(&jit->addr_hash, 64, NULL, NULL, 0);
memset(jit->stub_addr, 0, sizeof(jit->stub_addr));
ir_START();
}
static int zend_jit_free_ctx(zend_jit_ctx *jit)
{
if (jit->name) {
zend_string_release(jit->name);
}
zend_hash_destroy(&jit->addr_hash);
ir_free(&jit->ctx);
return 1;
}
static void *zend_jit_ir_compile(ir_ctx *ctx, size_t *size, const char *name)
{
void *entry;
ir_code_buffer code_buffer;
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_SRC) {
if (name) fprintf(stderr, "%s: ; after folding\n", name);
ir_save(ctx, 0, stderr);
}
#if ZEND_DEBUG
ir_check(ctx);
#endif
ir_build_def_use_lists(ctx);
#if ZEND_DEBUG
ir_check(ctx);
#endif
#if 1
ir_sccp(ctx);
#endif
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCCP) {
if (name) fprintf(stderr, "%s: ; after SCCP\n", name);
ir_save(ctx, 0, stderr);
}
ir_build_cfg(ctx);
ir_build_dominators_tree(ctx);
ir_find_loops(ctx);
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_CFG) {
if (name) fprintf(stderr, "%s: ; after CFG\n", name);
ir_save(ctx, IR_SAVE_CFG, stderr);
}
ir_gcm(ctx);
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_GCM) {
if (name) fprintf(stderr, "%s: ; after GCM\n", name);
ir_save(ctx, IR_SAVE_CFG|IR_SAVE_CFG_MAP, stderr);
}
ir_schedule(ctx);
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE) {
if (name) fprintf(stderr, "%s: ; after schedule\n", name);
ir_save(ctx, IR_SAVE_CFG, stderr);
}
ir_match(ctx);
#if !defined(IR_TARGET_AARCH64)
ctx->flags &= ~IR_USE_FRAME_POINTER; /* don't use FRAME_POINTER even with ALLOCA, TODO: cleanup this ??? */
#endif
ir_assign_virtual_registers(ctx);
ir_compute_live_ranges(ctx);
ir_coalesce(ctx);
ir_reg_alloc(ctx);
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_REGS) {
if (name) fprintf(stderr, "%s: ; after register allocation\n", name);
ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
ir_dump_live_ranges(ctx, stderr);
}
ir_schedule_blocks(ctx);
if (JIT_G(debug) & (ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_CODEGEN) {
if (name) fprintf(stderr, "%s: ; codegen\n", name);
ir_dump_codegen(ctx, stderr);
} else {
if (name) fprintf(stderr, "%s: ; final\n", name);
ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
}
}
#if ZEND_DEBUG
ir_check(ctx);
#endif
code_buffer.start = dasm_buf;
code_buffer.end = dasm_end;
code_buffer.pos = *dasm_ptr;
ctx->code_buffer = &code_buffer;
entry = ir_emit_code(ctx, size);
*dasm_ptr = code_buffer.pos;
#if defined(IR_TARGET_AARCH64)
if (ctx->flags2 & IR_HAS_VENEERS) {
zend_jit_commit_veneers();
}
#endif
return entry;
}
static void zend_jit_setup_stubs(void)
{
zend_jit_ctx jit;
void *entry;
size_t size;
uint32_t i;
for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
zend_jit_init_ctx(&jit, zend_jit_stubs[i].flags);
if (!zend_jit_stubs[i].stub(&jit)) {
zend_jit_free_ctx(&jit);
zend_jit_stub_handlers[i] = NULL;
continue;
}
entry = zend_jit_ir_compile(&jit.ctx, &size, zend_jit_stubs[i].name);
if (!entry) {
zend_jit_free_ctx(&jit);
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile stub");
}
zend_jit_stub_handlers[i] = entry;
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
#ifdef HAVE_CAPSTONE
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
ir_disasm_add_symbol(zend_jit_stubs[i].name, (uintptr_t)entry, size);
}
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_STUBS) {
ir_disasm(zend_jit_stubs[i].name,
entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit.ctx, stderr);
}
#endif
#ifndef _WIN32
if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
// ir_mem_unprotect(entry, size);
ir_gdb_register(zend_jit_stubs[i].name, entry, size, 0, 0);
// ir_mem_protect(entry, size);
}
if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
ir_perf_map_register(zend_jit_stubs[i].name, entry, size);
if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
ir_perf_jitdump_register(zend_jit_stubs[i].name, entry, size);
}
}
#endif
}
zend_jit_free_ctx(&jit);
}
}
#define REGISTER_HELPER(n) \
ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)n, sizeof(void*));
#define REGISTER_DATA(n) \
ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)&n, sizeof(n));
static void zend_jit_setup_disasm(void)
{
#ifdef HAVE_CAPSTONE
ir_disasm_init();
if (zend_vm_kind() == ZEND_VM_KIND_HYBRID) {
zend_op opline;
memset(&opline, 0, sizeof(opline));
opline.opcode = ZEND_DO_UCALL;
opline.result_type = IS_UNUSED;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_UCALL;
opline.result_type = IS_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_FCALL_BY_NAME;
opline.result_type = IS_UNUSED;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_FCALL_BY_NAME;
opline.result_type = IS_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_FCALL;
opline.result_type = IS_UNUSED;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_FCALL;
opline.result_type = IS_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_RETURN;
opline.op1_type = IS_CONST;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_RETURN_SPEC_CONST_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_RETURN;
opline.op1_type = IS_TMP_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_RETURN_SPEC_TMP_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_RETURN;
opline.op1_type = IS_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_RETURN_SPEC_VAR_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_RETURN;
opline.op1_type = IS_CV;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_RETURN_SPEC_CV_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
ir_disasm_add_symbol("ZEND_HYBRID_HALT_LABEL", (uint64_t)(uintptr_t)zend_jit_halt_op->handler, sizeof(void*));
}
REGISTER_DATA(zend_jit_profile_counter);
REGISTER_HELPER(zend_runtime_jit);
REGISTER_HELPER(zend_jit_hot_func);
REGISTER_HELPER(zend_jit_trace_hot_root);
REGISTER_HELPER(zend_jit_trace_exit);
REGISTER_HELPER(zend_jit_array_free);
REGISTER_HELPER(zend_jit_undefined_op_helper);
REGISTER_HELPER(zend_jit_pre_inc_typed_ref);
REGISTER_HELPER(zend_jit_pre_dec_typed_ref);
REGISTER_HELPER(zend_jit_post_inc_typed_ref);
REGISTER_HELPER(zend_jit_post_dec_typed_ref);
REGISTER_HELPER(zend_jit_pre_inc);
REGISTER_HELPER(zend_jit_pre_dec);
REGISTER_HELPER(zend_jit_add_arrays_helper);
REGISTER_HELPER(zend_jit_fast_assign_concat_helper);
REGISTER_HELPER(zend_jit_fast_concat_helper);
REGISTER_HELPER(zend_jit_fast_concat_tmp_helper);
REGISTER_HELPER(zend_jit_assign_op_to_typed_ref_tmp);
REGISTER_HELPER(zend_jit_assign_op_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_const_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_var_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_const_to_typed_ref2);
REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref2);
REGISTER_HELPER(zend_jit_assign_var_to_typed_ref2);
REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref2);
REGISTER_HELPER(zend_jit_check_constant);
REGISTER_HELPER(zend_jit_get_constant);
REGISTER_HELPER(zend_jit_int_extend_stack_helper);
REGISTER_HELPER(zend_jit_extend_stack_helper);
REGISTER_HELPER(zend_jit_init_func_run_time_cache_helper);
REGISTER_HELPER(zend_jit_find_func_helper);
REGISTER_HELPER(zend_jit_find_ns_func_helper);
REGISTER_HELPER(zend_jit_jmp_frameless_helper);
REGISTER_HELPER(zend_jit_unref_helper);
REGISTER_HELPER(zend_jit_invalid_method_call);
REGISTER_HELPER(zend_jit_invalid_method_call_tmp);
REGISTER_HELPER(zend_jit_find_method_helper);
REGISTER_HELPER(zend_jit_find_method_tmp_helper);
REGISTER_HELPER(zend_jit_push_static_metod_call_frame);
REGISTER_HELPER(zend_jit_push_static_metod_call_frame_tmp);
REGISTER_HELPER(zend_jit_free_trampoline_helper);
REGISTER_HELPER(zend_jit_verify_return_slow);
REGISTER_HELPER(zend_jit_deprecated_helper);
REGISTER_HELPER(zend_jit_undefined_long_key);
REGISTER_HELPER(zend_jit_undefined_string_key);
REGISTER_HELPER(zend_jit_copy_extra_args_helper);
REGISTER_HELPER(zend_jit_vm_stack_free_args_helper);
REGISTER_HELPER(zend_free_extra_named_params);
REGISTER_HELPER(zend_jit_free_call_frame);
REGISTER_HELPER(zend_jit_exception_in_interrupt_handler_helper);
REGISTER_HELPER(zend_jit_verify_arg_slow);
REGISTER_HELPER(zend_missing_arg_error);
REGISTER_HELPER(zend_jit_only_vars_by_reference);
REGISTER_HELPER(zend_jit_leave_func_helper);
REGISTER_HELPER(zend_jit_leave_nested_func_helper);
REGISTER_HELPER(zend_jit_leave_top_func_helper);
REGISTER_HELPER(zend_jit_fetch_global_helper);
REGISTER_HELPER(zend_jit_hash_index_lookup_rw_no_packed);
REGISTER_HELPER(zend_jit_hash_index_lookup_rw);
REGISTER_HELPER(zend_jit_hash_lookup_rw);
REGISTER_HELPER(zend_jit_symtable_find);
REGISTER_HELPER(zend_jit_symtable_lookup_w);
REGISTER_HELPER(zend_jit_symtable_lookup_rw);
REGISTER_HELPER(zend_jit_fetch_dim_r_helper);
REGISTER_HELPER(zend_jit_fetch_dim_is_helper);
REGISTER_HELPER(zend_jit_fetch_dim_isset_helper);
REGISTER_HELPER(zend_jit_fetch_dim_rw_helper);
REGISTER_HELPER(zend_jit_fetch_dim_w_helper);
REGISTER_HELPER(zend_jit_fetch_dim_str_offset_r_helper);
REGISTER_HELPER(zend_jit_fetch_dim_str_r_helper);
REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper);
REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper);
REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper);
REGISTER_HELPER(zend_jit_invalid_array_access);
REGISTER_HELPER(zend_jit_zval_array_dup);
REGISTER_HELPER(zend_jit_prepare_assign_dim_ref);
REGISTER_HELPER(zend_jit_fetch_dim_obj_w_helper);
REGISTER_HELPER(zend_jit_fetch_dim_obj_rw_helper);
REGISTER_HELPER(zend_jit_isset_dim_helper);
REGISTER_HELPER(zend_jit_assign_dim_helper);
REGISTER_HELPER(zend_jit_assign_dim_op_helper);
REGISTER_HELPER(zend_jit_fetch_obj_w_slow);
REGISTER_HELPER(zend_jit_fetch_obj_r_slow);
REGISTER_HELPER(zend_jit_fetch_obj_r_slow_ex);
REGISTER_HELPER(zend_jit_fetch_obj_is_slow);
REGISTER_HELPER(zend_jit_fetch_obj_is_slow_ex);
REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic);
REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic_ex);
REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic);
REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic_ex);
REGISTER_HELPER(zend_jit_check_array_promotion);
REGISTER_HELPER(zend_jit_create_typed_ref);
REGISTER_HELPER(zend_jit_invalid_property_write);
REGISTER_HELPER(zend_jit_invalid_property_read);
REGISTER_HELPER(zend_jit_extract_helper);
REGISTER_HELPER(zend_jit_invalid_property_assign);
REGISTER_HELPER(zend_jit_assign_to_typed_prop);
REGISTER_HELPER(zend_jit_assign_obj_helper);
REGISTER_HELPER(zend_jit_invalid_property_assign_op);
REGISTER_HELPER(zend_jit_assign_op_to_typed_prop);
REGISTER_HELPER(zend_jit_assign_obj_op_helper);
REGISTER_HELPER(zend_jit_invalid_property_incdec);
REGISTER_HELPER(zend_jit_inc_typed_prop);
REGISTER_HELPER(zend_jit_dec_typed_prop);
REGISTER_HELPER(zend_jit_pre_inc_typed_prop);
REGISTER_HELPER(zend_jit_post_inc_typed_prop);
REGISTER_HELPER(zend_jit_pre_dec_typed_prop);
REGISTER_HELPER(zend_jit_post_dec_typed_prop);
REGISTER_HELPER(zend_jit_pre_inc_obj_helper);
REGISTER_HELPER(zend_jit_post_inc_obj_helper);
REGISTER_HELPER(zend_jit_pre_dec_obj_helper);
REGISTER_HELPER(zend_jit_post_dec_obj_helper);
REGISTER_HELPER(zend_jit_rope_end);
#ifndef ZTS
REGISTER_DATA(EG(current_execute_data));
REGISTER_DATA(EG(exception));
REGISTER_DATA(EG(opline_before_exception));
REGISTER_DATA(EG(vm_interrupt));
REGISTER_DATA(EG(timed_out));
REGISTER_DATA(EG(uninitialized_zval));
REGISTER_DATA(EG(zend_constants));
REGISTER_DATA(EG(jit_trace_num));
REGISTER_DATA(EG(vm_stack_top));
REGISTER_DATA(EG(vm_stack_end));
REGISTER_DATA(EG(exception_op));
REGISTER_DATA(EG(symbol_table));
REGISTER_DATA(CG(map_ptr_base));
#endif
#endif
}
static void zend_jit_calc_trace_prologue_size(void)
{
zend_jit_ctx jit_ctx;
zend_jit_ctx *jit = &jit_ctx;
void *entry;
size_t size;
zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
if (!GCC_GLOBAL_REGS) {
ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
jit_STORE_FP(jit, ref);
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
ir_UNREACHABLE();
entry = zend_jit_ir_compile(&jit->ctx, &size, "JIT$trace_prologue");
zend_jit_free_ctx(jit);
if (!entry) {
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile prologue");
}
zend_jit_trace_prologue_size = size;
}
#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
static uintptr_t zend_jit_hybrid_vm_sp_adj = 0;
typedef struct _Unwind_Context _Unwind_Context;
typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
typedef struct _zend_jit_unwind_arg {
int cnt;
uintptr_t cfa[3];
} zend_jit_unwind_arg;
static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a)
{
zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a;
arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx);
arg->cnt++;
if (arg->cnt == 3) {
return 5; // _URC_END_OF_STACK
}
return 0; // _URC_NO_REASON;
}
static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
{
zend_jit_unwind_arg arg;
memset(&arg, 0, sizeof(arg));
_Unwind_Backtrace(zend_jit_unwind_cb, &arg);
if (arg.cnt == 3) {
zend_jit_hybrid_vm_sp_adj = arg.cfa[2] - arg.cfa[1];
}
}
extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data);
static zend_never_inline void zend_jit_set_sp_adj_vm(void)
{
void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *);
orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data;
zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data;
execute_ex(NULL); // set sp_adj[SP_ADJ_VM]
zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data;
}
#endif
static void zend_jit_setup(void)
{
#if defined(IR_TARGET_X86)
if (!zend_cpu_supports_sse2()) {
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: CPU doesn't support SSE2");
}
#endif
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
allowed_opt_flags = 0;
if (zend_cpu_supports_avx()) {
allowed_opt_flags |= ZEND_JIT_CPU_AVX;
}
# if PHP_HAVE_BUILTIN_CPU_SUPPORTS && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
if (zend_cpu_supports_cldemote()) {
default_mflags |= IR_X86_CLDEMOTE;
}
# endif
#endif
#ifdef ZTS
#if defined(IR_TARGET_AARCH64)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
# ifdef __FreeBSD__
if (tsrm_ls_cache_tcb_offset == 0) {
TLSDescriptor **where;
__asm__(
"adrp %0, :tlsdesc:_tsrm_ls_cache\n"
"add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n"
: "=r" (where));
/* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst
* section "Relocations for thread-local storage".
* The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds
* a platform-specific offset or pointer. */
TLSDescriptor *tlsdesc = where[1];
tsrm_tls_offset = tlsdesc->offset;
/* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */
tsrm_tls_index = (tlsdesc->index + 1) * 8;
}
# else
ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0);
# endif
# elif defined(_WIN64)
tsrm_tls_index = _tls_index * sizeof(void*);
/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
/* Probably, it might be better solution */
do {
void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index];
void *val = _tsrm_ls_cache;
size_t offset = 0;
size_t size = (char*)&_tls_end - (char*)&_tls_start;
while (offset < size) {
if (*tls_mem == val) {
tsrm_tls_offset = offset;
break;
}
tls_mem++;
offset += sizeof(void*);
}
if (offset >= size) {
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
}
} while(0);
# elif defined(ZEND_WIN32)
tsrm_tls_index = _tls_index * sizeof(void*);
/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
/* Probably, it might be better solution */
do {
void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index];
void *val = _tsrm_ls_cache;
size_t offset = 0;
size_t size = (char*)&_tls_end - (char*)&_tls_start;
while (offset < size) {
if (*tls_mem == val) {
tsrm_tls_offset = offset;
break;
}
tls_mem++;
offset += sizeof(void*);
}
if (offset >= size) {
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
}
} while(0);
# elif defined(__APPLE__) && defined(__x86_64__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
size_t *ti;
__asm__(
"leaq __tsrm_ls_cache(%%rip),%0"
: "=r" (ti));
tsrm_tls_offset = ti[2];
tsrm_tls_index = ti[1] * 8;
}
# elif defined(__GNUC__) && defined(__x86_64__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && \
!defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
size_t ret;
asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0"
: "=r" (ret));
tsrm_ls_cache_tcb_offset = ret;
#elif defined(__MUSL__)
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=a" (ti));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 8;
#elif defined(__FreeBSD__)
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=a" (ti));
tsrm_tls_offset = ti[1];
/* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */
tsrm_tls_index = (ti[0] + 1) * 8;
#else
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=a" (ti));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 16;
#endif
}
# elif defined(__GNUC__) && defined(__i386__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
size_t ret;
asm ("leal _tsrm_ls_cache@ntpoff,%0\n"
: "=a" (ret));
tsrm_ls_cache_tcb_offset = ret;
#else
size_t *ti, _ebx, _ecx, _edx;
__asm__(
"call 1f\n"
".subsection 1\n"
"1:\tmovl (%%esp), %%ebx\n\t"
"ret\n"
".previous\n\t"
"addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t"
"leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t"
"call ___tls_get_addr@plt\n\t"
"leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n"
: "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 8;
#endif
}
# endif
#endif
#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
zend_jit_set_sp_adj_vm(); // set zend_jit_hybrid_vm_sp_adj
}
#endif
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
zend_jit_setup_disasm();
}
#ifndef _WIN32
if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
ir_perf_jitdump_open();
}
#endif
zend_long debug = JIT_G(debug);
if (!(debug & ZEND_JIT_DEBUG_ASM_STUBS)) {
JIT_G(debug) &= ~(ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_FINAL|
ZEND_JIT_DEBUG_IR_CODEGEN|
ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_CFG|ZEND_JIT_DEBUG_IR_AFTER_GCM|
ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS);
}
zend_jit_calc_trace_prologue_size();
zend_jit_setup_stubs();
JIT_G(debug) = debug;
}
static void zend_jit_shutdown_ir(void)
{
#ifndef _WIN32
if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
ir_perf_jitdump_close();
}
if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
ir_gdb_unregister_all();
}
#endif
#ifdef HAVE_CAPSTONE
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
ir_disasm_free();
}
#endif
}
/* PHP control flow reconstruction helpers */
static ir_ref jit_IF_ex(zend_jit_ctx *jit, ir_ref condition, ir_ref true_block)
{
ir_ref ref = ir_IF(condition);
/* op3 is used as a temporary storage for PHP BB number to reconstruct PHP control flow.
*
* It's used in jit_IF_TRUE_FALSE_ex() to select IF_TRUE or IF_FALSE instructions
* to start target block
*/
ir_set_op(&jit->ctx, ref, 3, true_block);
return ref;
}
static void jit_IF_TRUE_FALSE_ex(zend_jit_ctx *jit, ir_ref if_ref, ir_ref true_block)
{
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
ZEND_ASSERT(if_ref);
ZEND_ASSERT(jit->ctx.ir_base[if_ref].op == IR_IF);
ZEND_ASSERT(jit->ctx.ir_base[if_ref].op3);
if (jit->ctx.ir_base[if_ref].op3 == true_block) {
ir_IF_TRUE(if_ref);
} else {
ir_IF_FALSE(if_ref);
}
}
static void _zend_jit_add_predecessor_ref(zend_jit_ctx *jit, int b, int pred, ir_ref ref)
{
int i, *p;
zend_basic_block *bb;
ir_ref *r, header;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
bb = &jit->ssa->cfg.blocks[b];
p = &jit->ssa->cfg.predecessors[bb->predecessor_offset];
r = &jit->bb_edges[jit->bb_predecessors[b]];
for (i = 0; i < bb->predecessors_count; i++, p++, r++) {
if (*p == pred) {
ZEND_ASSERT(*r == IR_UNUSED || *r == ref);
header = jit->bb_start_ref[b];
if (header) {
/* this is back edge */
ZEND_ASSERT(jit->ctx.ir_base[header].op == IR_LOOP_BEGIN);
if (jit->ctx.ir_base[ref].op == IR_END) {
jit->ctx.ir_base[ref].op = IR_LOOP_END;
} else if (jit->ctx.ir_base[ref].op == IR_IF) {
jit_IF_TRUE_FALSE_ex(jit, ref, b);
ref = ir_LOOP_END();
} else if (jit->ctx.ir_base[ref].op == IR_UNREACHABLE) {
ir_BEGIN(ref);
ref = ir_LOOP_END();
} else {
ZEND_UNREACHABLE();
}
ir_MERGE_SET_OP(header, i + 1, ref);
}
*r = ref;
return;
}
}
ZEND_UNREACHABLE();
}
static void _zend_jit_merge_smart_branch_inputs(zend_jit_ctx *jit,
uint32_t true_label,
uint32_t false_label,
ir_ref true_inputs,
ir_ref false_inputs)
{
ir_ref true_path = IR_UNUSED, false_path = IR_UNUSED;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
if (true_inputs) {
ZEND_ASSERT(jit->ctx.ir_base[true_inputs].op == IR_END);
if (!jit->ctx.ir_base[true_inputs].op2) {
true_path = true_inputs;
} else {
ir_MERGE_list(true_inputs);
true_path = ir_END();
}
}
if (false_inputs) {
ZEND_ASSERT(jit->ctx.ir_base[false_inputs].op == IR_END);
if (!jit->ctx.ir_base[false_inputs].op2) {
false_path = false_inputs;
} else {
ir_MERGE_list(false_inputs);
false_path = ir_END();
}
}
if (true_label == false_label && true_path && false_path) {
ir_MERGE_2(true_path, false_path);
_zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
} else if (!true_path && !false_path) {
/* dead code */
true_path = ir_END();
_zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
} else {
if (true_path) {
_zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
}
if (false_path) {
_zend_jit_add_predecessor_ref(jit, false_label, jit->b, false_path);
}
}
jit->b = -1;
}
static void _zend_jit_fix_merges(zend_jit_ctx *jit)
{
int i, count;
ir_ref j, k, n, *p, *q, *r;
ir_ref ref;
ir_insn *insn, *phi;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
count = jit->ssa->cfg.blocks_count;
for (i = 0, p = jit->bb_start_ref; i < count; i++, p++) {
ref = *p;
if (ref) {
insn = &jit->ctx.ir_base[ref];
if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) {
n = insn->inputs_count;
/* Remove IS_UNUSED inputs */
for (j = k = 0, q = r = insn->ops + 1; j < n; j++, q++) {
if (*q) {
if (q != r) {
*r = *q;
phi = insn + 1 + (n >> 2);
while (phi->op == IR_PI) {
phi++;
}
while (phi->op == IR_PHI) {
ir_insn_set_op(phi, k + 2, ir_insn_op(phi, j + 2));
phi += 1 + ((n + 1) >> 2);
}
}
k++;
r++;
}
}
if (k != n) {
ir_ref n2, k2;
if (k <= 1) {
insn->op = IR_BEGIN;
insn->inputs_count = 0;
} else {
insn->inputs_count = k;
}
n2 = 1 + (n >> 2);
k2 = 1 + (k >> 2);
while (k2 != n2) {
(insn+k2)->optx = IR_NOP;
k2++;
}
phi = insn + 1 + (n >> 2);
while (phi->op == IR_PI) {
phi++;
}
while (phi->op == IR_PHI) {
if (k <= 1) {
phi->op = IR_COPY;
phi->op1 = phi->op2;
phi->op2 = 1;
phi->inputs_count = 0;
} else {
phi->inputs_count = k + 1;
}
n2 = 1 + ((n + 1) >> 2);
k2 = 1 + ((k + 1) >> 2);
while (k2 != n2) {
(phi+k2)->optx = IR_NOP;
k2++;
}
phi += 1 + ((n + 1) >> 2);
}
}
}
}
}
}
static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref)
{
zend_basic_block *bb = &jit->ssa->cfg.blocks[switch_b];
const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
if (opline->opcode == ZEND_SWITCH_LONG
|| opline->opcode == ZEND_SWITCH_STRING
|| opline->opcode == ZEND_MATCH) {
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
int default_b = jit->ssa->cfg.map[default_opline - jit->op_array->opcodes];
zval *zv;
ir_ref list = IR_UNUSED, idx;
bool first = 1;
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
int b = jit->ssa->cfg.map[target - jit->op_array->opcodes];
if (b == case_b) {
if (!first) {
ir_END_list(list);
}
if (HT_IS_PACKED(jumptable)) {
idx = ir_CONST_LONG(zv - jumptable->arPacked);
} else {
idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
}
ir_CASE_VAL(switch_ref, idx);
first = 0;
}
} ZEND_HASH_FOREACH_END();
if (default_b == case_b) {
if (!first) {
ir_END_list(list);
}
if (jit->ctx.ir_base[switch_ref].op3) {
/* op3 may contain a list of additional "default" path inputs for MATCH */
ir_ref ref = jit->ctx.ir_base[switch_ref].op3;
jit->ctx.ir_base[switch_ref].op3 = IS_UNDEF;
ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
ir_ref end = ref;
while (jit->ctx.ir_base[end].op2) {
ZEND_ASSERT(jit->ctx.ir_base[end].op == IR_END);
end = jit->ctx.ir_base[end].op2;
}
jit->ctx.ir_base[end].op2 = list;
list = ref;
}
ir_CASE_DEFAULT(switch_ref);
}
if (list) {
ir_END_list(list);
ir_MERGE_list(list);
}
} else {
ZEND_UNREACHABLE();
}
}
static int zend_jit_bb_start(zend_jit_ctx *jit, int b)
{
zend_basic_block *bb;
int i, n, *p, pred;
ir_ref ref, bb_start;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
ZEND_ASSERT(b < jit->ssa->cfg.blocks_count);
bb = &jit->ssa->cfg.blocks[b];
ZEND_ASSERT((bb->flags & ZEND_BB_REACHABLE) != 0);
n = bb->predecessors_count;
if (n == 0) {
/* pass */
ZEND_ASSERT(jit->ctx.control);
#if ZEND_DEBUG
ref = jit->ctx.control;
ir_insn *insn = &jit->ctx.ir_base[ref];
while (insn->op >= IR_CALL && insn->op <= IR_TRAP) {
ref = insn->op1;
insn = &jit->ctx.ir_base[ref];
}
ZEND_ASSERT(insn->op == IR_START);
ZEND_ASSERT(ref == 1);
#endif
bb_start = 1;
if (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) {
/* prvent END/BEGIN merging */
jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ir_END());
bb_start = jit->ctx.control;
}
} else if (n == 1) {
ZEND_ASSERT(!jit->ctx.control);
pred = jit->ssa->cfg.predecessors[bb->predecessor_offset];
ref = jit->bb_edges[jit->bb_predecessors[b]];
if (ref == IR_UNUSED) {
if (!jit->ctx.control) {
ir_BEGIN(IR_UNUSED); /* unreachable block */
}
} else {
ir_op op = jit->ctx.ir_base[ref].op;
if (op == IR_IF) {
if (!jit->ctx.control) {
jit_IF_TRUE_FALSE_ex(jit, ref, b);
} else {
ir_ref entry_path = ir_END();
jit_IF_TRUE_FALSE_ex(jit, ref, b);
ir_MERGE_WITH(entry_path);
}
} else if (op == IR_SWITCH) {
zend_jit_case_start(jit, pred, b, ref);
} else {
if (!jit->ctx.control) {
ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
if ((jit->ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY)
&& (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)) {
/* prvent END/BEGIN merging */
jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ref);
} else {
ir_BEGIN(ref);
}
} else {
ir_MERGE_WITH(ref);
}
}
}
bb_start = jit->ctx.control;
} else {
int forward_edges_count = 0;
int back_edges_count = 0;
ir_ref *pred_refs;
ir_ref entry_path = IR_UNUSED;
ALLOCA_FLAG(use_heap);
ZEND_ASSERT(!jit->ctx.control);
if (jit->ctx.control) {
entry_path = ir_END();
}
pred_refs = (ir_ref *)do_alloca(sizeof(ir_ref) * n, use_heap);
for (i = 0, p = jit->ssa->cfg.predecessors + bb->predecessor_offset; i < n; p++, i++) {
pred = *p;
if (jit->bb_start_ref[pred]) {
/* forward edge */
forward_edges_count++;
ref = jit->bb_edges[jit->bb_predecessors[b] + i];
if (ref == IR_UNUSED) {
/* dead edge */
pred_refs[i] = IR_UNUSED;
} else {
ir_op op = jit->ctx.ir_base[ref].op;
if (op == IR_IF) {
jit_IF_TRUE_FALSE_ex(jit, ref, b);
pred_refs[i] = ir_END();
} else if (op == IR_SWITCH) {
zend_jit_case_start(jit, pred, b, ref);
pred_refs[i] = ir_END();
} else {
ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
pred_refs[i] = ref;
}
}
} else {
/* backward edge */
back_edges_count++;
pred_refs[i] = IR_UNUSED;
}
}
if (bb->flags & ZEND_BB_LOOP_HEADER) {
ZEND_ASSERT(back_edges_count != 0);
ZEND_ASSERT(forward_edges_count != 0);
ir_MERGE_N(n, pred_refs);
jit->ctx.ir_base[jit->ctx.control].op = IR_LOOP_BEGIN;
bb_start = jit->ctx.control;
if (entry_path) {
ir_MERGE_WITH(entry_path);
}
} else {
// ZEND_ASSERT(back_edges_count != 0);
/* edges from exceptional blocks may be counted as back edges */
ir_MERGE_N(n, pred_refs);
bb_start = jit->ctx.control;
if (entry_path) {
ir_MERGE_WITH(entry_path);
}
}
free_alloca(pred_refs, use_heap);
}
jit->b = b;
jit->bb_start_ref[b] = bb_start;
if ((bb->flags & ZEND_BB_ENTRY) || (bb->idom >= 0 && jit->bb_start_ref[bb->idom] < jit->ctx.fold_cse_limit)) {
jit->ctx.fold_cse_limit = bb_start;
}
return 1;
}
static int zend_jit_bb_end(zend_jit_ctx *jit, int b)
{
int succ;
zend_basic_block *bb;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
if (jit->b != b) {
return 1;
}
bb = &jit->ssa->cfg.blocks[b];
ZEND_ASSERT(bb->successors_count != 0);
if (bb->successors_count == 1) {
succ = bb->successors[0];
} else {
const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
/* Use only the following successor of SWITCH and FE_RESET_R */
ZEND_ASSERT(opline->opcode == ZEND_SWITCH_LONG
|| opline->opcode == ZEND_SWITCH_STRING
|| opline->opcode == ZEND_MATCH
|| opline->opcode == ZEND_FE_RESET_R);
succ = b + 1;
}
_zend_jit_add_predecessor_ref(jit, succ, b, ir_END());
jit->b = -1;
return 1;
}
static int jit_CMP_IP(zend_jit_ctx *jit, ir_op op, const zend_op *next_opline)
{
ir_ref ref;
#if 1
if (GCC_GLOBAL_REGS) {
ref = jit_IP32(jit);
} else {
ref = ir_LOAD_U32(jit_EX(opline));
}
ref = ir_CMP_OP(op, ref, ir_CONST_U32((uint32_t)(uintptr_t)next_opline));
#else
if (GCC_GLOBAL_REGS) {
ref = jit_IP(jit);
} else {
ref = ir_LOAD_A(jit_EX(opline));
}
ref = ir_CMP_OP(op, ref, ir_CONST_ADDR(next_opline));
#endif
return ref;
}
static int zend_jit_jmp_frameless(
zend_jit_ctx *jit,
const zend_op *opline,
const void *exit_addr,
zend_jmp_fl_result guard
) {
ir_ref ref, if_ref, cache_result, function_result, phi_result, cache_slot_ref;
zend_basic_block *bb;
// JIT: CACHED_PTR(opline->extended_value)
cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
cache_result = ir_LOAD_L(cache_slot_ref);
// JIT: if (UNEXPECTED(!result))
if_ref = ir_IF(cache_result);
ir_IF_FALSE_cold(if_ref);
zval *func_name_zv = RT_CONSTANT(opline, opline->op1);
function_result = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_jit_jmp_frameless_helper),
ir_CONST_ADDR(func_name_zv),
cache_slot_ref);
ir_MERGE_WITH_EMPTY_TRUE(if_ref);
phi_result = ir_PHI_2(IR_LONG, function_result, cache_result);
if (exit_addr) {
ir_GUARD(ir_EQ(phi_result, ir_CONST_LONG(guard)), ir_CONST_ADDR(exit_addr));
} else {
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
// JIT: if (result == ZEND_JMP_FL_HIT)
ref = jit_IF_ex(jit, ir_EQ(phi_result, ir_CONST_LONG(ZEND_JMP_FL_HIT)), bb->successors[0]);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
jit->b = -1;
}
return 1;
}
static int zend_jit_cond_jmp(zend_jit_ctx *jit, const zend_op *next_opline, int target_block)
{
ir_ref ref;
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
ZEND_ASSERT(bb->successors_count == 2);
if (bb->successors[0] == bb->successors[1]) {
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
jit->b = -1;
zend_jit_set_last_valid_opline(jit, next_opline);
return 1;
}
ref = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, next_opline), target_block);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
jit->b = -1;
zend_jit_set_last_valid_opline(jit, next_opline);
return 1;
}
static int zend_jit_set_cond(zend_jit_ctx *jit, const zend_op *next_opline, uint32_t var)
{
ir_ref ref;
ref = ir_ADD_U32(ir_ZEXT_U32(jit_CMP_IP(jit, IR_EQ, next_opline)), ir_CONST_U32(IS_FALSE));
// EX_VAR(var) = ...
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), var + offsetof(zval, u1.type_info)), ref);
zend_jit_reset_last_valid_opline(jit);
return zend_jit_set_ip(jit, next_opline - 1);
}
/* PHP JIT handlers */
static void zend_jit_check_exception(zend_jit_ctx *jit)
{
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
static void zend_jit_check_exception_undef_result(zend_jit_ctx *jit, const zend_op *opline)
{
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit,
(opline->result_type & (IS_TMP_VAR|IS_VAR)) ? jit_stub_exception_handler_undef : jit_stub_exception_handler));
}
static void zend_jit_type_check_undef(zend_jit_ctx *jit,
ir_ref type,
uint32_t var,
const zend_op *opline,
bool check_exception,
bool in_cold_path,
bool undef_result)
{
ir_ref if_def = ir_IF(type);
if (!in_cold_path) {
ir_IF_FALSE_cold(if_def);
} else {
ir_IF_FALSE(if_def);
}
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
if (check_exception) {
if (undef_result) {
zend_jit_check_exception_undef_result(jit, opline);
} else {
zend_jit_check_exception(jit);
}
}
ir_MERGE_WITH_EMPTY_TRUE(if_def);
}
static ir_ref zend_jit_zval_check_undef(zend_jit_ctx *jit,
ir_ref ref,
uint32_t var,
const zend_op *opline,
bool check_exception)
{
ir_ref if_def, ref2;
if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
ir_IF_FALSE_cold(if_def);
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
if (check_exception) {
zend_jit_check_exception(jit);
}
ref2 = jit_EG(uninitialized_zval);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
return ir_PHI_2(IR_ADDR, ref2, ref);
}
static void zend_jit_recv_entry(zend_jit_ctx *jit, int b)
{
zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
int pred;
ir_ref ref;
ZEND_ASSERT(bb->predecessors_count > 0);
pred = jit->bb_predecessors[b];
ref = jit->bb_edges[pred];
ZEND_ASSERT(ref);
ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
/* Insert a MERGE block with additional ENTRY input between predecessor and this one */
ir_ENTRY(ref, bb->start);
if (!GCC_GLOBAL_REGS) {
/* 2 is hardcoded reference to IR_PARAM */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
jit_STORE_FP(jit, 2);
}
ir_MERGE_WITH(ref);
jit->bb_edges[pred] = ir_END();
}
static void zend_jit_osr_entry(zend_jit_ctx *jit, int b)
{
zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
ir_ref ref = ir_END();
/* Insert a MERGE block with additional ENTRY input between predecessor and this one */
ir_ENTRY(ref, bb->start);
if (!GCC_GLOBAL_REGS) {
/* 2 is hardcoded reference to IR_PARAM */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
jit_STORE_FP(jit, 2);
}
ir_MERGE_WITH(ref);
}
static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label)
{
ir_ENTRY(src, label);
if (!GCC_GLOBAL_REGS) {
/* 2 is hardcoded reference to IR_PARAM */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
jit_STORE_FP(jit, 2);
}
return ir_END();
}
static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw)
{
ir_ref ref;
const void *handler;
zend_jit_set_ip(jit, opline);
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
handler = zend_get_opcode_handler_func(opline);
} else {
handler = opline->handler;
}
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
} else {
ref = jit_FP(jit);
ref = ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(handler), ref);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
/* Skip the following OP_DATA */
switch (opline->opcode) {
case ZEND_ASSIGN_DIM:
case ZEND_ASSIGN_OBJ:
case ZEND_ASSIGN_STATIC_PROP:
case ZEND_ASSIGN_DIM_OP:
case ZEND_ASSIGN_OBJ_OP:
case ZEND_ASSIGN_STATIC_PROP_OP:
case ZEND_ASSIGN_STATIC_PROP_REF:
case ZEND_ASSIGN_OBJ_REF:
zend_jit_set_last_valid_opline(jit, opline + 2);
break;
default:
zend_jit_set_last_valid_opline(jit, opline + 1);
break;
}
return 1;
}
static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline)
{
const void *handler;
ir_ref ref;
zend_basic_block *bb;
zend_jit_set_ip(jit, opline);
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_RETURN) {
/* Use inlined HYBRID VM handler */
handler = opline->handler;
ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
} else {
handler = zend_get_opcode_handler_func(opline);
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
ref = ir_LOAD_A(jit_IP(jit));
ir_TAILCALL(IR_VOID, ref);
}
} else {
handler = opline->handler;
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
} else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)
&& (opline->opcode == ZEND_CATCH
|| opline->opcode == ZEND_FAST_CALL
|| opline->opcode == ZEND_FAST_RET
|| opline->opcode == ZEND_MATCH_ERROR
|| opline->opcode == ZEND_THROW
|| opline->opcode == ZEND_VERIFY_NEVER_TYPE)) {
ref = jit_FP(jit);
ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
ir_RETURN(ir_CONST_I32(1));
} else {
ref = jit_FP(jit);
ir_TAILCALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
}
}
if (jit->b >= 0) {
bb = &jit->ssa->cfg.blocks[jit->b];
if (bb->successors_count > 0
&& (opline->opcode == ZEND_DO_FCALL
|| opline->opcode == ZEND_DO_UCALL
|| opline->opcode == ZEND_DO_FCALL_BY_NAME
|| opline->opcode == ZEND_INCLUDE_OR_EVAL
|| opline->opcode == ZEND_GENERATOR_CREATE
|| opline->opcode == ZEND_YIELD
|| opline->opcode == ZEND_YIELD_FROM
|| opline->opcode == ZEND_FAST_CALL)) {
/* Add a fake control edge from UNREACHABLE to the following ENTRY */
int succ;
if (bb->successors_count == 1) {
succ = bb->successors[0];
ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
} else {
/* Use only the following successor of FAST_CALL */
ZEND_ASSERT(opline->opcode == ZEND_FAST_CALL);
succ = jit->b + 1;
/* we need an entry */
jit->ssa->cfg.blocks[succ].flags |= ZEND_BB_ENTRY;
}
ref = jit->ctx.insns_count - 1;
ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN);
ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
_zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
}
jit->b = -1;
zend_jit_reset_last_valid_opline(jit);
}
return 1;
}
static int zend_jit_call(zend_jit_ctx *jit, const zend_op *opline, unsigned int next_block)
{
return zend_jit_tail_handler(jit, opline);
}
static int zend_jit_spill_store(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type)
{
ZEND_ASSERT(Z_MODE(src) == IS_REG);
ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_spill_store_inv(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
ZEND_ASSERT(Z_MODE(src) == IS_REG);
ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
} else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG) {
/* invalidate memory type */
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
} else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE) {
/* invalidate memory type */
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_load_reg(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
ZEND_ASSERT(Z_MODE(dst) == IS_REG);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
zend_jit_def_reg(jit, dst, jit_Z_LVAL(jit, src));
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
zend_jit_def_reg(jit, dst, jit_Z_DVAL(jit, src));
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var, bool set_type)
{
zend_jit_addr src = ZEND_ADDR_REG(ssa_var);
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
return zend_jit_spill_store(jit, src, dst, info, set_type);
}
static int zend_jit_store_ref(zend_jit_ctx *jit, uint32_t info, int var, int32_t src, bool set_type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
jit_set_Z_LVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg)
{
ir_ref ref = jit->ctx.control;
ir_insn *insn;
while (1) {
insn = &jit->ctx.ir_base[ref];
if (insn->op == IR_RLOAD && insn->op2 == reg) {
ZEND_ASSERT(insn->type == type);
return ref;
} else if (insn->op == IR_START) {
break;
}
ref = insn->op1;
}
return ir_RLOAD(type, reg);
}
static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
ir_ref src = ir_CONST_LONG(val);
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
}
jit_set_Z_LVAL(jit, dst, src);
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
return 1;
}
static int zend_jit_store_const_double(zend_jit_ctx *jit, int var, double val)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
ir_ref src = ir_CONST_DOUBLE(val);
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
}
jit_set_Z_DVAL(jit, dst, src);
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
return 1;
}
static int zend_jit_store_type(zend_jit_ctx *jit, int var, uint8_t type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
ZEND_ASSERT(type <= IS_DOUBLE);
jit_set_Z_TYPE_INFO(jit, dst, type);
return 1;
}
static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, bool in_mem, bool set_type)
{
zend_jit_addr src;
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
ir_type type;
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
type = IR_LONG;
src = zend_jit_deopt_rload(jit, type, reg);
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
} else if (!in_mem) {
jit_set_Z_LVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
type = IR_DOUBLE;
src = zend_jit_deopt_rload(jit, type, reg);
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
} else if (!in_mem) {
jit_set_Z_DVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t offset, bool set_type)
{
zend_jit_addr src;
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
src = ir_LOAD_L(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
} else {
jit_set_Z_LVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
src = ir_LOAD_D(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
} else {
jit_set_Z_DVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
jit_set_Z_TYPE_INFO(jit, dst, type);
return 1;
}
static int zend_jit_zval_try_addref(zend_jit_ctx *jit, zend_jit_addr var_addr)
{
ir_ref if_refcounted, end1;
if_refcounted = jit_if_REFCOUNTED(jit, var_addr);
ir_IF_FALSE(if_refcounted);
end1 = ir_END();
ir_IF_TRUE(if_refcounted);
jit_GC_ADDREF(jit, jit_Z_PTR(jit, var_addr));
ir_MERGE_WITH(end1);
return 1;
}
static int zend_jit_store_var_if_necessary(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
return zend_jit_spill_store(jit, src, dst, info, 1);
}
return 1;
}
static int zend_jit_store_var_if_necessary_ex(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
bool set_type = 1;
if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
(old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
if (JIT_G(current_frame)) {
uint32_t mem_type = STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var));
if (mem_type != IS_UNKNOWN
&& (info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (1 << mem_type)) {
set_type = 0;
}
} else {
set_type = 0;
}
}
}
return zend_jit_spill_store(jit, src, dst, info, set_type);
}
return 1;
}
static int zend_jit_load_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var)
{
zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
zend_jit_addr dst = ZEND_ADDR_REG(ssa_var);
return zend_jit_load_reg(jit, src, dst, info);
}
static int zend_jit_invalidate_var_if_necessary(zend_jit_ctx *jit, uint8_t op_type, zend_jit_addr addr, znode_op op)
{
if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
/* Invalidate operand type to prevent incorrect destuction by exception_handler_free_op1_op2() */
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
jit_set_Z_TYPE_INFO(jit, dst, IS_UNDEF);
}
return 1;
}
static int zend_jit_update_regs(zend_jit_ctx *jit, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
if (!zend_jit_same_addr(src, dst)) {
if (Z_MODE(src) == IS_REG) {
if (Z_MODE(dst) == IS_REG) {
zend_jit_def_reg(jit, dst, zend_jit_use_reg(jit, src));
if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(jit, dst, var_addr, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else if (Z_MODE(dst) == IS_MEM_ZVAL) {
if (!Z_LOAD(src) && !Z_STORE(src)) {
if (!zend_jit_spill_store(jit, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(src) == IS_MEM_ZVAL) {
if (Z_MODE(dst) == IS_REG) {
if (!zend_jit_load_reg(jit, src, dst, info)) {
return 0;
}
} else {
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(jit, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
return 1;
}
struct jit_observer_fcall_is_unobserved_data {
ir_ref if_unobserved;
ir_ref ir_end_inputs;
};
static struct jit_observer_fcall_is_unobserved_data jit_observer_fcall_is_unobserved_start(zend_jit_ctx *jit, const zend_function *func, ir_ref *observer_handler, ir_ref rx, ir_ref func_ref) {
ir_ref run_time_cache;
struct jit_observer_fcall_is_unobserved_data data = { .ir_end_inputs = IR_UNUSED };
if (func) {
ZEND_ASSERT((func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0);
} else {
// JIT: if (function->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) {
ZEND_ASSERT(rx != IR_UNUSED);
ir_ref if_trampoline_or_generator = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)));
ir_IF_TRUE(if_trampoline_or_generator);
ir_END_list(data.ir_end_inputs);
ir_IF_FALSE(if_trampoline_or_generator);
}
if (func && (func->common.fn_flags & ZEND_ACC_CLOSURE) == 0 && ZEND_MAP_PTR_IS_OFFSET(func->common.run_time_cache)) {
// JIT: ZEND_MAP_PTR_GET_IMM(func->common.runtime_cache)
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)), (uintptr_t)ZEND_MAP_PTR(func->common.run_time_cache)));
} else {
ZEND_ASSERT(rx != IR_UNUSED);
// Closures may be duplicated and have a different runtime cache. Use the regular run_time_cache access pattern for these
if (func && ZEND_USER_CODE(func->type)) { // not a closure and definitely not an internal function
run_time_cache = ir_LOAD_A(jit_CALL(rx, run_time_cache));
} else {
// JIT: ZEND_MAP_PTR_GET(func->common.runtime_cache)
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_op_array, run_time_cache__ptr)));
ir_ref if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1)));
ir_IF_TRUE(if_odd);
ir_ref run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base))));
ir_ref if_odd_end = ir_END();
ir_IF_FALSE(if_odd);
// JIT: if (func->common.runtime_cache != NULL) {
ir_ref if_rt_cache = ir_IF(ir_EQ(run_time_cache, IR_NULL));
ir_IF_TRUE(if_rt_cache);
ir_END_list(data.ir_end_inputs);
ir_IF_FALSE(if_rt_cache);
ir_MERGE_WITH(if_odd_end);
run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache, run_time_cache2);
}
}
// JIT: observer_handler = runtime_cache + ZEND_OBSERVER_HANDLE(function)
if (func) {
*observer_handler = ir_ADD_OFFSET(run_time_cache, ZEND_OBSERVER_HANDLE(func) * sizeof(void *));
} else {
// JIT: (func->type == ZEND_INTERNAL_FUNCTION ? zend_observer_fcall_internal_function_extension : zend_observer_fcall_op_array_extension) * sizeof(void *)
ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
ir_ref if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(ZEND_INTERNAL_FUNCTION)));
ir_IF_TRUE(if_internal_func);
ir_ref observer_handler_internal = ir_ADD_OFFSET(run_time_cache, zend_observer_fcall_internal_function_extension * sizeof(void *));
ir_ref if_internal_func_end = ir_END();
ir_IF_FALSE(if_internal_func);
ir_ref observer_handler_user = ir_ADD_OFFSET(run_time_cache, zend_observer_fcall_op_array_extension * sizeof(void *));
ir_MERGE_WITH(if_internal_func_end);
*observer_handler = ir_PHI_2(IR_ADDR, observer_handler_internal, observer_handler_user);
}
// JIT: if (*observer_handler == ZEND_OBSERVER_NONE_OBSERVED) {
data.if_unobserved = ir_IF(ir_EQ(ir_LOAD_A(*observer_handler), ir_CONST_ADDR(ZEND_OBSERVER_NONE_OBSERVED)));
ir_IF_FALSE(data.if_unobserved);
return data;
}
/* For frameless the true branch of if_unobserved is used and this function not called. */
static void jit_observer_fcall_is_unobserved_end(zend_jit_ctx *jit, struct jit_observer_fcall_is_unobserved_data *data) {
ir_END_list(data->ir_end_inputs);
ir_IF_TRUE(data->if_unobserved);
ir_END_list(data->ir_end_inputs);
ir_MERGE_list(data->ir_end_inputs);
}
static void jit_observer_fcall_begin(zend_jit_ctx *jit, ir_ref rx, ir_ref observer_handler) {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_begin_prechecked), rx, observer_handler);
}
static void jit_observer_fcall_end(zend_jit_ctx *jit, ir_ref rx, ir_ref res_ref) {
// JIT: if (execute_data == EG(current_observed_frame)) {
ir_ref has_end_observer = ir_IF(ir_EQ(rx, ir_LOAD_A(jit_EG(current_observed_frame))));
ir_IF_TRUE(has_end_observer);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_end_prechecked),
rx, res_ref);
ir_MERGE_WITH_EMPTY_FALSE(has_end_observer);
}
static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
ir_ref if_long = IR_UNUSED;
ir_ref op1_lval_ref = IR_UNUSED;
ir_ref ref;
ir_op op;
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_long);
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
jit_set_Z_LVAL(jit, res_addr, op1_lval_ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL
&& Z_MODE(op1_addr) == IS_REG
&& !Z_LOAD(op1_addr)
&& !Z_STORE(op1_addr)) {
jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_LONG);
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
op = may_overflow ? IR_ADD_OV : IR_ADD;
} else {
op = may_overflow ? IR_SUB_OV : IR_SUB;
}
if (!op1_lval_ref) {
op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
}
ref = ir_BINARY_OP_L(op, op1_lval_ref, ir_CONST_LONG(1));
if (op1_def_info & MAY_BE_LONG) {
jit_set_Z_LVAL(jit, op1_def_addr, ref);
}
if (may_overflow &&
(((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) ||
((opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD))))) {
int32_t exit_point;
const void *exit_addr;
zend_jit_trace_stack *stack;
uint32_t old_op1_info, old_res_info = 0;
stack = JIT_G(current_frame)->stack;
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else {
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
}
if (opline->result_type != IS_UNUSED) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
if (opline->opcode == ZEND_PRE_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else if (opline->opcode == ZEND_PRE_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
} else if (opline->opcode == ZEND_POST_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MAX));
} else if (opline->opcode == ZEND_POST_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MIN));
}
}
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
if (opline->result_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
} else if (may_overflow) {
ir_ref if_overflow;
ir_ref merge_inputs = IR_UNUSED;
if (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
|| (opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
int32_t exit_point;
const void *exit_addr;
zend_jit_trace_stack *stack;
uint32_t old_res_info = 0;
stack = JIT_G(current_frame)->stack;
if (opline->result_type != IS_UNUSED) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
}
}
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
if_overflow = ir_IF(ir_OVERFLOW(ref));
ir_IF_FALSE_cold(if_overflow);
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
ir_IF_TRUE(if_overflow);
} else {
ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
}
if (opline->result_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
} else {
if_overflow = ir_IF(ir_OVERFLOW(ref));
ir_IF_FALSE(if_overflow);
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
ir_END_list(merge_inputs);
/* overflow => cold path */
ir_IF_TRUE_cold(if_overflow);
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (Z_MODE(op1_def_addr) == IS_REG) {
jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
}
} else {
if (Z_MODE(op1_def_addr) == IS_REG) {
jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
}
}
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (Z_MODE(res_addr) == IS_REG) {
jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
} else {
if (Z_MODE(res_addr) == IS_REG) {
jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
}
if (merge_inputs) {
ir_END_list(merge_inputs);
ir_MERGE_list(merge_inputs);
}
} else {
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
ir_ref merge_inputs = ir_END();
/* !is_long => cold path */
ir_IF_FALSE_cold(if_long);
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
jit_SET_EX_OPLINE(jit, opline);
if (op1_info & MAY_BE_UNDEF) {
ir_ref if_def;
if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
// zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_NULL);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
op1_info |= MAY_BE_NULL;
}
ref = jit_ZVAL_ADDR(jit, op1_addr);
if (op1_info & MAY_BE_REF) {
ir_ref if_ref, if_typed, func, ref2, arg2;
if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
ir_IF_TRUE(if_ref);
ref2 = jit_Z_PTR_ref(jit, ref);
if_typed = jit_if_TYPED_REF(jit, ref2);
ir_IF_TRUE(if_typed);
if (RETURN_VALUE_USED(opline)) {
ZEND_ASSERT(Z_MODE(res_addr) != IS_REG);
arg2 = jit_ZVAL_ADDR(jit, res_addr);
} else {
arg2 = IR_NULL;
}
if (opline->opcode == ZEND_PRE_INC) {
func = ir_CONST_FC_FUNC(zend_jit_pre_inc_typed_ref);
} else if (opline->opcode == ZEND_PRE_DEC) {
func = ir_CONST_FC_FUNC(zend_jit_pre_dec_typed_ref);
} else if (opline->opcode == ZEND_POST_INC) {
func = ir_CONST_FC_FUNC(zend_jit_post_inc_typed_ref);
} else if (opline->opcode == ZEND_POST_DEC) {
func = ir_CONST_FC_FUNC(zend_jit_post_dec_typed_ref);
} else {
ZEND_UNREACHABLE();
}
ir_CALL_2(IR_VOID, func, ref2, arg2);
zend_jit_check_exception(jit);
ir_END_list(merge_inputs);
ir_IF_FALSE(if_typed);
ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
jit_ZVAL_COPY(jit,
res_addr,
res_use_info,
ZEND_ADDR_REF_ZVAL(ref), op1_info, 1);
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc), ref, arg2);
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function), ref);
}
} else {
if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec), ref, arg2);
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function), ref);
}
}
if (may_throw) {
zend_jit_check_exception(jit);
}
} else {
ref = jit_Z_DVAL(jit, op1_addr);
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
jit_set_Z_DVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
op = IR_ADD;
} else {
op = IR_SUB;
}
ref = ir_BINARY_OP_D(op, ref, ir_CONST_DOUBLE(1.0));
jit_set_Z_DVAL(jit, op1_def_addr, ref);
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
jit_set_Z_DVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
ir_END_list(merge_inputs);
ir_MERGE_list(merge_inputs);
}
if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
return 0;
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
static int zend_jit_math_long_long(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow)
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
ir_op op;
ir_ref op1, op2, ref, if_overflow = IR_UNUSED;
if (opcode == ZEND_ADD) {
op = may_overflow ? IR_ADD_OV : IR_ADD;
} else if (opcode == ZEND_SUB) {
op = may_overflow ? IR_SUB_OV : IR_SUB;
} else if (opcode == ZEND_MUL) {
op = may_overflow ? IR_MUL_OV : IR_MUL;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_LVAL(jit, op1_addr);
op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
ref = ir_BINARY_OP_L(op, op1, op2);
if (may_overflow) {
if (res_info & MAY_BE_GUARD) {
if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_res_info;
int32_t exit_point;
const void *exit_addr;
if (opline->opcode == ZEND_ADD
&& Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
} else if (opline->opcode == ZEND_SUB
&& Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
} else {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
}
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
may_overflow = 0;
} else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
} else {
ZEND_UNREACHABLE();
}
} else {
if_overflow = ir_IF(ir_OVERFLOW(ref));
ir_IF_FALSE(if_overflow);
}
}
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if (!zend_jit_same_addr(op1_addr, res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
}
}
if (may_overflow) {
ir_ref fast_path = IR_UNUSED;
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
fast_path = ir_END();
ir_IF_TRUE_cold(if_overflow);
}
if (opcode == ZEND_ADD) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
if (Z_MODE(res_addr) == IS_REG) {
jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
ir_MERGE_WITH(fast_path);
}
return 1;
}
op = IR_ADD;
} else if (opcode == ZEND_SUB) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
if (Z_MODE(res_addr) == IS_REG) {
jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
ir_MERGE_WITH(fast_path);
}
return 1;
}
op = IR_SUB;
} else if (opcode == ZEND_MUL) {
op = IR_MUL;
} else {
ZEND_UNREACHABLE();
}
#if 1
/* reload */
op1 = jit_Z_LVAL(jit, op1_addr);
op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
#endif
#if 1
/* disable CSE */
ir_ref old_cse_limit = jit->ctx.fold_cse_limit;
jit->ctx.fold_cse_limit = 0x7fffffff;
#endif
op1 = ir_INT2D(op1);
op2 = ir_INT2D(op2);
#if 1
jit->ctx.fold_cse_limit = old_cse_limit;
#endif
ref = ir_BINARY_OP_D(op, op1, op2);
jit_set_Z_DVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
ir_MERGE_WITH(fast_path);
}
}
return 1;
}
static int zend_jit_math_long_double(zend_jit_ctx *jit,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
ir_op op;
ir_ref op1, op2, ref;
if (opcode == ZEND_ADD) {
op = IR_ADD;
} else if (opcode == ZEND_SUB) {
op = IR_SUB;
} else if (opcode == ZEND_MUL) {
op = IR_MUL;
} else if (opcode == ZEND_DIV) {
op = IR_DIV;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_LVAL(jit, op1_addr);
op2 = jit_Z_DVAL(jit, op2_addr);
ref = ir_BINARY_OP_D(op, ir_INT2D(op1), op2);
jit_set_Z_DVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
return 1;
}
static int zend_jit_math_double_long(zend_jit_ctx *jit,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
ir_op op;
ir_ref op1, op2, ref;
if (opcode == ZEND_ADD) {
op = IR_ADD;
} else if (opcode == ZEND_SUB) {
op = IR_SUB;
} else if (opcode == ZEND_MUL) {
op = IR_MUL;
} else if (opcode == ZEND_DIV) {
op = IR_DIV;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_DVAL(jit, op1_addr);
op2 = jit_Z_LVAL(jit, op2_addr);
ref = ir_BINARY_OP_D(op, op1, ir_INT2D(op2));
jit_set_Z_DVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if (!zend_jit_same_addr(op1_addr, res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
}
return 1;
}
static int zend_jit_math_double_double(zend_jit_ctx *jit,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
ir_op op;
ir_ref op1, op2, ref;
if (opcode == ZEND_ADD) {
op = IR_ADD;
} else if (opcode == ZEND_SUB) {
op = IR_SUB;
} else if (opcode == ZEND_MUL) {
op = IR_MUL;
} else if (opcode == ZEND_DIV) {
op = IR_DIV;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_DVAL(jit, op1_addr);
op2 = (same_ops) ? op1 : jit_Z_DVAL(jit, op2_addr);
ref = ir_BINARY_OP_D(op, op1, op2);
jit_set_Z_DVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if (!zend_jit_same_addr(op1_addr, res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
}
return 1;
}
static int zend_jit_math_helper(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t opcode,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow,
int may_throw)
{
ir_ref if_op1_long = IR_UNUSED;
ir_ref if_op1_double = IR_UNUSED;
ir_ref if_op2_double = IR_UNUSED;
ir_ref if_op1_long_op2_long = IR_UNUSED;
ir_ref if_op1_long_op2_double = IR_UNUSED;
ir_ref if_op1_double_op2_double = IR_UNUSED;
ir_ref if_op1_double_op2_long = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
ir_refs *end_inputs;
ir_refs *res_inputs;
ir_refs_init(end_inputs, 6);
ir_refs_init(res_inputs, 6);
if (Z_MODE(op1_addr) == IS_REG) {
if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op1_addr);
}
} else if (Z_MODE(op2_addr) == IS_REG) {
if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op2_addr);
}
}
if (Z_MODE(res_addr) == IS_REG) {
jit->delay_var = Z_SSA_VAR(res_addr);
jit->delay_refs = res_inputs;
}
if ((res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG) && (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_op1_long);
}
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_TRUE(if_op1_long_op2_long);
}
if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_long) {
ir_IF_FALSE_cold(if_op1_long);
ir_END_list(slow_inputs);
}
if (if_op1_long_op2_long) {
ir_IF_FALSE_cold(if_op1_long_op2_long);
ir_END_list(slow_inputs);
}
} else if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_op1_long);
}
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_long_op2_long);
if (op2_info & MAY_BE_DOUBLE) {
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_long_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_long_op2_double);
}
if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
} else {
ir_END_list(slow_inputs);
}
ir_IF_TRUE(if_op1_long_op2_long);
}
if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_long) {
ir_IF_FALSE_cold(if_op1_long);
}
if (op1_info & MAY_BE_DOUBLE) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double);
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops) {
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_double_op2_long);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double_op2_long);
}
if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
} else if (if_op1_long) {
ir_END_list(slow_inputs);
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double);
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_double_op2_long);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double_op2_long);
}
if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op2_double);
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_long_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_long_op2_double);
}
if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
ir_ref func, arg1, arg2, arg3;
if (slow_inputs) {
ir_MERGE_list(slow_inputs);
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
op2_addr = real_addr;
}
if (Z_MODE(res_addr) == IS_REG) {
arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
} else {
arg1 = jit_ZVAL_ADDR(jit, res_addr);
}
arg2 = jit_ZVAL_ADDR(jit, op1_addr);
arg3 = jit_ZVAL_ADDR(jit, op2_addr);
jit_SET_EX_OPLINE(jit, opline);
if (opcode == ZEND_ADD) {
func = ir_CONST_FC_FUNC(add_function);
} else if (opcode == ZEND_SUB) {
func = ir_CONST_FC_FUNC(sub_function);
} else if (opcode == ZEND_MUL) {
func = ir_CONST_FC_FUNC(mul_function);
} else if (opcode == ZEND_DIV) {
func = ir_CONST_FC_FUNC(div_function);
} else {
ZEND_UNREACHABLE();
}
ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(jit, opline);
} else {
zend_jit_check_exception(jit);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_refs_add(end_inputs, ir_END());
}
if (end_inputs->count) {
ir_MERGE_N(end_inputs->count, end_inputs->refs);
}
if (Z_MODE(res_addr) == IS_REG) {
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(end_inputs->count == res_inputs->count);
jit->delay_var = -1;
jit->delay_refs = NULL;
if (res_inputs->count == 1) {
zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
} else {
ir_ref phi = ir_PHI_N((res_info & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE, res_inputs->count, res_inputs->refs);
zend_jit_def_reg(jit, res_addr, phi);
}
}
return 1;
}
static int zend_jit_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
if (!zend_jit_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_add_arrays(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr)
{
ir_ref ref;
ir_ref arg1 = jit_Z_PTR(jit, op1_addr);
ir_ref arg2 = jit_Z_PTR(jit, op2_addr);
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_add_arrays_helper), arg1, arg2);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_ARRAY_EX);
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
return 1;
}
static int zend_jit_long_math_helper(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t opcode,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
zend_ssa_range *op1_range,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_throw)
{
ir_ref ref = IR_UNUSED;
ir_ref if_long1 = IR_UNUSED;
ir_ref if_long2 = IR_UNUSED;
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
ir_refs *res_inputs;
ir_refs_init(res_inputs, 2);
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if_long1 = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_long1);
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if_long2 = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_TRUE(if_long2);
}
if (opcode == ZEND_SL) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
ref = ir_CONST_LONG(0);
} else {
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
jit_SET_EX_OPLINE(jit, opline);
ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
if (Z_MODE(res_addr) == IS_REG) {
ref = ir_CONST_LONG(0); // dead code
}
}
} else {
ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
}
} else {
ref = jit_Z_LVAL(jit, op2_addr);
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
ir_ref if_wrong, cold_path, ref2, if_ok;
ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
ir_IF_TRUE_cold(if_wrong);
if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
ir_IF_FALSE(if_ok);
jit_SET_EX_OPLINE(jit, opline);
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
ir_IF_TRUE(if_ok);
ref2 = ir_CONST_LONG(0);
cold_path = ir_END();
ir_IF_FALSE(if_wrong);
ref = ir_SHL_L(op1_ref, ref);
ir_MERGE_WITH(cold_path);
ref = ir_PHI_2(IR_LONG, ref, ref2);
} else {
ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ref);
}
}
} else if (opcode == ZEND_SR) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
ref = ir_SAR_L(
jit_Z_LVAL(jit, op1_addr),
ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1));
} else {
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
jit_SET_EX_OPLINE(jit, opline);
ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
if (Z_MODE(res_addr) == IS_REG) {
ref = ir_CONST_LONG(0); // dead code
}
}
} else {
ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
}
} else {
ref = jit_Z_LVAL(jit, op2_addr);
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
ir_ref if_wrong, cold_path, ref2, if_ok;
if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
ir_IF_TRUE_cold(if_wrong);
if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
ir_IF_FALSE(if_ok);
jit_SET_EX_OPLINE(jit, opline);
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
ir_IF_TRUE(if_ok);
ref2 = ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1);
cold_path = ir_END();
ir_IF_FALSE(if_wrong);
ir_MERGE_WITH(cold_path);
ref = ir_PHI_2(IR_LONG, ref, ref2);
}
ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ref);
}
} else if (opcode == ZEND_MOD) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (op2_lval == 0) {
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
jit_SET_EX_OPLINE(jit, opline);
ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
if (Z_MODE(res_addr) == IS_REG) {
ref = ir_CONST_LONG(0); // dead code
}
} else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
ref = ir_AND_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval - 1));
} else {
ref = ir_MOD_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
}
} else {
ir_ref zero_path = 0;
ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
ref = jit_Z_LVAL(jit, op2_addr);
if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
ir_ref if_ok = ir_IF(ref);
ir_IF_FALSE(if_ok);
jit_SET_EX_OPLINE(jit, opline);
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
ir_IF_TRUE(if_ok);
}
/* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
ir_ref if_minus_one = ir_IF(ir_EQ(ref, ir_CONST_LONG(-1)));
ir_IF_TRUE_cold(if_minus_one);
zero_path = ir_END();
ir_IF_FALSE(if_minus_one);
}
ref = ir_MOD_L(op1_ref, ref);
if (zero_path) {
ir_MERGE_WITH(zero_path);
ref = ir_PHI_2(IR_LONG, ref, ir_CONST_LONG(0));
}
}
} else {
ir_op op;
ir_ref op1, op2;
if (opcode == ZEND_BW_OR) {
op = IR_OR;
} else if (opcode == ZEND_BW_AND) {
op = IR_AND;
} else if (opcode == ZEND_BW_XOR) {
op = IR_XOR;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_LVAL(jit, op1_addr);
op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
ref = ir_BINARY_OP_L(op, op1, op2);
}
if (ref) {
if (Z_MODE(res_addr) == IS_REG
&& ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))
|| (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
jit->delay_var = Z_SSA_VAR(res_addr);
jit->delay_refs = res_inputs;
}
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if (!zend_jit_same_addr(op1_addr, res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
}
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
ir_ref fast_path = ir_END();
ir_ref func, arg1, arg2, arg3;
if (if_long2 && if_long1) {
ir_ref ref;
ir_IF_FALSE_cold(if_long2);
ref = ir_END();
ir_IF_FALSE_cold(if_long1);
ir_MERGE_2(ref, ir_END());
} else if (if_long1) {
ir_IF_FALSE_cold(if_long1);
} else if (if_long2) {
ir_IF_FALSE_cold(if_long2);
}
if (op1_info & MAY_BE_UNDEF) {
ir_ref if_def, ref, ref2;
ref = jit_ZVAL_ADDR(jit, op1_addr);
if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
// zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
ref2 = jit_EG(uninitialized_zval);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op2_info & MAY_BE_UNDEF) {
ir_ref if_def, ref, ref2;
ref = jit_ZVAL_ADDR(jit, op2_addr);
if_def = jit_if_not_Z_TYPE(jit, op2_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
// zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)));
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
ref2 = jit_EG(uninitialized_zval);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
op2_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
op2_addr = real_addr;
}
if (Z_MODE(res_addr) == IS_REG) {
arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
} else {
arg1 = jit_ZVAL_ADDR(jit, res_addr);
}
arg2 = jit_ZVAL_ADDR(jit, op1_addr);
arg3 = jit_ZVAL_ADDR(jit, op2_addr);
jit_SET_EX_OPLINE(jit, opline);
if (opcode == ZEND_BW_OR) {
func = ir_CONST_FC_FUNC(bitwise_or_function);
} else if (opcode == ZEND_BW_AND) {
func = ir_CONST_FC_FUNC(bitwise_and_function);
} else if (opcode == ZEND_BW_XOR) {
func = ir_CONST_FC_FUNC(bitwise_xor_function);
} else if (opcode == ZEND_SL) {
func = ir_CONST_FC_FUNC(shift_left_function);
} else if (opcode == ZEND_SR) {
func = ir_CONST_FC_FUNC(shift_right_function);
} else if (opcode == ZEND_MOD) {
func = ir_CONST_FC_FUNC(mod_function);
} else {
ZEND_UNREACHABLE();
}
ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
/* compound assignment may decrement "op2" refcount */
op2_info |= MAY_BE_RC1;
}
jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(jit, opline);
} else {
zend_jit_check_exception(jit);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_MERGE_2(fast_path, ir_END());
if (Z_MODE(res_addr) == IS_REG) {
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(res_inputs->count == 2);
jit->delay_var = -1;
jit->delay_refs = NULL;
if (res_inputs->count == 1) {
zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
} else {
ir_ref phi = ir_PHI_N(IR_LONG, res_inputs->count, res_inputs->refs);
zend_jit_def_reg(jit, res_addr, phi);
}
}
}
return 1;
}
static int zend_jit_long_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));
if (!zend_jit_long_math_helper(jit, opline, opline->opcode,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_concat_helper(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_jit_addr res_addr,
int may_throw)
{
ir_ref if_op1_string = IR_UNUSED;
ir_ref if_op2_string = IR_UNUSED;
ir_ref fast_path = IR_UNUSED;
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
if_op1_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_TRUE(if_op1_string);
}
if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
if_op2_string = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
ir_IF_TRUE(if_op2_string);
}
if (zend_jit_same_addr(op1_addr, res_addr)) {
ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
ir_ref arg2 = jit_ZVAL_ADDR(jit, op2_addr);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_assign_concat_helper), arg1, arg2);
/* concatenation with itself may reduce refcount */
op2_info |= MAY_BE_RC1;
} else {
ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
if (op1_type == IS_CV || op1_type == IS_CONST) {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_helper), arg1, arg2, arg3);
} else {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_tmp_helper), arg1, arg2, arg3);
}
}
/* concatenation with empty string may increase refcount */
op2_info |= MAY_BE_RCN;
jit_FREE_OP(jit, op2_type, op2, op2_info, opline);
if (if_op1_string || if_op2_string) {
fast_path = ir_END();
}
}
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
(op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
if (if_op1_string && if_op2_string) {
ir_IF_FALSE(if_op1_string);
ir_MERGE_WITH_EMPTY_FALSE(if_op2_string);
} else if (if_op1_string) {
ir_IF_FALSE_cold(if_op1_string);
} else if (if_op2_string) {
ir_IF_FALSE_cold(if_op2_string);
}
}
ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(concat_function), arg1, arg2, arg3);
/* concatenation with empty string may increase refcount */
op1_info |= MAY_BE_RCN;
op2_info |= MAY_BE_RCN;
jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(jit, opline);
} else {
zend_jit_check_exception(jit);
}
}
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
ir_MERGE_WITH(fast_path);
}
}
return 1;
}
static int zend_jit_concat(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
{
zend_jit_addr op1_addr, op2_addr;
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));
op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
return zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
}
static int zend_jit_assign_op(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_ssa_range *op1_range,
uint32_t op1_def_info,
zend_jit_addr op1_def_addr,
uint32_t op1_mem_info,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
int may_overflow,
int may_throw)
{
int result = 1;
ir_ref slow_path = IR_UNUSED;
ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
if (op1_info & MAY_BE_REF) {
ir_ref ref, ref2, arg2, op1_noref_path;
ir_ref if_op1_ref = IR_UNUSED;
ir_ref if_op1_typed = IR_UNUSED;
binary_op_type binary_op = get_binary_op(opline->extended_value);
ref = jit_ZVAL_ADDR(jit, op1_addr);
if_op1_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
ir_IF_FALSE(if_op1_ref);
op1_noref_path = ir_END();
ir_IF_TRUE(if_op1_ref);
ref2 = jit_Z_PTR_ref(jit, ref);
if_op1_typed = jit_if_TYPED_REF(jit, ref2);
ir_IF_TRUE_cold(if_op1_typed);
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
arg2 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
jit_SET_EX_OPLINE(jit, opline);
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref_tmp),
ref2, arg2, ir_CONST_FC_FUNC(binary_op));
} else {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
ref2, arg2, ir_CONST_FC_FUNC(binary_op));
}
zend_jit_check_exception(jit);
slow_path = ir_END();
ir_IF_FALSE(if_op1_typed);
ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
ir_MERGE_WITH(op1_noref_path);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
ZEND_ASSERT(op1_addr == op1_def_addr);
op1_def_addr = op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
result = zend_jit_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_overflow, may_throw);
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
result = zend_jit_long_math_helper(jit, opline, opline->extended_value,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_throw);
break;
case ZEND_CONCAT:
result = zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_def_addr, may_throw);
break;
default:
ZEND_UNREACHABLE();
}
if (op1_info & MAY_BE_REF) {
ir_MERGE_WITH(slow_path);
}
return result;
}
static ir_ref jit_ZVAL_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref if_ref, ref2;
if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_REFERENCE)));
ir_IF_TRUE(if_ref);
ref2 = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
return ir_PHI_2(IR_ADDR, ref2, ref);
}
static zend_jit_addr jit_ZVAL_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
{
ir_ref ref = jit_ZVAL_ADDR(jit, addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
return ZEND_ADDR_REF_ZVAL(ref);
}
static ir_ref jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref if_ref, ref2;
if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_INDIRECT)));
ir_IF_TRUE(if_ref);
ref2 = jit_Z_PTR_ref(jit, ref);
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
return ir_PHI_2(IR_ADDR, ref2, ref);
}
static zend_jit_addr jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
{
ir_ref ref = jit_ZVAL_ADDR(jit, addr);
ref = jit_ZVAL_INDIRECT_DEREF_ref(jit, ref);
return ZEND_ADDR_REF_ZVAL(ref);
}
static int zend_jit_simple_assign(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
bool check_exception)
{
ir_ref end_inputs = IR_UNUSED;
if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
zval *zv = Z_ZV(val_addr);
if (!res_addr) {
jit_ZVAL_COPY_CONST(jit,
var_addr,
var_info, var_def_info,
zv, 1);
} else {
jit_ZVAL_COPY_CONST(jit,
var_addr,
var_info, var_def_info,
zv, 1);
jit_ZVAL_COPY_CONST(jit,
res_addr,
-1, var_def_info,
zv, 1);
}
} else {
if (val_info & MAY_BE_UNDEF) {
ir_ref if_def, ret;
if_def = jit_if_not_Z_TYPE(jit, val_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
jit_set_Z_TYPE_INFO(jit, var_addr, IS_NULL);
if (res_addr) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
}
jit_SET_EX_OPLINE(jit, opline);
ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL);
// zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
if (check_exception) {
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
}
ir_END_list(end_inputs);
ir_IF_TRUE(if_def);
}
if (val_info & MAY_BE_REF) {
if (val_type == IS_CV) {
ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
val_addr = ZEND_ADDR_REF_ZVAL(ref);
} else {
ir_ref ref, type, if_ref, ref2, refcount, if_not_zero;
ref = jit_ZVAL_ADDR(jit, val_addr);
type = jit_Z_TYPE_ref(jit, ref);
if_ref = ir_IF(ir_EQ(type, ir_CONST_U8(IS_REFERENCE)));
ir_IF_TRUE_cold(if_ref);
ref = jit_Z_PTR_ref(jit, ref);
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
if (!res_addr) {
jit_ZVAL_COPY(jit,
var_addr,
var_info,
ZEND_ADDR_REF_ZVAL(ref2), val_info, 1);
} else {
jit_ZVAL_COPY_2(jit,
res_addr,
var_addr,
var_info,
ZEND_ADDR_REF_ZVAL(ref2), val_info, 2);
}
refcount = jit_GC_DELREF(jit, ref);
if_not_zero = ir_IF(refcount);
ir_IF_FALSE(if_not_zero);
// TODO: instead of dtor() call and ADDREF above, we may call efree() and move addref at "true" path ???
// This is related to GH-10168 (keep this before GH-10168 is completely closed)
// jit_EFREE(jit, ref, sizeof(zend_reference), NULL, NULL);
jit_ZVAL_DTOR(jit, ref, val_info, opline);
ir_END_list(end_inputs);
ir_IF_TRUE(if_not_zero);
ir_END_list(end_inputs);
ir_IF_FALSE(if_ref);
}
}
if (!res_addr) {
jit_ZVAL_COPY(jit,
var_addr,
var_info,
val_addr, val_info, val_type == IS_CV);
} else {
jit_ZVAL_COPY_2(jit,
res_addr,
var_addr,
var_info,
val_addr, val_info, val_type == IS_CV ? 2 : 1);
}
}
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
return 1;
}
static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr __var_use_addr,
zend_jit_addr var_addr,
uint32_t __var_info,
uint32_t __var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr __res_addr,
bool __check_exception)
{
jit_stub_id func;
ir_ref undef_path = IR_UNUSED;
if (val_info & MAY_BE_UNDEF) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_not_Z_TYPE(jit, val_addr, IS_UNDEF, exit_addr);
} else {
ir_ref if_def;
ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
if_def = ir_IF(jit_Z_TYPE(jit, val_addr));
ir_IF_FALSE_cold(if_def);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_assign_const, IR_FASTCALL_FUNC),
jit_ZVAL_ADDR(jit, var_addr),
jit_EG(uninitialized_zval));
undef_path = ir_END();
ir_IF_TRUE(if_def);
}
}
if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
func = jit_stub_assign_tmp;
} else if (val_type == IS_CONST) {
func = jit_stub_assign_const;
} else if (val_type == IS_TMP_VAR) {
func = jit_stub_assign_tmp;
} else if (val_type == IS_VAR) {
if (!(val_info & MAY_BE_REF)) {
func = jit_stub_assign_tmp;
} else {
func = jit_stub_assign_var;
}
} else if (val_type == IS_CV) {
if (!(val_info & MAY_BE_REF)) {
func = jit_stub_assign_cv_noref;
} else {
func = jit_stub_assign_cv;
}
} else {
ZEND_UNREACHABLE();
}
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, func, IR_FASTCALL_FUNC),
jit_ZVAL_ADDR(jit, var_addr),
jit_ZVAL_ADDR(jit, val_addr));
if (undef_path) {
ir_MERGE_WITH(undef_path);
}
return 1;
}
static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr var_use_addr,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
zend_jit_addr ref_addr,
bool check_exception)
{
ir_ref if_refcounted = IR_UNUSED;
ir_ref simple_inputs = IR_UNUSED;
bool done = 0;
zend_jit_addr real_res_addr = 0;
ir_refs *end_inputs;
ir_refs *res_inputs;
ir_refs_init(end_inputs, 6);
ir_refs_init(res_inputs, 6);
if (Z_MODE(val_addr) == IS_REG && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, val_addr);
}
if (Z_MODE(var_addr) == IS_REG) {
jit->delay_var = Z_SSA_VAR(var_addr);
jit->delay_refs = res_inputs;
if (Z_MODE(res_addr) == IS_REG) {
real_res_addr = res_addr;
res_addr = 0;
}
} else if (Z_MODE(res_addr) == IS_REG) {
jit->delay_var = Z_SSA_VAR(res_addr);
jit->delay_refs = res_inputs;
}
if ((var_info & MAY_BE_REF) || ref_addr) {
ir_ref ref = 0, if_ref = 0, ref2, arg2, if_typed, non_ref_path;
uintptr_t func;
if (!ref_addr) {
ref = jit_ZVAL_ADDR(jit, var_use_addr);
if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
ir_IF_TRUE(if_ref);
ref2 = jit_Z_PTR_ref(jit, ref);
} else {
ref2 = jit_ZVAL_ADDR(jit, ref_addr);
}
if_typed = jit_if_TYPED_REF(jit, ref2);
ir_IF_TRUE_cold(if_typed);
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr;
if (opline->opcode == ZEND_ASSIGN_DIM || opline->opcode == ZEND_ASSIGN_OBJ) {
real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
} else {
ZEND_ASSERT(opline->opcode == ZEND_ASSIGN);
real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
}
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg2 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg2 = jit_ZVAL_ADDR(jit, val_addr);
}
if (!res_addr) {
if (val_type == IS_CONST) {
func = (uintptr_t)zend_jit_assign_const_to_typed_ref;
} else if (val_type == IS_TMP_VAR) {
func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref;
} else if (val_type == IS_VAR) {
func = (uintptr_t)zend_jit_assign_var_to_typed_ref;
} else if (val_type == IS_CV) {
func = (uintptr_t)zend_jit_assign_cv_to_typed_ref;
} else {
ZEND_UNREACHABLE();
}
ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2);
} else {
if (val_type == IS_CONST) {
func = (uintptr_t)zend_jit_assign_const_to_typed_ref2;
} else if (val_type == IS_TMP_VAR) {
func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref2;
} else if (val_type == IS_VAR) {
func = (uintptr_t)zend_jit_assign_var_to_typed_ref2;
} else if (val_type == IS_CV) {
func = (uintptr_t)zend_jit_assign_cv_to_typed_ref2;
} else {
ZEND_UNREACHABLE();
}
ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2, jit_ZVAL_ADDR(jit, res_addr));
}
if (check_exception) {
zend_jit_check_exception(jit);
}
ir_refs_add(end_inputs, ir_END());
if (!ref_addr) {
ir_IF_FALSE(if_ref);
non_ref_path = ir_END();
ir_IF_FALSE(if_typed);
ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
ir_MERGE_WITH(non_ref_path);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
var_addr = var_use_addr = ZEND_ADDR_REF_ZVAL(ref);
} else {
ir_IF_FALSE(if_typed);
}
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
ir_ref ref, counter, if_not_zero;
if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if_refcounted = jit_if_REFCOUNTED(jit, var_use_addr);
ir_IF_FALSE(if_refcounted);
ir_END_list(simple_inputs);
ir_IF_TRUE_cold(if_refcounted);
} else if (RC_MAY_BE_1(var_info)) {
done = 1;
}
ref = jit_Z_PTR(jit, var_use_addr);
if (RC_MAY_BE_1(var_info)) {
if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
counter = jit_GC_DELREF(jit, ref);
if_not_zero = ir_IF(counter);
ir_IF_FALSE(if_not_zero);
jit_ZVAL_DTOR(jit, ref, var_info, opline);
if (check_exception) {
zend_jit_check_exception(jit);
}
ir_refs_add(end_inputs, ir_END());
ir_IF_TRUE(if_not_zero);
if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
ir_IF_FALSE(if_may_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(res_inputs->count > 0);
ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
}
if (check_exception && (val_info & MAY_BE_UNDEF)) {
zend_jit_check_exception(jit);
}
ir_refs_add(end_inputs, ir_END());
ir_IF_TRUE(if_may_leak);
}
if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(res_inputs->count > 0);
ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
}
if (check_exception && (val_info & MAY_BE_UNDEF)) {
zend_jit_check_exception(jit);
}
ir_refs_add(end_inputs, ir_END());
} else /* if (RC_MAY_BE_N(var_info)) */ {
jit_GC_DELREF(jit, ref);
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
ir_IF_FALSE(if_may_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
ir_END_list(simple_inputs);
ir_IF_TRUE(if_may_leak);
}
ir_END_list(simple_inputs);
}
}
if (simple_inputs) {
ir_MERGE_list(simple_inputs);
}
if (!done) {
if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, check_exception)) {
return 0;
}
if (end_inputs->count) {
ir_refs_add(end_inputs, ir_END());
}
}
if (end_inputs->count) {
ir_MERGE_N(end_inputs->count, end_inputs->refs);
}
if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
ir_ref phi;
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(end_inputs->count == res_inputs->count || (end_inputs->count == 0 && res_inputs->count == 1));
jit->delay_var = -1;
jit->delay_refs = NULL;
if (res_inputs->count == 1) {
phi = res_inputs->refs[0];
} else {
phi = ir_PHI_N((var_def_info & MAY_BE_LONG & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
res_inputs->count, res_inputs->refs);
}
if (Z_MODE(var_addr) == IS_REG) {
if ((var_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || ref_addr) {
phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1);
}
zend_jit_def_reg(jit, var_addr, phi);
if (real_res_addr) {
if (var_def_info & MAY_BE_LONG) {
jit_set_Z_LVAL(jit, real_res_addr, jit_Z_LVAL(jit, var_addr));
} else {
jit_set_Z_DVAL(jit, real_res_addr, jit_Z_DVAL(jit, var_addr));
}
}
} else {
zend_jit_def_reg(jit, res_addr, phi);
}
}
return 1;
}
static int zend_jit_qm_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
{
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr = op1_def_addr;
}
}
if (!zend_jit_simple_assign(jit, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 1)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_assign(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_use_addr,
uint32_t op1_def_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_jit_addr op2_def_addr,
uint32_t res_info,
zend_jit_addr res_addr,
zend_jit_addr ref_addr,
int may_throw)
{
ZEND_ASSERT(opline->op1_type == IS_CV);
if (op2_addr != op2_def_addr) {
if (!zend_jit_update_regs(jit, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
return 0;
}
if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
op2_addr = op2_def_addr;
}
}
if (Z_MODE(op1_addr) != IS_REG
&& Z_MODE(op1_use_addr) == IS_REG
&& !Z_LOAD(op1_use_addr)
&& !Z_STORE(op1_use_addr)) {
/* Force type update */
op1_info |= MAY_BE_UNDEF;
}
if (!zend_jit_assign_to_variable(jit, opline, op1_use_addr, op1_addr, op1_info, op1_def_info,
opline->op2_type, op2_addr, op2_info, res_addr, ref_addr, may_throw)) {
return 0;
}
if (Z_MODE(op1_addr) == IS_REG) {
if (Z_STORE(op1_addr)) {
if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
return 0;
}
} else if ((op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& Z_MODE(op1_use_addr) == IS_MEM_ZVAL
&& Z_REG(op1_use_addr) == ZREG_FP
&& EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)) < jit->current_op_array->last_var) {
/* We have to update type of CV because it may be captured by exception backtrace or released on RETURN */
if ((op1_def_info & MAY_BE_ANY) == MAY_BE_LONG) {
jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_LONG);
if (JIT_G(current_frame)) {
SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_LONG, 1);
}
} else if ((op1_def_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_DOUBLE);
if (JIT_G(current_frame)) {
SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_DOUBLE, 1);
}
} else {
ZEND_UNREACHABLE();
}
}
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
static ir_op zend_jit_cmp_op(const zend_op *opline)
{
ir_op op;
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
op = IR_EQ;
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
op = IR_NE;
break;
case ZEND_IS_SMALLER:
op = IR_LT;
break;
case ZEND_IS_SMALLER_OR_EQUAL:
op = IR_LE;
break;
default:
ZEND_UNREACHABLE();
}
return op;
}
static ir_ref zend_jit_cmp_long_long(zend_jit_ctx *jit,
const zend_op *opline,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
ir_ref ref;
bool result;
if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
if (!smart_branch_opcode ||
smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
jit_set_Z_TYPE_INFO(jit, res_addr, result ? IS_TRUE : IS_FALSE);
}
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
return jit_IF_ex(jit, IR_FALSE, result ? target_label : target_label2);
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
return jit_IF_ex(jit, IR_TRUE, result ? target_label : target_label2);
} else {
ZEND_UNREACHABLE();
}
}
if (opline->opcode != ZEND_IS_IDENTICAL
&& opline->opcode != ZEND_IS_NOT_IDENTICAL
&& opline->opcode != ZEND_CASE_STRICT) {
return ir_END();
} else {
return IR_NULL; /* success */
}
}
ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_LVAL(jit, op1_addr), jit_Z_LVAL(jit, op2_addr));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
}
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
if (opline->opcode != ZEND_IS_IDENTICAL
&& opline->opcode != ZEND_IS_NOT_IDENTICAL
&& opline->opcode != ZEND_CASE_STRICT) {
return ir_END();
} else {
return IR_NULL; /* success */
}
}
static ir_ref zend_jit_cmp_long_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), ir_INT2D(jit_Z_LVAL(jit, op1_addr)), jit_Z_DVAL(jit, op2_addr));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
return ir_END();
}
static ir_ref zend_jit_cmp_double_long(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), ir_INT2D(jit_Z_LVAL(jit, op2_addr)));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
return ir_END();
}
static ir_ref zend_jit_cmp_double_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), jit_Z_DVAL(jit, op2_addr));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
}
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
if (opline->opcode != ZEND_IS_IDENTICAL
&& opline->opcode != ZEND_IS_NOT_IDENTICAL
&& opline->opcode != ZEND_CASE_STRICT) {
return ir_END();
} else {
return IR_NULL; /* success */
}
}
static ir_ref zend_jit_cmp_slow(zend_jit_ctx *jit, ir_ref ref, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
ref = ir_CMP_OP(zend_jit_cmp_op(opline), ref, ir_CONST_I32(0));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
return ir_END();
}
static int zend_jit_cmp(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
ir_ref ref = IR_UNUSED;
ir_ref if_op1_long = IR_UNUSED;
ir_ref if_op1_double = IR_UNUSED;
ir_ref if_op2_double = IR_UNUSED;
ir_ref if_op1_long_op2_long = IR_UNUSED;
ir_ref if_op1_long_op2_double = IR_UNUSED;
ir_ref if_op1_double_op2_double = IR_UNUSED;
ir_ref if_op1_double_op2_long = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
bool has_slow =
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
ir_refs *end_inputs;
ir_refs_init(end_inputs, 8);
if (Z_MODE(op1_addr) == IS_REG) {
if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op1_addr);
}
} else if (Z_MODE(op2_addr) == IS_REG) {
if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op2_addr);
}
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_op1_long);
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_long_op2_long);
if (op2_info & MAY_BE_DOUBLE) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_long_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_long_op2_double);
}
ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
} else {
ir_END_list(slow_inputs);
}
ir_IF_TRUE(if_op1_long_op2_long);
}
ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
if (if_op1_long) {
ir_IF_FALSE_cold(if_op1_long);
}
if (op1_info & MAY_BE_DOUBLE) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double);
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_double_op2_long);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double_op2_long);
}
ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
} else if (if_op1_long) {
ir_END_list(slow_inputs);
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double);
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_double_op2_long);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double_op2_long);
}
ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op2_double);
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_long_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_long_op2_double);
}
ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
}
if (has_slow ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
ir_ref op1, op2, ref;
if (slow_inputs) {
ir_MERGE_list(slow_inputs);
}
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
op2_addr = real_addr;
}
op1 = jit_ZVAL_ADDR(jit, op1_addr);
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, NULL, 0);
}
op2 = jit_ZVAL_ADDR(jit, op2_addr);
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, NULL, 0);
}
ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_compare), op1, op2);
if (opline->opcode != ZEND_CASE) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
ref = zend_jit_cmp_slow(jit, ref, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
}
if (end_inputs->count) {
uint32_t n = end_inputs->count;
if (smart_branch_opcode && !exit_addr) {
zend_basic_block *bb;
ir_ref ref;
uint32_t label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label2 : target_label;
uint32_t label2 = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label : target_label2;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
ZEND_ASSERT(bb->successors_count == 2);
if (UNEXPECTED(bb->successors[0] == bb->successors[1])) {
ir_ref merge_inputs = IR_UNUSED;
while (n) {
n--;
ir_IF_TRUE(end_inputs->refs[n]);
ir_END_list(merge_inputs);
ir_IF_FALSE(end_inputs->refs[n]);
ir_END_list(merge_inputs);
}
ir_MERGE_list(merge_inputs);
_zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
} else if (n == 1) {
ref = end_inputs->refs[0];
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
} else {
ir_ref true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
while (n) {
n--;
ir_IF_TRUE(end_inputs->refs[n]);
ir_END_list(true_inputs);
ir_IF_FALSE(end_inputs->refs[n]);
ir_END_list(false_inputs);
}
ir_MERGE_list(true_inputs);
_zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
ir_MERGE_list(false_inputs);
_zend_jit_add_predecessor_ref(jit, label2, jit->b, ir_END());
}
jit->b = -1;
} else {
ir_MERGE_N(n, end_inputs->refs);
}
} else if (smart_branch_opcode && !exit_addr) {
/* dead code */
_zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
jit->b = -1;
}
return 1;
}
static int zend_jit_identical(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
bool always_false = 0, always_true = 0;
ir_ref ref = IR_UNUSED;
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
ir_ref op1 = jit_ZVAL_ADDR(jit, op1_addr);
op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, opline, 0);
op1_info |= MAY_BE_NULL;
op1_addr = ZEND_ADDR_REF_ZVAL(op1);
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
ir_ref op2 = jit_ZVAL_ADDR(jit, op2_addr);
op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, opline, 0);
op2_info |= MAY_BE_NULL;
op2_addr = ZEND_ADDR_REF_ZVAL(op2);
}
if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
always_false = 1;
} else if (has_concrete_type(op1_info)
&& has_concrete_type(op2_info)
&& concrete_type(op1_info) == concrete_type(op2_info)
&& concrete_type(op1_info) <= IS_TRUE) {
always_true = 1;
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
always_true = 1;
} else {
always_false = 1;
}
}
if (always_true) {
if (opline->opcode != ZEND_CASE_STRICT) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!smart_branch_opcode
|| smart_branch_opcode == ZEND_JMPZ_EX
|| smart_branch_opcode == ZEND_JMPNZ_EX) {
jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
uint32_t label;
if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label : target_label2;
} else {
label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label2 : target_label;
}
_zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
jit->b = -1;
}
return 1;
} else if (always_false) {
if (opline->opcode != ZEND_CASE_STRICT) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!smart_branch_opcode
|| smart_branch_opcode == ZEND_JMPZ_EX
|| smart_branch_opcode == ZEND_JMPNZ_EX) {
jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
uint32_t label;
if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label2 : target_label;
} else {
label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label : target_label2;
}
_zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
jit->b = -1;
}
return 1;
}
if ((opline->op1_type & (IS_CV|IS_VAR)) && (op1_info & MAY_BE_REF)) {
ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if ((opline->op2_type & (IS_CV|IS_VAR)) && (op2_info & MAY_BE_REF)) {
ref = jit_ZVAL_ADDR(jit, op2_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op2_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
if (!ref) {
return 0;
}
} else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
} else {
if (opline->op1_type != IS_CONST) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
}
if (opline->op2_type != IS_CONST) {
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
}
}
if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op1_addr);
ref = ir_EQ(jit_Z_TYPE(jit, op2_addr), ir_CONST_U8(Z_TYPE_P(val)));
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op2_addr);
ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(Z_TYPE_P(val)));
} else {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
op2_addr = real_addr;
}
ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_is_identical),
jit_ZVAL_ADDR(jit, op1_addr),
jit_ZVAL_ADDR(jit, op2_addr));
}
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
}
if (opline->opcode != ZEND_CASE_STRICT) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
/* swap labels */
uint32_t tmp = target_label;
target_label = target_label2;
target_label2 = tmp;
}
ref = jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
}
if (smart_branch_opcode && !exit_addr) {
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
ZEND_ASSERT(bb->successors_count == 2);
if (bb->successors_count == 2 && bb->successors[0] == bb->successors[1]) {
ir_IF_TRUE(ref);
ir_MERGE_WITH_EMPTY_FALSE(ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
} else {
ZEND_ASSERT(bb->successors_count == 2);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
}
jit->b = -1;
}
return 1;
}
static int zend_jit_bool_jmpznz(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr)
{
uint32_t true_label = -1;
uint32_t false_label = -1;
bool set_bool = 0;
bool set_bool_not = 0;
bool always_true = 0, always_false = 0;
ir_ref ref, end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
ir_type type = IR_UNUSED;
if (branch_opcode == ZEND_BOOL) {
set_bool = 1;
} else if (branch_opcode == ZEND_BOOL_NOT) {
set_bool = 1;
set_bool_not = 1;
} else if (branch_opcode == ZEND_JMPZ) {
true_label = target_label2;
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
false_label = target_label2;
} else if (branch_opcode == ZEND_JMPZ_EX) {
set_bool = 1;
true_label = target_label2;
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ_EX) {
set_bool = 1;
true_label = target_label;
false_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
if (zend_is_true(Z_ZV(op1_addr))) {
always_true = 1;
} else {
always_false = 1;
}
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
always_true = 1;
} else if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
ref = jit_ZVAL_ADDR(jit, op1_addr);
zend_jit_zval_check_undef(jit, ref, opline->op1.var, opline, 0);
}
always_false = 1;
}
}
if (always_true) {
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
if (true_label != (uint32_t)-1) {
ZEND_ASSERT(exit_addr == NULL);
_zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
jit->b = -1;
}
return 1;
} else if (always_false) {
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
if (false_label != (uint32_t)-1) {
ZEND_ASSERT(exit_addr == NULL);
_zend_jit_add_predecessor_ref(jit, false_label, jit->b, ir_END());
jit->b = -1;
}
return 1;
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
type = jit_Z_TYPE(jit, op1_addr);
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
ir_ref if_type = ir_IF(ir_LT(type, ir_CONST_U8(IS_TRUE)));
ir_IF_TRUE_cold(if_type);
if (op1_info & MAY_BE_UNDEF) {
zend_jit_type_check_undef(jit,
type,
opline->op1.var,
opline, 1, 0, 1);
}
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
ir_END_list(end_inputs);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else if (false_label != (uint32_t)-1) {
ir_END_list(false_inputs);
} else {
ir_END_list(end_inputs);
}
ir_IF_FALSE(if_type);
}
if (op1_info & MAY_BE_TRUE) {
ir_ref if_type = IR_UNUSED;
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if_type = ir_IF(ir_EQ(type, ir_CONST_U8(IS_TRUE)));
ir_IF_TRUE(if_type);
}
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
ir_END_list(end_inputs);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else if (true_label != (uint32_t)-1) {
ir_END_list(true_inputs);
} else {
ir_END_list(end_inputs);
}
if (if_type) {
ir_IF_FALSE(if_type);
}
}
}
if (op1_info & MAY_BE_LONG) {
ir_ref if_long = IR_UNUSED;
ir_ref ref;
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
if (!type) {
type = jit_Z_TYPE(jit, op1_addr);
}
if_long = ir_IF(ir_EQ(type, ir_CONST_U8(IS_LONG)));
ir_IF_TRUE(if_long);
}
ref = jit_Z_LVAL(jit, op1_addr);
if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
ref = ir_NE(ref, ir_CONST_LONG(0));
if (set_bool_not) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
ir_END_list(end_inputs);
} else if (exit_addr) {
if (set_bool) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, ir_CONST_LONG(0))), ir_CONST_U32(IS_FALSE)));
}
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
ir_END_list(end_inputs);
} else {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
ir_END_list(false_inputs);
}
if (if_long) {
ir_IF_FALSE(if_long);
}
}
if (op1_info & MAY_BE_DOUBLE) {
ir_ref if_double = IR_UNUSED;
ir_ref ref;
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (!type) {
type = jit_Z_TYPE(jit, op1_addr);
}
if_double = ir_IF(ir_EQ(type, ir_CONST_U8(IS_DOUBLE)));
ir_IF_TRUE(if_double);
}
ref = ir_NE(jit_Z_DVAL(jit, op1_addr), ir_CONST_DOUBLE(0.0));
if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
if (set_bool_not) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
ir_END_list(end_inputs);
} else if (exit_addr) {
if (set_bool) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
ir_END_list(end_inputs);
} else {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
ir_END_list(false_inputs);
}
if (if_double) {
ir_IF_FALSE(if_double);
}
}
if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_is_true), jit_ZVAL_ADDR(jit, op1_addr));
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
if (set_bool_not) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (end_inputs) {
ir_END_list(end_inputs);
}
} else if (exit_addr) {
if (set_bool) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
if (end_inputs) {
ir_END_list(end_inputs);
}
} else {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
ir_END_list(false_inputs);
}
}
if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT || exit_addr) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
} else {
_zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
}
return 1;
}
static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t defined_label = (uint32_t)-1;
uint32_t undefined_label = (uint32_t)-1;
zval *zv = RT_CONSTANT(opline, opline->op1);
zend_jit_addr res_addr = 0;
ir_ref ref, ref2, if_set, if_zero, if_set2;
ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
defined_label = target_label2;
undefined_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
defined_label = target_label;
undefined_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
} else {
res_addr = RES_ADDR();
}
// if (CACHED_PTR(opline->extended_value)) {
ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
if_set = ir_IF(ref);
ir_IF_FALSE_cold(if_set);
if_zero = ir_END();
ir_IF_TRUE(if_set);
if_set2 = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
ir_IF_FALSE(if_set2);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(true_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
ir_IF_TRUE_cold(if_set2);
ref2 = jit_EG(zend_constants);
ref = ir_SHR_A(ref, ir_CONST_ADDR(1));
if (sizeof(void*) == 8) {
ref = ir_TRUNC_U32(ref);
}
ref2 = ir_EQ(ref, ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref2), offsetof(HashTable, nNumOfElements))));
ref2 = ir_IF(ref2);
ir_IF_TRUE(ref2);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
ir_END_list(end_inputs);
}
ir_IF_FALSE(ref2);
ir_MERGE_2(if_zero, ir_END());
jit_SET_EX_OPLINE(jit, opline);
ref2 = ir_NE(ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_check_constant), ir_CONST_ADDR(zv)), IR_NULL);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
ir_GUARD(ref2, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref2, ir_CONST_ADDR(exit_addr));
}
ir_END_list(end_inputs);
} else if (smart_branch_opcode) {
ref2 = ir_IF(ref2);
ir_IF_TRUE(ref2);
ir_END_list(true_inputs);
ir_IF_FALSE(ref2);
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref2), ir_CONST_U32(IS_FALSE)));
ir_END_list(end_inputs);
}
if (!smart_branch_opcode || exit_addr) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
} else {
_zend_jit_merge_smart_branch_inputs(jit, defined_label, undefined_label, true_inputs, false_inputs);
}
return 1;
}
static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg)
{
zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr));
ir_IF_FALSE_cold(if_def);
if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
if (!zend_jit_save_call_chain(jit, -1)) {
return 0;
}
}
if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
&& (opline-1)->opcode != ZEND_FETCH_LIST_R
&& ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);
zend_jit_zval_try_addref(jit, val_addr);
}
jit_LOAD_IP_ADDR(jit, opline - 1);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
ir_IF_TRUE(if_def);
return 1;
}
static int zend_jit_restore_zval(zend_jit_ctx *jit, int var, int8_t reg)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
// JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
jit_ZVAL_COPY(jit, var_addr, MAY_BE_ANY, reg_addr, MAY_BE_ANY, 1);
return 1;
}
static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr val_addr,
uint8_t type,
bool deref,
uint32_t flags,
bool op1_avoid_refcounting)
{
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
const void *res_exit_addr = NULL;
ir_ref end1 = IR_UNUSED, ref1 = IR_UNUSED;
ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
uint32_t old_op1_info = 0;
uint32_t old_info;
ir_ref old_ref;
if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
if (op1_avoid_refcounting
|| ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS))) {
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
}
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
if (deref) {
ir_ref if_type;
if (type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) {
if_type = ir_IF(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)));
} else {
if_type = jit_if_Z_TYPE(jit, val_addr, type);
}
ir_IF_TRUE(if_type);
end1 = ir_END();
ref1 = ref;
ir_IF_FALSE_cold(if_type);
SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!res_exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, val_addr, IS_REFERENCE, res_exit_addr);
ref = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
val_addr = ZEND_ADDR_REF_ZVAL(ref);
}
SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!res_exit_addr) {
return 0;
}
if (!deref && type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) {
ir_GUARD(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)), ir_CONST_ADDR(res_exit_addr));
} else {
jit_guard_Z_TYPE(jit, val_addr, type, res_exit_addr);
}
if (deref) {
ir_MERGE_WITH(end1);
ref = ir_PHI_2(IR_ADDR, ref, ref1);
}
val_addr = ZEND_ADDR_REF_ZVAL(ref);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), old_ref);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
}
return val_addr;
}
static int zend_jit_fetch_constant(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
zend_jit_addr res_addr)
{
zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
uint32_t res_info = RES_INFO();
ir_ref ref, ref2, if_set, if_special, not_set_path, special_path, fast_path;
// JIT: c = CACHED_PTR(opline->extended_value);
ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
// JIT: if (c != NULL)
if_set = ir_IF(ref);
if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) {
// JIT: if (!IS_SPECIAL_CACHE_VAL(c))
ir_IF_FALSE_cold(if_set);
not_set_path = ir_END();
ir_IF_TRUE(if_set);
if_special = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
ir_IF_TRUE_cold(if_special);
special_path = ir_END();
ir_IF_FALSE(if_special);
fast_path = ir_END();
ir_MERGE_2(not_set_path, special_path);
} else {
ir_IF_TRUE(if_set);
fast_path = ir_END();
ir_IF_FALSE_cold(if_set);
}
// JIT: zend_jit_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num);
jit_SET_EX_OPLINE(jit, opline);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_constant),
ir_CONST_ADDR(zv),
ir_CONST_U32(opline->op1.num));
ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH(fast_path);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
uint8_t type = concrete_type(res_info);
zend_jit_addr const_addr = ZEND_ADDR_REF_ZVAL(ref);
const_addr = zend_jit_guard_fetch_result_type(jit, opline, const_addr, type, 0, 0, 0);
if (!const_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
// JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, res_info, 1);
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
} else {
ir_ref const_addr = ZEND_ADDR_REF_ZVAL(ref);
// JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, 1);
}
return 1;
}
static int zend_jit_type_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t mask;
zend_jit_addr op1_addr = OP1_ADDR();
zend_jit_addr res_addr = 0;
uint32_t true_label = -1, false_label = -1;
ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
// TODO: support for is_resource() ???
ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
true_label = target_label2;
false_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
false_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
} else {
res_addr = RES_ADDR();
}
if (op1_info & MAY_BE_UNDEF) {
ir_ref if_def = IR_UNUSED;
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
}
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
zend_jit_check_exception_undef_result(jit, opline);
if (opline->extended_value & MAY_BE_NULL) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(true_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
} else {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
if (if_def) {
ir_END_list(end_inputs);
}
}
}
if (if_def) {
ir_IF_TRUE(if_def);
op1_info |= MAY_BE_NULL;
}
}
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
mask = opline->extended_value;
if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else if (end_inputs) {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(true_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
} else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else if (end_inputs) {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
ir_END_list(end_inputs);
}
} else {
ir_ref ref;
bool invert = 0;
uint8_t type;
switch (mask) {
case MAY_BE_NULL: type = IS_NULL; break;
case MAY_BE_FALSE: type = IS_FALSE; break;
case MAY_BE_TRUE: type = IS_TRUE; break;
case MAY_BE_LONG: type = IS_LONG; break;
case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
case MAY_BE_STRING: type = IS_STRING; break;
case MAY_BE_ARRAY: type = IS_ARRAY; break;
case MAY_BE_OBJECT: type = IS_OBJECT; break;
case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break;
case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break;
case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break;
case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break;
case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break;
case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break;
default:
type = 0;
}
if (op1_info & MAY_BE_REF) {
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (type == 0) {
ref = ir_AND_U32(ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(mask));
if (!smart_branch_opcode) {
ref = ir_NE(ref, ir_CONST_U32(0));
}
} else if (invert) {
ref = ir_NE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
} else {
ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
if (end_inputs) {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
ir_END_list(end_inputs);
}
}
}
if (!smart_branch_opcode || exit_addr) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
} else {
_zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
}
return 1;
}
static int zend_jit_isset_isempty_cv(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_jit_addr res_addr = RES_ADDR();
uint32_t true_label = -1, false_label = -1;
ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
// TODO: support for empty() ???
ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
true_label = target_label2;
false_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
false_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
} else {
res_addr = RES_ADDR();
}
if (op1_info & MAY_BE_REF) {
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
} else if (smart_branch_opcode) {
ir_END_list(true_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
} else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
} else if (smart_branch_opcode) {
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
ir_END_list(end_inputs);
}
} else {
ir_ref ref = ir_GT(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL));
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
ir_END_list(end_inputs);
}
}
if (!smart_branch_opcode || exit_addr) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
} else {
_zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
}
return 1;
}
/* copy of hidden zend_closure */
typedef struct _zend_closure {
zend_object std;
zend_function func;
zval this_ptr;
zend_class_entry *called_scope;
zif_handler orig_internal_handler;
} zend_closure;
static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
// JIT: if (EG(vm_stack_end) - EG(vm_stack_top) < used_stack)
ir_GUARD(
ir_UGE(
ir_SUB_A(ir_LOAD_A(jit_EG(vm_stack_end)), ir_LOAD_A(jit_EG(vm_stack_top))),
ir_CONST_ADDR(used_stack)),
ir_CONST_ADDR(exit_addr));
return 1;
}
static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg)
{
// JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
ir_ref func = ir_RLOAD_A(func_reg);
ir_ref if_trampoline = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))),
ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE)));
ir_IF_TRUE(if_trampoline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_trampoline_helper), func);
ir_MERGE_WITH_EMPTY_FALSE(if_trampoline);
return 1;
}
static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack, ir_ref func_ref, ir_ref this_ref)
{
uint32_t used_stack;
ir_ref used_stack_ref = IR_UNUSED;
bool stack_check = 1;
ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED;
ZEND_ASSERT(func_ref != IR_NULL);
if (func) {
used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
if ((int)used_stack <= checked_stack) {
stack_check = 0;
}
used_stack_ref = ir_CONST_ADDR(used_stack);
} else {
ir_ref num_args_ref;
ir_ref if_internal_func = IR_UNUSED;
used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval);
used_stack_ref = ir_CONST_ADDR(used_stack);
if (!is_closure) {
used_stack_ref = ir_HARD_COPY_A(used_stack_ref); /* load constant once */
// JIT: if (EXPECTED(ZEND_USER_CODE(func->type))) {
ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(1)));
ir_IF_FALSE(if_internal_func);
}
// JIT: used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
num_args_ref = ir_CONST_U32(opline->extended_value);
if (!is_closure) {
ref = ir_SUB_U32(
ir_SUB_U32(
ir_MIN_U32(
num_args_ref,
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.num_args)))),
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.last_var)))),
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.T))));
} else {
ref = ir_SUB_U32(
ir_SUB_U32(
ir_MIN_U32(
num_args_ref,
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.num_args)))),
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.last_var)))),
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.T))));
}
ref = ir_MUL_U32(ref, ir_CONST_U32(sizeof(zval)));
if (sizeof(void*) == 8) {
ref = ir_SEXT_A(ref);
}
ref = ir_SUB_A(used_stack_ref, ref);
if (is_closure) {
used_stack_ref = ref;
} else {
ir_MERGE_WITH_EMPTY_TRUE(if_internal_func);
used_stack_ref = ir_PHI_2(IR_ADDR, ref, used_stack_ref);
}
}
zend_jit_start_reuse_ip(jit);
// JIT: if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top)));
if (stack_check) {
// JIT: Check Stack Overflow
ref = ir_UGE(
ir_SUB_A(
ir_LOAD_A(jit_EG(vm_stack_end)),
jit_IP(jit)),
used_stack_ref);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
if_enough_stack = ir_IF(ref);
ir_IF_FALSE_cold(if_enough_stack);
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref);
} else {
if (!is_closure) {
ref = func_ref;
} else {
ref = ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func));
}
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper),
used_stack_ref, ref);
}
jit_STORE_IP(jit, ref);
cold_path = ir_END();
ir_IF_TRUE(if_enough_stack);
}
}
ref = jit_EG(vm_stack_top);
rx = jit_IP(jit);
#if !OPTIMIZE_FOR_SIZE
/* JIT: EG(vm_stack_top) = (zval*)((char*)call + used_stack);
* This vesions is longer but faster
* mov EG(vm_stack_top), %CALL
* lea size(%call), %tmp
* mov %tmp, EG(vm_stack_top)
*/
top = rx;
#else
/* JIT: EG(vm_stack_top) += used_stack;
* Use ir_emit() because ir_LOAD() makes load forwarding and doesn't allow load/store fusion
* mov EG(vm_stack_top), %CALL
* add $size, EG(vm_stack_top)
*/
top = jit->ctx.control = ir_emit2(&jit->ctx, IR_OPT(IR_LOAD, IR_ADDR), jit->ctx.control, ref);
#endif
ir_STORE(ref, ir_ADD_A(top, used_stack_ref));
// JIT: zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
// JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
ir_STORE(jit_CALL(rx, This.u1.type_info), ir_CONST_U32(IS_UNDEF | ZEND_CALL_NESTED_FUNCTION));
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
if (cold_path) {
ir_MERGE_WITH(cold_path);
rx = jit_IP(jit);
}
// JIT: call->func = func;
ir_STORE(jit_CALL(rx, func), func_ref);
} else {
if (!is_closure) {
// JIT: call->func = func;
ir_STORE(jit_CALL(rx, func), func_ref);
} else {
// JIT: call->func = &closure->func;
ir_STORE(jit_CALL(rx, func), ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
}
if (cold_path) {
ir_MERGE_WITH(cold_path);
rx = jit_IP(jit);
}
}
if (opline->opcode == ZEND_INIT_METHOD_CALL) {
// JIT: Z_PTR(call->This) = obj;
ZEND_ASSERT(this_ref != IR_NULL);
ir_STORE(jit_CALL(rx, This.value.ptr), this_ref);
if (opline->op1_type == IS_UNUSED || delayed_fetch_this) {
// JIT: call->call_info |= ZEND_CALL_HAS_THIS;
ref = jit_CALL(rx, This.u1.type_info);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS));
} else {
ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_HAS_THIS)));
}
} else {
if (opline->op1_type == IS_CV) {
// JIT: GC_ADDREF(obj);
jit_GC_ADDREF(jit, this_ref);
}
// JIT: call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
ref = jit_CALL(rx, This.u1.type_info);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS));
} else {
ir_STORE(ref,
ir_OR_U32(ir_LOAD_U32(ref),
ir_CONST_U32(ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)));
}
}
} else if (!is_closure) {
// JIT: Z_CE(call->This) = called_scope;
ir_STORE(jit_CALL(rx, This), IR_NULL);
} else {
ir_ref object_or_called_scope, call_info, call_info2, object, if_cond;
if (opline->op2_type == IS_CV) {
// JIT: GC_ADDREF(closure);
jit_GC_ADDREF(jit, func_ref);
}
// JIT: RX(object_or_called_scope) = closure->called_scope;
object_or_called_scope = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, called_scope)));
// JIT: call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
// (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
call_info = ir_OR_U32(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.common.fn_flags))),
ir_CONST_U32(ZEND_ACC_FAKE_CLOSURE)),
ir_CONST_U32(ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE));
// JIT: if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
if_cond = ir_IF(ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.u1.v.type))));
ir_IF_TRUE(if_cond);
// JIT: call_info |= ZEND_CALL_HAS_THIS;
call_info2 = ir_OR_U32(call_info, ir_CONST_U32(ZEND_CALL_HAS_THIS));
// JIT: object_or_called_scope = Z_OBJ(closure->this_ptr);
object = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.value.ptr)));
ir_MERGE_WITH_EMPTY_FALSE(if_cond);
call_info = ir_PHI_2(IR_U32, call_info2, call_info);
object_or_called_scope = ir_PHI_2(IR_ADDR, object, object_or_called_scope);
// JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
ref = jit_CALL(rx, This.u1.type_info);
ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), call_info));
// JIT: Z_PTR(call->This) = object_or_called_scope;
ir_STORE(jit_CALL(rx, This.value.ptr), object_or_called_scope);
// JIT: if (closure->func.op_array.run_time_cache__ptr)
if_cond = ir_IF(ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.run_time_cache__ptr))));
ir_IF_FALSE(if_cond);
// JIT: zend_jit_init_func_run_time_cache_helper(closure->func);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper),
ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
ir_MERGE_WITH_EMPTY_TRUE(if_cond);
}
// JIT: ZEND_CALL_NUM_ARGS(call) = num_args;
ir_STORE(jit_CALL(rx, This.u2.num_args), ir_CONST_U32(opline->extended_value));
return 1;
}
static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const zend_function *func, const zend_op *to_opline)
{
int32_t exit_point;
const void *exit_addr;
ir_ref call;
if (func->type == ZEND_INTERNAL_FUNCTION) {
#ifdef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
return 0;
#endif
} else if (func->type == ZEND_USER_FUNCTION) {
if (!zend_accel_in_shm(func->op_array.opcodes)) {
/* op_array and op_array->opcodes are not persistent. We can't link. */
return 0;
}
} else {
ZEND_UNREACHABLE();
return 0;
}
exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
// call = EX(call);
call = ir_LOAD_A(jit_EX(call));
while (level > 0) {
// call = call->prev_execute_data
call = ir_LOAD_A(jit_CALL(call, prev_execute_data));
level--;
}
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
// JIT: if (call->func.op_array.opcodes != opcodes) goto exit_addr;
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(call, func)), offsetof(zend_op_array, opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
} else {
// JIT: if (call->func != func) goto exit_addr;
ir_GUARD(ir_EQ(ir_LOAD_A(jit_CALL(call, func)), ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
}
return 1;
}
static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
ir_ref func_ref = IR_UNUSED;
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
if (!func
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL) {
#ifdef _WIN32
/* ASLR */
if (trace->func->type != ZEND_INTERNAL_FUNCTION) {
func = (zend_function*)trace->func;
}
#else
func = (zend_function*)trace->func;
#endif
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_INTERNAL_FUNCTION) {
#endif
/* load constant address later */
func_ref = ir_CONST_ADDR(func);
} else if (func && op_array == &func->op_array) {
/* recursive call */
if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
|| zend_jit_prefer_const_addr_load(jit, (uintptr_t)func)) {
func_ref = ir_LOAD_A(jit_EX(func));
} else {
func_ref = ir_CONST_ADDR(func);
}
} else {
ir_ref if_func, cache_slot_ref, ref;
// JIT: if (CACHED_PTR(opline->result.num))
cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num);
func_ref = ir_LOAD_A(cache_slot_ref);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& func
&& (func->common.fn_flags & ZEND_ACC_IMMUTABLE)
&& opline->opcode != ZEND_INIT_FCALL) {
/* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */
if_func = ir_IF(ir_EQ(func_ref, ir_CONST_ADDR(func)));
} else {
if_func = ir_IF(func_ref);
}
ir_IF_FALSE_cold(if_func);
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_USER_FUNCTION
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
ref = ir_HARD_COPY_A(ir_CONST_ADDR(func)); /* load constant once */
ir_STORE(cache_slot_ref, ref);
ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper), ref);
} else {
zval *zv = RT_CONSTANT(opline, opline->op2);
if (opline->opcode == ZEND_INIT_FCALL) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
ir_CONST_ADDR(Z_STR_P(zv)),
cache_slot_ref);
} else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
ir_CONST_ADDR(Z_STR_P(zv + 1)),
cache_slot_ref);
} else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_ns_func_helper),
ir_CONST_ADDR(zv),
cache_slot_ref);
} else {
ZEND_UNREACHABLE();
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline,
func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (!func || opline->opcode == ZEND_INIT_FCALL) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else if (func->type == ZEND_USER_FUNCTION
&& !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
const zend_op *opcodes = func->op_array.opcodes;
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_op_array, opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ir_EQ(ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
}
} else {
jit_SET_EX_OPLINE(jit, opline);
ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_undefined_function));
}
}
ir_MERGE_WITH_EMPTY_TRUE(if_func);
func_ref = ir_PHI_2(IR_ADDR, ref, func_ref);
}
if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, IR_UNUSED)) {
return 0;
}
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(jit, call_level)) {
return 0;
}
} else {
ZEND_ASSERT(call_level > 0);
jit->delayed_call_level = call_level;
delayed_call_chain = 1;
}
return 1;
}
static int zend_jit_init_method_call(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
zend_jit_trace_rec *trace,
int checked_stack,
int8_t func_reg,
int8_t this_reg,
bool polymorphic_side_trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
zval *function_name;
ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
function_name = RT_CONSTANT(opline, opline->op2);
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
if (polymorphic_side_trace) {
/* function is passed in r0 from parent_trace */
ZEND_ASSERT(func_reg >= 0 && this_reg >= 0);
func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
} else {
ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2;
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
this_ref = jit_Z_PTR(jit, this_addr);
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
// JIT: ZVAL_DEREF(op1)
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
} else {
ir_ref if_ref;
/* Hack: Convert reference to regular value to simplify JIT code */
ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, op1_addr));
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
}
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_OBJECT)),
ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_object = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_object);
jit_SET_EX_OPLINE(jit, opline);
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call_tmp),
jit_ZVAL_ADDR(jit, op1_addr));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call),
jit_ZVAL_ADDR(jit, op1_addr));
}
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_IF_TRUE(if_object);
}
}
this_ref = jit_Z_PTR(jit, op1_addr);
}
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
if (func) {
// JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num + sizeof(void*)));
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
fast_path = ir_END();
} else {
// JIT: if (CACHED_PTR(opline->result.num) == obj->ce)) {
run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ref = ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num)),
ir_LOAD_A(ir_ADD_OFFSET(this_ref, offsetof(zend_object, ce))));
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
// JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num + sizeof(void*)));
fast_path = ir_END();
}
ir_IF_FALSE_cold(if_found);
jit_SET_EX_OPLINE(jit, opline);
if (!jit->ctx.fixed_call_stack_size) {
// JIT: alloca(sizeof(void*));
this_ref2 = ir_ALLOCA(ir_CONST_ADDR(0x10));
} else {
this_ref2 = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
}
ir_STORE(this_ref2, this_ref);
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_tmp_helper),
this_ref,
ir_CONST_ADDR(function_name),
this_ref2);
} else {
ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_helper),
this_ref,
ir_CONST_ADDR(function_name),
this_ref2);
}
this_ref2 = ir_LOAD_A(ir_RLOAD_A(IR_REG_SP));
if (!jit->ctx.fixed_call_stack_size) {
// JIT: revert alloca
ir_AFREE(ir_CONST_ADDR(0x10));
}
ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH(fast_path);
func_ref = ir_PHI_2(IR_ADDR, ref2, ref);
this_ref = ir_PHI_2(IR_ADDR, this_ref2, this_ref);
}
if ((!func || zend_jit_may_be_modified(func, op_array))
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
#ifdef _WIN32
&& trace->func->type != ZEND_INTERNAL_FUNCTION
#endif
) {
int32_t exit_point;
const void *exit_addr;
exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
func = (zend_function*)trace->func;
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ir_EQ(func_ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
}
}
if (!func) {
// JIT: if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
if_static = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
ir_CONST_U32(ZEND_ACC_STATIC)));
ir_IF_TRUE_cold(if_static);
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
ir_ref ret;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_metod_call_frame_tmp),
this_ref,
func_ref,
ir_CONST_U32(opline->extended_value));
} else {
ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_metod_call_frame),
this_ref,
func_ref,
ir_CONST_U32(opline->extended_value));
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) {
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
jit_STORE_IP(jit, ret);
}
if (!func) {
cold_path = ir_END();
ir_IF_FALSE(if_static);
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_ref, this_ref)) {
return 0;
}
}
if (!func) {
ir_MERGE_WITH(cold_path);
}
zend_jit_start_reuse_ip(jit);
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(jit, call_level)) {
return 0;
}
} else {
ZEND_ASSERT(call_level > 0);
delayed_call_chain = 1;
jit->delayed_call_level = call_level;
}
return 1;
}
static int zend_jit_init_closure_call(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
zend_jit_trace_rec *trace,
int checked_stack)
{
zend_function *func = NULL;
zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
ir_ref ref;
ref = jit_Z_PTR(jit, op2_addr);
if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
&& !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_object, ce))),
ir_CONST_ADDR(zend_ce_closure)),
ir_CONST_ADDR(exit_addr));
if (ssa->var_info && ssa_op->op2_use >= 0) {
ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
}
}
if (trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
&& trace->func->type == ZEND_USER_FUNCTION) {
const zend_op *opcodes;
int32_t exit_point;
const void *exit_addr;
func = (zend_function*)trace->func;
opcodes = func->op_array.opcodes;
exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_closure, func.op_array.opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
}
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
if (!zend_jit_push_call_frame(jit, opline, NULL, func, 1, 0, checked_stack, ref, IR_UNUSED)) {
return 0;
}
if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(jit, call_level)) {
return 0;
}
} else {
ZEND_ASSERT(call_level > 0);
delayed_call_chain = 1;
jit->delayed_call_level = call_level;
}
if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_ip(jit, opline + 1)) {
return 0;
}
}
return 1;
}
static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);
if (!zend_jit_reuse_ip(jit)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAL_EX) {
uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
/* Don't generate code that always throws exception */
return 0;
}
} else {
ir_ref cond = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask));
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_pass_by_ref;
if_pass_by_ref = ir_IF(cond);
ir_IF_TRUE_cold(if_pass_by_ref);
if (Z_MODE(op1_addr) == IS_REG) {
/* set type to avoid zval_ptr_dtor() on uninitialized value */
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF);
}
jit_SET_EX_OPLINE(jit, opline);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref));
ir_IF_FALSE(if_pass_by_ref);
}
}
}
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
jit_ZVAL_COPY_CONST(jit,
arg_addr,
MAY_BE_ANY, MAY_BE_ANY,
zv, 1);
} else {
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
}
return 1;
}
static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
{
zend_jit_addr op1_addr, arg_addr, ref_addr;
ir_ref ref_path = IR_UNUSED;
op1_addr = OP1_ADDR();
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(jit)) {
return 0;
}
if (opline->op1_type == IS_VAR) {
if (op1_info & MAY_BE_INDIRECT) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
// JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
ir_ref if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE(if_def);
// JIT: ZVAL_NULL(op1)
jit_set_Z_TYPE_INFO(jit,op1_addr, IS_NULL);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
}
op1_info &= ~MAY_BE_UNDEF;
op1_info |= MAY_BE_NULL;
}
} else {
ZEND_UNREACHABLE();
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
ir_ref ref, ref2;
if (op1_info & MAY_BE_REF) {
ir_ref if_ref;
// JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
// JIT: ref = Z_PTR_P(op1)
ref = jit_Z_PTR(jit, op1_addr);
// JIT: GC_ADDREF(ref)
jit_GC_ADDREF(jit, ref);
// JIT: ZVAL_REFERENCE(arg, ref)
jit_set_Z_PTR(jit, arg_addr, ref);
jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
ref_path = ir_END();
ir_IF_FALSE(if_ref);
}
// JIT: ZVAL_NEW_REF(arg, varptr);
// JIT: ref = emalloc(sizeof(zend_reference));
ref = jit_EMALLOC(jit, sizeof(zend_reference), op_array, opline);
// JIT: GC_REFCOUNT(ref) = 2
jit_set_GC_REFCOUNT(jit, ref, 2);
// JIT: GC_TYPE(ref) = GC_REFERENCE
ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, gc.u.type_info)), ir_CONST_U32(GC_REFERENCE));
ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr)), IR_NULL);
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
// JIT: ZVAL_COPY_VALUE(&ref->val, op1)
jit_ZVAL_COPY(jit,
ref_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
// JIT: ZVAL_REFERENCE(arg, ref)
jit_set_Z_PTR(jit, op1_addr, ref);
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
// JIT: ZVAL_REFERENCE(arg, ref)
jit_set_Z_PTR(jit, arg_addr, ref);
jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
}
if (ref_path) {
ir_MERGE_WITH(ref_path);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
return 1;
}
static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ir_ref end_inputs = IR_UNUSED;
ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
arg_num <= MAX_ARG_FLAG_NUM);
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(jit)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAR_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
// JIT: if (RX->func->quick_arg_flags & mask)
ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask)));
ir_IF_TRUE_cold(if_send_by_ref);
if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
return 0;
}
ir_END_list(end_inputs);
ir_IF_FALSE(if_send_by_ref);
}
} else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
// JIT: ZVAL_COPY_VALUE(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!(op1_info & MAY_BE_REF)) {
/* Don't generate code that always throws exception */
return 0;
} else {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
// JIT: if (Z_TYPE_P(op1) != IS_REFERENCE)
ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U32(IS_REFERENCE)),
ir_CONST_ADDR(exit_addr));
}
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
ir_ref func, if_send_by_ref, if_prefer_ref;
// JIT: if (RX->func->quick_arg_flags & mask)
func = ir_LOAD_A(jit_RX(func));
if_send_by_ref = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask)));
ir_IF_TRUE_cold(if_send_by_ref);
mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
// JIT: ZVAL_COPY_VALUE(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
if (op1_info & MAY_BE_REF) {
ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
ir_END_list(end_inputs);
ir_IF_FALSE(if_ref);
}
// JIT: if (RX->func->quick_arg_flags & mask)
if_prefer_ref = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask)));
ir_IF_TRUE(if_prefer_ref);
ir_END_list(end_inputs);
ir_IF_FALSE(if_prefer_ref);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
jit_ZVAL_ADDR(jit, arg_addr));
zend_jit_check_exception(jit);
ir_END_list(end_inputs);
}
ir_IF_FALSE(if_send_by_ref);
}
} else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
// JIT: if (RX->This.u1.type_info & ZEND_CALL_SEND_ARG_BY_REF)
ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
ir_LOAD_U32(jit_RX(This.u1.type_info)),
ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
ir_IF_TRUE_cold(if_send_by_ref);
if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
return 0;
}
ir_END_list(end_inputs);
ir_IF_FALSE(if_send_by_ref);
}
}
if (op1_info & MAY_BE_UNDEF) {
ir_ref ref, if_def = IR_UNUSED;
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
}
// JIT: zend_jit_undefined_op_helper(opline->op1.var)
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
ir_CONST_U32(opline->op1.var));
// JIT: ZVAL_NULL(arg)
jit_set_Z_TYPE_INFO(jit, arg_addr, IS_NULL);
// JIT: check_exception
ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
ir_END_list(end_inputs);
ir_IF_TRUE(if_def);
} else {
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
return 1;
}
}
if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
// JIT: ZVAL_COPY_VALUE(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
if (op1_info & MAY_BE_REF) {
// JIT: if (Z_TYPE_P(arg) == IS_REFERENCE)
ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
ir_END_list(end_inputs);
ir_IF_FALSE(if_ref);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(IR_FALSE, ir_CONST_ADDR(exit_addr));
} else {
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
jit_ZVAL_ADDR(jit, arg_addr));
zend_jit_check_exception(jit);
}
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
ir_ref ref;
// JIT: ZVAL_DEREF(op1)
ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
// JIT: ZVAL_COPY(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 1);
} else {
ir_ref if_ref, ref, ref2, refcount, if_not_zero, if_refcounted;
zend_jit_addr ref_addr;
// JIT: if (Z_TYPE_P(op1) == IS_REFERENCE)
if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_TRUE_cold(if_ref);
// JIT: ref = Z_COUNTED_P(op1);
ref = jit_Z_PTR(jit, op1_addr);
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
// JIT: ZVAL_COPY_VALUE(arg, op1);
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
ref_addr, op1_info, 0);
// JIT: if (GC_DELREF(ref) != 0)
refcount = jit_GC_DELREF(jit, ref);
if_not_zero = ir_IF(refcount);
ir_IF_TRUE(if_not_zero);
// JIT: if (Z_REFCOUNTED_P(arg)
if_refcounted = jit_if_REFCOUNTED(jit, arg_addr);
ir_IF_TRUE(if_refcounted);
// JIT: Z_ADDREF_P(arg)
jit_GC_ADDREF(jit, jit_Z_PTR(jit, arg_addr));
ir_END_list(end_inputs);
ir_IF_FALSE(if_refcounted);
ir_END_list(end_inputs);
ir_IF_FALSE(if_not_zero);
// JIT: efree(ref)
jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
ir_END_list(end_inputs);
ir_IF_FALSE(if_ref);
// JIT: ZVAL_COPY_VALUE(arg, op1);
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
}
} else {
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr = op1_def_addr;
}
}
// JIT: ZVAL_COPY_VALUE(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, opline->op1_type == IS_CV);
}
}
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
return 1;
}
static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline)
{
uint32_t arg_num = opline->op2.num;
ir_ref ref;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
// JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
if (jit->reuse_ip) {
ref = jit_IP(jit);
} else {
ref = ir_LOAD_A(jit_EX(call));
}
ref = jit_CALL(ref, This.u1.type_info);
ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
}
} else {
if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
// JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
if (jit->reuse_ip) {
ref = jit_IP(jit);
} else {
ref = ir_LOAD_A(jit_EX(call));
}
ref = jit_CALL(ref, This.u1.type_info);
ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
}
}
} else {
// JIT: if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
ir_ref rx, if_ref, cold_path;
if (!zend_jit_reuse_ip(jit)) {
return 0;
}
rx = jit_IP(jit);
ref = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask));
if_ref = ir_IF(ref);
ir_IF_TRUE_cold(if_ref);
// JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
ref = jit_CALL(rx, This.u1.type_info);
ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
cold_path = ir_END();
ir_IF_FALSE(if_ref);
// JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
ref = jit_CALL(rx, This.u1.type_info);
ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
ir_MERGE_WITH(cold_path);
}
return 1;
}
static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline)
{
ir_ref call, if_may_have_undef, ret;
if (jit->reuse_ip) {
call = jit_IP(jit);
} else {
call = ir_LOAD_A(jit_EX(call));
}
if_may_have_undef = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(call, offsetof(zend_execute_data, This.u1.type_info) + 3)),
ir_CONST_U8(ZEND_CALL_MAY_HAVE_UNDEF >> 24)));
ir_IF_TRUE_cold(if_may_have_undef);
jit_SET_EX_OPLINE(jit, opline);
ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_handle_undef_args), call);
ir_GUARD_NOT(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH_EMPTY_FALSE(if_may_have_undef);
return 1;
}
static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
const zend_function *func = NULL;
uint32_t i;
uint32_t call_num_args = 0;
bool unknown_num_args = 0;
const void *exit_addr = NULL;
const zend_op *prev_opline;
ir_ref rx, func_ref = IR_UNUSED, if_user = IR_UNUSED, user_path = IR_UNUSED;
prev_opline = opline - 1;
while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
prev_opline--;
}
if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
unknown_num_args = 1;
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_call_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& !JIT_G(current_frame)->call->func) {
call_info = NULL; func = NULL; /* megamorphic call from trait */
}
}
if (!func) {
/* resolve function at run time */
} else if (func->type == ZEND_USER_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
call_num_args = call_info->num_args;
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
call_num_args = call_info->num_args;
} else {
ZEND_UNREACHABLE();
}
if (trace && !func) {
if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
#ifndef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
#endif
} else if (trace->op == ZEND_JIT_TRACE_ENTER) {
ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
}
}
}
bool may_have_extra_named_params =
opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
(!func || func->common.fn_flags & ZEND_ACC_VARIADIC);
if (!jit->reuse_ip) {
zend_jit_start_reuse_ip(jit);
// JIT: call = EX(call);
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
}
rx = jit_IP(jit);
zend_jit_stop_reuse_ip(jit);
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
func_ref = ir_LOAD_A(jit_CALL(rx, func));
ir_GUARD_NOT(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
ir_CONST_U32(ZEND_ACC_DEPRECATED)),
ir_CONST_ADDR(exit_addr));
}
}
}
if (!jit->delayed_call_level) {
// JIT: EX(call) = call->prev_execute_data;
ir_STORE(jit_EX(call),
(call_level == 1) ? IR_NULL : ir_LOAD_A(jit_CALL(rx, prev_execute_data)));
}
delayed_call_chain = 0;
jit->delayed_call_level = 0;
// JIT: call->prev_execute_data = execute_data;
ir_STORE(jit_CALL(rx, prev_execute_data), jit_FP(jit));
if (!func) {
if (!func_ref) {
func_ref = ir_LOAD_A(jit_CALL(rx, func));
}
}
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (!trace) {
ir_ref if_deprecated, ret;
if_deprecated = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
ir_CONST_U32(ZEND_ACC_DEPRECATED)));
ir_IF_TRUE_cold(if_deprecated);
if (GCC_GLOBAL_REGS) {
ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
} else {
ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
}
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH_EMPTY_FALSE(if_deprecated);
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
ir_ref ret;
if (GCC_GLOBAL_REGS) {
ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
} else {
ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
}
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
}
if (!func
&& opline->opcode != ZEND_DO_UCALL
&& opline->opcode != ZEND_DO_ICALL) {
ir_ref type_ref = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
if_user = ir_IF(ir_EQ(type_ref, ir_CONST_U8(ZEND_USER_FUNCTION)));
ir_IF_TRUE(if_user);
}
if ((!func || func->type == ZEND_USER_FUNCTION)
&& opline->opcode != ZEND_DO_ICALL) {
bool recursive_call_through_jmp = 0;
// JIT: EX(call) = NULL;
ir_STORE(jit_CALL(rx, call), IR_NULL);
// JIT: EX(return_value) = RETURN_VALUE_USED(opline) ? EX_VAR(opline->result.var) : 0;
ir_STORE(jit_CALL(rx, return_value),
RETURN_VALUE_USED(opline) ?
jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var)) :
IR_NULL);
// JIT: EX_LOAD_RUN_TIME_CACHE(op_array);
if (!func || func->op_array.cache_size) {
ir_ref run_time_cache;
if (func && op_array == &func->op_array) {
/* recursive call */
run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
} else if (func
&& !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)
&& ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)),
(uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)));
} else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) ||
(JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) {
/* Closures always use direct pointers */
ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
} else {
ir_ref if_odd, run_time_cache2;
ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1)));
ir_IF_TRUE(if_odd);
run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base))));
ir_MERGE_WITH_EMPTY_FALSE(if_odd);
run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache2, run_time_cache);
}
ir_STORE(jit_CALL(rx, run_time_cache), run_time_cache);
}
// JIT: EG(current_execute_data) = execute_data = call;
ir_STORE(jit_EG(current_execute_data), rx);
jit_STORE_FP(jit, rx);
// JIT: opline = op_array->opcodes;
if (func && !unknown_num_args) {
for (i = call_num_args; i < func->op_array.last_var; i++) {
uint32_t n = EX_NUM_TO_VAR(i);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, n);
jit_set_Z_TYPE_INFO_ex(jit, var_addr, ir_CONST_U32(IS_UNDEF));
}
if (call_num_args <= func->op_array.num_args) {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
uint32_t num_args;
if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
if (trace) {
num_args = 0;
} else if (call_info) {
num_args = skip_valid_arguments(op_array, ssa, call_info);
} else {
num_args = call_num_args;
}
} else {
num_args = call_num_args;
}
if (zend_accel_in_shm(func->op_array.opcodes)) {
jit_LOAD_IP_ADDR(jit, func->op_array.opcodes + num_args);
} else {
if (!func_ref) {
func_ref = ir_LOAD_A(jit_CALL(rx, func));
}
ir_ref ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
if (num_args) {
ip = ir_ADD_OFFSET(ip, num_args * sizeof(zend_op));
}
jit_LOAD_IP(jit, ip);
}
if (!trace && op_array == &func->op_array && call_num_args >= op_array->required_num_args) {
/* recursive call */
recursive_call_through_jmp = 1;
}
}
} else {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ir_ref ip;
if (zend_accel_in_shm(func->op_array.opcodes)) {
ip = ir_CONST_ADDR(func->op_array.opcodes);
} else {
if (!func_ref) {
func_ref = ir_LOAD_A(jit_CALL(rx, func));
}
ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
}
jit_LOAD_IP(jit, ip);
}
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
}
}
} else {
ir_ref ip;
ir_ref merge_inputs = IR_UNUSED;
// JIT: opline = op_array->opcodes
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
ip = ir_CONST_ADDR(func->op_array.opcodes);
} else {
if (!func_ref) {
func_ref = ir_LOAD_A(jit_CALL(rx, func));
}
ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
}
jit_LOAD_IP(jit, ip);
// JIT: num_args = EX_NUM_ARGS();
ir_ref num_args, first_extra_arg;
num_args = ir_LOAD_U32(jit_EX(This.u2.num_args));
if (func) {
first_extra_arg = ir_CONST_U32(func->op_array.num_args);
} else {
// JIT: first_extra_arg = op_array->num_args;
ZEND_ASSERT(func_ref);
first_extra_arg = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, num_args)));
}
// JIT: if (UNEXPECTED(num_args > first_extra_arg))
ir_ref if_extra_args = ir_IF(ir_GT(num_args, first_extra_arg));
ir_IF_TRUE_cold(if_extra_args);
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
}
ir_END_list(merge_inputs);
ir_IF_FALSE(if_extra_args);
if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
if (!func) {
// JIT: if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
ir_ref if_has_type_hints = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
ir_CONST_U32(ZEND_ACC_HAS_TYPE_HINTS)));
ir_IF_TRUE(if_has_type_hints);
ir_END_list(merge_inputs);
ir_IF_FALSE(if_has_type_hints);
}
// JIT: opline += num_args;
ir_ref ref = ir_MUL_U32(num_args, ir_CONST_U32(sizeof(zend_op)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ir_ADD_A(jit_IP(jit), ref));
} else {
ir_ref addr = jit_EX(opline);
ir_STORE(addr, ir_ADD_A(ir_LOAD_A(addr), ref));
}
}
ir_END_list(merge_inputs);
ir_MERGE_list(merge_inputs);
// JIT: if (EXPECTED((int)num_args < op_array->last_var)) {
ir_ref last_var;
if (func) {
last_var = ir_CONST_U32(func->op_array.last_var);
} else {
ZEND_ASSERT(func_ref);
last_var = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, last_var)));
}
ir_ref idx = ir_SUB_U32(last_var, num_args);
ir_ref if_need = ir_IF(ir_GT(idx, ir_CONST_U32(0)));
ir_IF_TRUE(if_need);
// JIT: zval *var = EX_VAR_NUM(num_args);
if (sizeof(void*) == 8) {
num_args = ir_ZEXT_A(num_args);
}
ir_ref var_ref = ir_ADD_OFFSET(
ir_ADD_A(jit_FP(jit), ir_MUL_A(num_args, ir_CONST_ADDR(sizeof(zval)))),
(ZEND_CALL_FRAME_SLOT * sizeof(zval)) + offsetof(zval, u1.type_info));
ir_ref loop = ir_LOOP_BEGIN(ir_END());
var_ref = ir_PHI_2(IR_ADDR, var_ref, IR_UNUSED);
idx = ir_PHI_2(IR_U32, idx, IR_UNUSED);
ir_STORE(var_ref, ir_CONST_I32(IS_UNDEF));
ir_PHI_SET_OP(var_ref, 2, ir_ADD_OFFSET(var_ref, sizeof(zval)));
ir_ref idx2 = ir_SUB_U32(idx, ir_CONST_U32(1));
ir_PHI_SET_OP(idx, 2, idx2);
ir_ref if_not_zero = ir_IF(idx2);
ir_IF_TRUE(if_not_zero);
ir_MERGE_SET_OP(loop, 2, ir_LOOP_END());
ir_IF_FALSE(if_not_zero);
ir_MERGE_WITH_EMPTY_FALSE(if_need);
}
if (ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0)) {
ir_ref observer_handler;
ir_ref rx = jit_FP(jit);
struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref);
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
jit_SET_EX_OPLINE(jit, trace[1].opline);
} else if (GCC_GLOBAL_REGS) {
// EX(opline) = opline
ir_STORE(jit_EX(opline), jit_IP(jit));
}
jit_observer_fcall_begin(jit, rx, observer_handler);
jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
}
if (trace) {
if (!func && (opline->opcode != ZEND_DO_UCALL)) {
user_path = ir_END();
}
} else {
zend_basic_block *bb;
do {
if (recursive_call_through_jmp) {
ir_ref begin, end;
ir_insn *insn;
/* attempt to convert direct recursive call into loop */
begin = jit->bb_start_ref[call_num_args];
ZEND_ASSERT(begin != IR_UNUSED);
insn = &jit->ctx.ir_base[begin];
if (insn->op == IR_BEGIN) {
end = ir_LOOP_END();
insn = &jit->ctx.ir_base[begin];
insn->op = IR_LOOP_BEGIN;
insn->inputs_count = 2;
insn->op2 = end;
break;
} else if ((insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN)
&& insn->inputs_count == 2) {
end = ir_LOOP_END();
insn = &jit->ctx.ir_base[begin];
insn->op = IR_LOOP_BEGIN;
insn->inputs_count = 3;
insn->op3 = end;
break;
} else if (insn->op == IR_LOOP_BEGIN && insn->inputs_count == 3) {
ZEND_ASSERT(jit->ctx.ir_base[insn->op3].op == IR_LOOP_END);
jit->ctx.ir_base[insn->op3].op = IR_END;
ir_MERGE_2(insn->op3, ir_END());
end = ir_LOOP_END();
insn = &jit->ctx.ir_base[begin];
insn->op3 = end;
break;
}
}
/* fallback to indirect JMP or RETURN */
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(1));
}
} while (0);
bb = &jit->ssa->cfg.blocks[jit->b];
if (bb->successors_count > 0) {
int succ;
ir_ref ref;
ZEND_ASSERT(bb->successors_count == 1);
succ = bb->successors[0];
/* Add a fake control edge from UNREACHABLE/RETURN to the following ENTRY */
ref = jit->ctx.insns_count - 1;
ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE
|| jit->ctx.ir_base[ref].op == IR_RETURN
|| jit->ctx.ir_base[ref].op == IR_LOOP_END);
ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
if (func || (opline->opcode == ZEND_DO_UCALL)) {
_zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
jit->b = -1;
} else {
user_path = ref;
}
}
}
}
if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
&& (opline->opcode != ZEND_DO_UCALL)) {
if (!func && (opline->opcode != ZEND_DO_ICALL)) {
ir_IF_FALSE(if_user);
}
if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ZEND_ASSERT(func_ref);
ir_GUARD_NOT(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
ir_CONST_U32(ZEND_ACC_DEPRECATED)),
ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_deprecated, ret;
if_deprecated = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
ir_CONST_U32(ZEND_ACC_DEPRECATED)));
ir_IF_TRUE_cold(if_deprecated);
if (GCC_GLOBAL_REGS) {
ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
} else {
ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
}
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH_EMPTY_FALSE(if_deprecated);
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
ir_ref ret;
if (GCC_GLOBAL_REGS) {
ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
} else {
ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
}
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
}
// JIT: EG(current_execute_data) = execute_data;
ir_STORE(jit_EG(current_execute_data), rx);
bool may_have_observer = ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0);
if (may_have_observer) {
ir_ref observer_handler;
struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func)));
jit_observer_fcall_begin(jit, rx, observer_handler);
jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
}
// JIT: ZVAL_NULL(EX_VAR(opline->result.var));
ir_ref res_addr = IR_UNUSED, func_ptr;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
} else {
/* CPU stack allocated temporary zval */
ir_ref ptr;
if (!jit->ctx.fixed_call_stack_size) {
// JIT: alloca(sizeof(void*));
ptr = ir_ALLOCA(ir_CONST_ADDR(sizeof(zval)));
} else {
ptr = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
}
res_addr = ZEND_ADDR_REF_ZVAL(ptr);
}
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
zend_jit_reset_last_valid_opline(jit);
// JIT: (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret);
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
if (zend_execute_internal) {
ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_execute_internal), rx, res_ref);
} else {
if (func) {
func_ptr = ir_CONST_FC_FUNC(func->internal_function.handler);
} else {
func_ptr = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_internal_function, handler)));
#if defined(IR_TARGET_X86)
func_ptr = ir_CAST_FC_FUNC(func_ptr);
#endif
}
ir_CALL_2(IR_VOID, func_ptr, rx, res_ref);
}
if (may_have_observer) {
jit_observer_fcall_end(jit, rx, res_ref);
}
// JIT: EG(current_execute_data) = execute_data;
ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
// JIT: zend_vm_stack_free_args(call);
if (func && !unknown_num_args) {
for (i = 0; i < call_num_args; i++ ) {
if (zend_jit_needs_arg_dtor(func, i, call_info)) {
uint32_t offset = EX_NUM_TO_VAR(i);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset);
jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline);
}
}
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_vm_stack_free_args_helper), rx);
}
if (may_have_extra_named_params) {
// JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS))
ir_ref if_has_named = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 3)),
ir_CONST_U8(ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24)));
ir_IF_TRUE_cold(if_has_named);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_extra_named_params),
ir_LOAD_A(jit_CALL(rx, extra_named_params)));
ir_MERGE_WITH_EMPTY_FALSE(if_has_named);
}
if (opline->opcode == ZEND_DO_FCALL) {
// TODO: optimize ???
// JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
ir_ref if_release_this = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
ir_CONST_U8(ZEND_CALL_RELEASE_THIS >> 16)));
ir_IF_TRUE_cold(if_release_this);
// JIT: OBJ_RELEASE(Z_OBJ(RX->This));
jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_CALL(rx, This.value.obj)));
ir_MERGE_WITH_EMPTY_FALSE(if_release_this);
}
ir_ref allocated_path = IR_UNUSED;
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!JIT_G(current_frame)->call ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
prev_opline->opcode == ZEND_SEND_UNPACK ||
prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
// JIT: zend_vm_stack_free_call_frame(call);
// JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_ALLOCATED))
ir_ref if_allocated = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
ir_CONST_U8(ZEND_CALL_ALLOCATED >> 16)));
ir_IF_TRUE_cold(if_allocated);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_call_frame), rx);
allocated_path = ir_END();
ir_IF_FALSE(if_allocated);
}
ir_STORE(jit_EG(vm_stack_top), rx);
if (allocated_path) {
ir_MERGE_WITH(allocated_path);
}
if (!RETURN_VALUE_USED(opline)) {
zend_class_entry *ce;
bool ce_is_instanceof;
uint32_t func_info = call_info ?
zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
(MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
/* If an exception is thrown, the return_value may stay at the
* original value of null. */
func_info |= MAY_BE_NULL;
if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
ir_ref sp = ir_RLOAD_A(IR_REG_SP);
res_addr = ZEND_ADDR_REF_ZVAL(sp);
jit_ZVAL_PTR_DTOR(jit, res_addr, func_info, 1, opline);
}
if (!jit->ctx.fixed_call_stack_size) {
// JIT: revert alloca
ir_AFREE(ir_CONST_ADDR(sizeof(zval)));
}
}
// JIT: if (UNEXPECTED(EG(exception) != NULL)) {
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_icall_throw));
// TODO: Can we avoid checking for interrupts after each call ???
if (trace && jit->last_valid_opline != opline) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else {
exit_addr = NULL;
}
if (!zend_jit_check_timeout(jit, opline + 1, exit_addr)) {
return 0;
}
if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
jit_LOAD_IP_ADDR(jit, opline + 1);
} else if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
jit_LOAD_IP_ADDR(jit, opline + 1);
}
}
if (user_path) {
ir_MERGE_WITH(user_path);
}
return 1;
}
static int zend_jit_constructor(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, int next_block)
{
ir_ref if_skip_constructor = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, opline), next_block);
ir_IF_FALSE(if_skip_constructor);
if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
if (!zend_jit_tail_handler(jit, opline)) {
return 0;
}
} else {
if (!zend_jit_do_fcall(jit, opline, op_array, ssa, call_level, next_block, NULL)) {
return 0;
}
}
/* override predecessors of the next block */
ZEND_ASSERT(jit->ssa->cfg.blocks[next_block].predecessors_count == 1);
if (!jit->ctx.control) {
ZEND_ASSERT(jit->bb_edges[jit->bb_predecessors[next_block]]);
ir_IF_TRUE(if_skip_constructor);
ir_MERGE_2(jit->bb_edges[jit->bb_predecessors[next_block]], ir_END());
jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
} else {
ZEND_ASSERT(!jit->bb_edges[jit->bb_predecessors[next_block]]);
/* merge current control path with the true branch of constructor skip condition */
ir_MERGE_WITH_EMPTY_TRUE(if_skip_constructor);
jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
jit->b = -1;
}
return 1;
}
static int zend_jit_verify_arg_type(zend_jit_ctx *jit, const zend_op *opline, zend_arg_info *arg_info, bool check_exception)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
ir_ref ref, fast_path = IR_UNUSED;
ref = jit_ZVAL_ADDR(jit, res_addr);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->prev) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
if (type != IS_UNKNOWN && (type_mask & (1u << type))) {
return 1;
}
}
if (ZEND_ARG_SEND_MODE(arg_info)) {
if (opline->opcode == ZEND_RECV_INIT) {
ref = jit_ZVAL_DEREF_ref(jit, ref);
} else {
ref = jit_Z_PTR_ref(jit, ref);
ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
}
}
if (type_mask != 0) {
if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
ir_ref if_ok = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(type_code));
ir_IF_TRUE(if_ok);
fast_path = ir_END();
ir_IF_FALSE_cold(if_ok);
} else {
ir_ref if_ok = ir_IF(ir_AND_U32(
ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE_ref(jit, ref)),
ir_CONST_U32(type_mask)));
ir_IF_TRUE(if_ok);
fast_path = ir_END();
ir_IF_FALSE_cold(if_ok);
}
}
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_verify_arg_slow),
ref, ir_CONST_ADDR(arg_info));
if (check_exception) {
ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
if (fast_path) {
ir_MERGE_WITH(fast_path);
}
return 1;
}
static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array)
{
uint32_t arg_num = opline->op1.num;
zend_arg_info *arg_info = NULL;
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
if (EXPECTED(arg_num <= op_array->num_args)) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
arg_info = &op_array->arg_info[op_array->num_args];
}
if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
arg_info = NULL;
}
}
if (arg_info || (opline+1)->opcode != ZEND_RECV) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 ||
arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)),
ir_CONST_ADDR(exit_addr));
}
} else {
ir_ref if_ok =ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
ir_IF_FALSE_cold(if_ok);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_missing_arg_error), jit_FP(jit));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_IF_TRUE(if_ok);
}
}
if (arg_info) {
if (!zend_jit_verify_arg_type(jit, opline, arg_info, 1)) {
return 0;
}
}
return 1;
}
static int zend_jit_recv_init(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw)
{
uint32_t arg_num = opline->op1.num;
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ir_ref ref, if_fail, skip_path = IR_UNUSED;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) {
if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
jit_ZVAL_COPY_CONST(jit,
res_addr,
-1, -1,
zv, 1);
}
} else {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
ir_ref if_skip = ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
ir_IF_TRUE(if_skip);
skip_path = ir_END();
ir_IF_FALSE(if_skip);
}
jit_ZVAL_COPY_CONST(jit,
res_addr,
-1, -1,
zv, 1);
}
if (Z_CONSTANT_P(zv)) {
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zval_update_constant_ex),
jit_ZVAL_ADDR(jit, res_addr),
ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), offsetof(zend_op_array, scope))));
if_fail = ir_IF(ref);
ir_IF_TRUE_cold(if_fail);
jit_ZVAL_PTR_DTOR(jit, res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, opline);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_IF_FALSE(if_fail);
}
if (skip_path) {
ir_MERGE_WITH(skip_path);
}
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
do {
zend_arg_info *arg_info;
if (arg_num <= op_array->num_args) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
arg_info = &op_array->arg_info[op_array->num_args];
} else {
break;
}
if (!ZEND_TYPE_IS_SET(arg_info->type)) {
break;
}
if (!zend_jit_verify_arg_type(jit, opline, arg_info, may_throw)) {
return 0;
}
} while (0);
}
return 1;
}
static bool zend_jit_verify_return_type(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
{
zend_arg_info *arg_info = &op_array->arg_info[-1];
ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
zend_jit_addr op1_addr = OP1_ADDR();
bool needs_slow_check = 1;
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
ir_ref fast_path = IR_UNUSED;
if (type_mask != 0) {
if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
/* pass */
} else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
needs_slow_check = 0;
} else if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
ir_ref if_ok = jit_if_Z_TYPE(jit, op1_addr, type_code);
ir_IF_TRUE(if_ok);
fast_path = ir_END();
ir_IF_FALSE_cold(if_ok);
} else {
ir_ref if_ok = ir_IF(ir_AND_U32(
ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)),
ir_CONST_U32(type_mask)));
ir_IF_TRUE(if_ok);
fast_path = ir_END();
ir_IF_FALSE_cold(if_ok);
}
}
if (needs_slow_check) {
ir_ref ref;
jit_SET_EX_OPLINE(jit, opline);
ref = jit_ZVAL_ADDR(jit, op1_addr);
if (op1_info & MAY_BE_UNDEF) {
ref = zend_jit_zval_check_undef(jit, ref, opline->op1.var, NULL, 1);
}
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_verify_return_slow),
ref,
ir_LOAD_A(jit_EX(func)),
ir_CONST_ADDR(arg_info),
ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->op2.num));
zend_jit_check_exception(jit);
if (fast_path) {
ir_MERGE_WITH(fast_path);
}
}
return 1;
}
static int zend_jit_leave_frame(zend_jit_ctx *jit)
{
// JIT: EG(current_execute_data) = EX(prev_execute_data);
ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
return 1;
}
static int zend_jit_free_cvs(zend_jit_ctx *jit)
{
// JIT: EG(current_execute_data) = EX(prev_execute_data);
ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
// JIT: zend_free_compiled_variables(execute_data);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_compiled_variables), jit_FP(jit));
return 1;
}
static int zend_jit_free_cv(zend_jit_ctx *jit, uint32_t info, uint32_t var)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
jit_ZVAL_PTR_DTOR(jit, var_addr, info, 1, NULL);
}
return 1;
}
static int zend_jit_free_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t info, uint32_t var_offset)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, opline);
}
return 1;
}
static int zend_jit_leave_func(zend_jit_ctx *jit,
const zend_op_array *op_array,
const zend_op *opline,
uint32_t op1_info,
bool left_frame,
zend_jit_trace_rec *trace,
zend_jit_trace_info *trace_info,
int indirect_var_access,
int may_throw)
{
bool may_be_top_frame =
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
bool may_need_call_helper =
indirect_var_access || /* may have symbol table */
!op_array->function_name || /* may have symbol table */
may_be_top_frame ||
(op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
(uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
bool may_need_release_this =
!(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
op_array->scope &&
!(op_array->fn_flags & ZEND_ACC_STATIC) &&
(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));
ir_ref call_info = IR_UNUSED, ref, cold_path = IR_UNUSED;
if (may_need_call_helper) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(jit)) {
return 0;
}
}
/* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
ref = ir_AND_U32(call_info,
ir_CONST_U32(ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE));
if (trace && trace->op != ZEND_JIT_TRACE_END) {
ir_ref if_slow = ir_IF(ref);
ir_IF_TRUE_cold(if_slow);
if (!GCC_GLOBAL_REGS) {
ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_func_helper), jit_FP(jit));
} else {
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_func_helper));
}
if (may_be_top_frame) {
// TODO: try to avoid this check ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
/* this check should be handled by the following OPLINE guard */
| cmp IP, zend_jit_halt_op
| je ->trace_halt
#endif
} else if (GCC_GLOBAL_REGS) {
ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
} else {
ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
}
}
if (!GCC_GLOBAL_REGS) {
// execute_data = EG(current_execute_data)
jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
}
cold_path = ir_END();
ir_IF_FALSE(if_slow);
} else {
ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_leave_function_handler));
}
}
if ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) == ZEND_ACC_CLOSURE) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(jit)) {
return 0;
}
}
// JIT: OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
jit_OBJ_RELEASE(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), -sizeof(zend_object)));
} else if (may_need_release_this) {
ir_ref if_release, fast_path = IR_UNUSED;
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(jit)) {
return 0;
}
}
if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) {
// JIT: if (call_info & ZEND_CALL_RELEASE_THIS)
if (!call_info) {
call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
}
if_release = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_RELEASE_THIS)));
ir_IF_FALSE(if_release);
fast_path = ir_END();
ir_IF_TRUE(if_release);
}
// JIT: OBJ_RELEASE(execute_data->This))
jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_EX(This.value.obj)));
if (fast_path) {
ir_MERGE_WITH(fast_path);
}
// TODO: avoid EG(excption) check for $this->foo() calls
may_throw = 1;
}
// JIT: EG(vm_stack_top) = (zval*)execute_data
ir_STORE(jit_EG(vm_stack_top), jit_FP(jit));
// JITL execute_data = EX(prev_execute_data)
jit_STORE_FP(jit, ir_LOAD_A(jit_EX(prev_execute_data)));
if (!left_frame) {
// JIT: EG(current_execute_data) = execute_data
ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
}
if (trace) {
if (trace->op != ZEND_JIT_TRACE_END
&& (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
zend_jit_reset_last_valid_opline(jit);
} else {
if (GCC_GLOBAL_REGS) {
/* We add extra RLOAD and RSTORE to make fusion for persistent register
* mov (%FP), %IP
* add $0x1c, %IP
* The naive (commented) code leads to extra register allocation and move.
* mov (%FP), %tmp
* add $0x1c, %tmp
* mov %tmp, %FP
*/
#if 0
jit_STORE_IP(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(opline)), sizeof(zend_op)));
#else
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
#endif
} else {
ir_ref ref = jit_EX(opline);
ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
}
}
if (cold_path) {
ir_MERGE_WITH(cold_path);
}
if (trace->op == ZEND_JIT_TRACE_BACK
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
const zend_op *next_opline = trace->opline;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
/* exception might be thrown during destruction of unused return value */
// JIT: if (EG(exception))
ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
}
do {
trace++;
} while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
next_opline = trace->opline;
ZEND_ASSERT(next_opline != NULL);
if (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
trace_info->flags |= ZEND_JIT_TRACE_LOOP;
ir_ref if_eq = ir_IF(jit_CMP_IP(jit, IR_EQ, next_opline));
ir_IF_TRUE(if_eq);
ZEND_ASSERT(jit->trace_loop_ref);
ZEND_ASSERT(jit->ctx.ir_base[jit->trace_loop_ref].op2 == IR_UNUSED);
ir_MERGE_SET_OP(jit->trace_loop_ref, 2, ir_END());
ir_IF_FALSE(if_eq);
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
#else
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
#endif
} else {
ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), jit_STUB_ADDR(jit, jit_stub_trace_escape));
}
zend_jit_set_last_valid_opline(jit, trace->opline);
return 1;
} else if (may_throw ||
(((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
// JIT: if (EG(exception))
ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
}
return 1;
} else {
// JIT: if (EG(exception))
ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
// JIT: opline = EX(opline) + 1
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
} else {
ir_ref ref = jit_EX(opline);
ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
}
}
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
}
jit->b = -1;
return 1;
}
static void zend_jit_common_return(zend_jit_ctx *jit)
{
ZEND_ASSERT(jit->return_inputs);
ir_MERGE_list(jit->return_inputs);
}
static int zend_jit_return(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
{
zend_jit_addr ret_addr;
int8_t return_value_used = -1;
ir_ref return_value = IR_UNUSED, ref, refcount, if_return_value_used = IR_UNUSED;
ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
jit->return_inputs = IR_UNUSED;
if (JIT_G(current_frame)) {
if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
return_value_used = 1;
} else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
return_value_used = 0;
} else {
return_value_used = -1;
}
}
}
if (ZEND_OBSERVER_ENABLED) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, dst, op1_info)) {
return 0;
}
op1_addr = dst;
}
jit_observer_fcall_end(jit, jit_FP(jit), jit_ZVAL_ADDR(jit, op1_addr));
}
// JIT: if (!EX(return_value))
return_value = ir_LOAD_A(jit_EX(return_value));
ret_addr = ZEND_ADDR_REF_ZVAL(return_value);
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (return_value_used == -1) {
if_return_value_used = ir_IF(return_value);
ir_IF_FALSE_cold(if_return_value_used);
}
if (return_value_used != 1) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_ref if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
ir_IF_FALSE(if_refcounted);
ir_END_list(jit->return_inputs);
ir_IF_TRUE(if_refcounted);
}
ref = jit_Z_PTR(jit, op1_addr);
refcount = jit_GC_DELREF(jit, ref);
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
ir_ref if_non_zero = ir_IF(refcount);
ir_IF_TRUE(if_non_zero);
ir_END_list(jit->return_inputs);
ir_IF_FALSE(if_non_zero);
}
jit_ZVAL_DTOR(jit, ref, op1_info, opline);
}
if (return_value_used == -1) {
ir_END_list(jit->return_inputs);
}
}
} else if (return_value_used == -1) {
if_return_value_used = ir_IF(return_value);
ir_IF_FALSE_cold(if_return_value_used);
ir_END_list(jit->return_inputs);
}
if (if_return_value_used) {
ir_IF_TRUE(if_return_value_used);
}
if (return_value_used == 0) {
if (jit->return_inputs) {
ZEND_ASSERT(JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE);
ir_END_list(jit->return_inputs);
ir_MERGE_list(jit->return_inputs);
jit->return_inputs = IR_UNUSED;
}
return 1;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
jit_ZVAL_COPY_CONST(jit, ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
} else if (opline->op1_type == IS_TMP_VAR) {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_REF) {
ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
!op_array->function_name) {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 1);
} else if (return_value_used != 1) {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
// JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_NULL);
} else {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
}
} else {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
}
} else {
if (op1_info & MAY_BE_REF) {
ir_ref if_ref, ref2, if_non_zero;
zend_jit_addr ref_addr;
if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_TRUE_cold(if_ref);
// JIT: zend_refcounted *ref = Z_COUNTED_P(retval_ptr)
ref = jit_Z_PTR(jit, op1_addr);
// JIT: ZVAL_COPY_VALUE(return_value, &ref->value)
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, ref_addr, op1_info, 0);
ref2 = jit_GC_DELREF(jit, ref);
if_non_zero = ir_IF(ref2);
ir_IF_TRUE(if_non_zero);
// JIT: if (IS_REFCOUNTED())
ir_ref if_refcounted = jit_if_REFCOUNTED(jit, ret_addr);
ir_IF_FALSE(if_refcounted);
ir_END_list(jit->return_inputs);
ir_IF_TRUE(if_refcounted);
// JIT: ADDREF
ref2 = jit_Z_PTR(jit, ret_addr);
jit_GC_ADDREF(jit, ref2);
ir_END_list(jit->return_inputs);
ir_IF_FALSE(if_non_zero);
jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
ir_END_list(jit->return_inputs);
ir_IF_FALSE(if_ref);
}
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (jit->return_inputs) {
ir_END_list(jit->return_inputs);
ir_MERGE_list(jit->return_inputs);
jit->return_inputs = IR_UNUSED;
}
} else {
ir_END_list(jit->return_inputs);
jit->b = -1;
}
return 1;
}
static int zend_jit_bind_global(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr op1_addr = OP1_ADDR();
zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));
ir_ref cache_slot_ref, idx_ref, num_used_ref, bucket_ref, ref, ref2;
ir_ref if_fit, if_reference, if_same_key, fast_path;
ir_ref slow_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
// JIT: idx = (uintptr_t)CACHED_PTR(opline->extended_value) - 1;
cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
idx_ref = ir_SUB_A(ir_LOAD_A(cache_slot_ref), ir_CONST_ADDR(1));
// JIT: if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
num_used_ref = ir_MUL_U32(ir_LOAD_U32(jit_EG(symbol_table.nNumUsed)),
ir_CONST_U32(sizeof(Bucket)));
if (sizeof(void*) == 8) {
num_used_ref = ir_ZEXT_A(num_used_ref);
}
if_fit = ir_IF(ir_ULT(idx_ref, num_used_ref));
ir_IF_FALSE_cold(if_fit);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_fit);
// JIT: Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
bucket_ref = ir_ADD_A(ir_LOAD_A(jit_EG(symbol_table.arData)), idx_ref);
if_reference = jit_if_Z_TYPE_ref(jit, bucket_ref, ir_CONST_U8(IS_REFERENCE));
ir_IF_FALSE_cold(if_reference);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_reference);
// JIT: (EXPECTED(p->key == varname))
if_same_key = ir_IF(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(bucket_ref, offsetof(Bucket, key))), ir_CONST_ADDR(varname)));
ir_IF_FALSE_cold(if_same_key);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same_key);
// JIT: GC_ADDREF(Z_PTR(p->val))
ref = jit_Z_PTR_ref(jit, bucket_ref);
jit_GC_ADDREF(jit, ref);
fast_path = ir_END();
ir_MERGE_list(slow_inputs);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_global_helper),
ir_CONST_ADDR(varname),
cache_slot_ref);
ir_MERGE_WITH(fast_path);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
ir_ref if_refcounted = IR_UNUSED, refcount, if_non_zero, if_may_not_leak;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
// JIT: if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
ir_IF_TRUE_cold(if_refcounted);
}
// JIT:zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
ref2 = jit_Z_PTR(jit, op1_addr);
// JIT: ZVAL_REF(variable_ptr, ref)
jit_set_Z_PTR(jit, op1_addr, ref);
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
// JIT: if (GC_DELREF(garbage) == 0)
refcount = jit_GC_DELREF(jit, ref2);
if_non_zero = ir_IF(refcount);
if (!(op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
ir_IF_TRUE(if_non_zero);
ir_END_list(end_inputs);
}
ir_IF_FALSE(if_non_zero);
jit_ZVAL_DTOR(jit, ref2, op1_info, opline);
if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
ir_END_list(end_inputs);
ir_IF_TRUE(if_non_zero);
// JIT: GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref2);
ir_IF_TRUE(if_may_not_leak);
ir_END_list(end_inputs);
ir_IF_FALSE(if_may_not_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref2);
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_END_list(end_inputs);
ir_IF_FALSE(if_refcounted);
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
// JIT: ZVAL_REF(variable_ptr, ref)
jit_set_Z_PTR(jit, op1_addr, ref);
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
}
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
return 1;
}
static int zend_jit_free(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, int may_throw)
{
zend_jit_addr op1_addr = OP1_ADDR();
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
if (may_throw) {
jit_SET_EX_OPLINE(jit, opline);
}
if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
ir_ref ref, if_array, if_exists, end_inputs = IR_UNUSED;
if (op1_info & MAY_BE_ARRAY) {
if_array = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
ir_IF_TRUE(if_array);
ir_END_list(end_inputs);
ir_IF_FALSE(if_array);
}
ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_iter_idx)));
if_exists = ir_IF(ir_EQ(ref, ir_CONST_U32(-1)));
ir_IF_TRUE(if_exists);
ir_END_list(end_inputs);
ir_IF_FALSE(if_exists);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_hash_iterator_del), ref);
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
}
return 1;
}
static int zend_jit_echo(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
if (len > 0) {
const char *str = Z_STRVAL_P(zv);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
ir_CONST_ADDR(str), ir_CONST_ADDR(len));
zend_jit_check_exception(jit);
}
} else {
zend_jit_addr op1_addr = OP1_ADDR();
ir_ref ref;
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
jit_SET_EX_OPLINE(jit, opline);
ref = jit_Z_PTR(jit, op1_addr);
ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
ir_ADD_OFFSET(ref, offsetof(zend_string, val)),
ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_string, len))));
if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
}
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_strlen(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(len));
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
} else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
ir_ref ref;
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
ref = jit_Z_PTR(jit, op1_addr);
ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(zend_string, len)));
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) == IS_REG) {
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
return 1;
}
static int zend_jit_count(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
zend_long count;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY);
count = zend_hash_num_elements(Z_ARRVAL_P(zv));
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(count));
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
} else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
ir_ref ref;
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY);
// Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+.
ref = jit_Z_PTR(jit, op1_addr);
if (sizeof(void*) == 8) {
ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
ref = ir_ZEXT_L(ref);
} else {
ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
}
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) == IS_REG) {
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_in_array(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ir_ref ref;
ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);
// JIT: result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
if (opline->op1_type != IS_CONST) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find),
ir_CONST_ADDR(ht),
jit_Z_PTR(jit, op1_addr));
} else {
zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash),
ir_CONST_ADDR(ht), ir_CONST_ADDR(str));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
ZEND_ASSERT(bb->successors_count == 2);
ref = jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ) ? target_label2 : target_label);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
jit->b = -1;
} else {
jit_set_Z_TYPE_INFO_ex(jit, res_addr,
ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, IR_NULL)), ir_CONST_U32(IS_FALSE)));
}
return 1;
}
static int zend_jit_rope(zend_jit_ctx *jit, const zend_op *opline, uint32_t op2_info)
{
uint32_t offset;
offset = (opline->opcode == ZEND_ROPE_INIT) ?
opline->result.var :
opline->op1.var + opline->extended_value * sizeof(zend_string*);
if (opline->op2_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_string *str;
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
str = Z_STR_P(zv);
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ir_CONST_ADDR(str));
} else {
zend_jit_addr op2_addr = OP2_ADDR();
ir_ref ref;
ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
ref = jit_Z_PTR(jit, op2_addr);
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ref);
if (opline->op2_type == IS_CV) {
ir_ref if_refcounted, long_path;
if_refcounted = jit_if_REFCOUNTED(jit, op2_addr);
ir_IF_TRUE(if_refcounted);
jit_GC_ADDREF(jit, ref);
long_path = ir_END();
ir_IF_FALSE(if_refcounted);
ir_MERGE_WITH(long_path);
}
}
if (opline->opcode == ZEND_ROPE_END) {
zend_jit_addr res_addr = RES_ADDR();
ir_ref ref;
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_rope_end),
ir_ADD_OFFSET(jit_FP(jit), opline->op1.var),
ir_CONST_U32(opline->extended_value));
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
}
return 1;
}
static int zend_jit_zval_copy_deref(zend_jit_ctx *jit, zend_jit_addr res_addr, zend_jit_addr val_addr, ir_ref type)
{
ir_ref if_refcounted, if_reference, if_refcounted2, ptr, val2, ptr2, type2;
ir_refs *merge_inputs, *types, *ptrs;
#if SIZEOF_ZEND_LONG == 4
ir_ref val = jit_ZVAL_ADDR(jit, val_addr);
ir_refs *values; /* we need this only for zval.w2 copy */
#endif
ir_refs_init(merge_inputs, 4);
ir_refs_init(types, 4);
ir_refs_init(ptrs, 4);
#if SIZEOF_ZEND_LONG == 4
ir_refs_init(values, 4);
#endif
// JIT: ptr = Z_PTR_P(val);
ptr = jit_Z_PTR(jit, val_addr);
// JIT: if (Z_OPT_REFCOUNTED_P(val)) {
if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
ir_IF_FALSE_cold(if_refcounted);
ir_refs_add(merge_inputs, ir_END());
ir_refs_add(types, type);
ir_refs_add(ptrs, ptr);
#if SIZEOF_ZEND_LONG == 4
ir_refs_add(values, val);
#endif
ir_IF_TRUE(if_refcounted);
// JIT: if (UNEXPECTED(Z_OPT_ISREF_P(val))) {
if_reference = ir_IF(ir_EQ(type, ir_CONST_U32(IS_REFERENCE_EX)));
// if_reference = ir_IF(ir_EQ(ir_TRUNC_U8(type), ir_CONST_U8(IS_REFERENCE))); // TODO: fix IR to avoid need for extra register ???
ir_IF_TRUE(if_reference);
// JIT: val = Z_REFVAL_P(val);
val2 = ir_ADD_OFFSET(ptr, offsetof(zend_reference, val));
type2 = jit_Z_TYPE_INFO_ref(jit, val2);
ptr2 = jit_Z_PTR_ref(jit, val2);
// JIT: if (Z_OPT_REFCOUNTED_P(val)) {
if_refcounted2 = ir_IF(ir_AND_U32(type2, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
ir_IF_FALSE_cold(if_refcounted2);
ir_refs_add(merge_inputs, ir_END());
ir_refs_add(types, type2);
ir_refs_add(ptrs, ptr2);
#if SIZEOF_ZEND_LONG == 4
ir_refs_add(values, val2);
#endif
ir_IF_TRUE(if_refcounted2);
ir_MERGE_WITH_EMPTY_FALSE(if_reference);
type = ir_PHI_2(IR_U32, type2, type);
ptr = ir_PHI_2(IR_ADDR, ptr2, ptr);
#if SIZEOF_ZEND_LONG == 4
val = ir_PHI_2(IR_ADDR, val2, val);
#endif
// JIT: Z_ADDREF_P(val);
jit_GC_ADDREF(jit, ptr);
ir_refs_add(merge_inputs, ir_END());
ir_refs_add(types, type);
ir_refs_add(ptrs, ptr);
#if SIZEOF_ZEND_LONG == 4
ir_refs_add(values, val);
#endif
ir_MERGE_N(merge_inputs->count, merge_inputs->refs);
type = ir_PHI_N(IR_U32, types->count, types->refs);
ptr = ir_PHI_N(IR_ADDR, ptrs->count, ptrs->refs);
#if SIZEOF_ZEND_LONG == 4
val = ir_PHI_N(IR_ADDR, values->count, values->refs);
val_addr = ZEND_ADDR_REF_ZVAL(val);
#endif
// JIT: Z_PTR_P(res) = ptr;
jit_set_Z_PTR(jit, res_addr, ptr);
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_W2(jit, res_addr, jit_Z_W2(jit, val_addr));
#endif
jit_set_Z_TYPE_INFO_ex(jit, res_addr, type);
return 1;
}
static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t type,
uint32_t op1_info,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint8_t dim_type,
const void *found_exit_addr,
const void *not_found_exit_addr,
const void *exit_addr,
bool result_type_guard,
ir_ref ht_ref,
ir_refs *found_inputs,
ir_refs *found_vals,
ir_ref *end_inputs,
ir_ref *not_found_inputs)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ir_ref ref = IR_UNUSED, cond, if_found;
ir_ref if_type = IS_UNUSED;
ir_refs *test_zval_inputs, *test_zval_values;
ir_refs_init(test_zval_inputs, 4);
ir_refs_init(test_zval_values, 4);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& type == BP_VAR_R
&& !exit_addr) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (op2_info & MAY_BE_LONG) {
bool op2_loaded = 0;
bool packed_loaded = 0;
bool bad_packed_key = 0;
ir_ref if_packed = IS_UNDEF;
ir_ref h = IR_UNUSED;
ir_ref idx_not_found_inputs = IR_UNUSED;
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
// JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
if_type = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_TRUE(if_type);
}
if (op1_info & MAY_BE_PACKED_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
cond = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
ir_CONST_U32(HASH_FLAG_PACKED));
if (op1_info & MAY_BE_ARRAY_PACKED) {
ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
}
}
if (type == BP_VAR_W) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
op2_loaded = 1;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
zend_long val = -1;
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
val = Z_LVAL_P(Z_ZV(op2_addr));
if (val >= 0 && val < HT_MAX_SIZE) {
packed_loaded = 1;
} else {
bad_packed_key = 1;
}
h = ir_CONST_LONG(val);
} else {
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
op2_loaded = 1;
}
packed_loaded = 1;
}
if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) {
/* don't generate "fast" code for packed array */
packed_loaded = 0;
}
if (packed_loaded) {
// JIT: ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
if_packed = ir_IF(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
ir_CONST_U32(HASH_FLAG_PACKED)));
ir_IF_TRUE(if_packed);
}
// JIT: if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
ref = ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed)));
#if SIZEOF_ZEND_LONG == 8
if ((Z_MODE(op2_addr) == IS_CONST_ZVAL && val >= 0 && val <= UINT32_MAX)
|| (op2_range && op2_range->min >= 0 && op2_range->max <= UINT32_MAX)) {
/* comapre only the lower 32-bits to allow load fusion on x86_64 */
cond = ir_ULT(ir_TRUNC_U32(h), ref);
} else {
cond = ir_ULT(h, ir_ZEXT_L(ref));
}
#else
cond = ir_ULT(h, ref);
#endif
if (type == BP_JIT_IS) {
if (not_found_exit_addr) {
ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
} else {
ir_ref if_fit = ir_IF(cond);
ir_IF_FALSE(if_fit);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_fit);
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_RW && not_found_exit_addr) {
ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_IS && result_type_guard) {
ir_ref if_fit = ir_IF(cond);
ir_IF_FALSE(if_fit);
ir_END_list(*not_found_inputs);
ir_IF_TRUE(if_fit);
} else {
ir_ref if_fit = ir_IF(cond);
ir_IF_FALSE(if_fit);
ir_END_list(idx_not_found_inputs);
ir_IF_TRUE(if_fit);
}
// JIT: _ret = &_ht->arPacked[h];
ref = ir_MUL_L(h, ir_CONST_LONG(sizeof(zval)));
ref = ir_BITCAST_A(ref);
ref = ir_ADD_A(ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))), ref);
if (type == BP_JIT_IS) {
ir_refs_add(test_zval_values, ref);
ir_refs_add(test_zval_inputs, ir_END());
}
}
}
switch (type) {
case BP_JIT_IS:
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
if (if_packed) {
ir_IF_FALSE(if_packed);
if_packed = IR_UNUSED;
}
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
}
if (packed_loaded) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
}
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(test_zval_values, ref);
ir_refs_add(test_zval_inputs, ir_END());
} else if (!not_found_exit_addr && !packed_loaded) {
ir_END_list(*end_inputs);
}
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (packed_loaded) {
ir_ref type_ref = jit_Z_TYPE_ref(jit, ref);
if (result_type_guard) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
ir_GUARD(type_ref, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
ir_GUARD(type_ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
ir_ref if_def = ir_IF(type_ref);
ir_IF_FALSE(if_def);
ir_END_list(idx_not_found_inputs);
ir_IF_TRUE(if_def);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
}
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
if (if_packed) {
ir_IF_FALSE(if_packed);
if_packed = IR_UNUSED;
}
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
}
if (packed_loaded) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_IS && result_type_guard) {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*not_found_inputs);
ir_IF_TRUE(if_found);
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(idx_not_found_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
} else if (!packed_loaded) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_IS && result_type_guard) {
ir_END_list(*not_found_inputs);
} else {
ir_END_list(idx_not_found_inputs);
}
}
if (idx_not_found_inputs) {
ir_MERGE_list(idx_not_found_inputs);
switch (type) {
case BP_VAR_R:
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
// JIT: zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
// JIT: retval = &EG(uninitialized_zval);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_offset, IR_FASTCALL_FUNC));
ir_END_list(*end_inputs);
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
if (!not_found_exit_addr) {
// JIT: retval = &EG(uninitialized_zval);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_END_list(*end_inputs);
}
break;
default:
ZEND_UNREACHABLE();
}
}
break;
case BP_VAR_RW:
if (packed_loaded) {
if (not_found_exit_addr) {
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
ir_IF_TRUE(if_def);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
ir_IF_FALSE_cold(if_def);
ir_END_list(idx_not_found_inputs);
}
}
if (!packed_loaded ||
!not_found_exit_addr ||
(op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
if (if_packed) {
ir_IF_FALSE(if_packed);
if_packed = IR_UNUSED;
ir_END_list(idx_not_found_inputs);
} else if (!packed_loaded) {
ir_END_list(idx_not_found_inputs);
}
ir_MERGE_list(idx_not_found_inputs);
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
}
if (packed_loaded) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw_no_packed),
ht_ref, h);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw), ht_ref, h);
}
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
}
break;
case BP_VAR_W:
if (packed_loaded) {
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
ir_IF_TRUE_cold(if_def);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
ir_IF_FALSE(if_def);
ir_END_list(idx_not_found_inputs);
}
if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) {
if (if_packed) {
ir_IF_FALSE(if_packed);
if_packed = IR_UNUSED;
ir_END_list(idx_not_found_inputs);
} else if (!packed_loaded) {
ir_END_list(idx_not_found_inputs);
}
ir_MERGE_list(idx_not_found_inputs);
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
}
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_lookup), ht_ref, h);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
}
break;
default:
ZEND_UNREACHABLE();
}
}
if (op2_info & MAY_BE_STRING) {
ir_ref key;
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IS_UNUSED;
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
// JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
if_type = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
ir_IF_TRUE(if_type);
}
// JIT: offset_key = Z_STR_P(dim);
key = jit_Z_PTR(jit, op2_addr);
// JIT: retval = zend_hash_find(ht, offset_key);
switch (type) {
case BP_JIT_IS:
if (opline->op2_type != IS_CONST) {
ir_ref if_num, end1, ref2;
if_num = ir_IF(
ir_ULE(
ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
ir_CONST_CHAR('9')));
ir_IF_TRUE_cold(if_num);
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
end1 = ir_END();
ir_IF_FALSE(if_num);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
ir_MERGE_WITH(end1);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
}
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(test_zval_values, ref);
ir_refs_add(test_zval_inputs, ir_END());
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (opline->op2_type != IS_CONST) {
ir_ref if_num, end1, ref2;
if_num = ir_IF(
ir_ULE(
ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
ir_CONST_CHAR('9')));
ir_IF_TRUE_cold(if_num);
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
end1 = ir_END();
ir_IF_FALSE(if_num);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
ir_MERGE_WITH(end1);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_IS && result_type_guard) {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*not_found_inputs);
ir_IF_TRUE(if_found);
} else {
if_found = ir_IF(ref);
switch (type) {
case BP_VAR_R:
ir_IF_FALSE_cold(if_found);
// JIT: zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
jit_SET_EX_OPLINE(jit, opline);
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_key, IR_FASTCALL_FUNC));
ir_END_list(*end_inputs);
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
ir_IF_FALSE(if_found);
// JIT: retval = &EG(uninitialized_zval);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_END_list(*end_inputs);
break;
default:
ZEND_UNREACHABLE();
}
ir_IF_TRUE(if_found);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
break;
case BP_VAR_RW:
if (opline->op2_type != IS_CONST) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_rw), ht_ref, key);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_lookup_rw), ht_ref, key);
}
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
break;
case BP_VAR_W:
if (opline->op2_type != IS_CONST) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_w), ht_ref, key);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_lookup), ht_ref, key);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
break;
default:
ZEND_UNREACHABLE();
}
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNDEF;
}
if (type != BP_VAR_RW) {
jit_SET_EX_OPLINE(jit, opline);
}
ref = jit_ZVAL_ADDR(jit, op2_addr);
switch (type) {
case BP_VAR_R:
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_r_helper),
ht_ref,
ref,
jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(*end_inputs);
break;
case BP_JIT_IS:
ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_fetch_dim_isset_helper), ht_ref, ref);
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
ir_refs_add(found_inputs, ir_END());
} else if (found_exit_addr) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(found_exit_addr));
ir_END_list(*end_inputs);
} else {
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
ir_refs_add(found_inputs, ir_END());
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_is_helper),
ht_ref,
ref,
jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(*end_inputs);
break;
case BP_VAR_RW:
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_rw_helper), ht_ref, ref);
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
break;
case BP_VAR_W:
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_w_helper), ht_ref, ref);
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
break;
default:
ZEND_UNREACHABLE();
}
}
if (type == BP_JIT_IS
&& !(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
/* dead code */
ir_END_list(*end_inputs);
} else if (type == BP_JIT_IS
&& (op1_info & MAY_BE_ARRAY)
&& (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
&& test_zval_inputs->count) {
ir_MERGE_N(test_zval_inputs->count, test_zval_inputs->refs);
ref = ir_PHI_N(IR_ADDR, test_zval_values->count, test_zval_values->refs);
if (op1_info & MAY_BE_ARRAY_OF_REF) {
ref = jit_ZVAL_DEREF_ref(jit, ref);
}
cond = ir_GT(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_NULL));
if (not_found_exit_addr) {
ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
ir_refs_add(found_inputs, ir_END());
} else if (found_exit_addr) {
ir_GUARD_NOT(cond, ir_CONST_ADDR(found_exit_addr));
ir_END_list(*end_inputs);
} else {
ir_ref if_set = ir_IF(cond);
ir_IF_FALSE(if_set);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_set);
ir_refs_add(found_inputs, ir_END());
}
}
return 1;
}
static int zend_jit_fetch_dim_read(zend_jit_ctx *jit,
const zend_op *opline,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_avoid_refcounting,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint32_t res_info,
zend_jit_addr res_addr,
uint8_t dim_type)
{
zend_jit_addr orig_op1_addr;
const void *exit_addr = NULL;
const void *not_found_exit_addr = NULL;
bool result_type_guard = 0;
bool result_avoid_refcounting = 0;
uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
int may_throw = 0;
ir_ref if_type = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref not_found_inputs = IR_UNUSED;
orig_op1_addr = OP1_ADDR();
if (opline->opcode != ZEND_FETCH_DIM_IS
&& JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& !has_concrete_type(op1_info)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if ((res_info & MAY_BE_GUARD)
&& JIT_G(current_frame)
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
result_type_guard = 1;
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& (opline->opcode == ZEND_FETCH_LIST_R
|| !(opline->op1_type & (IS_VAR|IS_TMP_VAR))
|| op1_avoid_refcounting)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
if (opline->opcode == ZEND_FETCH_DIM_IS
&& !(res_info & MAY_BE_NULL)) {
uint32_t flags = 0;
uint32_t old_op1_info = 0;
uint32_t old_info;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
if (opline->opcode != ZEND_FETCH_LIST_R
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !op1_avoid_refcounting) {
flags |= ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
flags |= ZEND_JIT_EXIT_FREE_OP2;
}
if (op1_avoid_refcounting) {
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_TYPE_ONLY);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
if (op1_avoid_refcounting) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
}
}
}
if (op1_info & MAY_BE_REF) {
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op1_info & MAY_BE_ARRAY) {
ir_ref ht_ref, ref;
zend_jit_addr val_addr;
ir_refs *found_inputs, *found_vals;
ir_refs_init(found_inputs, 10);
ir_refs_init(found_vals, 10);
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
jit_guard_Z_TYPE(jit, op1_addr, IS_ARRAY, exit_addr);
} else {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
ir_IF_TRUE(if_type);
}
}
ht_ref = jit_Z_PTR(jit, op1_addr);
if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) ||
(opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) {
may_throw = 1;
}
if (!zend_jit_fetch_dimension_address_inner(jit, opline,
(opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS,
op1_info, op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, exit_addr,
result_type_guard, ht_ref, found_inputs, found_vals,
&end_inputs, &not_found_inputs)) {
return 0;
}
if (found_inputs->count) {
ir_MERGE_N(found_inputs->count, found_inputs->refs);
ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
val_addr = ZEND_ADDR_REF_ZVAL(ref);
if (result_type_guard) {
uint8_t type = concrete_type(res_info);
uint32_t flags = 0;
if (opline->opcode != ZEND_FETCH_LIST_R
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !op1_avoid_refcounting) {
flags |= ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
flags |= ZEND_JIT_EXIT_FREE_OP2;
}
val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
(op1_info & MAY_BE_ARRAY_OF_REF) != 0, flags, op1_avoid_refcounting);
if (!val_addr) {
return 0;
}
if (not_found_inputs) {
ir_END_list(not_found_inputs);
ir_MERGE_list(not_found_inputs);
}
// ZVAL_COPY
jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
if (Z_MODE(res_addr) != IS_REG) {
} else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
} else if (op1_info & MAY_BE_ARRAY_OF_REF) {
// ZVAL_COPY_DEREF
ir_ref type_info = jit_Z_TYPE_INFO(jit, val_addr);
if (!zend_jit_zval_copy_deref(jit, res_addr, val_addr, type_info)) {
return 0;
}
} else {
// ZVAL_COPY
jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, 1);
}
ir_END_list(end_inputs);
} else if (not_found_inputs) {
ir_MERGE_list(not_found_inputs);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_END_list(end_inputs);
} else if (!end_inputs && jit->ctx.control) {
ir_END_list(end_inputs); /* dead code */
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNDEF;
}
if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
ir_ref str_ref;
may_throw = 1;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, exit_addr);
} else {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_TRUE(if_type);
}
}
jit_SET_EX_OPLINE(jit, opline);
str_ref = jit_Z_PTR(jit, op1_addr);
if (opline->opcode != ZEND_FETCH_DIM_IS) {
ir_ref ref;
if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_offset_r_helper),
str_ref, jit_Z_LVAL(jit, op2_addr));
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_r_helper),
str_ref, jit_ZVAL_ADDR(jit, op2_addr));
}
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
} else {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_is_helper),
str_ref,
jit_ZVAL_ADDR(jit, op2_addr),
jit_ZVAL_ADDR(jit, res_addr));
}
ir_END_list(end_inputs);
}
if (op1_info & MAY_BE_OBJECT) {
ir_ref arg2;
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNDEF;
}
may_throw = 1;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
if (exit_addr) {
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_TRUE(if_type);
}
}
jit_SET_EX_OPLINE(jit, opline);
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
if (opline->opcode != ZEND_FETCH_DIM_IS) {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_r_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, res_addr));
} else {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_is_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, res_addr));
}
ir_END_list(end_inputs);
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
&& (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNDEF;
}
if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
may_throw = 1;
zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op1_addr), opline->op1.var, NULL, 0, 1, 0);
}
if (op2_info & MAY_BE_UNDEF) {
may_throw = 1;
zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op2_addr), opline->op2.var, NULL, 0, 1, 0);
}
}
if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
ir_ref ref;
may_throw = 1;
if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
} else {
jit_SET_EX_OPLINE(jit, opline);
ref = jit_ZVAL_ADDR(jit, op1_addr);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_array_access), ref);
}
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_END_list(end_inputs);
}
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetGet() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) {
if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) {
may_throw = 1;
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
}
if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
}
if (may_throw) {
zend_jit_check_exception(jit);
}
} else if (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) {
ir_BEGIN(IR_UNUSED); /* unreachable tail */
}
return 1;
}
static zend_jit_addr zend_jit_prepare_array_update(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
ir_ref *if_type,
ir_ref *ht_ref,
int *may_throw)
{
ir_ref ref = IR_UNUSED;
ir_ref array_reference_end = IR_UNUSED, array_reference_ref = IR_UNUSED;
ir_refs *array_inputs, *array_values;
ir_refs_init(array_inputs, 4);
ir_refs_init(array_values, 4);
ref = jit_ZVAL_ADDR(jit, op1_addr);
if (op1_info & MAY_BE_REF) {
ir_ref if_reference, if_array, end1, ref2;
*may_throw = 1;
if_reference = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_FALSE(if_reference);
end1 = ir_END();
ir_IF_TRUE_cold(if_reference);
array_reference_ref = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
if_array = jit_if_Z_TYPE_ref(jit, array_reference_ref, ir_CONST_U8(IS_ARRAY));
ir_IF_TRUE(if_array);
array_reference_end = ir_END();
ir_IF_FALSE_cold(if_array);
if (opline->opcode != ZEND_FETCH_DIM_RW && opline->opcode != ZEND_ASSIGN_DIM_OP) {
jit_SET_EX_OPLINE(jit, opline);
}
ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_prepare_assign_dim_ref), ref);
ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
ir_MERGE_WITH(end1);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op1_info & MAY_BE_ARRAY) {
ir_ref op1_ref = ref;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
*if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
ir_IF_TRUE(*if_type);
}
if (array_reference_end) {
ir_MERGE_WITH(array_reference_end);
op1_ref = ir_PHI_2(IR_ADDR, ref, array_reference_ref);
}
// JIT: SEPARATE_ARRAY()
ref = jit_Z_PTR_ref(jit, op1_ref);
if (RC_MAY_BE_N(op1_info)) {
if (RC_MAY_BE_1(op1_info)) {
ir_ref if_refcount_1 = ir_IF(ir_EQ(jit_GC_REFCOUNT(jit, ref), ir_CONST_U32(1)));
ir_IF_TRUE(if_refcount_1);
ir_refs_add(array_inputs, ir_END());
ir_refs_add(array_values, ref);
ir_IF_FALSE(if_refcount_1);
}
ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_zval_array_dup), op1_ref);
}
if (array_inputs->count || (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
ir_refs_add(array_inputs, ir_END());
ir_refs_add(array_values, ref);
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
if (*if_type) {
ir_IF_FALSE_cold(*if_type);
*if_type = IR_UNUSED;
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
*if_type = ir_IF(ir_LE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)));
ir_IF_TRUE(*if_type);
}
if ((op1_info & MAY_BE_UNDEF)
&& (opline->opcode == ZEND_FETCH_DIM_RW || opline->opcode == ZEND_ASSIGN_DIM_OP)) {
ir_ref end1 = IR_UNUSED;
*may_throw = 1;
if (op1_info & MAY_BE_NULL) {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
ir_IF_TRUE(if_def);
end1 = ir_END();
ir_IF_FALSE(if_def);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
if (end1) {
ir_MERGE_WITH(end1);
}
}
// JIT: ZVAL_ARR(container, zend_new_array(8));
ref = ir_CALL_1(IR_ADDR,
jit_STUB_FUNC_ADDR(jit, jit_stub_new_array, IR_FASTCALL_FUNC),
jit_ZVAL_ADDR(jit, op1_addr));
if (array_inputs->count) {
ir_refs_add(array_inputs, ir_END());
ir_refs_add(array_values, ref);
}
}
if (array_inputs->count) {
ir_MERGE_N(array_inputs->count, array_inputs->refs);
ref = ir_PHI_N(IR_ADDR, array_values->count, array_values->refs);
}
*ht_ref = ref;
return op1_addr;
}
static int zend_jit_fetch_dim(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
zend_jit_addr res_addr,
uint8_t dim_type)
{
int may_throw = 0;
ir_ref end_inputs = IR_UNUSED;
ir_ref ref, if_type = IR_UNUSED, ht_ref;
if (opline->opcode == ZEND_FETCH_DIM_RW) {
jit_SET_EX_OPLINE(jit, opline);
}
op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
ir_refs *found_inputs, *found_vals;
ir_refs_init(found_inputs, 8);
ir_refs_init(found_vals, 8);
if (opline->op2_type == IS_UNUSED) {
ir_ref if_ok;
may_throw = 1;
// JIT:var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
ht_ref, jit_EG(uninitialized_zval));
// JIT: if (UNEXPECTED(!var_ptr)) {
if_ok = ir_IF(ref);
ir_IF_FALSE_cold(if_ok);
if (opline->opcode != ZEND_FETCH_DIM_RW) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
ir_END_list(end_inputs);
ir_IF_TRUE(if_ok);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
ir_END_list(end_inputs);
} else {
uint32_t type;
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
type = BP_VAR_W;
break;
case ZEND_FETCH_DIM_RW:
may_throw = 1;
type = BP_VAR_RW;
break;
case ZEND_FETCH_DIM_UNSET:
type = BP_VAR_UNSET;
break;
default:
ZEND_UNREACHABLE();
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
may_throw = 1;
}
if (!zend_jit_fetch_dimension_address_inner(jit, opline, type, op1_info,
op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL,
0, ht_ref, found_inputs, found_vals, &end_inputs, NULL)) {
return 0;
}
if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
end_inputs = ir_END();
}
} else if (!(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
/* impossible dead path */
end_inputs = ir_END();
} else {
ZEND_ASSERT(end_inputs == IR_UNUSED);
}
if (found_inputs->count) {
ir_MERGE_N(found_inputs->count, found_inputs->refs);
ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
ir_END_list(end_inputs);
}
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
ir_ref arg2;
may_throw = 1;
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IR_UNUSED;
}
if (opline->opcode != ZEND_FETCH_DIM_RW) {
jit_SET_EX_OPLINE(jit, opline);
}
if (opline->op2_type == IS_UNUSED) {
arg2 = IR_NULL;
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_w_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, res_addr));
break;
case ZEND_FETCH_DIM_RW:
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_rw_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, res_addr));
break;
// case ZEND_FETCH_DIM_UNSET:
// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0
// break;
default:
ZEND_UNREACHABLE();
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
ir_END_list(end_inputs);
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & MAY_HAVE_DTOR)
&& (op2_info & MAY_BE_RC1)) {
may_throw = 1;
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_isset_isempty_dim(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_avoid_refcounting,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint8_t dim_type,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr)
{
zend_jit_addr res_addr;
ir_ref if_type = IR_UNUSED;
ir_ref false_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
ir_refs *true_inputs;
ir_refs_init(true_inputs, 8);
// TODO: support for empty() ???
ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (op1_info & MAY_BE_REF) {
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op1_info & MAY_BE_ARRAY) {
const void *found_exit_addr = NULL;
const void *not_found_exit_addr = NULL;
ir_ref ht_ref;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
ir_IF_TRUE(if_type);
}
ht_ref = jit_Z_PTR(jit, op1_addr);
if (exit_addr
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
&& !may_throw
&& (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
&& (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
if (smart_branch_opcode == ZEND_JMPNZ) {
found_exit_addr = exit_addr;
} else {
not_found_exit_addr = exit_addr;
}
}
if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_JIT_IS, op1_info,
op2_info, op2_addr, op2_range, dim_type, found_exit_addr, not_found_exit_addr, NULL,
0, ht_ref, true_inputs, NULL, &false_inputs, NULL)) {
return 0;
}
if (found_exit_addr) {
ir_MERGE_list(false_inputs);
return 1;
} else if (not_found_exit_addr) {
ir_MERGE_N(true_inputs->count, true_inputs->refs);
return 1;
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IR_UNUSED;
}
if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
ir_ref ref, arg1, arg2, if_true;
jit_SET_EX_OPLINE(jit, opline);
arg1 = jit_ZVAL_ADDR(jit, op1_addr);
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_isset_dim_helper), arg1, arg2);
if_true = ir_IF(ref);
ir_IF_TRUE(if_true);
ir_refs_add(true_inputs, ir_END());
ir_IF_FALSE(if_true);
ir_END_list(false_inputs);
} else {
if (op2_info & MAY_BE_UNDEF) {
ir_ref end1 = IR_UNUSED;
if (op2_info & MAY_BE_ANY) {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op2_addr));
ir_IF_TRUE(if_def);
end1 = ir_END();
ir_IF_FALSE(if_def);
}
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
if (end1) {
ir_MERGE_WITH(end1);
}
}
ir_END_list(false_inputs);
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetExists() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (true_inputs->count) {
ir_MERGE_N(true_inputs->count, true_inputs->refs);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!op1_avoid_refcounting) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
_zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
} else if (smart_branch_opcode == ZEND_JMPNZ) {
_zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
} else {
ZEND_UNREACHABLE();
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
} else {
ZEND_UNREACHABLE(); // TODO: support for empty()
}
}
ir_MERGE_list(false_inputs);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!op1_avoid_refcounting) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
_zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
} else if (smart_branch_opcode == ZEND_JMPNZ) {
_zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
} else {
ZEND_UNREACHABLE();
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
ir_END_list(end_inputs);
}
} else {
ZEND_UNREACHABLE(); // TODO: support for empty()
}
if (!exit_addr && smart_branch_opcode) {
jit->b = -1;
} else {
ir_MERGE_list(end_inputs);
}
return 1;
}
static int zend_jit_assign_dim(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint32_t val_info,
zend_jit_addr op3_addr,
zend_jit_addr op3_def_addr,
zend_jit_addr res_addr,
uint8_t dim_type,
int may_throw)
{
ir_ref if_type = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED, ht_ref;
if (op3_addr != op3_def_addr && op3_def_addr) {
if (!zend_jit_update_regs(jit, (opline+1)->op1.var, op3_addr, op3_def_addr, val_info)) {
return 0;
}
if (Z_MODE(op3_def_addr) == IS_REG && Z_MODE(op3_addr) != IS_REG) {
op3_addr = op3_def_addr;
}
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_not_Z_TYPE(jit, op3_addr, IS_UNDEF, exit_addr);
val_info &= ~MAY_BE_UNDEF;
}
op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
if (opline->op2_type == IS_UNUSED) {
uint32_t var_info = MAY_BE_NULL;
ir_ref if_ok, ref;
zend_jit_addr var_addr;
// JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
ht_ref, jit_EG(uninitialized_zval));
// JIT: if (UNEXPECTED(!var_ptr)) {
if_ok = ir_IF(ref);
ir_IF_FALSE_cold(if_ok);
// JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
jit_SET_EX_OPLINE(jit, opline);
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
ir_END_list(end_inputs);
ir_IF_TRUE(if_ok);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
zend_jit_addr var_addr;
ir_ref ref;
ir_refs *found_inputs, *found_values;
ir_refs_init(found_inputs, 8);
ir_refs_init(found_values, 8);
if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_W, op1_info,
op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL,
0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
return 0;
}
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
if (found_inputs->count) {
ir_MERGE_N(found_inputs->count, found_inputs->refs);
ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
// JIT: value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
if (opline->op1_type == IS_VAR
&& Z_MODE(op3_addr) != IS_REG
&& (res_addr == 0 || Z_MODE(res_addr) != IS_REG)) {
ZEND_ASSERT(opline->result_type == IS_UNUSED);
if (!zend_jit_assign_to_variable_call(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) {
return 0;
}
}
}
}
ir_END_list(end_inputs);
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
ir_ref arg2, arg4;
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IR_UNUSED;
}
jit_SET_EX_OPLINE(jit, opline);
if (opline->op2_type == IS_UNUSED) {
arg2 = IR_NULL;
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
if (opline->result_type == IS_UNUSED) {
arg4 = IR_NULL;
} else {
arg4 = jit_ZVAL_ADDR(jit, res_addr);
}
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, op3_addr),
arg4);
#ifdef ZEND_JIT_USE_RC_INFERENCE
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
/* ASSIGN_DIM may increase refcount of the value */
val_info |= MAY_BE_RCN;
}
#endif
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, NULL);
ir_END_list(end_inputs);
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
ir_MERGE_list(end_inputs);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_assign_dim_op(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
uint32_t op1_def_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint32_t op1_data_info,
zend_jit_addr op3_addr,
zend_ssa_range *op1_data_range,
uint8_t dim_type,
int may_throw)
{
zend_jit_addr var_addr = IS_UNUSED;
const void *not_found_exit_addr = NULL;
uint32_t var_info = MAY_BE_NULL;
ir_ref if_type = IS_UNUSED;
ir_ref end_inputs = IR_UNUSED, ht_ref;
bool emit_fast_path = 1;
ZEND_ASSERT(opline->result_type == IS_UNUSED);
if (may_throw) {
jit_SET_EX_OPLINE(jit, opline);
}
op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);
if (opline->op2_type == IS_UNUSED) {
var_info = MAY_BE_NULL;
ir_ref if_ok, ref;
// JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
ht_ref, jit_EG(uninitialized_zval));
// JIT: if (UNEXPECTED(!var_ptr)) {
if_ok = ir_IF(ref);
ir_IF_FALSE_cold(if_ok);
// JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
ir_END_list(end_inputs);
ir_IF_TRUE(if_ok);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
} else {
ir_ref ref;
ir_refs *found_inputs, *found_values;
ir_refs_init(found_inputs, 8);
ir_refs_init(found_values, 8);
var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
if (dim_type != IS_UNKNOWN
&& dim_type != IS_UNDEF
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
&& (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
}
if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_RW, op1_info,
op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, NULL,
0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
return 0;
}
if (found_inputs->count) {
ir_MERGE_N(found_inputs->count, found_inputs->refs);
ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
if (not_found_exit_addr && dim_type != IS_REFERENCE) {
jit_guard_Z_TYPE(jit, var_addr, dim_type, not_found_exit_addr);
var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
binary_op_type binary_op = get_binary_op(opline->extended_value);
ir_ref if_ref, if_typed, noref_path, ref_path, ref, reference, ref2, arg2;
ref = jit_ZVAL_ADDR(jit, var_addr);
if_ref = jit_if_Z_TYPE(jit, var_addr, IS_REFERENCE);
ir_IF_FALSE(if_ref);
noref_path = ir_END();
ir_IF_TRUE(if_ref);
reference = jit_Z_PTR_ref(jit, ref);
ref2 = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
if_typed = jit_if_TYPED_REF(jit, reference);
ir_IF_FALSE(if_typed);
ref_path = ir_END();
ir_IF_TRUE_cold(if_typed);
arg2 = jit_ZVAL_ADDR(jit, op3_addr);
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
reference, arg2, ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
ir_MERGE_2(noref_path, ref_path);
ref = ir_PHI_2(IR_ADDR, ref, ref2);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
}
} else {
emit_fast_path = 0;
}
}
if (emit_fast_path) {
uint8_t val_op_type = (opline+1)->op1_type;
if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
/* prevent FREE_OP in the helpers */
val_op_type = IS_CV;
}
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, may_throw)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
val_op_type, (opline+1)->op1, op3_addr, op1_data_info,
op1_data_range,
0, var_addr, var_def_info, var_info, may_throw)) {
return 0;
}
break;
case ZEND_CONCAT:
if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
may_throw)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
ir_END_list(end_inputs);
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
binary_op_type binary_op;
ir_ref arg2;
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNUSED;
}
if (opline->op2_type == IS_UNUSED) {
arg2 = IR_NULL;
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
binary_op = get_binary_op(opline->extended_value);
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_op_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, op3_addr),
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
// JIT: ZVAL_COPY(res, value);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
jit_ZVAL_COPY_CONST(jit, res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
} else {
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
jit_ZVAL_COPY(jit, res_addr, -1, op1_addr, op1_info, opline->op1_type == IS_CV);
}
// JIT: Z_FE_POS_P(res) = 0;
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->result.var + offsetof(zval, u2.fe_pos)), ir_CONST_U32(0));
return 1;
}
static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t op_info)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
ir_ref ref;
if (!exit_addr) {
return 0;
}
ref = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(jit_Z_PTR(jit, addr), offsetof(zend_array, u.flags))),
ir_CONST_U32(HASH_FLAG_PACKED));
if (op_info & MAY_BE_ARRAY_PACKED) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
return 1;
}
static int zend_jit_fe_fetch(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr)
{
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
ir_ref ref, ht_ref, hash_pos_ref, packed_pos_ref, hash_p_ref = IR_UNUSED, packed_p_ref = IR_UNUSED, if_packed = IR_UNUSED;
ir_ref if_def_hash = IR_UNUSED, if_def_packed = IR_UNUSED;
ir_ref exit_inputs = IR_UNUSED;
if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) {
/* empty array */
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else {
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
jit->b = -1;
}
return 1;
}
// JIT: array = EX_VAR(opline->op1.var);
// JIT: fe_ht = Z_ARRVAL_P(array);
ht_ref = jit_Z_PTR(jit, op1_addr);
if (op1_info & MAY_BE_PACKED_GUARD) {
if (!zend_jit_packed_guard(jit, opline, opline->op1.var, op1_info)) {
return 0;
}
}
// JIT: pos = Z_FE_POS_P(array);
hash_pos_ref = packed_pos_ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)));
if (MAY_BE_HASH(op1_info)) {
ir_ref loop_ref, pos2_ref, p2_ref;
if (MAY_BE_PACKED(op1_info)) {
ref = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
ir_CONST_U32(HASH_FLAG_PACKED));
if_packed = ir_IF(ref);
ir_IF_FALSE(if_packed);
}
// JIT: p = fe_ht->arData + pos;
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(hash_pos_ref);
} else {
ref = ir_BITCAST_A(hash_pos_ref);
}
hash_p_ref = ir_ADD_A(
ir_MUL_A(ref, ir_CONST_ADDR(sizeof(Bucket))),
ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arData))));
loop_ref = ir_LOOP_BEGIN(ir_END());
hash_pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, IR_UNUSED);
hash_p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, IR_UNUSED);
// JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
ref = ir_ULT(hash_pos_ref,
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
// JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
// JIT: ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_fit = ir_IF(ref);
ir_IF_FALSE(if_fit);
ir_END_list(exit_inputs);
ir_IF_TRUE(if_fit);
}
} else {
ir_ref if_fit = ir_IF(ref);
ir_IF_FALSE(if_fit);
ir_END_list(exit_inputs);
ir_IF_TRUE(if_fit);
}
// JIT: pos++;
pos2_ref = ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1));
// JIT: value_type = Z_TYPE_INFO_P(value);
// JIT: if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
if_def_hash = ir_IF(jit_Z_TYPE_ref(jit, hash_p_ref));
ir_IF_FALSE(if_def_hash);
} else {
ir_GUARD_NOT(jit_Z_TYPE_ref(jit, hash_p_ref), ir_CONST_ADDR(exit_addr));
}
// JIT: p++;
p2_ref = ir_ADD_OFFSET(hash_p_ref, sizeof(Bucket));
ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
ir_PHI_SET_OP(hash_pos_ref, 2, pos2_ref);
ir_PHI_SET_OP(hash_p_ref, 2, p2_ref);
if (MAY_BE_PACKED(op1_info)) {
ir_IF_TRUE(if_packed);
}
}
if (MAY_BE_PACKED(op1_info)) {
ir_ref loop_ref, pos2_ref, p2_ref;
// JIT: p = fe_ht->arPacked + pos;
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(packed_pos_ref);
} else {
ref = ir_BITCAST_A(packed_pos_ref);
}
packed_p_ref = ir_ADD_A(
ir_MUL_A(ref, ir_CONST_ADDR(sizeof(zval))),
ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))));
loop_ref = ir_LOOP_BEGIN(ir_END());
packed_pos_ref = ir_PHI_2(IR_U32, packed_pos_ref, IR_UNUSED);
packed_p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, IR_UNUSED);
// JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
ref = ir_ULT(packed_pos_ref,
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
// JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
// JIT: ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_fit = ir_IF(ref);
ir_IF_FALSE(if_fit);
ir_END_list(exit_inputs);
ir_IF_TRUE(if_fit);
}
} else {
ir_ref if_fit = ir_IF(ref);
ir_IF_FALSE(if_fit);
ir_END_list(exit_inputs);
ir_IF_TRUE(if_fit);
}
// JIT: pos++;
pos2_ref = ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1));
// JIT: value_type = Z_TYPE_INFO_P(value);
// JIT: if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
if_def_packed = ir_IF(jit_Z_TYPE_ref(jit, packed_p_ref));
ir_IF_FALSE(if_def_packed);
} else {
ir_GUARD_NOT(jit_Z_TYPE_ref(jit, packed_p_ref), ir_CONST_ADDR(exit_addr));
}
// JIT: p++;
p2_ref = ir_ADD_OFFSET(packed_p_ref, sizeof(zval));
ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
ir_PHI_SET_OP(packed_pos_ref, 2, pos2_ref);
ir_PHI_SET_OP(packed_p_ref, 2, p2_ref);
}
if (!exit_addr || exit_opcode == ZEND_JMP) {
zend_jit_addr val_addr;
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
uint32_t val_info;
ir_ref p_ref = IR_UNUSED, hash_path = IR_UNUSED;
if (RETURN_VALUE_USED(opline)) {
zend_jit_addr res_addr = RES_ADDR();
if (MAY_BE_HASH(op1_info)) {
ir_ref key_ref = IR_UNUSED, if_key = IR_UNUSED, key_path = IR_UNUSED;
ZEND_ASSERT(if_def_hash);
ir_IF_TRUE(if_def_hash);
// JIT: Z_FE_POS_P(array) = pos + 1;
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1)));
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
key_ref = ir_LOAD_A(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, key)));
}
if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
&& (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
// JIT: if (!p->key) {
if_key = ir_IF(key_ref);
ir_IF_TRUE(if_key);
}
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
ir_ref if_interned, interned_path;
// JIT: ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
jit_set_Z_PTR(jit, res_addr, key_ref);
ref = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(key_ref, offsetof(zend_refcounted, gc.u.type_info))),
ir_CONST_U32(IS_STR_INTERNED));
if_interned = ir_IF(ref);
ir_IF_TRUE(if_interned);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
interned_path = ir_END();
ir_IF_FALSE(if_interned);
jit_GC_ADDREF(jit, key_ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
ir_MERGE_WITH(interned_path);
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
key_path = ir_END();
}
}
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
ir_IF_FALSE(if_key);
}
// JIT: ZVAL_LONG(EX_VAR(opline->result.var), p->h);
ref = ir_LOAD_L(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, h)));
jit_set_Z_LVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
ir_MERGE_WITH(key_path);
}
}
if (MAY_BE_PACKED(op1_info)) {
hash_path = ir_END();
} else {
p_ref = hash_p_ref;
}
}
if (MAY_BE_PACKED(op1_info)) {
ZEND_ASSERT(if_def_packed);
ir_IF_TRUE(if_def_packed);
// JIT: Z_FE_POS_P(array) = pos + 1;
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1)));
// JIT: ZVAL_LONG(EX_VAR(opline->result.var), pos);
if (sizeof(zend_long) == 8) {
packed_pos_ref = ir_ZEXT_L(packed_pos_ref);
} else {
packed_pos_ref = ir_BITCAST_L(packed_pos_ref);
}
jit_set_Z_LVAL(jit, res_addr, packed_pos_ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
if (MAY_BE_HASH(op1_info)) {
ir_MERGE_WITH(hash_path);
p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, hash_p_ref);
} else {
p_ref = packed_p_ref;
}
}
} else {
ir_ref pos_ref = IR_UNUSED;
if (if_def_hash && if_def_packed) {
ir_IF_TRUE(if_def_hash);
ir_MERGE_WITH_EMPTY_TRUE(if_def_packed);
pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, packed_pos_ref);
p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, packed_p_ref);
} else if (if_def_hash) {
ir_IF_TRUE(if_def_hash);
pos_ref = hash_pos_ref;
p_ref = hash_p_ref;
} else if (if_def_packed) {
ir_IF_TRUE(if_def_packed);
pos_ref = packed_pos_ref;
p_ref = packed_p_ref;
} else {
ZEND_UNREACHABLE();
}
// JIT: Z_FE_POS_P(array) = pos + 1;
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
ir_ADD_U32(pos_ref, ir_CONST_U32(1)));
}
val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
if (val_info & MAY_BE_ARRAY) {
val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
}
if (op1_info & MAY_BE_ARRAY_OF_REF) {
val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
} else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1 | MAY_BE_RCN;
}
val_addr = ZEND_ADDR_REF_ZVAL(p_ref);
if (opline->op2_type == IS_CV) {
// JIT: zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 0, 1)) {
return 0;
}
} else {
// JIT: ZVAL_COPY(res, value);
jit_ZVAL_COPY(jit, var_addr, -1, val_addr, val_info, 1);
}
if (!exit_addr) {
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ir_END());
ZEND_ASSERT(exit_inputs);
if (!jit->ctx.ir_base[exit_inputs].op2) {
ref = exit_inputs;
} else {
ir_MERGE_list(exit_inputs);
ref = ir_END();
}
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
jit->b = -1;
}
} else {
ZEND_ASSERT(exit_inputs);
ir_MERGE_list(exit_inputs);
}
return 1;
}
static int zend_jit_load_this(zend_jit_ctx *jit, uint32_t var)
{
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
ir_ref ref = jit_Z_PTR(jit, this_addr);
jit_set_Z_PTR(jit, var_addr, ref);
jit_set_Z_TYPE_INFO(jit, var_addr, IS_OBJECT_EX);
jit_GC_ADDREF(jit, ref);
return 1;
}
static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool check_only)
{
if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
!TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, this_addr, IS_OBJECT, exit_addr);
if (JIT_G(current_frame)) {
TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
}
}
} else {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
ir_ref if_object = jit_if_Z_TYPE(jit, this_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_object);
jit_SET_EX_OPLINE(jit, opline);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_invalid_this));
ir_IF_TRUE(if_object);
}
}
if (!check_only) {
if (!zend_jit_load_this(jit, opline->result.var)) {
return 0;
}
}
return 1;
}
static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_class_entry *ce)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))), ir_CONST_ADDR(ce)),
ir_CONST_ADDR(exit_addr));
return 1;
}
static int zend_jit_fetch_obj(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
bool op1_avoid_refcounting,
zend_class_entry *trace_ce,
zend_jit_addr res_addr,
uint8_t prop_type,
int may_throw)
{
zval *member;
zend_property_info *prop_info;
bool may_be_dynamic = 1;
zend_jit_addr prop_addr;
uint32_t res_info = RES_INFO();
ir_ref prop_type_ref = IR_UNUSED;
ir_ref obj_ref = IR_UNUSED;
ir_ref prop_ref = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
ir_ref end_values = IR_UNUSED;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename);
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
obj_ref = jit_Z_PTR(jit, this_addr);
} else {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_obj);
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
ir_ref op1_ref = IR_UNUSED;
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode != ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_UNDEF)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
ir_ref fast_path = IR_UNUSED;
if (op1_info & MAY_BE_ANY) {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
ir_IF_TRUE(if_def);
fast_path = ir_END();
ir_IF_FALSE_cold(if_def);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
ir_CONST_U32(opline->op1.var));
if (fast_path) {
ir_MERGE_WITH(fast_path);
}
op1_ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
} else {
op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
}
if (opline->opcode == ZEND_FETCH_OBJ_W) {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_write),
op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
} else {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_read),
op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
}
ir_END_list(end_inputs);
ir_IF_TRUE(if_obj);
}
}
obj_ref = jit_Z_PTR(jit, op1_addr);
}
ZEND_ASSERT(obj_ref);
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
ir_ref if_same = ir_IF(ir_EQ(ref,
ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
ir_IF_FALSE_cold(if_same);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same);
ir_ref offset_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
if (may_be_dynamic) {
ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
if (opline->opcode == ZEND_FETCH_OBJ_W) {
ir_IF_TRUE_cold(if_dynamic);
ir_END_list(slow_inputs);
} else {
ir_IF_TRUE_cold(if_dynamic);
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
if (((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info)
|| Z_MODE(res_addr) == IS_REG) {
ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic_ex),
obj_ref, offset_ref);
ir_END_PHI_list(end_values, val_addr);
} else {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic),
obj_ref, offset_ref);
ir_END_list(end_inputs);
}
} else {
if (((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info)
|| Z_MODE(res_addr) == IS_REG) {
ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic_ex),
obj_ref, offset_ref);
ir_END_PHI_list(end_values, val_addr);
} else {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic),
obj_ref, offset_ref);
ir_END_list(end_inputs);
}
}
}
ir_IF_FALSE(if_dynamic);
}
prop_ref = ir_ADD_A(obj_ref, offset_ref);
prop_type_ref = jit_Z_TYPE_ref(jit, prop_ref);
ir_ref if_def = ir_IF(prop_type_ref);
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
ir_ref prop_info_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
ir_ref if_has_prop_info = ir_IF(prop_info_ref);
ir_IF_TRUE_cold(if_has_prop_info);
ir_ref if_readonly = ir_IF(
ir_AND_U32(ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, flags))),
ir_CONST_U32(ZEND_ACC_READONLY)));
ir_IF_TRUE(if_readonly);
ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
ir_IF_TRUE(if_prop_obj);
ref = jit_Z_PTR(jit, prop_addr);
jit_GC_ADDREF(jit, ref);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
ir_END_list(end_inputs);
ir_IF_FALSE_cold(if_prop_obj);
ir_ref extra_addr = ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, prop_addr), offsetof(zval, u2.extra));
ir_ref extra = ir_LOAD_U32(extra_addr);
ir_ref if_reinitable = ir_IF(ir_AND_U32(extra, ir_CONST_U32(IS_PROP_REINITABLE)));
ir_IF_TRUE(if_reinitable);
ir_STORE(extra_addr, ir_AND_U32(extra, ir_CONST_U32(~IS_PROP_REINITABLE)));
ir_ref reinit_path = ir_END();
ir_IF_FALSE(if_reinitable);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_modification_error), prop_info_ref);
jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
ir_END_list(end_inputs);
if (flags == ZEND_FETCH_DIM_WRITE) {
ir_IF_FALSE_cold(if_readonly);
ir_MERGE_WITH(reinit_path);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
prop_ref, prop_info_ref);
ir_END_list(end_inputs);
ir_IF_FALSE(if_has_prop_info);
} else if (flags == ZEND_FETCH_REF) {
ir_IF_FALSE_cold(if_readonly);
ir_MERGE_WITH(reinit_path);
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
prop_ref,
prop_info_ref,
jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(end_inputs);
ir_IF_FALSE(if_has_prop_info);
} else {
ir_ref list = reinit_path;
ZEND_ASSERT(flags == 0);
ir_IF_FALSE(if_has_prop_info);
ir_END_list(list);
ir_IF_FALSE(if_readonly);
ir_END_list(list);
ir_MERGE_list(list);
}
}
} else {
prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
ir_GUARD(prop_type_ref, ir_CONST_ADDR(exit_addr));
}
} else {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
ir_ref if_def = ir_IF(prop_type_ref);
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) {
if (!prop_type_ref) {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
}
ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
ir_IF_TRUE(if_prop_obj);
ir_ref ref = jit_Z_PTR(jit, prop_addr);
jit_GC_ADDREF(jit, ref);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
ir_END_list(end_inputs);
ir_IF_FALSE_cold(if_prop_obj);
ir_ref extra_addr = ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, prop_addr), offsetof(zval, u2.extra));
ir_ref extra = ir_LOAD_U32(extra_addr);
ir_ref if_reinitable = ir_IF(ir_AND_U32(extra, ir_CONST_U32(IS_PROP_REINITABLE)));
ir_IF_FALSE(if_reinitable);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_modification_error), ir_CONST_ADDR(prop_info));
jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
ir_END_list(end_inputs);
ir_IF_TRUE(if_reinitable);
ir_STORE(extra_addr, ir_AND_U32(extra, ir_CONST_U32(~IS_PROP_REINITABLE)));
}
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
&& ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
if (flags == ZEND_FETCH_DIM_WRITE) {
if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) {
if (!prop_type_ref) {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
}
ir_ref if_null_or_flase = ir_IF(ir_LE(prop_type_ref, ir_CONST_U32(IR_FALSE)));
ir_IF_TRUE_cold(if_null_or_flase);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
prop_ref, ir_CONST_ADDR(prop_info));
ir_END_list(end_inputs);
ir_IF_FALSE(if_null_or_flase);
}
} else if (flags == ZEND_FETCH_REF) {
ir_ref ref;
if (!prop_type_ref) {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
}
ir_ref if_reference = ir_IF(ir_EQ(prop_type_ref, ir_CONST_U32(IS_REFERENCE_EX)));
ir_IF_FALSE(if_reference);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
ref = ir_CONST_ADDR(prop_info);
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
}
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
prop_ref,
ref,
jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(end_inputs);
ir_IF_TRUE(if_reference);
} else {
ZEND_UNREACHABLE();
}
}
}
if (opline->opcode == ZEND_FETCH_OBJ_W) {
ZEND_ASSERT(prop_ref);
jit_set_Z_PTR(jit, res_addr, prop_ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
ssa->var_info[ssa_op->result_def].indirect_reference = 1;
}
ir_END_list(end_inputs);
} else {
if (((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info)
|| Z_MODE(res_addr) == IS_REG) {
ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr));
} else {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
if (!zend_jit_zval_copy_deref(jit, res_addr, prop_addr, prop_type_ref)) {
return 0;
}
ir_END_list(end_inputs);
}
}
if (op1_avoid_refcounting) {
SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
ir_MERGE_list(slow_inputs);
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode == ZEND_FETCH_OBJ_W) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_w_slow), obj_ref);
ir_END_list(end_inputs);
} else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
if (Z_MODE(res_addr) == IS_REG) {
ir_ref val_ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow_ex), obj_ref);
ir_END_PHI_list(end_values, val_ref);
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow), obj_ref);
ir_END_list(end_inputs);
}
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_slow), obj_ref);
ir_END_list(end_inputs);
}
}
if (end_values) {
ir_ref val_ref = ir_PHI_list(end_values);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val_ref);
bool result_avoid_refcounting = 0;
ZEND_ASSERT(opline->opcode == ZEND_FETCH_OBJ_R
|| opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG
|| opline->opcode == ZEND_FETCH_OBJ_IS);
ZEND_ASSERT(end_inputs == IR_UNUSED);
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
uint8_t type = concrete_type(res_info);
uint32_t flags = 0;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !delayed_fetch_this
&& !op1_avoid_refcounting) {
flags = ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
1, flags, op1_avoid_refcounting);
if (!val_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
}
// ZVAL_COPY
jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
} else {
ir_MERGE_list(end_inputs);
}
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_RC1)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
ir_ref if_refcounted, ptr, refcount, if_non_zero;
ir_ref merge_inputs = IR_UNUSED;
if_refcounted = jit_if_REFCOUNTED(jit, orig_op1_addr);
ir_IF_FALSE( if_refcounted);
ir_END_list(merge_inputs);
ir_IF_TRUE( if_refcounted);
ptr = jit_Z_PTR(jit, orig_op1_addr);
refcount = jit_GC_DELREF(jit, ptr);
if_non_zero = ir_IF(refcount);
ir_IF_TRUE( if_non_zero);
ir_END_list(merge_inputs);
ir_IF_FALSE( if_non_zero);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_extract_helper), ptr);
ir_END_list(merge_inputs);
ir_MERGE_list(merge_inputs);
} else if (!op1_avoid_refcounting) {
if (on_this) {
op1_info &= ~MAY_BE_RC1;
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& prop_info
&& (opline->opcode != ZEND_FETCH_OBJ_W ||
!(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
!ZEND_TYPE_IS_SET(prop_info->type))
&& (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) {
may_throw = 0;
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_assign_obj(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
zend_jit_addr val_addr,
zend_jit_addr val_def_addr,
zend_jit_addr res_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type,
int may_throw)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr prop_addr;
ir_ref obj_ref = IR_UNUSED;
ir_ref prop_ref = IR_UNUSED;
ir_ref delayed_end_input = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
uint32_t res_info = RES_INFO();
if (val_addr != val_def_addr && val_def_addr) {
if (!zend_jit_update_regs(jit, (opline+1)->op1.var, val_addr, val_def_addr, val_info)) {
return 0;
}
if (Z_MODE(val_def_addr) == IS_REG && Z_MODE(val_addr) != IS_REG) {
val_addr = val_def_addr;
}
}
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
obj_ref = jit_Z_PTR(jit, this_addr);
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_obj);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
jit_ZVAL_ADDR(jit, op1_addr),
ir_CONST_ADDR(ZSTR_VAL(name)));
if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
}
ir_END_list(end_inputs);
ir_IF_TRUE(if_obj);
}
}
obj_ref = jit_Z_PTR(jit, op1_addr);
}
ZEND_ASSERT(obj_ref);
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
ir_IF_FALSE_cold(if_same);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same);
ir_ref offset_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
ir_IF_TRUE_cold(if_dynamic);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_dynamic);
prop_ref = ir_ADD_A(obj_ref, offset_ref);
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
ir_ref arg3, arg4;
ir_ref prop_info_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
ir_ref if_has_prop_info = ir_IF(prop_info_ref);
ir_IF_TRUE_cold(if_has_prop_info);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg3 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg3 = jit_ZVAL_ADDR(jit, val_addr);
}
if (!RETURN_VALUE_USED(opline)) {
arg4 = IR_NULL;
} else if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
arg4 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg4 = jit_ZVAL_ADDR(jit, res_addr);
}
// JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
prop_ref,
prop_info_ref,
arg3,
arg4);
if ((opline+1)->op1_type == IS_CONST) {
// TODO: ???
// if (Z_TYPE_P(value) == orig_type) {
// CACHE_PTR_EX(cache_slot + 2, NULL);
}
if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_END_list(end_inputs);
ir_IF_FALSE(if_has_prop_info);
}
} else {
prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) {
// Undefined property with magic __get()/__set()
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
ir_ref ref, arg3, arg4;
// JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
jit_SET_EX_OPLINE(jit, opline);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
ref = ir_CONST_ADDR(prop_info);
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
}
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg3 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg3 = jit_ZVAL_ADDR(jit, val_addr);
}
if (!RETURN_VALUE_USED(opline)) {
arg4 = IR_NULL;
} else if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
arg4 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg4 = jit_ZVAL_ADDR(jit, res_addr);
}
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
prop_ref,
ref,
arg3,
arg4);
if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_END_list(end_inputs);
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
if (Z_MODE(val_addr) != IS_REG
&& (res_addr == 0 || Z_MODE(res_addr) != IS_REG)
&& opline->result_type == IS_UNUSED) {
if (!zend_jit_assign_to_variable_call(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0, 0)) {
return 0;
}
}
if (end_inputs || slow_inputs) {
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
/* skip FREE_OP_DATA() */
delayed_end_input = ir_END();
} else {
ir_END_list(end_inputs);
}
}
}
if (slow_inputs) {
ir_ref arg3, arg5;
ir_MERGE_list(slow_inputs);
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg3 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg3 = jit_ZVAL_ADDR(jit, val_addr);
}
if (!RETURN_VALUE_USED(opline)) {
arg5 = IR_NULL;
} else if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
arg5 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg5 = jit_ZVAL_ADDR(jit, res_addr);
}
// JIT: value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_helper),
obj_ref,
ir_CONST_ADDR(name),
arg3,
ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
arg5);
if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_END_list(end_inputs);
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
if (delayed_end_input) {
ir_MERGE_WITH(delayed_end_input);
}
}
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_assign_obj_op(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
zend_jit_addr val_addr,
zend_ssa_range *val_range,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr prop_addr;
bool use_prop_guard = 0;
bool may_throw = 0;
binary_op_type binary_op = get_binary_op(opline->extended_value);
ir_ref obj_ref = IR_UNUSED;
ir_ref prop_ref = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
ZEND_ASSERT(opline->result_type == IS_UNUSED);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
obj_ref = jit_Z_PTR(jit, this_addr);
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_obj);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID,
(op1_info & MAY_BE_UNDEF) ?
ir_CONST_FC_FUNC(zend_jit_invalid_property_assign_op) :
ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
jit_ZVAL_ADDR(jit, op1_addr),
ir_CONST_ADDR(ZSTR_VAL(name)));
may_throw = 1;
ir_END_list(end_inputs);
ir_IF_TRUE(if_obj);
}
}
obj_ref = jit_Z_PTR(jit, op1_addr);
}
ZEND_ASSERT(obj_ref);
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
use_prop_guard = (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
if (!prop_info) {
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
ir_IF_FALSE_cold(if_same);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same);
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
ir_ref prop_info_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
ir_ref if_has_prop_info = ir_IF(prop_info_ref);
ir_IF_TRUE_cold(if_has_prop_info);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_has_prop_info);
}
ir_ref offset_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
ir_IF_TRUE_cold(if_dynamic);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_dynamic);
prop_ref = ir_ADD_A(obj_ref, offset_ref);
if (!use_prop_guard) {
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
} else {
prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2;
may_throw = 1;
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg2 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg2 = jit_ZVAL_ADDR(jit, val_addr);
}
if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
ir_IF_FALSE(if_ref);
noref_path = ir_END();
ir_IF_TRUE(if_ref);
reference = jit_Z_PTR(jit, prop_addr);
ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
if_typed = jit_if_TYPED_REF(jit, reference);
ir_IF_FALSE(if_typed);
ref_path = ir_END();
ir_IF_TRUE_cold(if_typed);
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
reference,
arg2,
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
ir_MERGE_2(noref_path, ref_path);
prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
// JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
ref = ir_CONST_ADDR(prop_info);
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
}
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_prop),
prop_ref,
ref,
arg2,
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
zend_jit_addr var_addr = prop_addr;
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2;
may_throw = 1;
if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
ir_IF_FALSE(if_ref);
noref_path = ir_END();
ir_IF_TRUE(if_ref);
reference = jit_Z_PTR(jit, var_addr);
ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
if_typed = jit_if_TYPED_REF(jit, reference);
ir_IF_FALSE(if_typed);
ref_path = ir_END();
ir_IF_TRUE_cold(if_typed);
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg2 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg2 = jit_ZVAL_ADDR(jit, val_addr);
}
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
reference,
arg2,
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
ir_MERGE_2(noref_path, ref_path);
prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
var_info &= ~MAY_BE_REF;
}
uint8_t val_op_type = (opline+1)->op1_type;
if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
/* prevent FREE_OP in the helpers */
val_op_type = IS_CV;
}
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (opline->extended_value != ZEND_ADD ||
(var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
(val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
may_throw = 1;
}
}
if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, 0)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
(val_info & MAY_BE_ANY) != MAY_BE_STRING) {
may_throw = 1;
}
}
goto long_math;
case ZEND_SL:
case ZEND_SR:
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
}
if (val_op_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
may_throw = 1;
}
goto long_math;
case ZEND_MOD:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
}
if (val_op_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
may_throw = 1;
}
long_math:
if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
val_op_type, (opline+1)->op1, val_addr, val_info,
val_range,
0, var_addr, var_def_info, var_info, /* may throw */ 1)) {
return 0;
}
break;
case ZEND_CONCAT:
may_throw = 1;
if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, var_addr,
0)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
if (end_inputs || slow_inputs) {
ir_END_list(end_inputs);
}
}
if (slow_inputs) {
ir_ref arg3;
ir_MERGE_list(slow_inputs);
may_throw = 1;
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg3 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg3 = jit_ZVAL_ADDR(jit, val_addr);
}
jit_SET_EX_OPLINE(jit, opline);
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_op_helper),
obj_ref,
ir_CONST_ADDR(name),
arg3,
ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
// JIT: FREE_OP_DATA();
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_incdec_obj(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr res_addr = 0;
zend_jit_addr prop_addr;
bool use_prop_guard = 0;
bool may_throw = 0;
uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0;
ir_ref obj_ref = IR_UNUSED;
ir_ref prop_ref = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
if (opline->result_type != IS_UNUSED) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
obj_ref = jit_Z_PTR(jit, this_addr);
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_obj);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_incdec),
jit_ZVAL_ADDR(jit, op1_addr),
ir_CONST_ADDR(ZSTR_VAL(name)));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_IF_TRUE(if_obj);
}
}
obj_ref = jit_Z_PTR(jit, op1_addr);
}
ZEND_ASSERT(obj_ref);
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
use_prop_guard = (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
if (!prop_info) {
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
ir_IF_FALSE_cold(if_same);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same);
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
ir_ref prop_info_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
ir_ref if_has_prop_info = ir_IF(prop_info_ref);
ir_IF_TRUE_cold(if_has_prop_info);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_has_prop_info);
}
ir_ref offset_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
ir_IF_TRUE_cold(if_dynamic);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_dynamic);
prop_ref = ir_ADD_A(obj_ref, offset_ref);
if (!use_prop_guard) {
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
} else {
prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
const void *func;
ir_ref ref;
may_throw = 1;
jit_SET_EX_OPLINE(jit, opline);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
ref = ir_CONST_ADDR(prop_info);
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
}
if (opline->result_type == IS_UNUSED) {
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
case ZEND_POST_INC_OBJ:
func = zend_jit_inc_typed_prop;
break;
case ZEND_PRE_DEC_OBJ:
case ZEND_POST_DEC_OBJ:
func = zend_jit_dec_typed_prop;
break;
default:
ZEND_UNREACHABLE();
}
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func), prop_ref, ref);
} else {
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
func = zend_jit_pre_inc_typed_prop;
break;
case ZEND_PRE_DEC_OBJ:
func = zend_jit_pre_dec_typed_prop;
break;
case ZEND_POST_INC_OBJ:
func = zend_jit_post_inc_typed_prop;
break;
case ZEND_POST_DEC_OBJ:
func = zend_jit_post_dec_typed_prop;
break;
default:
ZEND_UNREACHABLE();
}
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(func),
prop_ref,
ref,
jit_ZVAL_ADDR(jit, res_addr));
}
ir_END_list(end_inputs);
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
zend_jit_addr var_addr = prop_addr;
ir_ref if_long = IR_UNUSED;
ir_ref if_overflow = IR_UNUSED;
if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
const void *func;
ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref;
if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
ir_IF_FALSE(if_ref);
noref_path = ir_END();
ir_IF_TRUE(if_ref);
reference = jit_Z_PTR(jit, var_addr);
ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
if_typed = jit_if_TYPED_REF(jit, reference);
ir_IF_FALSE(if_typed);
ref_path = ir_END();
ir_IF_TRUE_cold(if_typed);
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
func = zend_jit_pre_inc_typed_ref;
break;
case ZEND_PRE_DEC_OBJ:
func = zend_jit_pre_dec_typed_ref;
break;
case ZEND_POST_INC_OBJ:
func = zend_jit_post_inc_typed_ref;
break;
case ZEND_POST_DEC_OBJ:
func = zend_jit_post_dec_typed_ref;
break;
default:
ZEND_UNREACHABLE();
}
may_throw = 1;
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func),
reference,
(opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(end_inputs);
ir_MERGE_2(noref_path, ref_path);
prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
var_info &= ~MAY_BE_REF;
}
if (var_info & MAY_BE_LONG) {
ir_ref addr, ref;
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
if_long = jit_if_Z_TYPE(jit, var_addr, IS_LONG);
ir_IF_TRUE(if_long);
}
addr = jit_ZVAL_ADDR(jit, var_addr);
ref = ir_LOAD_L(addr);
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
ref = ir_ADD_OV_L(ref, ir_CONST_LONG(1));
} else {
ref = ir_SUB_OV_L(ref, ir_CONST_LONG(1));
}
ir_STORE(addr, ref);
if_overflow = ir_IF(ir_OVERFLOW(ref));
ir_IF_FALSE(if_overflow);
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
ir_END_list(end_inputs);
}
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
may_throw = 1;
}
if (if_long) {
ir_IF_FALSE_cold(if_long);
}
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
jit_ZVAL_COPY(jit, res_addr, -1, var_addr, var_info, 1);
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc),
jit_ZVAL_ADDR(jit, var_addr),
jit_ZVAL_ADDR(jit, res_addr));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function),
jit_ZVAL_ADDR(jit, var_addr));
}
} else {
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec),
jit_ZVAL_ADDR(jit, var_addr),
jit_ZVAL_ADDR(jit, res_addr));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function),
jit_ZVAL_ADDR(jit, var_addr));
}
}
ir_END_list(end_inputs);
}
if (var_info & MAY_BE_LONG) {
ir_IF_TRUE_cold(if_overflow);
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
if (opline->result_type != IS_UNUSED
&& (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ)
&& prop_info
&& !ZEND_TYPE_IS_SET(prop_info->type)
&& (res_info & MAY_BE_GUARD)
&& (res_info & MAY_BE_LONG)) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
int32_t exit_point;
const void *exit_addr;
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD;
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
}
}
if (slow_inputs) {
const void *func;
ir_MERGE_list(slow_inputs);
// JIT: zend_jit_pre_inc_obj_helper(zobj, name, CACHE_ADDR(opline->extended_value), result);
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
func = zend_jit_pre_inc_obj_helper;
break;
case ZEND_PRE_DEC_OBJ:
func = zend_jit_pre_dec_obj_helper;
break;
case ZEND_POST_INC_OBJ:
func = zend_jit_post_inc_obj_helper;
break;
case ZEND_POST_DEC_OBJ:
func = zend_jit_post_dec_obj_helper;
break;
default:
ZEND_UNREACHABLE();
}
may_throw = 1;
jit_SET_EX_OPLINE(jit, opline);
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(func),
obj_ref,
ir_CONST_ADDR(name),
ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
(opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(end_inputs);
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
{
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
const zend_op *next_opline = NULL;
ir_refs *slow_inputs;
ir_refs_init(slow_inputs, 8);
if (trace) {
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
ZEND_ASSERT(trace->opline != NULL);
next_opline = trace->opline;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
zval *jump_zv = NULL;
int b;
if (opline->opcode == ZEND_SWITCH_LONG) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
}
} else if (opline->opcode == ZEND_MATCH) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
} else if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
}
} else {
ZEND_UNREACHABLE();
}
if (next_opline) {
const zend_op *target;
if (jump_zv != NULL) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
}
ZEND_ASSERT(target == next_opline);
} else {
if (jump_zv != NULL) {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
} else {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
}
_zend_jit_add_predecessor_ref(jit, b, jit->b, ir_END());
jit->b = -1;
}
} else {
zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
uint32_t op1_info = OP1_INFO();
zend_jit_addr op1_addr = OP1_ADDR();
const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
const zend_op *target;
int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
int b;
int32_t exit_point;
const void *exit_addr;
const void *default_label = NULL;
zval *zv;
if (next_opline) {
if (next_opline != default_opline) {
exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
default_label = zend_jit_trace_get_exit_addr(exit_point);
if (!default_label) {
return 0;
}
}
}
if (opline->opcode == ZEND_SWITCH_LONG) {
if (op1_info & MAY_BE_LONG) {
const void *fallback_label = NULL;
if (next_opline) {
if (next_opline != opline + 1) {
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
fallback_label = zend_jit_trace_get_exit_addr(exit_point);
if (!fallback_label) {
return 0;
}
}
}
if (op1_info & MAY_BE_REF) {
ir_ref ref, if_long, fast_path, ref2;
ref = jit_ZVAL_ADDR(jit, op1_addr);
if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_long);
fast_path = ir_END();
ir_IF_FALSE_cold(if_long);
// JIT: ZVAL_DEREF(op)
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
} else {
ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_FALSE_cold(if_ref);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_ref);
}
ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
} else {
if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE_cold(if_long);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_long);
}
ir_MERGE_2(fast_path, ir_END());
ref = ir_PHI_2(IR_ADDR, ref, ref2);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
} else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
} else {
ir_ref if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE_cold(if_long);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_long);
}
}
ir_ref ref = jit_Z_LVAL(jit, op1_addr);
if (!HT_IS_PACKED(jumptable)) {
ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
ir_CONST_ADDR(jumptable), ref);
ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
/* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
if (sizeof(Bucket) == 32) {
ref = ir_SHR_L(ref, ir_CONST_LONG(5));
} else {
ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
}
}
ref = ir_SWITCH(ref);
if (next_opline) {
ir_ref continue_list = IR_UNUSED;
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
ir_ref idx;
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
if (HT_IS_PACKED(jumptable)) {
idx = ir_CONST_LONG(zv - jumptable->arPacked);
} else {
idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
}
ir_CASE_VAL(ref, idx);
if (target == next_opline) {
ir_END_list(continue_list);
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} ZEND_HASH_FOREACH_END();
ir_CASE_DEFAULT(ref);
if (next_opline == default_opline) {
ir_END_list(continue_list);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
}
if (continue_list) {
ir_MERGE_list(continue_list);
} else {
ZEND_ASSERT(slow_inputs->count);
ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
}
} else {
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
b = ssa->cfg.map[target - op_array->opcodes];
_zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
} ZEND_HASH_FOREACH_END();
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
if (slow_inputs->count) {
ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
_zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
}
jit->b = -1;
}
} else if (!next_opline) {
_zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
jit->b = -1;
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (op1_info & MAY_BE_STRING) {
const void *fallback_label = NULL;
if (next_opline) {
if (next_opline != opline + 1) {
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
fallback_label = zend_jit_trace_get_exit_addr(exit_point);
if (!fallback_label) {
return 0;
}
}
}
if (op1_info & MAY_BE_REF) {
ir_ref ref, if_string, fast_path, ref2;
ref = jit_ZVAL_ADDR(jit, op1_addr);
if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_TRUE(if_string);
fast_path = ir_END();
ir_IF_FALSE_cold(if_string);
// JIT: ZVAL_DEREF(op)
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
} else {
ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE_cold(if_ref);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_ref);
}
ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
} else {
if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE_cold(if_string);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_string);
}
ir_MERGE_2(fast_path, ir_END());
ref = ir_PHI_2(IR_ADDR, ref, ref2);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
} else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, fallback_label);
} else {
ir_ref if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE_cold(if_string);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_string);
}
}
ir_ref ref = jit_Z_PTR(jit, op1_addr);
ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
ir_CONST_ADDR(jumptable), ref);
ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
/* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
if (sizeof(Bucket) == 32) {
ref = ir_SHR_L(ref, ir_CONST_LONG(5));
} else {
ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
}
ref = ir_SWITCH(ref);
if (next_opline) {
ir_ref continue_list = IR_UNUSED;
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
ir_ref idx;
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
if (HT_IS_PACKED(jumptable)) {
idx = ir_CONST_LONG(zv - jumptable->arPacked);
} else {
idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
}
ir_CASE_VAL(ref, idx);
if (target == next_opline) {
ir_END_list(continue_list);
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} ZEND_HASH_FOREACH_END();
ir_CASE_DEFAULT(ref);
if (next_opline == default_opline) {
ir_END_list(continue_list);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
}
if (continue_list) {
ir_MERGE_list(continue_list);
} else {
ZEND_ASSERT(slow_inputs->count);
ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
}
} else {
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
b = ssa->cfg.map[target - op_array->opcodes];
_zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
} ZEND_HASH_FOREACH_END();
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
if (slow_inputs->count) {
ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
_zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
}
jit->b = -1;
}
} else if (!next_opline) {
_zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
jit->b = -1;
}
} else if (opline->opcode == ZEND_MATCH) {
ir_ref if_type = IR_UNUSED, default_input_list = IR_UNUSED, ref = IR_UNUSED;
ir_ref continue_list = IR_UNUSED;
if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
ir_ref long_path = IR_UNUSED;
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_LONG) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (op1_info & (MAY_BE_STRING|MAY_BE_UNDEF)) {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_type);
} else if (default_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, default_label);
} else if (next_opline) {
ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE(if_type);
ir_END_list(continue_list);
ir_IF_TRUE(if_type);
} else {
ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE(if_type);
ir_END_list(default_input_list);
ir_IF_TRUE(if_type);
}
}
ref = jit_Z_LVAL(jit, op1_addr);
ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
ir_CONST_ADDR(jumptable), ref);
if (op1_info & MAY_BE_STRING) {
long_path = ir_END();
}
}
if (op1_info & MAY_BE_STRING) {
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IS_UNUSED;
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
if (op1_info & MAY_BE_UNDEF) {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_TRUE(if_type);
} else if (default_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, default_label);
} else if (next_opline) {
ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE(if_type);
ir_END_list(continue_list);
ir_IF_TRUE(if_type);
} else {
ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE(if_type);
ir_END_list(default_input_list);
ir_IF_TRUE(if_type);
}
}
ir_ref ref2 = jit_Z_PTR(jit, op1_addr);
ref2 = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
ir_CONST_ADDR(jumptable), ref2);
if (op1_info & MAY_BE_LONG) {
ir_MERGE_WITH(long_path);
ref = ir_PHI_2(IR_LONG, ref2, ref);
} else {
ref = ref2;
}
}
ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
/* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
if (HT_IS_PACKED(jumptable)) {
ZEND_ASSERT(sizeof(zval) == 16);
ref = ir_SHR_L(ref, ir_CONST_LONG(4));
} else {
if (sizeof(Bucket) == 32) {
ref = ir_SHR_L(ref, ir_CONST_LONG(5));
} else {
ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
}
}
ref = ir_SWITCH(ref);
if (next_opline) {
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
ir_ref idx;
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
if (HT_IS_PACKED(jumptable)) {
idx = ir_CONST_LONG(zv - jumptable->arPacked);
} else {
idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
}
ir_CASE_VAL(ref, idx);
if (target == next_opline) {
ir_END_list(continue_list);
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} ZEND_HASH_FOREACH_END();
ir_CASE_DEFAULT(ref);
if (next_opline == default_opline) {
ir_END_list(continue_list);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
}
} else {
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
b = ssa->cfg.map[target - op_array->opcodes];
_zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
} ZEND_HASH_FOREACH_END();
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
}
} else if (!(op1_info & MAY_BE_UNDEF)) {
if (next_opline) {
if (next_opline == default_opline) {
ir_END_list(continue_list);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
}
} else {
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
}
}
if (op1_info & MAY_BE_UNDEF) {
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IS_UNUSED;
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
if (default_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_UNDEF, default_label);
} else if (next_opline) {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
ir_IF_TRUE(if_def);
ir_END_list(continue_list);
ir_IF_FALSE_cold(if_def);
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
ir_IF_TRUE(if_def);
ir_END_list(default_input_list);
ir_IF_FALSE_cold(if_def);
}
}
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
ir_CONST_U32(opline->op1.var));
zend_jit_check_exception_undef_result(jit, opline);
if (default_label) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
} else if (next_opline) {
ir_END_list(continue_list);
} else {
ir_END_list(default_input_list);
}
}
if (next_opline) {
ZEND_ASSERT(continue_list);
ir_MERGE_list(continue_list);
} else {
if (default_input_list) {
if (jit->ctx.ir_base[ref].op == IR_SWITCH) {
ZEND_ASSERT(jit->ctx.ir_base[ref].op3 == IR_UNUSED);
jit->ctx.ir_base[ref].op3 = default_input_list;
} else {
ir_MERGE_list(default_input_list);
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
}
}
jit->b = -1;
}
} else {
ZEND_UNREACHABLE();
}
}
return 1;
}
static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa)
{
int i, count;
zend_basic_block *bb;
zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_BR_TARGET));
jit->ctx.spill_base = ZREG_FP;
jit->op_array = jit->current_op_array = op_array;
jit->ssa = ssa;
jit->bb_start_ref = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count * 2, sizeof(ir_ref));
jit->bb_predecessors = jit->bb_start_ref + ssa->cfg.blocks_count;
count = 0;
for (i = 0, bb = ssa->cfg.blocks; i < ssa->cfg.blocks_count; i++, bb++) {
jit->bb_predecessors[i] = count;
count += bb->predecessors_count;
}
jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref));
if (!GCC_GLOBAL_REGS) {
ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
jit_STORE_FP(jit, ref);
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
return 1;
}
static void *zend_jit_finish(zend_jit_ctx *jit)
{
void *entry;
size_t size;
zend_string *str = NULL;
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP|
ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_SCCP|
ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS|ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
if (jit->name) {
str = zend_string_copy(jit->name);
} else {
str = zend_jit_func_name(jit->op_array);
}
}
if (jit->op_array) {
/* Only for function JIT */
_zend_jit_fix_merges(jit);
#if defined(IR_TARGET_AARCH64)
} else if (jit->trace) {
jit->ctx.deoptimization_exits = jit->trace->exit_count;
jit->ctx.get_exit_addr = zend_jit_trace_get_exit_addr;
#endif
} else {
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
jit->ctx.flags |= IR_GEN_CACHE_DEMOTE;
#endif
}
entry = zend_jit_ir_compile(&jit->ctx, &size, str ? ZSTR_VAL(str) : NULL);
if (entry) {
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
#ifdef HAVE_CAPSTONE
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
if (str) {
ir_disasm_add_symbol(ZSTR_VAL(str), (uintptr_t)entry, size);
}
ir_disasm(str ? ZSTR_VAL(str) : "unknown",
entry, size,
(JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0,
&jit->ctx, stderr);
}
#endif
#ifndef _WIN32
if (str) {
if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
uintptr_t sp_offset = 0;
// ir_mem_unprotect(entry, size);
if (!(jit->ctx.flags & IR_FUNCTION)
&& zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
sp_offset = zend_jit_hybrid_vm_sp_adj;
#else
sp_offset = sizeof(void*);
#endif
} else {
sp_offset = sizeof(void*);
}
ir_gdb_register(ZSTR_VAL(str), entry, size, sp_offset, 0);
// ir_mem_protect(entry, size);
}
if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
ir_perf_map_register(ZSTR_VAL(str), entry, size);
if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
ir_perf_jitdump_register(ZSTR_VAL(str), entry, size);
}
}
}
#endif
}
if (jit->op_array) {
/* Only for function JIT */
const zend_op_array *op_array = jit->op_array;
zend_op *opline = (zend_op*)op_array->opcodes;
if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
while (opline->opcode == ZEND_RECV) {
opline++;
}
}
opline->handler = entry;
if (jit->ctx.entries_count) {
/* For all entries */
int i = jit->ctx.entries_count;
do {
ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]];
op_array->opcodes[insn->op2].handler = (char*)entry + insn->op3;
} while (i != 0);
}
} else {
/* Only for tracing JIT */
zend_jit_trace_info *t = jit->trace;
zend_jit_trace_stack *stack;
uint32_t i;
if (t) {
for (i = 0; i < t->stack_map_size; i++) {
stack = t->stack_map + i;
if (stack->flags & ZREG_SPILL_SLOT) {
stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP;
stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref);
}
}
}
zend_jit_trace_add_code(entry, size);
}
}
if (str) {
zend_string_release(str);
}
return entry;
}
static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
{
const void *entry;
size_t size;
ir_code_buffer code_buffer;
code_buffer.start = dasm_buf;
code_buffer.end = dasm_end;
code_buffer.pos = *dasm_ptr;
entry = ir_emit_exitgroup(n, ZEND_JIT_EXIT_POINTS_PER_GROUP, zend_jit_stub_handlers[jit_stub_trace_exit],
&code_buffer, &size);
*dasm_ptr = code_buffer.pos;
if (entry) {
#ifdef HAVE_CAPSTONE
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
uint32_t i;
char name[32];
for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
snprintf(name, sizeof(name), "jit$$trace_exit_%d", n + i);
ir_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
}
}
#endif
}
return entry;
}
static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
return 1;
}
static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_LT(jit_Z_TYPE(jit, addr), ir_CONST_U8(IS_STRING)), ir_CONST_ADDR(exit_addr));
return 1;
}
static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr)
{
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_NE(jit_Z_TYPE(jit, var_addr), ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
return 1;
}
static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline)
{
uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_CMP_IP(jit, IR_EQ, opline), ir_CONST_ADDR(exit_addr));
zend_jit_set_last_valid_opline(jit, opline);
return 1;
}
static bool zend_jit_guard_reference(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr *var_addr_ptr,
zend_jit_addr *ref_addr_ptr,
bool add_ref_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
const void *exit_addr = NULL;
ir_ref ref;
if (add_ref_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ref = jit_Z_TYPE(jit, var_addr);
ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
}
ref = jit_Z_PTR(jit, var_addr);
*ref_addr_ptr = ZEND_ADDR_REF_ZVAL(ref);
ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
var_addr = ZEND_ADDR_REF_ZVAL(ref);
*var_addr_ptr = var_addr;
return 1;
}
static bool zend_jit_fetch_reference(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t var_type,
uint32_t *var_info_ptr,
zend_jit_addr *var_addr_ptr,
bool add_ref_guard,
bool add_type_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
const void *exit_addr = NULL;
ir_ref ref;
if (add_ref_guard || add_type_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (add_ref_guard) {
ref = jit_Z_TYPE(jit, var_addr);
ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
}
if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
/* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper),
jit_ZVAL_ADDR(jit, var_addr));
*var_addr_ptr = var_addr;
} else {
ref = jit_Z_PTR(jit, var_addr);
ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
var_addr = ZEND_ADDR_REF_ZVAL(ref);
*var_addr_ptr = var_addr;
}
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (add_type_guard
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
ref = jit_Z_TYPE(jit, var_addr);
ir_GUARD(ir_EQ(ref, ir_CONST_U8(var_type)), ir_CONST_ADDR(exit_addr));
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
} else {
var_info &= ~MAY_BE_REF;
*var_info_ptr = var_info;
}
*var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */
return 1;
}
static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
int32_t exit_point;
const void *exit_addr;
ir_ref ref = IR_UNUSED;
if (add_indirect_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, var_addr, IS_INDIRECT, exit_addr);
ref = jit_Z_PTR(jit, var_addr);
} else {
/* This LOAD of INDIRECT VAR, stored by the previous FETCH_(DIM/OBJ)_W,
* is eliminated by store forwarding (S2L) */
ref = jit_Z_PTR(jit, var_addr);
}
*var_info_ptr &= ~MAY_BE_INDIRECT;
var_addr = ZEND_ADDR_REF_ZVAL(ref);
*var_addr_ptr = var_addr;
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (!(var_type & IS_TRACE_REFERENCE)
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, var_addr, var_type, exit_addr);
//var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
}
return 1;
}
static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
{
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
size_t offset = jit_extension->offset;
const void *handler =
(zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
ir_ref ref;
zend_jit_set_ip(jit, opline);
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
} else {
ref = jit_FP(jit);
ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
}
if (may_throw
&& opline->opcode != ZEND_RETURN
&& opline->opcode != ZEND_RETURN_BY_REF) {
zend_jit_check_exception(jit);
}
while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
trace++;
}
if (!GCC_GLOBAL_REGS
&& (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_GENERATOR_CREATE) {
ir_ref addr = jit_EG(current_execute_data);
jit_STORE_FP(jit, ir_LOAD_A(addr));
}
}
if (zend_jit_trace_may_exit(op_array, opline)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_GENERATOR_CREATE) {
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
/* this check may be handled by the following OPLINE guard or jmp [IP] */
ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
jit_STUB_ADDR(jit, jit_stub_trace_halt));
}
} else if (GCC_GLOBAL_REGS) {
ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
} else {
ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
}
} else if (opline->opcode == ZEND_EXIT ||
opline->opcode == ZEND_GENERATOR_RETURN ||
opline->opcode == ZEND_YIELD ||
opline->opcode == ZEND_YIELD_FROM) {
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_halt));
ir_BEGIN(IR_UNUSED); /* unreachable block */
}
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
const zend_op *next_opline = trace->opline;
const zend_op *exit_opline = NULL;
uint32_t exit_point;
const void *exit_addr;
uint32_t old_info = 0;
uint32_t old_res_info = 0;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
if (zend_is_smart_branch(opline)) {
bool exit_if_true = 0;
exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
} else {
switch (opline->opcode) {
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
case ZEND_JMP_SET:
case ZEND_COALESCE:
case ZEND_JMP_NULL:
case ZEND_FE_RESET_R:
case ZEND_FE_RESET_RW:
exit_opline = (trace->opline == opline + 1) ?
OP_JMP_ADDR(opline, opline->op2) :
opline + 1;
break;
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
exit_opline = (trace->opline == opline + 1) ?
ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
opline + 1;
break;
}
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
}
break;
case ZEND_BIND_INIT_STATIC_OR_JMP:
if (opline->op1_type == IS_CV) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
}
break;
}
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
}
exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
}
break;
case ZEND_BIND_INIT_STATIC_OR_JMP:
if (opline->op1_type == IS_CV) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
}
break;
}
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), ir_CONST_ADDR(exit_addr));
}
}
zend_jit_set_last_valid_opline(jit, trace->opline);
return 1;
}
static int zend_jit_deoptimizer_start(zend_jit_ctx *jit,
zend_string *name,
uint32_t trace_num,
uint32_t exit_num)
{
zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
jit->ctx.spill_base = ZREG_FP;
jit->op_array = NULL;
jit->ssa = NULL;
jit->name = zend_string_copy(name);
jit->ctx.flags |= IR_SKIP_PROLOGUE;
return 1;
}
static int zend_jit_trace_start(zend_jit_ctx *jit,
const zend_op_array *op_array,
zend_ssa *ssa,
zend_string *name,
uint32_t trace_num,
zend_jit_trace_info *parent,
uint32_t exit_num)
{
zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
jit->ctx.spill_base = ZREG_FP;
jit->op_array = NULL;
jit->current_op_array = op_array;
jit->ssa = ssa;
jit->name = zend_string_copy(name);
if (!GCC_GLOBAL_REGS) {
if (!parent) {
ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
jit_STORE_FP(jit, ref);
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
}
if (parent) {
jit->ctx.flags |= IR_SKIP_PROLOGUE;
}
if (parent) {
int i;
int parent_vars_count = parent->exit_info[exit_num].stack_size;
zend_jit_trace_stack *parent_stack = parent_vars_count == 0 ? NULL :
parent->stack_map +
parent->exit_info[exit_num].stack_offset;
/* prevent clobbering of registers used for deoptimization */
for (i = 0; i < parent_vars_count; i++) {
if (STACK_FLAGS(parent_stack, i) != ZREG_CONST
&& STACK_REG(parent_stack, i) != ZREG_NONE) {
int32_t reg = STACK_REG(parent_stack, i);
ir_type type;
if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
type = IR_ADDR;
} else if (STACK_TYPE(parent_stack, i) == IS_LONG) {
type = IR_LONG;
} else if (STACK_TYPE(parent_stack, i) == IS_DOUBLE) {
type = IR_DOUBLE;
} else {
ZEND_UNREACHABLE();
}
if (ssa && ssa->vars[i].no_val) {
/* pass */
} else {
ir_ref ref = ir_RLOAD(type, reg);
if (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) {
/* op3 is used as a flag that the value is already stored in memory.
* In case the IR framework decides to spill the result of IR_LOAD,
* it doesn't have to store the value once again.
*
* See: insn->op3 check in ir_emit_rload()
*/
ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i));
}
}
}
}
}
if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 && parent->exit_info[exit_num].poly_this_reg >= 0);
ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg);
ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg);
}
ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num));
return 1;
}
static int zend_jit_trace_begin_loop(zend_jit_ctx *jit)
{
return ir_LOOP_BEGIN(ir_END());
}
static void zend_jit_trace_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
{
int dst_var = phi->ssa_var;
int src_var = phi->sources[0];
ir_ref ref;
ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
ZEND_ASSERT(jit->ra[src_var].ref != IR_UNUSED && jit->ra[src_var].ref != IR_NULL);
ref = ir_PHI_2(
(jit->ssa->var_info[src_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)), IR_UNUSED);
src_var = phi->sources[1];
ZEND_ASSERT(jit->ra[src_var].ref == IR_NULL);
jit->ra[src_var].flags |= ZREG_FORWARD;
zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
}
static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *timeout_exit_addr)
{
if (timeout_exit_addr) {
zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
}
ZEND_ASSERT(jit->ctx.ir_base[loop_ref].op2 == IR_UNUSED);
ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
return 1;
}
static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline)
{
if (GCC_GLOBAL_REGS) {
if (!original_handler) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_TAILCALL(IR_VOID, zend_jit_orig_opline_handler(jit));
}
} else {
if (original_handler) {
ir_ref ref;
ir_ref addr = zend_jit_orig_opline_handler(jit);
#if defined(IR_TARGET_X86)
addr = ir_CAST_FC_FUNC(addr);
#endif
ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
if (opline &&
(opline->opcode == ZEND_RETURN
|| opline->opcode == ZEND_RETURN_BY_REF
|| opline->opcode == ZEND_GENERATOR_RETURN
|| opline->opcode == ZEND_GENERATOR_CREATE
|| opline->opcode == ZEND_YIELD
|| opline->opcode == ZEND_YIELD_FROM)) {
ir_RETURN(ref);
return 1;
}
}
ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
}
return 1;
}
static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
{
return ir_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
}
static int zend_jit_trace_link_to_root(zend_jit_ctx *jit, zend_jit_trace_info *t, const void *timeout_exit_addr)
{
const void *link_addr;
/* Skip prologue. */
ZEND_ASSERT(zend_jit_trace_prologue_size != (size_t)-1);
link_addr = (const void*)((const char*)t->code_start + zend_jit_trace_prologue_size);
if (timeout_exit_addr) {
zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
}
ir_IJMP(ir_CONST_ADDR(link_addr));
return 1;
}
static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
{
uint32_t op1_info, op2_info;
switch (opline->opcode) {
case ZEND_SEND_VAR:
case ZEND_SEND_VAL:
case ZEND_SEND_VAL_EX:
return (opline->op2_type != IS_CONST) && (opline->opcode != ZEND_SEND_VAL_EX || opline->op2.num <= MAX_ARG_FLAG_NUM);
case ZEND_QM_ASSIGN:
case ZEND_IS_SMALLER:
case ZEND_IS_SMALLER_OR_EQUAL:
case ZEND_IS_EQUAL:
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_IS_NOT_IDENTICAL:
case ZEND_CASE:
return 1;
case ZEND_RETURN:
return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
case ZEND_ASSIGN:
return (opline->op1_type == IS_CV);
case ZEND_ASSIGN_OP:
if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
return 0;
}
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return zend_jit_supported_binary_op(opline->extended_value, op1_info, op2_info);
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
return 0;
}
if (trace && trace->op1_type != IS_UNKNOWN) {
op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
}
if (trace && trace->op2_type != IS_UNKNOWN) {
op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
}
return !(op1_info & MAY_BE_UNDEF)
&& !(op2_info & MAY_BE_UNDEF)
&& (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
&& (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace && trace->op1_type != IS_UNKNOWN) {
op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
}
if (trace && trace->op2_type != IS_UNKNOWN) {
op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
}
return (op1_info & MAY_BE_LONG)
&& (op2_info & MAY_BE_LONG);
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
op1_info = OP1_INFO();
return opline->op1_type == IS_CV
&& (op1_info & MAY_BE_LONG)
&& !(op1_info & MAY_BE_REF);
case ZEND_STRLEN:
op1_info = OP1_INFO();
return (opline->op1_type & (IS_CV|IS_CONST))
&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING;
case ZEND_COUNT:
op1_info = OP1_INFO();
return (opline->op1_type & (IS_CV|IS_CONST))
&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY;
case ZEND_JMPZ:
case ZEND_JMPNZ:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if (!ssa->cfg.map) {
return 0;
}
if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
return 0;
}
}
ZEND_FALLTHROUGH;
case ZEND_BOOL:
case ZEND_BOOL_NOT:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
return 1;
case ZEND_FETCH_CONSTANT:
return 1;
case ZEND_ISSET_ISEMPTY_DIM_OBJ:
if ((opline->extended_value & ZEND_ISEMPTY)) {
return 0;
}
ZEND_FALLTHROUGH;
case ZEND_FETCH_DIM_R:
case ZEND_FETCH_DIM_IS:
case ZEND_FETCH_LIST_R:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace
&& trace->op1_type != IS_UNKNOWN
&& (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
}
return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING));
case ZEND_ASSIGN_DIM_OP:
if (opline->result_type != IS_UNUSED) {
return 0;
}
if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
return 0;
}
ZEND_FALLTHROUGH;
case ZEND_ASSIGN_DIM:
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_DIM_RW:
case ZEND_FETCH_LIST_W:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace) {
if (opline->op1_type == IS_CV) {
if ((opline->opcode == ZEND_ASSIGN_DIM
|| opline->opcode == ZEND_ASSIGN_DIM_OP)
&& (opline+1)->op1_type == IS_CV
&& (opline+1)->op1.var == opline->op1.var) {
/* skip $a[x] = $a; */
return 0;
}
} else if (opline->op1_type == IS_VAR) {
if (trace->op1_type == IS_UNKNOWN
|| !(trace->op1_type & IS_TRACE_INDIRECT)
|| opline->result_type != IS_UNUSED) {
return 0;
}
}
if (trace->op1_type != IS_UNKNOWN
&& (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
}
} else {
if (opline->op1_type != IS_CV) {
return 0;
}
}
return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING));
case ZEND_ASSIGN_OBJ_OP:
if (opline->result_type != IS_UNUSED) {
return 0;
}
if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
return 0;
}
ZEND_FALLTHROUGH;
case ZEND_FETCH_OBJ_R:
case ZEND_ASSIGN_OBJ:
if (opline->op2_type != IS_CONST
|| Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
|| Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
return 0;
}
op1_info = OP1_INFO();
return opline->op1_type == IS_UNUSED || (op1_info & MAY_BE_OBJECT);
}
return 0;
}
static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
{
if (ssa->vars[var].no_val) {
/* we don't need the value */
return 0;
}
if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
/* Disable global register allocation,
* register allocation for SSA variables connected through Phi functions
*/
if (ssa->vars[var].definition_phi) {
return 0;
}
if (ssa->vars[var].phi_use_chain) {
zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
do {
if (!ssa->vars[phi->ssa_var].no_val) {
return 0;
}
phi = zend_ssa_next_use_phi(ssa, var, phi);
} while (phi);
}
}
if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
/* bad type */
return 0;
}
return 1;
}
static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
{
if (!zend_jit_var_supports_reg(ssa, var)) {
return 0;
}
if (ssa->vars[var].definition >= 0) {
uint32_t def = ssa->vars[var].definition;
if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
return 0;
}
}
if (ssa->vars[var].use_chain >= 0) {
int use = ssa->vars[var].use_chain;
do {
if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
return 0;
}
use = zend_ssa_next_use(ssa->ops, var, use);
} while (use >= 0);
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
int def_block, use_block, b, use, j;
zend_basic_block *bb;
zend_ssa_phi *p;
bool ret = 1;
zend_worklist worklist;
ALLOCA_FLAG(use_heap)
/* Check if live range is split by ENTRY block */
if (ssa->vars[var].definition >= 0) {
def_block =ssa->cfg.map[ssa->vars[var].definition];
} else {
ZEND_ASSERT(ssa->vars[var].definition_phi);
def_block = ssa->vars[var].definition_phi->block;
}
ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
if (ssa->vars[var].use_chain >= 0) {
use = ssa->vars[var].use_chain;
do {
use_block = ssa->cfg.map[use];
if (use_block != def_block) {
zend_worklist_push(&worklist, use_block);
}
use = zend_ssa_next_use(ssa->ops, var, use);
} while (use >= 0);
}
p = ssa->vars[var].phi_use_chain;
while (p) {
use_block = p->block;
if (use_block != def_block) {
bb = &ssa->cfg.blocks[use_block];
for (j = 0; j < bb->predecessors_count; j++) {
if (p->sources[j] == var) {
use_block = ssa->cfg.predecessors[bb->predecessor_offset + j];
if (use_block != def_block) {
zend_worklist_push(&worklist, use_block);
}
}
}
}
p = zend_ssa_next_use_phi(ssa, var, p);
}
while (zend_worklist_len(&worklist) != 0) {
b = zend_worklist_pop(&worklist);
bb = &ssa->cfg.blocks[b];
if (bb->flags & (ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
ret = 0;
break;
}
for (j = 0; j < bb->predecessors_count; j++) {
b = ssa->cfg.predecessors[bb->predecessor_offset + j];
if (b != def_block) {
zend_worklist_push(&worklist, b);
}
}
}
ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
return ret;
}
return 1;
}
static ir_ref jit_frameless_observer(zend_jit_ctx *jit, const zend_op *opline) {
// JIT: zend_observer_handler_is_unobserved(ZEND_OBSERVER_DATA(fbc))
ir_ref observer_handler;
zend_function *fbc = ZEND_FLF_FUNC(opline);
// Not need for runtime cache or generator checks here, we just need if_unobserved
ir_ref if_unobserved = jit_observer_fcall_is_unobserved_start(jit, fbc, &observer_handler, IR_UNUSED, IR_UNUSED).if_unobserved;
// Call zend_frameless_observed_call for the main logic.
ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)zend_frameless_observed_call), jit_FP(jit));
ir_ref skip = ir_END();
ir_IF_TRUE(if_unobserved);
return skip;
}
static void jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline)
{
jit_SET_EX_OPLINE(jit, opline);
void *function = ZEND_FLF_HANDLER(opline);
zend_jit_addr res_addr = RES_ADDR();
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_ref skip_observer = IR_UNUSED;
if (ZEND_OBSERVER_ENABLED) {
skip_observer = jit_frameless_observer(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref);
if (skip_observer != IR_UNUSED) {
ir_MERGE_WITH(skip_observer);
}
zend_jit_check_exception(jit);
}
static void jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
{
jit_SET_EX_OPLINE(jit, opline);
/* Avoid dropping RC check in case op escapes. */
if (op1_info & MAY_BE_RC1) {
op1_info |= MAY_BE_RCN;
}
void *function = ZEND_FLF_HANDLER(opline);
zend_jit_addr res_addr = RES_ADDR();
zend_jit_addr op1_addr = OP1_ADDR();
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
}
if (op1_info & MAY_BE_REF) {
op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
}
ir_ref skip_observer = IR_UNUSED;
if (ZEND_OBSERVER_ENABLED) {
skip_observer = jit_frameless_observer(jit, opline);
}
ir_CALL_2(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref);
if (skip_observer != IR_UNUSED) {
ir_MERGE_WITH(skip_observer);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
zend_jit_check_exception(jit);
}
static void jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info)
{
jit_SET_EX_OPLINE(jit, opline);
/* Avoid dropping RC check in case op escapes. */
if (op1_info & MAY_BE_RC1) {
op1_info |= MAY_BE_RCN;
}
if (op2_info & MAY_BE_RC1) {
op2_info |= MAY_BE_RCN;
}
void *function = ZEND_FLF_HANDLER(opline);
zend_jit_addr res_addr = RES_ADDR();
zend_jit_addr op1_addr = OP1_ADDR();
zend_jit_addr op2_addr = OP2_ADDR();
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
}
if (op1_info & MAY_BE_REF) {
op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
}
if (op2_info & MAY_BE_REF) {
op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
}
ir_ref skip_observer = IR_UNUSED;
if (ZEND_OBSERVER_ENABLED) {
skip_observer = jit_frameless_observer(jit, opline);
}
ir_CALL_3(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref);
if (skip_observer != IR_UNUSED) {
ir_MERGE_WITH(skip_observer);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
/* Set OP1 to UNDEF in case FREE_OP2() throws. */
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) != 0 && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0) {
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
zend_jit_check_exception(jit);
}
static void jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, uint32_t op1_data_info)
{
jit_SET_EX_OPLINE(jit, opline);
/* Avoid dropping RC check in case op escapes. */
if (op1_info & MAY_BE_RC1) {
op1_info |= MAY_BE_RCN;
}
if (op2_info & MAY_BE_RC1) {
op2_info |= MAY_BE_RCN;
}
if (op1_data_info & MAY_BE_RC1) {
op1_data_info |= MAY_BE_RCN;
}
void *function = ZEND_FLF_HANDLER(opline);
uint8_t op_data_type = (opline + 1)->op1_type;
zend_jit_addr res_addr = RES_ADDR();
zend_jit_addr op1_addr = OP1_ADDR();
zend_jit_addr op2_addr = OP2_ADDR();
zend_jit_addr op3_addr = OP1_DATA_ADDR();
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
ir_ref op3_ref = jit_ZVAL_ADDR(jit, op3_addr);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
}
if ((opline+1)->op1_type == IS_CV && (op1_data_info & MAY_BE_UNDEF)) {
zend_jit_zval_check_undef(jit, op3_ref, (opline+1)->op1.var, opline, 1);
}
if (op1_info & MAY_BE_REF) {
op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
}
if (op2_info & MAY_BE_REF) {
op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
}
if (op1_data_info & MAY_BE_REF) {
op3_ref = jit_ZVAL_DEREF_ref(jit, op3_ref);
}
ir_ref skip_observer = IR_UNUSED;
if (ZEND_OBSERVER_ENABLED) {
skip_observer = jit_frameless_observer(jit, opline);
}
ir_CALL_4(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref, op3_ref);
if (skip_observer != IR_UNUSED) {
ir_MERGE_WITH(skip_observer);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
/* Set OP1 to UNDEF in case FREE_OP2() throws. */
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
|| (op_data_type & (IS_VAR|IS_TMP_VAR)))) {
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
/* If OP1 is a TMP|VAR, we don't need to set OP2 to UNDEF on free because
* zend_fetch_debug_backtrace aborts when it encounters the first UNDEF TMP|VAR. */
if (!(opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0
&& (op_data_type & (IS_VAR|IS_TMP_VAR)) != 0) {
jit_set_Z_TYPE_INFO(jit, op2_addr, IS_UNDEF);
}
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
zend_jit_check_exception(jit);
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* indent-tabs-mode: t
* End:
*/
Reference in a new issue View git blame Copy permalink