From 77bf144d5287bd39f0d380cd63e9414ee65563aa Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 14 Jan 2020 13:47:10 +0300 Subject: [PATCH] Inline hot parts of bitwise instructions into hybrid VM --- Zend/zend_vm_def.h | 29 +++++++---- Zend/zend_vm_execute.h | 111 ++++++++++++++++------------------------ Zend/zend_vm_handlers.h | 6 +-- Zend/zend_vm_opcodes.c | 2 +- 4 files changed, 66 insertions(+), 82 deletions(-) diff --git a/Zend/zend_vm_def.h b/Zend/zend_vm_def.h index faa8f6174bb..e41f571ba71 100644 --- a/Zend/zend_vm_def.h +++ b/Zend/zend_vm_def.h @@ -801,7 +801,7 @@ ZEND_VM_HELPER(zend_bw_or_helper, ANY, ANY, zval *op_1, zval *op_2) ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); } -ZEND_VM_COLD_CONSTCONST_HANDLER(9, ZEND_BW_OR, CONST|TMPVARCV, CONST|TMPVARCV, SPEC(COMMUTATIVE)) +ZEND_VM_HOT_NOCONSTCONST_HANDLER(9, ZEND_BW_OR, CONST|TMPVARCV, CONST|TMPVARCV, SPEC(COMMUTATIVE)) { USE_OPLINE zval *op1, *op2; @@ -840,7 +840,7 @@ ZEND_VM_HELPER(zend_bw_and_helper, ANY, ANY, zval *op_1, zval *op_2) ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); } -ZEND_VM_COLD_CONSTCONST_HANDLER(10, ZEND_BW_AND, CONST|TMPVARCV, CONST|TMPVARCV, SPEC(COMMUTATIVE)) +ZEND_VM_HOT_NOCONSTCONST_HANDLER(10, ZEND_BW_AND, CONST|TMPVARCV, CONST|TMPVARCV, SPEC(COMMUTATIVE)) { USE_OPLINE zval *op1, *op2; @@ -879,7 +879,7 @@ ZEND_VM_HELPER(zend_bw_xor_helper, ANY, ANY, zval *op_1, zval *op_2) ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); } -ZEND_VM_COLD_CONSTCONST_HANDLER(11, ZEND_BW_XOR, CONST|TMPVARCV, CONST|TMPVARCV, SPEC(COMMUTATIVE)) +ZEND_VM_HOT_NOCONSTCONST_HANDLER(11, ZEND_BW_XOR, CONST|TMPVARCV, CONST|TMPVARCV, SPEC(COMMUTATIVE)) { USE_OPLINE zval *op1, *op2; @@ -911,7 +911,20 @@ ZEND_VM_COLD_CONSTCONST_HANDLER(15, ZEND_BOOL_XOR, CONST|TMPVAR|CV, CONST|TMPVAR ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); } -ZEND_VM_COLD_CONST_HANDLER(13, ZEND_BW_NOT, CONST|TMPVAR|CV, ANY) +ZEND_VM_HELPER(zend_bw_not_helper, ANY, ANY, zval *op_1) +{ + USE_OPLINE + + SAVE_OPLINE(); + if (UNEXPECTED(Z_TYPE_P(op_1) == IS_UNDEF)) { + op_1 = ZVAL_UNDEFINED_OP1(); + } + bitwise_not_function(EX_VAR(opline->result.var), op_1); + FREE_OP1(); + ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); +} + +ZEND_VM_HOT_NOCONST_HANDLER(13, ZEND_BW_NOT, CONST|TMPVARCV, ANY) { USE_OPLINE zval *op1; @@ -922,13 +935,7 @@ ZEND_VM_COLD_CONST_HANDLER(13, ZEND_BW_NOT, CONST|TMPVAR|CV, ANY) ZEND_VM_NEXT_OPCODE(); } - SAVE_OPLINE(); - if (OP1_TYPE == IS_CV && UNEXPECTED(Z_TYPE_P(op1) == IS_UNDEF)) { - op1 = ZVAL_UNDEFINED_OP1(); - } - bitwise_not_function(EX_VAR(opline->result.var), op1); - FREE_OP1(); - ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); + ZEND_VM_DISPATCH_TO_HELPER(zend_bw_not_helper, op_1, op1); } ZEND_VM_COLD_CONST_HANDLER(14, ZEND_BOOL_NOT, CONST|TMPVAR|CV, ANY) diff --git a/Zend/zend_vm_execute.h b/Zend/zend_vm_execute.h index 9454ad2d687..65daa03b146 100644 --- a/Zend/zend_vm_execute.h +++ b/Zend/zend_vm_execute.h @@ -727,6 +727,19 @@ static zend_never_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_bw_xor_helpe ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); } +static zend_never_inline ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_bw_not_helper_SPEC(zval *op_1 ZEND_OPCODE_HANDLER_ARGS_DC) +{ + USE_OPLINE + + SAVE_OPLINE(); + if (UNEXPECTED(Z_TYPE_P(op_1) == IS_UNDEF)) { + op_1 = ZVAL_UNDEFINED_OP1(); + } + bitwise_not_function(EX_VAR(opline->result.var), op_1); + FREE_OP(opline->op1_type, opline->op1.var); + ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); +} + static zend_never_inline ZEND_COLD ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_this_not_in_object_context_helper_SPEC(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE @@ -3201,13 +3214,7 @@ static ZEND_VM_COLD ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_NOT_SPEC_CONST ZEND_VM_NEXT_OPCODE(); } - SAVE_OPLINE(); - if (IS_CONST == IS_CV && UNEXPECTED(Z_TYPE_P(op1) == IS_UNDEF)) { - op1 = ZVAL_UNDEFINED_OP1(); - } - bitwise_not_function(EX_VAR(opline->result.var), op1); - - ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); + ZEND_VM_TAIL_CALL(zend_bw_not_helper_SPEC(op1 ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); } static ZEND_VM_COLD ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BOOL_NOT_SPEC_CONST_HANDLER(ZEND_OPCODE_HANDLER_ARGS) @@ -10733,6 +10740,20 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_YIELD_SPEC_CONST_CV_HANDLER(ZE ZEND_VM_RETURN(); } +static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_NOT_SPEC_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) +{ + USE_OPLINE + zval *op1; + + op1 = EX_VAR(opline->op1.var); + if (EXPECTED(Z_TYPE_INFO_P(op1) == IS_LONG)) { + ZVAL_LONG(EX_VAR(opline->result.var), ~Z_LVAL_P(op1)); + ZEND_VM_NEXT_OPCODE(); + } + + ZEND_VM_TAIL_CALL(zend_bw_not_helper_SPEC(op1 ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); +} + static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_QM_ASSIGN_LONG_SPEC_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE @@ -11217,7 +11238,7 @@ is_smaller_or_equal_double: ZEND_VM_TAIL_CALL(zend_is_smaller_or_equal_helper_SPEC(op1, op2 ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); } -static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_OR_SPEC_TMPVARCV_CONST_HANDLER(ZEND_OPCODE_HANDLER_ARGS) +static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_OR_SPEC_TMPVARCV_CONST_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE zval *op1, *op2; @@ -11235,7 +11256,7 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_OR_SPEC_TMPVARCV_CONST_HAND ZEND_VM_TAIL_CALL(zend_bw_or_helper_SPEC(op1, op2 ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); } -static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_AND_SPEC_TMPVARCV_CONST_HANDLER(ZEND_OPCODE_HANDLER_ARGS) +static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_AND_SPEC_TMPVARCV_CONST_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE zval *op1, *op2; @@ -11253,7 +11274,7 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_AND_SPEC_TMPVARCV_CONST_HAN ZEND_VM_TAIL_CALL(zend_bw_and_helper_SPEC(op1, op2 ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); } -static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_XOR_SPEC_TMPVARCV_CONST_HANDLER(ZEND_OPCODE_HANDLER_ARGS) +static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_XOR_SPEC_TMPVARCV_CONST_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE zval *op1, *op2; @@ -12196,7 +12217,7 @@ is_smaller_or_equal_double: ZEND_VM_TAIL_CALL(zend_is_smaller_or_equal_helper_SPEC(op1, op2 ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); } -static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_OR_SPEC_TMPVARCV_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) +static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_OR_SPEC_TMPVARCV_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE zval *op1, *op2; @@ -12214,7 +12235,7 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_OR_SPEC_TMPVARCV_TMPVARCV_H ZEND_VM_TAIL_CALL(zend_bw_or_helper_SPEC(op1, op2 ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); } -static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_AND_SPEC_TMPVARCV_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) +static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_AND_SPEC_TMPVARCV_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE zval *op1, *op2; @@ -12232,7 +12253,7 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_AND_SPEC_TMPVARCV_TMPVARCV_ ZEND_VM_TAIL_CALL(zend_bw_and_helper_SPEC(op1, op2 ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC)); } -static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_XOR_SPEC_TMPVARCV_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) +static ZEND_VM_HOT ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_XOR_SPEC_TMPVARCV_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE zval *op1, *op2; @@ -12672,26 +12693,6 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_FETCH_LIST_R_SPEC_TMPVARCV_CV_ ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); } -static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_NOT_SPEC_TMPVAR_HANDLER(ZEND_OPCODE_HANDLER_ARGS) -{ - USE_OPLINE - zval *op1; - - op1 = _get_zval_ptr_var(opline->op1.var EXECUTE_DATA_CC); - if (EXPECTED(Z_TYPE_INFO_P(op1) == IS_LONG)) { - ZVAL_LONG(EX_VAR(opline->result.var), ~Z_LVAL_P(op1)); - ZEND_VM_NEXT_OPCODE(); - } - - SAVE_OPLINE(); - if ((IS_TMP_VAR|IS_VAR) == IS_CV && UNEXPECTED(Z_TYPE_P(op1) == IS_UNDEF)) { - op1 = ZVAL_UNDEFINED_OP1(); - } - bitwise_not_function(EX_VAR(opline->result.var), op1); - zval_ptr_dtor_nogc(EX_VAR(opline->op1.var)); - ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); -} - static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BOOL_NOT_SPEC_TMPVAR_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE @@ -35127,26 +35128,6 @@ static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_YIELD_SPEC_UNUSED_CV_HANDLER(Z ZEND_VM_RETURN(); } -static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BW_NOT_SPEC_CV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) -{ - USE_OPLINE - zval *op1; - - op1 = EX_VAR(opline->op1.var); - if (EXPECTED(Z_TYPE_INFO_P(op1) == IS_LONG)) { - ZVAL_LONG(EX_VAR(opline->result.var), ~Z_LVAL_P(op1)); - ZEND_VM_NEXT_OPCODE(); - } - - SAVE_OPLINE(); - if (IS_CV == IS_CV && UNEXPECTED(Z_TYPE_P(op1) == IS_UNDEF)) { - op1 = ZVAL_UNDEFINED_OP1(); - } - bitwise_not_function(EX_VAR(opline->result.var), op1); - - ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); -} - static ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL ZEND_BOOL_NOT_SPEC_CV_HANDLER(ZEND_OPCODE_HANDLER_ARGS) { USE_OPLINE @@ -48793,10 +48774,10 @@ ZEND_API void execute_ex(zend_execute_data *ex) (void*)&&ZEND_NULL_LABEL, (void*)&&ZEND_POW_SPEC_CV_CV_LABEL, (void*)&&ZEND_BW_NOT_SPEC_CONST_LABEL, - (void*)&&ZEND_BW_NOT_SPEC_TMPVAR_LABEL, - (void*)&&ZEND_BW_NOT_SPEC_TMPVAR_LABEL, + (void*)&&ZEND_BW_NOT_SPEC_TMPVARCV_LABEL, + (void*)&&ZEND_BW_NOT_SPEC_TMPVARCV_LABEL, (void*)&&ZEND_NULL_LABEL, - (void*)&&ZEND_BW_NOT_SPEC_CV_LABEL, + (void*)&&ZEND_BW_NOT_SPEC_TMPVARCV_LABEL, (void*)&&ZEND_BOOL_NOT_SPEC_CONST_LABEL, (void*)&&ZEND_BOOL_NOT_SPEC_TMPVAR_LABEL, (void*)&&ZEND_BOOL_NOT_SPEC_TMPVAR_LABEL, @@ -52814,6 +52795,10 @@ zend_leave_helper_SPEC_LABEL: VM_TRACE(ZEND_YIELD_SPEC_CONST_CV) ZEND_YIELD_SPEC_CONST_CV_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); HYBRID_BREAK(); + HYBRID_CASE(ZEND_BW_NOT_SPEC_TMPVARCV): + VM_TRACE(ZEND_BW_NOT_SPEC_TMPVARCV) + ZEND_BW_NOT_SPEC_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); + HYBRID_BREAK(); HYBRID_CASE(ZEND_QM_ASSIGN_LONG_SPEC_TMPVARCV): VM_TRACE(ZEND_QM_ASSIGN_LONG_SPEC_TMPVARCV) ZEND_QM_ASSIGN_LONG_SPEC_TMPVARCV_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); @@ -53230,10 +53215,6 @@ zend_leave_helper_SPEC_LABEL: VM_TRACE(ZEND_FETCH_LIST_R_SPEC_TMPVARCV_CV) ZEND_FETCH_LIST_R_SPEC_TMPVARCV_CV_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); HYBRID_BREAK(); - HYBRID_CASE(ZEND_BW_NOT_SPEC_TMPVAR): - VM_TRACE(ZEND_BW_NOT_SPEC_TMPVAR) - ZEND_BW_NOT_SPEC_TMPVAR_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); - HYBRID_BREAK(); HYBRID_CASE(ZEND_BOOL_NOT_SPEC_TMPVAR): VM_TRACE(ZEND_BOOL_NOT_SPEC_TMPVAR) ZEND_BOOL_NOT_SPEC_TMPVAR_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); @@ -54880,10 +54861,6 @@ zend_leave_helper_SPEC_LABEL: VM_TRACE(ZEND_YIELD_SPEC_UNUSED_CV) ZEND_YIELD_SPEC_UNUSED_CV_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); HYBRID_BREAK(); - HYBRID_CASE(ZEND_BW_NOT_SPEC_CV): - VM_TRACE(ZEND_BW_NOT_SPEC_CV) - ZEND_BW_NOT_SPEC_CV_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); - HYBRID_BREAK(); HYBRID_CASE(ZEND_BOOL_NOT_SPEC_CV): VM_TRACE(ZEND_BOOL_NOT_SPEC_CV) ZEND_BOOL_NOT_SPEC_CV_HANDLER(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); @@ -56329,10 +56306,10 @@ void zend_vm_init(void) ZEND_NULL_HANDLER, ZEND_POW_SPEC_CV_CV_HANDLER, ZEND_BW_NOT_SPEC_CONST_HANDLER, - ZEND_BW_NOT_SPEC_TMPVAR_HANDLER, - ZEND_BW_NOT_SPEC_TMPVAR_HANDLER, + ZEND_BW_NOT_SPEC_TMPVARCV_HANDLER, + ZEND_BW_NOT_SPEC_TMPVARCV_HANDLER, ZEND_NULL_HANDLER, - ZEND_BW_NOT_SPEC_CV_HANDLER, + ZEND_BW_NOT_SPEC_TMPVARCV_HANDLER, ZEND_BOOL_NOT_SPEC_CONST_HANDLER, ZEND_BOOL_NOT_SPEC_TMPVAR_HANDLER, ZEND_BOOL_NOT_SPEC_TMPVAR_HANDLER, diff --git a/Zend/zend_vm_handlers.h b/Zend/zend_vm_handlers.h index 04977853151..8a6dbe9c199 100644 --- a/Zend/zend_vm_handlers.h +++ b/Zend/zend_vm_handlers.h @@ -180,9 +180,9 @@ _(298, ZEND_POW_SPEC_CV_TMPVAR) \ _(300, ZEND_POW_SPEC_CV_CV) \ _(301, ZEND_BW_NOT_SPEC_CONST) \ - _(302, ZEND_BW_NOT_SPEC_TMPVAR) \ - _(303, ZEND_BW_NOT_SPEC_TMPVAR) \ - _(305, ZEND_BW_NOT_SPEC_CV) \ + _(302, ZEND_BW_NOT_SPEC_TMPVARCV) \ + _(303, ZEND_BW_NOT_SPEC_TMPVARCV) \ + _(305, ZEND_BW_NOT_SPEC_TMPVARCV) \ _(306, ZEND_BOOL_NOT_SPEC_CONST) \ _(307, ZEND_BOOL_NOT_SPEC_TMPVAR) \ _(308, ZEND_BOOL_NOT_SPEC_TMPVAR) \ diff --git a/Zend/zend_vm_opcodes.c b/Zend/zend_vm_opcodes.c index 6f7252c8f5c..1d96e1f0938 100644 --- a/Zend/zend_vm_opcodes.c +++ b/Zend/zend_vm_opcodes.c @@ -234,7 +234,7 @@ static uint32_t zend_vm_opcodes_flags[195] = { 0x80000b0b, 0x80000b0b, 0x00000707, - 0x00000007, + 0x0000000b, 0x00000007, 0x80000707, 0x80000303,