From bad5d2c78a5a8da843bf0ac22d37bfd7c404db6e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 26 Apr 2024 02:40:42 +0300 Subject: [PATCH] Update IR IR commit: ae34ae52b7b1c359afa03a0a1f45cbf689a64471 --- ext/opcache/jit/ir/ir.c | 67 +- ext/opcache/jit/ir/ir_aarch64.dasc | 4 +- ext/opcache/jit/ir/ir_cfg.c | 926 ++---------------------- ext/opcache/jit/ir/ir_fold.h | 51 +- ext/opcache/jit/ir/ir_gcm.c | 32 + ext/opcache/jit/ir/ir_private.h | 1 + ext/opcache/jit/ir/ir_sccp.c | 1069 +++++++++++++++++++++++++++- ext/opcache/jit/ir/ir_x86.dasc | 9 +- 8 files changed, 1225 insertions(+), 934 deletions(-) diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index 211d2b8c866..c8f179d4af2 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -1860,22 +1860,30 @@ static ir_ref ir_find_aliasing_load(ir_ctx *ctx, ir_ref ref, ir_type type, ir_re while (ref > limit) { insn = &ctx->ir_base[ref]; if (insn->op == IR_LOAD) { - if (insn->type == type && insn->op2 == addr) { - return ref; /* load forwarding (L2L) */ + if (insn->op2 == addr) { + if (insn->type == type) { + return ref; /* load forwarding (L2L) */ + } else if (ir_type_size[insn->type] == ir_type_size[type]) { + return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), ref); /* load forwarding with bitcast (L2L) */ + } else if (ir_type_size[insn->type] > ir_type_size[type] + && IR_IS_TYPE_INT(type) && IR_IS_TYPE_INT(insn->type)) { + return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), ref); /* partial load forwarding (L2L) */ + } } } else if (insn->op == IR_STORE) { ir_type type2 = ctx->ir_base[insn->op3].type; if (insn->op2 == addr) { - if (type2 == type) { - ref = insn->op3; - insn = &ctx->ir_base[ref]; - if (insn->op == IR_RLOAD && (modified_regset & (1 << insn->op2))) { - /* anti-dependency */ - return IR_UNUSED; - } - return ref; /* store forwarding (S2L) */ - } else if (IR_IS_TYPE_INT(type) && ir_type_size[type2] > ir_type_size[type]) { + if (ctx->ir_base[insn->op3].op == IR_RLOAD + && (modified_regset & (1 << ctx->ir_base[insn->op3].op2))) { + /* anti-dependency */ + return IR_UNUSED; + } else if (type2 == type) { + return insn->op3; /* store forwarding (S2L) */ + } else if (ir_type_size[type2] == ir_type_size[type]) { + return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), insn->op3); /* store forwarding with bitcast (S2L) */ + } else if (ir_type_size[type2] > ir_type_size[type] + && IR_IS_TYPE_INT(type) && IR_IS_TYPE_INT(type2)) { return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), insn->op3); /* partial store forwarding (S2L) */ } else { return IR_UNUSED; @@ -2664,6 +2672,43 @@ void _ir_AFREE(ir_ctx *ctx, ir_ref size) ir_ref _ir_VLOAD(ir_ctx *ctx, ir_type type, ir_ref var) { + ir_ref ref = ctx->control; + ir_insn *insn; + + while (ref > var) { + insn = &ctx->ir_base[ref]; + if (insn->op == IR_VLOAD) { + if (insn->op2 == var) { + if (insn->type == type) { + return ref; /* load forwarding (L2L) */ + } else if (ir_type_size[insn->type] == ir_type_size[type]) { + return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), ref); /* load forwarding with bitcast (L2L) */ + } else if (ir_type_size[insn->type] > ir_type_size[type] + && IR_IS_TYPE_INT(type) && IR_IS_TYPE_INT(insn->type)) { + return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), ref); /* partial load forwarding (L2L) */ + } + } + } else if (insn->op == IR_VSTORE) { + ir_type type2 = ctx->ir_base[insn->op3].type; + + if (insn->op2 == var) { + if (type2 == type) { + return insn->op3; /* store forwarding (S2L) */ + } else if (ir_type_size[type2] == ir_type_size[type]) { + return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), insn->op3); /* store forwarding with bitcast (S2L) */ + } else if (ir_type_size[type2] > ir_type_size[type] + && IR_IS_TYPE_INT(type) && IR_IS_TYPE_INT(type2)) { + return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), insn->op3); /* partial store forwarding (S2L) */ + } else { + break; + } + } + } else if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN || insn->op == IR_CALL || insn->op == IR_STORE) { + break; + } + ref = insn->op1; + } + IR_ASSERT(ctx->control); return ctx->control = ir_emit2(ctx, IR_OPT(IR_VLOAD, type), ctx->control, var); } diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 4184d003444..777c16f0fc6 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -982,9 +982,7 @@ binop_fp: } else if ((ir_op_flags[op_insn->op] & IR_OP_FLAG_COMMUTATIVE) && ctx->ir_base[op_insn->op2].op == IR_RLOAD && ctx->ir_base[op_insn->op2].op2 == insn->op3) { - ir_ref tmp = op_insn->op1; - op_insn->op1 = op_insn->op2; - op_insn->op2 = tmp; + SWAP_REFS(op_insn->op1, op_insn->op2); ctx->rules[insn->op2] = IR_FUSED | IR_BINOP_INT; ctx->rules[op_insn->op1] = IR_SKIPPED | IR_RLOAD; return IR_REG_BINOP_INT; diff --git a/ext/opcache/jit/ir/ir_cfg.c b/ext/opcache/jit/ir/ir_cfg.c index a6fd641d242..b9983131b66 100644 --- a/ext/opcache/jit/ir/ir_cfg.c +++ b/ext/opcache/jit/ir/ir_cfg.c @@ -8,854 +8,6 @@ #include "ir.h" #include "ir_private.h" -static void ir_get_true_false_refs(const ir_ctx *ctx, ir_ref if_ref, ir_ref *if_true_ref, ir_ref *if_false_ref) -{ - ir_use_list *use_list = &ctx->use_lists[if_ref]; - ir_ref *p = &ctx->use_edges[use_list->refs]; - - IR_ASSERT(use_list->count == 2); - if (ctx->ir_base[*p].op == IR_IF_TRUE) { - IR_ASSERT(ctx->ir_base[*(p + 1)].op == IR_IF_FALSE); - *if_true_ref = *p; - *if_false_ref = *(p + 1); - } else { - IR_ASSERT(ctx->ir_base[*p].op == IR_IF_FALSE); - IR_ASSERT(ctx->ir_base[*(p + 1)].op == IR_IF_TRUE); - *if_false_ref = *p; - *if_true_ref = *(p + 1); - } -} - -static ir_ref _ir_merge_blocks(ir_ctx *ctx, ir_ref end, ir_ref begin) -{ - ir_ref prev, next; - ir_use_list *use_list; - - IR_ASSERT(ctx->ir_base[begin].op == IR_BEGIN); - IR_ASSERT(ctx->ir_base[end].op == IR_END); - IR_ASSERT(ctx->ir_base[begin].op1 == end); - IR_ASSERT(ctx->use_lists[end].count == 1); - - prev = ctx->ir_base[end].op1; - - use_list = &ctx->use_lists[begin]; - IR_ASSERT(use_list->count == 1); - next = ctx->use_edges[use_list->refs]; - - /* remove BEGIN and END */ - MAKE_NOP(&ctx->ir_base[begin]); CLEAR_USES(begin); - MAKE_NOP(&ctx->ir_base[end]); CLEAR_USES(end); - - /* connect their predecessor and successor */ - ctx->ir_base[next].op1 = prev; - ir_use_list_replace_all(ctx, prev, end, next); - return next; -} - -static ir_ref ir_try_remove_empty_diamond(ir_ctx *ctx, ir_ref ref, ir_insn *insn) -{ - if (insn->inputs_count == 2) { - ir_ref end1_ref = insn->op1, end2_ref = insn->op2; - ir_insn *end1 = &ctx->ir_base[end1_ref]; - ir_insn *end2 = &ctx->ir_base[end2_ref]; - - if (end1->op != IR_END || end2->op != IR_END) { - return IR_UNUSED; - } - - ir_ref start1_ref = end1->op1, start2_ref = end2->op1; - ir_insn *start1 = &ctx->ir_base[start1_ref]; - ir_insn *start2 = &ctx->ir_base[start2_ref]; - - if (start1->op1 != start2->op1) { - return IR_UNUSED; - } - - ir_ref root_ref = start1->op1; - ir_insn *root = &ctx->ir_base[root_ref]; - - if (root->op != IR_IF - && !(root->op == IR_SWITCH && ctx->use_lists[root_ref].count == 2)) { - return IR_UNUSED; - } - - /* Empty Diamond - * - * prev prev - * | condition | condition - * | / | - * IF | - * | \ | - * | +-----+ | - * | IF_FALSE | - * IF_TRUE | => | - * | END | - * END / | - * | +---+ | - * | / | - * MERGE | - * | | - * next next - */ - - ir_ref next_ref = ctx->use_edges[ctx->use_lists[ref].refs]; - ir_insn *next = &ctx->ir_base[next_ref]; - - IR_ASSERT(ctx->use_lists[start1_ref].count == 1); - IR_ASSERT(ctx->use_lists[start2_ref].count == 1); - - next->op1 = root->op1; - ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); - if (!IR_IS_CONST_REF(root->op2)) { - ir_use_list_remove_all(ctx, root->op2, root_ref); - } - - MAKE_NOP(root); CLEAR_USES(root_ref); - MAKE_NOP(start1); CLEAR_USES(start1_ref); - MAKE_NOP(start2); CLEAR_USES(start2_ref); - MAKE_NOP(end1); CLEAR_USES(end1_ref); - MAKE_NOP(end2); CLEAR_USES(end2_ref); - MAKE_NOP(insn); CLEAR_USES(ref); - - return next_ref; - } else { - ir_ref i, count = insn->inputs_count, *ops = insn->ops + 1; - ir_ref root_ref = IR_UNUSED; - - for (i = 0; i < count; i++) { - ir_ref end_ref, start_ref; - ir_insn *end, *start; - - end_ref = ops[i]; - end = &ctx->ir_base[end_ref]; - if (end->op != IR_END) { - return IR_UNUSED; - } - start_ref = end->op1; - start = &ctx->ir_base[start_ref]; - if (start->op != IR_CASE_VAL && start->op != IR_CASE_DEFAULT) { - return IR_UNUSED; - } - IR_ASSERT(ctx->use_lists[start_ref].count == 1); - if (!root_ref) { - root_ref = start->op1; - if (ctx->use_lists[root_ref].count != count) { - return IR_UNUSED; - } - } else if (start->op1 != root_ref) { - return IR_UNUSED; - } - } - - /* Empty N-Diamond */ - ir_ref next_ref = ctx->use_edges[ctx->use_lists[ref].refs]; - ir_insn *next = &ctx->ir_base[next_ref]; - ir_insn *root = &ctx->ir_base[root_ref]; - - next->op1 = root->op1; - ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); - ir_use_list_remove_all(ctx, root->op2, root_ref); - - MAKE_NOP(root); CLEAR_USES(root_ref); - - for (i = 0; i < count; i++) { - ir_ref end_ref = ops[i]; - ir_insn *end = &ctx->ir_base[end_ref]; - ir_ref start_ref = end->op1; - ir_insn *start = &ctx->ir_base[start_ref]; - - MAKE_NOP(start); CLEAR_USES(start_ref); - MAKE_NOP(end); CLEAR_USES(end_ref); - } - - MAKE_NOP(insn); CLEAR_USES(ref); - - return next_ref; - } -} - -static bool ir_is_zero(ir_ctx *ctx, ir_ref ref) -{ - return IR_IS_CONST_REF(ref) - && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) - && ctx->ir_base[ref].val.u32 == 0; -} - -static ir_ref ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_ref ref, ir_insn *insn) -{ - IR_ASSERT(insn->inputs_count == 3); - IR_ASSERT(ctx->use_lists[merge_ref].count == 2); - - ir_ref end1_ref = merge->op1, end2_ref = merge->op2; - ir_insn *end1 = &ctx->ir_base[end1_ref]; - ir_insn *end2 = &ctx->ir_base[end2_ref]; - - if (end1->op == IR_END && end2->op == IR_END) { - ir_ref start1_ref = end1->op1, start2_ref = end2->op1; - ir_insn *start1 = &ctx->ir_base[start1_ref]; - ir_insn *start2 = &ctx->ir_base[start2_ref]; - - if (start1->op1 == start2->op1) { - ir_ref root_ref = start1->op1; - ir_insn *root = &ctx->ir_base[root_ref]; - - if (root->op == IR_IF && ctx->use_lists[root->op2].count == 1) { - ir_ref cond_ref = root->op2; - ir_insn *cond = &ctx->ir_base[cond_ref]; - ir_type type = insn->type; - bool is_cmp, is_less; - - if (IR_IS_TYPE_FP(type)) { - is_cmp = (cond->op == IR_LT || cond->op == IR_LE || cond->op == IR_GT || cond->op == IR_GE || - cond->op == IR_ULT || cond->op == IR_ULE || cond->op == IR_UGT || cond->op == IR_UGE); - is_less = (cond->op == IR_LT || cond->op == IR_LE || - cond->op == IR_ULT || cond->op == IR_ULE); - } else if (IR_IS_TYPE_SIGNED(type)) { - is_cmp = (cond->op == IR_LT || cond->op == IR_LE || cond->op == IR_GT || cond->op == IR_GE); - is_less = (cond->op == IR_LT || cond->op == IR_LE); - } else { - IR_ASSERT(IR_IS_TYPE_UNSIGNED(type)); - is_cmp = (cond->op == IR_ULT || cond->op == IR_ULE || cond->op == IR_UGT || cond->op == IR_UGE); - is_less = (cond->op == IR_ULT || cond->op == IR_ULE); - } - - if (is_cmp - && ((insn->op2 == cond->op1 && insn->op3 == cond->op2) - || (insn->op2 == cond->op2 && insn->op3 == cond->op1))) { - /* MAX/MIN - * - * prev prev - * | LT(A, B) | - * | / | - * IF | - * | \ | - * | +-----+ | - * | IF_FALSE | - * IF_TRUE | => | - * | END | - * END / | - * | +---+ | - * | / | - * MERGE | - * | \ | - * | PHI(A, B) | MIN(A, B) - * next next - */ - ir_ref next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs]; - ir_insn *next; - - if (next_ref == ref) { - next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs + 1]; - } - next = &ctx->ir_base[next_ref]; - - IR_ASSERT(ctx->use_lists[start1_ref].count == 1); - IR_ASSERT(ctx->use_lists[start2_ref].count == 1); - - insn->op = ( - (is_less ? cond->op1 : cond->op2) - == - ((start1->op == IR_IF_TRUE) ? insn->op2 : insn->op3) - ) ? IR_MIN : IR_MAX; - insn->inputs_count = 2; - if (insn->op2 > insn->op3) { - insn->op1 = insn->op2; - insn->op2 = insn->op3; - } else { - insn->op1 = insn->op3; - } - insn->op3 = IR_UNUSED; - - next->op1 = root->op1; - ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); - if (!IR_IS_CONST_REF(insn->op1)) { - ir_use_list_remove_all(ctx, insn->op1, cond_ref); - } - if (!IR_IS_CONST_REF(insn->op2)) { - ir_use_list_remove_all(ctx, insn->op2, cond_ref); - } - - MAKE_NOP(cond); CLEAR_USES(cond_ref); - MAKE_NOP(root); CLEAR_USES(root_ref); - MAKE_NOP(start1); CLEAR_USES(start1_ref); - MAKE_NOP(start2); CLEAR_USES(start2_ref); - MAKE_NOP(end1); CLEAR_USES(end1_ref); - MAKE_NOP(end2); CLEAR_USES(end2_ref); - MAKE_NOP(merge); CLEAR_USES(merge_ref); - - return next_ref; - } else if (is_cmp - && ((ctx->ir_base[insn->op2].op == IR_NEG - && ctx->use_lists[insn->op2].count == 1 - && ctx->ir_base[insn->op2].op1 == insn->op3 - && ((cond->op1 == insn->op3 - && ir_is_zero(ctx, cond->op2) - && is_less == (start1->op == IR_IF_TRUE)) - || (cond->op2 == insn->op3 - && ir_is_zero(ctx, cond->op1) - && is_less != (start1->op == IR_IF_TRUE)))) - || (ctx->ir_base[insn->op3].op == IR_NEG - && ctx->use_lists[insn->op3].count == 1 - && ctx->ir_base[insn->op3].op1 == insn->op2 - && ((cond->op1 == insn->op2 - && ir_is_zero(ctx, cond->op2) - && is_less != (start1->op == IR_IF_TRUE)) - || (cond->op2 == insn->op2 - && ir_is_zero(ctx, cond->op1) - && is_less == (start1->op == IR_IF_TRUE)))))) { - /* ABS - * - * prev prev - * | LT(A, 0) | - * | / | - * IF | - * | \ | - * | +-----+ | - * | IF_FALSE | - * IF_TRUE | => | - * | END | - * END / | - * | +---+ | - * | / | - * MERGE | - * | \ | - * | PHI(A, NEG(A)) | ABS(A) - * next next - */ - ir_ref neg_ref; - ir_ref next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs]; - ir_insn *next; - - if (next_ref == ref) { - next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs + 1]; - } - next = &ctx->ir_base[next_ref]; - - IR_ASSERT(ctx->use_lists[start1_ref].count == 1); - IR_ASSERT(ctx->use_lists[start2_ref].count == 1); - - insn->op = IR_ABS; - insn->inputs_count = 1; - if (ctx->ir_base[insn->op2].op == IR_NEG) { - neg_ref = insn->op2; - insn->op1 = insn->op3; - } else { - neg_ref = insn->op3; - insn->op1 = insn->op2; - } - insn->op2 = IR_UNUSED; - insn->op3 = IR_UNUSED; - - next->op1 = root->op1; - ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); - ir_use_list_remove_one(ctx, insn->op1, neg_ref); - if (!IR_IS_CONST_REF(insn->op1)) { - ir_use_list_remove_all(ctx, insn->op1, cond_ref); - } - - MAKE_NOP(cond); CLEAR_USES(cond_ref); - MAKE_NOP(root); CLEAR_USES(root_ref); - MAKE_NOP(start1); CLEAR_USES(start1_ref); - MAKE_NOP(start2); CLEAR_USES(start2_ref); - MAKE_NOP(end1); CLEAR_USES(end1_ref); - MAKE_NOP(end2); CLEAR_USES(end2_ref); - MAKE_NOP(merge); CLEAR_USES(merge_ref); - MAKE_NOP(&ctx->ir_base[neg_ref]); CLEAR_USES(neg_ref); - - return next_ref; -#if 0 - } else { - /* COND - * - * prev prev - * | cond | - * | / | - * IF | - * | \ | - * | +-----+ | - * | IF_FALSE | - * IF_TRUE | => | - * | END | - * END / | - * | +---+ | - * | / | - * MERGE | - * | \ | - * | PHI(A, B) | COND(cond, A, B) - * next next - */ - ir_ref next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs]; - ir_insn *next; - - if (next_ref == ref) { - next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs + 1]; - } - next = &ctx->ir_base[next_ref]; - - IR_ASSERT(ctx->use_lists[start1_ref].count == 1); - IR_ASSERT(ctx->use_lists[start2_ref].count == 1); - - insn->op = IR_COND; - insn->inputs_count = 3; - insn->op1 = cond_ref; - if (start1->op == IR_IF_FALSE) { - SWAP_REFS(insn->op2, insn->op3); - } - - next->op1 = root->op1; - ir_use_list_replace_one(ctx, cond_ref, root_ref, ref); - ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); - ir_use_list_remove_all(ctx, root->op2, root_ref); - - MAKE_NOP(root); CLEAR_USES(root_ref); - MAKE_NOP(start1); CLEAR_USES(start1_ref); - MAKE_NOP(start2); CLEAR_USES(start2_ref); - MAKE_NOP(end1); CLEAR_USES(end1_ref); - MAKE_NOP(end2); CLEAR_USES(end2_ref); - MAKE_NOP(merge); CLEAR_USES(merge_ref); - - return next_ref; -#endif - } - } - } - } - - return IR_UNUSED; -} - -static bool ir_cmp_is_true(ir_op op, ir_insn *op1, ir_insn *op2) -{ - IR_ASSERT(op1->type == op2->type); - if (IR_IS_TYPE_INT(op1->type)) { - if (op == IR_EQ) { - return op1->val.u64 == op2->val.u64; - } else if (op == IR_NE) { - return op1->val.u64 != op2->val.u64; - } else if (op == IR_LT) { - if (IR_IS_TYPE_SIGNED(op1->type)) { - return op1->val.i64 < op2->val.i64; - } else { - return op1->val.u64 < op2->val.u64; - } - } else if (op == IR_GE) { - if (IR_IS_TYPE_SIGNED(op1->type)) { - return op1->val.i64 >= op2->val.i64; - } else { - return op1->val.u64 >= op2->val.u64; - } - } else if (op == IR_LE) { - if (IR_IS_TYPE_SIGNED(op1->type)) { - return op1->val.i64 <= op2->val.i64; - } else { - return op1->val.u64 <= op2->val.u64; - } - } else if (op == IR_GT) { - if (IR_IS_TYPE_SIGNED(op1->type)) { - return op1->val.i64 > op2->val.i64; - } else { - return op1->val.u64 > op2->val.u64; - } - } else if (op == IR_ULT) { - return op1->val.u64 < op2->val.u64; - } else if (op == IR_UGE) { - return op1->val.u64 >= op2->val.u64; - } else if (op == IR_ULE) { - return op1->val.u64 <= op2->val.u64; - } else if (op == IR_UGT) { - return op1->val.u64 > op2->val.u64; - } else { - IR_ASSERT(0); - return 0; - } - } else if (op1->type == IR_DOUBLE) { - if (op == IR_EQ) { - return op1->val.d == op2->val.d; - } else if (op == IR_NE) { - return op1->val.d != op2->val.d; - } else if (op == IR_LT) { - return op1->val.d < op2->val.d; - } else if (op == IR_GE) { - return op1->val.d >= op2->val.d; - } else if (op == IR_LE) { - return op1->val.d <= op2->val.d; - } else if (op == IR_GT) { - return op1->val.d > op2->val.d; - } else if (op == IR_ULT) { - return !(op1->val.d >= op2->val.d); - } else if (op == IR_UGE) { - return !(op1->val.d < op2->val.d); - } else if (op == IR_ULE) { - return !(op1->val.d > op2->val.d); - } else if (op == IR_UGT) { - return !(op1->val.d <= op2->val.d); - } else { - IR_ASSERT(0); - return 0; - } - } else { - IR_ASSERT(op1->type == IR_FLOAT); - if (op == IR_EQ) { - return op1->val.f == op2->val.f; - } else if (op == IR_NE) { - return op1->val.f != op2->val.f; - } else if (op == IR_LT) { - return op1->val.f < op2->val.f; - } else if (op == IR_GE) { - return op1->val.f >= op2->val.f; - } else if (op == IR_LE) { - return op1->val.f <= op2->val.f; - } else if (op == IR_GT) { - return op1->val.f > op2->val.f; - } else if (op == IR_ULT) { - return !(op1->val.f >= op2->val.f); - } else if (op == IR_UGE) { - return !(op1->val.f < op2->val.f); - } else if (op == IR_ULE) { - return !(op1->val.f > op2->val.f); - } else if (op == IR_UGT) { - return !(op1->val.f <= op2->val.f); - } else { - IR_ASSERT(0); - return 0; - } - } -} - -static ir_ref ir_try_split_if(ir_ctx *ctx, ir_ref ref, ir_insn *insn) -{ - ir_ref cond_ref = insn->op2; - ir_insn *cond = &ctx->ir_base[cond_ref]; - - if (cond->op == IR_PHI - && cond->inputs_count == 3 - && cond->op1 == insn->op1 - && ((IR_IS_CONST_REF(cond->op2) && !IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op)) - || (IR_IS_CONST_REF(cond->op3) && !IR_IS_SYM_CONST(ctx->ir_base[cond->op3].op)))) { - ir_ref merge_ref = insn->op1; - ir_insn *merge = &ctx->ir_base[merge_ref]; - - if (ctx->use_lists[merge_ref].count == 2) { - ir_ref end1_ref = merge->op1, end2_ref = merge->op2; - ir_insn *end1 = &ctx->ir_base[end1_ref]; - ir_insn *end2 = &ctx->ir_base[end2_ref]; - - if (end1->op == IR_END && end2->op == IR_END) { - ir_ref if_true_ref, if_false_ref; - ir_insn *if_true, *if_false; - ir_op op = IR_IF_FALSE; - - ir_get_true_false_refs(ctx, ref, &if_true_ref, &if_false_ref); - - if (!IR_IS_CONST_REF(cond->op2) || IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op)) { - IR_ASSERT(IR_IS_CONST_REF(cond->op3)); - SWAP_REFS(cond->op2, cond->op3); - SWAP_REFS(merge->op1, merge->op2); - SWAP_REFS(end1_ref, end2_ref); - SWAP_INSNS(end1, end2); - } - if (ir_const_is_true(&ctx->ir_base[cond->op2])) { - SWAP_REFS(if_true_ref, if_false_ref); - op = IR_IF_TRUE; - } - if_true = &ctx->ir_base[if_true_ref]; - if_false = &ctx->ir_base[if_false_ref]; - - /* Simple IF Split - * - * | | | | - * | END | IF(X) - * END / END / \ - * | +---+ | +--+ + - * | / | / | - * MERGE | IF_FALSE | - * | \ | | | - * | PHI(false, X) | | | - * | / | | | - * IF => | END | - * | \ | | | - * | +------+ | | | - * | IF_TRUE | | IF_TRUE - * IF_FALSE | MERGE - * | | - */ - ir_use_list_remove_all(ctx, merge_ref, cond_ref); - ir_use_list_remove_all(ctx, ref, if_true_ref); - if (!IR_IS_CONST_REF(cond->op3)) { - ir_use_list_replace_one(ctx, cond->op3, cond_ref, end2_ref); - } - ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); - ir_use_list_add(ctx, end2_ref, if_true_ref); - - end2->optx = IR_OPTX(IR_IF, IR_VOID, 2); - end2->op2 = cond->op3; - - merge->optx = IR_OPTX(op, IR_VOID, 1); - merge->op1 = end2_ref; - merge->op2 = IR_UNUSED; - - MAKE_NOP(cond); - CLEAR_USES(cond_ref); - - insn->optx = IR_OPTX(IR_END, IR_VOID, 1); - insn->op1 = merge_ref; - insn->op2 = IR_UNUSED; - - if_true->op1 = end2_ref; - - if_false->optx = IR_OPTX(IR_MERGE, IR_VOID, 2); - if_false->op1 = end1_ref; - if_false->op2 = ref; - - return ref; - } - } - } - - return IR_UNUSED; -} - -static ir_ref ir_try_split_if_cmp(ir_ctx *ctx, ir_worklist *worklist, ir_ref ref, ir_insn *insn) -{ - ir_ref cond_ref = insn->op2; - ir_insn *cond = &ctx->ir_base[cond_ref]; - - if (cond->op >= IR_EQ && cond->op <= IR_UGT - && IR_IS_CONST_REF(cond->op2) - && !IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) - && ctx->use_lists[insn->op2].count == 1) { - ir_ref phi_ref = cond->op1; - ir_insn *phi = &ctx->ir_base[phi_ref]; - - if (phi->op == IR_PHI - && phi->inputs_count == 3 - && phi->op1 == insn->op1 - && ctx->use_lists[phi_ref].count == 1 - && ((IR_IS_CONST_REF(phi->op2) && !IR_IS_SYM_CONST(ctx->ir_base[phi->op2].op)) - || (IR_IS_CONST_REF(phi->op3) && !IR_IS_SYM_CONST(ctx->ir_base[phi->op3].op)))) { - ir_ref merge_ref = insn->op1; - ir_insn *merge = &ctx->ir_base[merge_ref]; - - if (ctx->use_lists[merge_ref].count == 2) { - ir_ref end1_ref = merge->op1, end2_ref = merge->op2; - ir_insn *end1 = &ctx->ir_base[end1_ref]; - ir_insn *end2 = &ctx->ir_base[end2_ref]; - - if (end1->op == IR_END && end2->op == IR_END) { - ir_ref if_true_ref, if_false_ref; - ir_insn *if_true, *if_false; - ir_op op = IR_IF_FALSE; - - ir_get_true_false_refs(ctx, ref, &if_true_ref, &if_false_ref); - - if (!IR_IS_CONST_REF(phi->op2) || IR_IS_SYM_CONST(ctx->ir_base[phi->op2].op)) { - IR_ASSERT(IR_IS_CONST_REF(phi->op3)); - SWAP_REFS(phi->op2, phi->op3); - SWAP_REFS(merge->op1, merge->op2); - SWAP_REFS(end1_ref, end2_ref); - SWAP_INSNS(end1, end2); - } - if (ir_cmp_is_true(cond->op, &ctx->ir_base[phi->op2], &ctx->ir_base[cond->op2])) { - SWAP_REFS(if_true_ref, if_false_ref); - op = IR_IF_TRUE; - } - if_true = &ctx->ir_base[if_true_ref]; - if_false = &ctx->ir_base[if_false_ref]; - - if (IR_IS_CONST_REF(phi->op3) && !IR_IS_SYM_CONST(ctx->ir_base[phi->op3].op)) { - if (ir_cmp_is_true(cond->op, &ctx->ir_base[phi->op3], &ctx->ir_base[cond->op2]) ^ (op == IR_IF_TRUE)) { - /* IF Split - * - * | | | | - * | END | END - * END / END | - * | +---+ | | - * | / | | - * MERGE | | - * | \ | | - * | PHI(C1, X) | | - * | | | | - * | CMP(_, C2) | | - * | / | | - * IF => | | - * | \ | | - * | +------+ | | - * | IF_TRUE | BEGIN - * IF_FALSE | BEGIN | - * | | - */ - - ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); - ir_use_list_replace_one(ctx, end2_ref, merge_ref, if_true_ref); - - MAKE_NOP(merge); CLEAR_USES(merge_ref); - MAKE_NOP(phi); CLEAR_USES(phi_ref); - MAKE_NOP(cond); CLEAR_USES(cond_ref); - MAKE_NOP(insn); CLEAR_USES(ref); - - if_false->optx = IR_OPTX(IR_BEGIN, IR_VOID, 1); - if_false->op1 = end1_ref; - - if_true->optx = IR_OPTX(IR_BEGIN, IR_VOID, 1); - if_true->op1 = end2_ref; - - ir_worklist_push(worklist, end1_ref); - ir_worklist_push(worklist, end2_ref); - - return IR_NULL; - } else { - /* IF Split - * - * | | | | - * | END | END - * END / END | - * | +---+ | | - * | / | | - * MERGE | | - * | \ | | - * | PHI(C1, X) | | - * | | | + - * | CMP(_, C2) | / - * | / | / - * IF => | / - * | \ | / - * | +------+ | / - * | IF_TRUE | / BEGIN(unreachable) - * IF_FALSE | MERGE | - * | | - */ - - ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); - ir_use_list_replace_one(ctx, end2_ref, merge_ref, if_false_ref); - - MAKE_NOP(merge); CLEAR_USES(merge_ref); - MAKE_NOP(phi); CLEAR_USES(phi_ref); - MAKE_NOP(cond); CLEAR_USES(cond_ref); - MAKE_NOP(insn); CLEAR_USES(ref); - - if_false->optx = IR_OPTX(IR_MERGE, IR_VOID, 2); - if_false->op1 = end1_ref; - if_false->op2 = end2_ref; - - if_true->optx = IR_BEGIN; - if_true->op1 = IR_UNUSED; - - ctx->flags2 &= ~IR_SCCP_DONE; - - ir_worklist_push(worklist, end1_ref); - ir_worklist_push(worklist, end2_ref); - - return IR_NULL; - } - } else { - /* IF Split - * - * | | | | - * | END | IF<----+ - * END / END / \ | - * | +---+ | +--+ + | - * | / | / | | - * MERGE | IF_FALSE | | - * | \ | | | | - * | PHI(C1, X) | | | | - * | | | | | | - * | CMP(_, C2) | | | CMP(X, C2) - * | / | | | - * IF => | END | - * | \ | | | - * | +------+ | | | - * | IF_TRUE | | IF_TRUE - * IF_FALSE | MERGE - * | | - */ - - ir_use_list_remove_all(ctx, merge_ref, phi_ref); - ir_use_list_remove_all(ctx, ref, if_true_ref); - if (!IR_IS_CONST_REF(phi->op3)) { - ir_use_list_replace_one(ctx, phi->op3, phi_ref, insn->op2); - } - ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); - ir_use_list_replace_one(ctx, cond_ref, ref, end2_ref); - ir_use_list_add(ctx, end2_ref, if_true_ref); - - end2->optx = IR_OPTX(IR_IF, IR_VOID, 2); - end2->op2 = insn->op2; - - merge->optx = IR_OPTX(op, IR_VOID, 1); - merge->op1 = end2_ref; - merge->op2 = IR_UNUSED; - - cond->op1 = phi->op3; - MAKE_NOP(phi); - CLEAR_USES(phi_ref); - - insn->optx = IR_OPTX(IR_END, IR_VOID, 1); - insn->op1 = merge_ref; - insn->op2 = IR_UNUSED; - - if_true->op1 = end2_ref; - - if_false->optx = IR_OPTX(IR_MERGE, IR_VOID, 2); - if_false->op1 = end1_ref; - if_false->op2 = ref; - - ir_worklist_push(worklist, end1_ref); - - return ref; - } - } - } - } - } - - return IR_UNUSED; -} - -static ir_ref ir_optimize_merge(ir_ctx *ctx, ir_worklist *worklist, ir_ref merge_ref, ir_insn *merge) -{ - ir_use_list *use_list = &ctx->use_lists[merge_ref]; - - if (use_list->count == 1) { - return ir_try_remove_empty_diamond(ctx, merge_ref, merge); - } else if (use_list->count == 2) { - if (merge->inputs_count == 2) { - ir_ref phi_ref = ctx->use_edges[use_list->refs]; - ir_insn *phi = &ctx->ir_base[phi_ref]; - - ir_ref next_ref = ctx->use_edges[use_list->refs + 1]; - ir_insn *next = &ctx->ir_base[next_ref]; - IR_ASSERT(next->op != IR_PHI); - - if (phi->op == IR_PHI) { - if (next->op == IR_IF && next->op1 == merge_ref && ctx->use_lists[phi_ref].count == 1) { - if (next->op2 == phi_ref) { - ir_ref ref = ir_try_split_if(ctx, next_ref, next); - if (ref) { - return ref; - } - } else { - ir_insn *cmp = &ctx->ir_base[next->op2]; - - if (cmp->op >= IR_EQ && cmp->op <= IR_UGT - && cmp->op1 == phi_ref - && IR_IS_CONST_REF(cmp->op2) - && !IR_IS_SYM_CONST(ctx->ir_base[cmp->op2].op) - && ctx->use_lists[next->op2].count == 1) { - ir_ref ref = ir_try_split_if_cmp(ctx, worklist, next_ref, next); - if (ref) { - return ref; - } - } - } - } - return ir_optimize_phi(ctx, merge_ref, merge, phi_ref, phi); - } - } - } - - return IR_UNUSED; -} - IR_ALWAYS_INLINE void _ir_add_successors(const ir_ctx *ctx, ir_ref ref, ir_worklist *worklist) { ir_use_list *use_list = &ctx->use_lists[ref]; @@ -932,7 +84,6 @@ int ir_build_cfg(ir_ctx *ctx) ref = ctx->ir_base[ref].op3; } -next: while (ir_worklist_len(&worklist)) { ref = ir_worklist_pop(&worklist); insn = &ctx->ir_base[ref]; @@ -957,25 +108,6 @@ next: while (1) { insn = &ctx->ir_base[ref]; if (IR_IS_BB_START(insn->op)) { - if (ctx->flags & IR_OPT_CFG) { - if (insn->op == IR_BEGIN) { - if (ctx->ir_base[insn->op1].op == IR_END - && ctx->use_lists[ref].count == 1) { - ref = _ir_merge_blocks(ctx, insn->op1, ref); - ref = ctx->ir_base[ref].op1; - continue; - } - } else if (insn->op == IR_MERGE) { - ir_ref prev = ir_optimize_merge(ctx, &worklist, ref, insn); - if (prev) { - if (prev == IR_NULL) { - goto next; - } - ref = ctx->ir_base[prev].op1; - continue; - } - } - } break; } ref = insn->op1; // follow connected control blocks untill BB start @@ -1025,20 +157,7 @@ next: } IR_ASSERT(next != IR_UNUSED); ref = next; -next_successor: if (IR_IS_BB_END(insn->op)) { - if (insn->op == IR_END && (ctx->flags & IR_OPT_CFG)) { - use_list = &ctx->use_lists[ref]; - IR_ASSERT(use_list->count == 1); - next = ctx->use_edges[use_list->refs]; - - if (ctx->ir_base[next].op == IR_BEGIN - && ctx->use_lists[next].count == 1) { - ref = _ir_merge_blocks(ctx, ref, next); - insn = &ctx->ir_base[ref]; - goto next_successor; - } - } break; } } @@ -1059,7 +178,7 @@ next_successor: bb = blocks + 1; count = 0; /* SCCP already removed UNREACHABKE blocks, otherwise all blocks are marked as UNREACHABLE first */ - bb_init_falgs = (ctx->flags2 & IR_SCCP_DONE) ? 0 : IR_BB_UNREACHABLE; + bb_init_falgs = (ctx->flags2 & IR_CFG_REACHABLE) ? 0 : IR_BB_UNREACHABLE; IR_BITSET_FOREACH(bb_starts, len, start) { insn = &ctx->ir_base[start]; if (insn->op == IR_NOP) { @@ -1146,7 +265,7 @@ next_successor: ctx->cfg_edges = edges; ctx->cfg_map = _blocks; - if (!(ctx->flags2 & IR_SCCP_DONE)) { + if (!(ctx->flags2 & IR_CFG_REACHABLE)) { uint32_t reachable_count = 0; /* Mark reachable blocks */ @@ -2658,10 +1777,51 @@ static int ir_schedule_blocks_top_down(ir_ctx *ctx) int ir_schedule_blocks(ir_ctx *ctx) { + ir_ref ref; + if (ctx->cfg_blocks_count <= 2) { return 1; } + /* Mark "stop" blocks termintated with UNREACHABLE as "unexpected" */ + ref = ctx->ir_base[1].op1; + while (ref) { + ir_insn *insn = &ctx->ir_base[ref]; + + if (insn->op == IR_UNREACHABLE && ctx->ir_base[insn->op1].op != IR_TAILCALL) { + ir_block *bb = &ctx->cfg_blocks[ctx->cfg_map[ref]]; + uint32_t n = bb->predecessors_count; + + if (n == 1) { + ir_insn *start_insn = &ctx->ir_base[bb->start]; + if (start_insn->op == IR_IF_TRUE + || start_insn->op == IR_IF_FALSE + || start_insn->op == IR_CASE_DEFAULT) { + if (!start_insn->op2) start_insn->op2 = 1; + } else if (start_insn->op == IR_CASE_VAL) { + if (!start_insn->op3) start_insn->op3 = 1; + } + } else if (n > 1) { + uint32_t *p = &ctx->cfg_edges[bb->predecessors]; + + for (; n > 0; p++, n--) { + bb = &ctx->cfg_blocks[*p]; + if (bb->predecessors_count == 1) { + ir_insn *start_insn = &ctx->ir_base[bb->start]; + if (start_insn->op == IR_IF_TRUE + || start_insn->op == IR_IF_FALSE + || start_insn->op == IR_CASE_DEFAULT) { + if (!start_insn->op2) start_insn->op2 = 1; + } else if (start_insn->op == IR_CASE_VAL) { + if (!start_insn->op3) start_insn->op3 = 1; + } + } + } + } + } + ref = insn->op3; + } + /* The bottom-up Pettis-Hansen algorithm is expensive - O(n^3), * use it only for relatively small functions. * diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index cb65ca505ac..9d36bc7b3cc 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -2405,9 +2405,22 @@ IR_FOLD(TRUNC(SEXT)) IR_FOLD(TRUNC(BITCAST)) IR_FOLD(ZEXT(BITCAST)) IR_FOLD(SEXT(BITCAST)) +{ + if (IR_IS_TYPE_INT(ctx->ir_base[op1_insn->op1].type)) { + op1 = op1_insn->op1; + IR_FOLD_RESTART; + } + IR_FOLD_NEXT; +} + IR_FOLD(BITCAST(BITCAST)) { - if (IR_IS_TYPE_INT(op1_insn->type)) { + ir_type dst_type = IR_OPT_TYPE(opt); + ir_type src_type = ctx->ir_base[op1_insn->op1].type; + + if (src_type == dst_type) { + IR_FOLD_COPY(op1_insn->op1); + } else if (IR_IS_TYPE_INT(src_type) == IR_IS_TYPE_INT(dst_type)) { op1 = op1_insn->op1; IR_FOLD_RESTART; } @@ -2422,6 +2435,26 @@ IR_FOLD(SEXT(SEXT)) IR_FOLD_RESTART; } +IR_FOLD(SEXT(ZEXT)) +{ + op1 = op1_insn->op1; + opt = IR_OPT(IR_ZEXT, IR_OPT_TYPE(opt)); + IR_FOLD_RESTART; +} + +IR_FOLD(SEXT(AND)) +{ + if (IR_IS_CONST_REF(op1_insn->op2) + && !IR_IS_SYM_CONST(ctx->ir_base[op1_insn->op2].op) + && !(ctx->ir_base[op1_insn->op2].val.u64 + & (1ULL << ((ir_type_size[op1_insn->type] * 8) - 1)))) { + /* SEXT(AND(_, 0b0*)) -> ZEXT(AND(_, 0b0*)) */ + opt = IR_OPT(IR_ZEXT, IR_OPT_TYPE(opt)); + IR_FOLD_RESTART; + } + IR_FOLD_NEXT; +} + IR_FOLD(TRUNC(AND)) { if (IR_IS_CONST_REF(op1_insn->op2)) { @@ -2838,9 +2871,7 @@ IR_FOLD(MUL(_, _)) IR_FOLD_NAMED(swap_ops) { if (op1 < op2) { /* move lower ref to op2 */ - ir_ref tmp = op1; - op1 = op2; - op2 = tmp; + SWAP_REFS(op1, op2); IR_FOLD_RESTART; } IR_FOLD_NEXT; @@ -2850,9 +2881,7 @@ IR_FOLD(ADD_OV(_, _)) IR_FOLD(MUL_OV(_, _)) { if (op1 < op2) { /* move lower ref to op2 */ - ir_ref tmp = op1; - op1 = op2; - op2 = tmp; + SWAP_REFS(op1, op2); IR_FOLD_RESTART; } /* skip CSE ??? */ @@ -2921,9 +2950,7 @@ IR_FOLD(GT(_, _)) IR_FOLD_BOOL((opt ^ (opt >> 1)) & 1); } } else if (op1 < op2) { /* move lower ref to op2 */ - ir_ref tmp = op1; - op1 = op2; - op2 = tmp; + SWAP_REFS(op1, op2); opt ^= 3; /* [U]LT <-> [U]GT, [U]LE <-> [U]GE */ IR_FOLD_RESTART; } @@ -2939,9 +2966,7 @@ IR_FOLD(UGT(_, _)) /* a >= a => true (two low bits are differ) */ IR_FOLD_BOOL((opt ^ (opt >> 1)) & 1); } else if (op1 < op2) { /* move lower ref to op2 */ - ir_ref tmp = op1; - op1 = op2; - op2 = tmp; + SWAP_REFS(op1, op2); opt ^= 3; /* [U]LT <-> [U]GT, [U]LE <-> [U]GE */ } IR_FOLD_NEXT; diff --git a/ext/opcache/jit/ir/ir_gcm.c b/ext/opcache/jit/ir/ir_gcm.c index 8a48fed4ce1..adbe77a3f5d 100644 --- a/ext/opcache/jit/ir/ir_gcm.c +++ b/ext/opcache/jit/ir/ir_gcm.c @@ -15,6 +15,7 @@ #define IR_GCM_EARLY_BLOCK(b) ((uint32_t)-((int32_t)(b))) #define IR_GCM_SPLIT 1 +#define IR_SCHEDULE_SWAP_OPS 1 static uint32_t ir_gcm_schedule_early(ir_ctx *ctx, ir_ref ref, ir_list *queue_late) { @@ -1131,6 +1132,37 @@ restart: new_insn->op1 = _xlat[insn->op1]; new_insn->op2 = _xlat[insn->op2]; new_insn->op3 = insn->op3; +#if IR_SCHEDULE_SWAP_OPS + /* Swap operands according to folding rules */ + if (new_insn->op1 < new_insn->op2) { + switch (new_insn->op) { + case IR_EQ: + case IR_NE: + case IR_ADD: + case IR_MUL: + case IR_ADD_OV: + case IR_MUL_OV: + case IR_OR: + case IR_AND: + case IR_XOR: + case IR_MIN: + case IR_MAX: + SWAP_REFS(new_insn->op1, new_insn->op2); + break; + case IR_LT: + case IR_GE: + case IR_LE: + case IR_GT: + case IR_ULT: + case IR_UGE: + case IR_ULE: + case IR_UGT: + SWAP_REFS(new_insn->op1, new_insn->op2); + new_insn->op ^= 3; /* [U]LT <-> [U]GT, [U]LE <-> [U]GE */ + break; + } + } +#endif break; case 3: new_insn->op1 = _xlat[insn->op1]; diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index ed3ba9777e8..e46daac16f3 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -987,6 +987,7 @@ IR_ALWAYS_INLINE uint32_t ir_insn_len(const ir_insn *insn) /* Temporary: SCCP -> CFG */ #define IR_SCCP_DONE (1<<25) +#define IR_CFG_REACHABLE (1<<26) /* Temporary: Dominators -> Loops */ #define IR_NO_LOOPS (1<<25) diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 5e19e2a0a3d..fbd31755edc 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -251,6 +251,28 @@ next: } } +static bool ir_is_dead_load(ir_ctx *ctx, ir_ref ref) +{ + if (ctx->use_lists[ref].count == 1) { + if ((ir_op_flags[ctx->ir_base[ref].op] & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD)) { + return 1; + } else if (ctx->ir_base[ref].op == IR_ALLOCA) { + return 1; + } + } + return 0; +} + +static bool ir_is_dead(ir_ctx *ctx, ir_ref ref) +{ + if (ctx->use_lists[ref].count == 0) { + return IR_IS_FOLDABLE_OP(ctx->ir_base[ref].op); + } else { + return ir_is_dead_load(ctx, ref); + } + return 0; +} + static bool ir_sccp_is_true(ir_ctx *ctx, ir_insn *_values, ir_ref a) { ir_insn *v = IR_IS_CONST_REF(a) ? &ctx->ir_base[a] : &_values[a]; @@ -323,6 +345,8 @@ static void ir_sccp_remove_insn2(ir_ctx *ctx, ir_ref ref, ir_bitqueue *worklist) || ((ir_op_flags[ctx->ir_base[input].op] & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD) && ctx->use_lists[input].count == 1)) { ir_bitqueue_add(worklist, input); + } else if (ctx->ir_base[input].op == IR_PHI && ctx->use_lists[input].count == 1) { + ir_bitqueue_add(worklist, ctx->ir_base[input].op1); } } } @@ -587,7 +611,7 @@ static void ir_sccp_remove_unfeasible_merge_inputs(ir_ctx *ctx, ir_insn *_values IR_ASSERT(!next); next = use; next_insn = use_insn; - } else { + } else if (use_insn->op != IR_NOP) { IR_ASSERT(use_insn->op1 == ref); use_insn->op1 = prev; ir_use_list_add(ctx, prev, use); @@ -1005,8 +1029,29 @@ static ir_ref ir_ext_const(ir_ctx *ctx, ir_insn *val_insn, ir_op op, ir_type typ static ir_ref ir_ext_ref(ir_ctx *ctx, ir_ref var_ref, ir_ref src_ref, ir_op op, ir_type type) { - ir_ref new_ext_ref = ir_emit1(ctx, IR_OPTX(op, type, 1), src_ref); + uint32_t optx = IR_OPTX(op, type, 1); + ir_ref new_ext_ref; + if (!IR_IS_CONST_REF(src_ref)) { + ir_use_list *use_list = &ctx->use_lists[src_ref]; + ir_ref *p, n = use_list->count; + + for (p = ctx->use_edges + use_list->refs; n > 0; p++, n--) { + ir_ref use = *p; + ir_insn *use_insn = &ctx->ir_base[use]; + + if (use_insn->optx == optx) { + IR_ASSERT(use_insn->op1 == src_ref); + ir_use_list_add(ctx, use, var_ref); + if (!IR_IS_CONST_REF(src_ref)) { + ir_use_list_remove_one(ctx, src_ref, var_ref); + } + return use; + } + } + } + + new_ext_ref = ir_emit1(ctx, optx, src_ref); ctx->use_lists = ir_mem_realloc(ctx->use_lists, ctx->insns_count * sizeof(ir_use_list)); ctx->use_lists[new_ext_ref].count = 0; ctx->use_lists[new_ext_ref].refs = IR_UNUSED; @@ -1037,8 +1082,7 @@ static bool ir_try_promote_ext(ir_ctx *ctx, ir_ref ext_ref, ir_insn *insn, ir_bi op_ref = phi_insn->op3; op_insn = &ctx->ir_base[op_ref]; if ((op_insn->op != IR_ADD && op_insn->op != IR_SUB && op_insn->op != IR_MUL) - || op_insn->op1 != ref - || op_insn->op2 == ref + || (op_insn->op1 != ref && op_insn->op2 != ref) || ctx->use_lists[op_ref].count != 1) { return 0; } @@ -1054,8 +1098,7 @@ static bool ir_try_promote_ext(ir_ctx *ctx, ir_ref ext_ref, ir_insn *insn, ir_bi ir_insn *use_insn = &ctx->ir_base[use]; if ((use_insn->op >= IR_EQ && use_insn->op <= IR_UGT) - && use_insn->op1 == ref - && use_insn->op2 != ref) { + && (use_insn->op1 == ref || use_insn->op2 == ref)) { continue; } else if (use_insn->op == IR_IF) { continue; @@ -1082,13 +1125,24 @@ static bool ir_try_promote_ext(ir_ctx *ctx, ir_ref ext_ref, ir_insn *insn, ir_bi } IR_ASSERT(((use_insn->op >= IR_EQ && use_insn->op <= IR_UGT) || use_insn->op == IR_ADD || use_insn->op == IR_SUB || use_insn->op == IR_MUL) - && use_insn->op1 == ref - && use_insn->op2 != ref); - if (IR_IS_CONST_REF(use_insn->op2) - && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op2].op)) { - ctx->ir_base[use].op2 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op2], op, type); - } else { - ctx->ir_base[use].op2 = ir_ext_ref(ctx, use, use_insn->op2, op, type); + && (use_insn->op1 == ref || use_insn->op2 == ref)); + if (use_insn->op1 != ref) { + if (IR_IS_CONST_REF(use_insn->op1) + && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op1].op)) { + ctx->ir_base[use].op1 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op1], op, type); + } else { + ctx->ir_base[use].op1 = ir_ext_ref(ctx, use, use_insn->op1, op, type); + ir_bitqueue_add(worklist, ctx->ir_base[use].op1); + } + } + if (use_insn->op2 != ref) { + if (IR_IS_CONST_REF(use_insn->op2) + && !IR_IS_SYM_CONST(ctx->ir_base[use_insn->op2].op)) { + ctx->ir_base[use].op2 = ir_ext_const(ctx, &ctx->ir_base[use_insn->op2], op, type); + } else { + ctx->ir_base[use].op2 = ir_ext_ref(ctx, use, use_insn->op2, op, type); + ir_bitqueue_add(worklist, ctx->ir_base[use].op2); + } } } } @@ -1101,11 +1155,973 @@ static bool ir_try_promote_ext(ir_ctx *ctx, ir_ref ext_ref, ir_insn *insn, ir_bi ctx->ir_base[ref].op2 = ir_ext_const(ctx, &ctx->ir_base[phi_insn->op2], op, type); } else { ctx->ir_base[ref].op2 = ir_ext_ref(ctx, ref, phi_insn->op2, op, type); + ir_bitqueue_add(worklist, ctx->ir_base[ref].op2); } return 1; } +static void ir_get_true_false_refs(const ir_ctx *ctx, ir_ref if_ref, ir_ref *if_true_ref, ir_ref *if_false_ref) +{ + ir_use_list *use_list = &ctx->use_lists[if_ref]; + ir_ref *p = &ctx->use_edges[use_list->refs]; + + IR_ASSERT(use_list->count == 2); + if (ctx->ir_base[*p].op == IR_IF_TRUE) { + IR_ASSERT(ctx->ir_base[*(p + 1)].op == IR_IF_FALSE); + *if_true_ref = *p; + *if_false_ref = *(p + 1); + } else { + IR_ASSERT(ctx->ir_base[*p].op == IR_IF_FALSE); + IR_ASSERT(ctx->ir_base[*(p + 1)].op == IR_IF_TRUE); + *if_false_ref = *p; + *if_true_ref = *(p + 1); + } +} + +static void ir_merge_blocks(ir_ctx *ctx, ir_ref end, ir_ref begin, ir_bitqueue *worklist2) +{ + ir_ref prev, next; + ir_use_list *use_list; + + IR_ASSERT(ctx->ir_base[begin].op == IR_BEGIN); + IR_ASSERT(ctx->ir_base[end].op == IR_END); + IR_ASSERT(ctx->ir_base[begin].op1 == end); + IR_ASSERT(ctx->use_lists[end].count == 1); + + prev = ctx->ir_base[end].op1; + + use_list = &ctx->use_lists[begin]; + IR_ASSERT(use_list->count == 1); + next = ctx->use_edges[use_list->refs]; + + /* remove BEGIN and END */ + MAKE_NOP(&ctx->ir_base[begin]); CLEAR_USES(begin); + MAKE_NOP(&ctx->ir_base[end]); CLEAR_USES(end); + + /* connect their predecessor and successor */ + ctx->ir_base[next].op1 = prev; + ir_use_list_replace_all(ctx, prev, end, next); + + if (ctx->ir_base[prev].op == IR_BEGIN || ctx->ir_base[prev].op == IR_MERGE) { + ir_bitqueue_add(worklist2, prev); + } +} + +static bool ir_try_remove_empty_diamond(ir_ctx *ctx, ir_ref ref, ir_insn *insn, ir_bitqueue *worklist) +{ + if (insn->inputs_count == 2) { + ir_ref end1_ref = insn->op1, end2_ref = insn->op2; + ir_insn *end1 = &ctx->ir_base[end1_ref]; + ir_insn *end2 = &ctx->ir_base[end2_ref]; + + if (end1->op != IR_END || end2->op != IR_END) { + return 0; + } + + ir_ref start1_ref = end1->op1, start2_ref = end2->op1; + ir_insn *start1 = &ctx->ir_base[start1_ref]; + ir_insn *start2 = &ctx->ir_base[start2_ref]; + + if (start1->op1 != start2->op1) { + return 0; + } + + ir_ref root_ref = start1->op1; + ir_insn *root = &ctx->ir_base[root_ref]; + + if (root->op != IR_IF + && !(root->op == IR_SWITCH && ctx->use_lists[root_ref].count == 2)) { + return 0; + } + + /* Empty Diamond + * + * prev prev + * | condition | condition + * | / | + * IF | + * | \ | + * | +-----+ | + * | IF_FALSE | + * IF_TRUE | => | + * | END | + * END / | + * | +---+ | + * | / | + * MERGE | + * | | + * next next + */ + + ir_ref next_ref = ctx->use_edges[ctx->use_lists[ref].refs]; + ir_insn *next = &ctx->ir_base[next_ref]; + + IR_ASSERT(ctx->use_lists[start1_ref].count == 1); + IR_ASSERT(ctx->use_lists[start2_ref].count == 1); + + next->op1 = root->op1; + ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); + if (!IR_IS_CONST_REF(root->op2)) { + ir_use_list_remove_all(ctx, root->op2, root_ref); + if (ir_is_dead(ctx, root->op2)) { + ir_bitqueue_add(worklist, root->op2); + } + } + + MAKE_NOP(root); CLEAR_USES(root_ref); + MAKE_NOP(start1); CLEAR_USES(start1_ref); + MAKE_NOP(start2); CLEAR_USES(start2_ref); + MAKE_NOP(end1); CLEAR_USES(end1_ref); + MAKE_NOP(end2); CLEAR_USES(end2_ref); + MAKE_NOP(insn); CLEAR_USES(ref); + + if (ctx->ir_base[next->op1].op == IR_BEGIN || ctx->ir_base[next->op1].op == IR_MERGE) { + ir_bitqueue_add(worklist, next->op1); + } + + return 1; + } else { + ir_ref i, count = insn->inputs_count, *ops = insn->ops + 1; + ir_ref root_ref = IR_UNUSED; + + for (i = 0; i < count; i++) { + ir_ref end_ref, start_ref; + ir_insn *end, *start; + + end_ref = ops[i]; + end = &ctx->ir_base[end_ref]; + if (end->op != IR_END) { + return 0; + } + start_ref = end->op1; + start = &ctx->ir_base[start_ref]; + if (start->op != IR_CASE_VAL && start->op != IR_CASE_DEFAULT) { + return 0; + } + IR_ASSERT(ctx->use_lists[start_ref].count == 1); + if (!root_ref) { + root_ref = start->op1; + if (ctx->use_lists[root_ref].count != count) { + return 0; + } + } else if (start->op1 != root_ref) { + return 0; + } + } + + /* Empty N-Diamond */ + ir_ref next_ref = ctx->use_edges[ctx->use_lists[ref].refs]; + ir_insn *next = &ctx->ir_base[next_ref]; + ir_insn *root = &ctx->ir_base[root_ref]; + + next->op1 = root->op1; + ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); + + if (!IR_IS_CONST_REF(root->op2)) { + ir_use_list_remove_all(ctx, root->op2, root_ref); + if (ir_is_dead(ctx, root->op2)) { + ir_bitqueue_add(worklist, root->op2); + } + } + + MAKE_NOP(root); CLEAR_USES(root_ref); + + for (i = 0; i < count; i++) { + ir_ref end_ref = ops[i]; + ir_insn *end = &ctx->ir_base[end_ref]; + ir_ref start_ref = end->op1; + ir_insn *start = &ctx->ir_base[start_ref]; + + MAKE_NOP(start); CLEAR_USES(start_ref); + MAKE_NOP(end); CLEAR_USES(end_ref); + } + + MAKE_NOP(insn); CLEAR_USES(ref); + + if (ctx->ir_base[next->op1].op == IR_BEGIN || ctx->ir_base[next->op1].op == IR_MERGE) { + ir_bitqueue_add(worklist, next->op1); + } + + return 1; + } +} + +static bool ir_is_zero(ir_ctx *ctx, ir_ref ref) +{ + return IR_IS_CONST_REF(ref) + && !IR_IS_SYM_CONST(ctx->ir_base[ref].op) + && ctx->ir_base[ref].val.u32 == 0; +} + +static bool ir_optimize_phi(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_ref ref, ir_insn *insn, ir_bitqueue *worklist) +{ + IR_ASSERT(insn->inputs_count == 3); + IR_ASSERT(ctx->use_lists[merge_ref].count == 2); + + ir_ref end1_ref = merge->op1, end2_ref = merge->op2; + ir_insn *end1 = &ctx->ir_base[end1_ref]; + ir_insn *end2 = &ctx->ir_base[end2_ref]; + + if (end1->op == IR_END && end2->op == IR_END) { + ir_ref start1_ref = end1->op1, start2_ref = end2->op1; + ir_insn *start1 = &ctx->ir_base[start1_ref]; + ir_insn *start2 = &ctx->ir_base[start2_ref]; + + if (start1->op1 == start2->op1) { + ir_ref root_ref = start1->op1; + ir_insn *root = &ctx->ir_base[root_ref]; + + if (root->op == IR_IF && ctx->use_lists[root->op2].count == 1) { + ir_ref cond_ref = root->op2; + ir_insn *cond = &ctx->ir_base[cond_ref]; + ir_type type = insn->type; + bool is_cmp, is_less; + + if (IR_IS_TYPE_FP(type)) { + is_cmp = (cond->op == IR_LT || cond->op == IR_LE || cond->op == IR_GT || cond->op == IR_GE || + cond->op == IR_ULT || cond->op == IR_ULE || cond->op == IR_UGT || cond->op == IR_UGE); + is_less = (cond->op == IR_LT || cond->op == IR_LE || + cond->op == IR_ULT || cond->op == IR_ULE); + } else if (IR_IS_TYPE_SIGNED(type)) { + is_cmp = (cond->op == IR_LT || cond->op == IR_LE || cond->op == IR_GT || cond->op == IR_GE); + is_less = (cond->op == IR_LT || cond->op == IR_LE); + } else { + IR_ASSERT(IR_IS_TYPE_UNSIGNED(type)); + is_cmp = (cond->op == IR_ULT || cond->op == IR_ULE || cond->op == IR_UGT || cond->op == IR_UGE); + is_less = (cond->op == IR_ULT || cond->op == IR_ULE); + } + + if (is_cmp + && ((insn->op2 == cond->op1 && insn->op3 == cond->op2) + || (insn->op2 == cond->op2 && insn->op3 == cond->op1))) { + /* MAX/MIN + * + * prev prev + * | LT(A, B) | + * | / | + * IF | + * | \ | + * | +-----+ | + * | IF_FALSE | + * IF_TRUE | => | + * | END | + * END / | + * | +---+ | + * | / | + * MERGE | + * | \ | + * | PHI(A, B) | MIN(A, B) + * next next + */ + ir_ref next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs]; + ir_insn *next; + + if (next_ref == ref) { + next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs + 1]; + } + next = &ctx->ir_base[next_ref]; + + IR_ASSERT(ctx->use_lists[start1_ref].count == 1); + IR_ASSERT(ctx->use_lists[start2_ref].count == 1); + + insn->op = ( + (is_less ? cond->op1 : cond->op2) + == + ((start1->op == IR_IF_TRUE) ? insn->op2 : insn->op3) + ) ? IR_MIN : IR_MAX; + insn->inputs_count = 2; + if (insn->op2 > insn->op3) { + insn->op1 = insn->op2; + insn->op2 = insn->op3; + } else { + insn->op1 = insn->op3; + } + insn->op3 = IR_UNUSED; + + next->op1 = root->op1; + ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); + if (!IR_IS_CONST_REF(insn->op1)) { + ir_use_list_remove_all(ctx, insn->op1, cond_ref); + } + if (!IR_IS_CONST_REF(insn->op2)) { + ir_use_list_remove_all(ctx, insn->op2, cond_ref); + } + + MAKE_NOP(cond); CLEAR_USES(cond_ref); + MAKE_NOP(root); CLEAR_USES(root_ref); + MAKE_NOP(start1); CLEAR_USES(start1_ref); + MAKE_NOP(start2); CLEAR_USES(start2_ref); + MAKE_NOP(end1); CLEAR_USES(end1_ref); + MAKE_NOP(end2); CLEAR_USES(end2_ref); + MAKE_NOP(merge); CLEAR_USES(merge_ref); + + if (ctx->ir_base[next->op1].op == IR_BEGIN || ctx->ir_base[next->op1].op == IR_MERGE) { + ir_bitqueue_add(worklist, next->op1); + } + + return 1; + } else if (is_cmp + && ((ctx->ir_base[insn->op2].op == IR_NEG + && ctx->use_lists[insn->op2].count == 1 + && ctx->ir_base[insn->op2].op1 == insn->op3 + && ((cond->op1 == insn->op3 + && ir_is_zero(ctx, cond->op2) + && is_less == (start1->op == IR_IF_TRUE)) + || (cond->op2 == insn->op3 + && ir_is_zero(ctx, cond->op1) + && is_less != (start1->op == IR_IF_TRUE)))) + || (ctx->ir_base[insn->op3].op == IR_NEG + && ctx->use_lists[insn->op3].count == 1 + && ctx->ir_base[insn->op3].op1 == insn->op2 + && ((cond->op1 == insn->op2 + && ir_is_zero(ctx, cond->op2) + && is_less != (start1->op == IR_IF_TRUE)) + || (cond->op2 == insn->op2 + && ir_is_zero(ctx, cond->op1) + && is_less == (start1->op == IR_IF_TRUE)))))) { + /* ABS + * + * prev prev + * | LT(A, 0) | + * | / | + * IF | + * | \ | + * | +-----+ | + * | IF_FALSE | + * IF_TRUE | => | + * | END | + * END / | + * | +---+ | + * | / | + * MERGE | + * | \ | + * | PHI(A, NEG(A)) | ABS(A) + * next next + */ + ir_ref neg_ref; + ir_ref next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs]; + ir_insn *next; + + if (next_ref == ref) { + next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs + 1]; + } + next = &ctx->ir_base[next_ref]; + + IR_ASSERT(ctx->use_lists[start1_ref].count == 1); + IR_ASSERT(ctx->use_lists[start2_ref].count == 1); + + insn->op = IR_ABS; + insn->inputs_count = 1; + if (ctx->ir_base[insn->op2].op == IR_NEG) { + neg_ref = insn->op2; + insn->op1 = insn->op3; + } else { + neg_ref = insn->op3; + insn->op1 = insn->op2; + } + insn->op2 = IR_UNUSED; + insn->op3 = IR_UNUSED; + + next->op1 = root->op1; + ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); + ir_use_list_remove_one(ctx, insn->op1, neg_ref); + if (!IR_IS_CONST_REF(insn->op1)) { + ir_use_list_remove_all(ctx, insn->op1, cond_ref); + } + + MAKE_NOP(cond); CLEAR_USES(cond_ref); + MAKE_NOP(root); CLEAR_USES(root_ref); + MAKE_NOP(start1); CLEAR_USES(start1_ref); + MAKE_NOP(start2); CLEAR_USES(start2_ref); + MAKE_NOP(end1); CLEAR_USES(end1_ref); + MAKE_NOP(end2); CLEAR_USES(end2_ref); + MAKE_NOP(merge); CLEAR_USES(merge_ref); + MAKE_NOP(&ctx->ir_base[neg_ref]); CLEAR_USES(neg_ref); + + if (ctx->ir_base[next->op1].op == IR_BEGIN || ctx->ir_base[next->op1].op == IR_MERGE) { + ir_bitqueue_add(worklist, next->op1); + } + + return 1; +#if 0 + } else { + /* COND + * + * prev prev + * | cond | + * | / | + * IF | + * | \ | + * | +-----+ | + * | IF_FALSE | + * IF_TRUE | => | + * | END | + * END / | + * | +---+ | + * | / | + * MERGE | + * | \ | + * | PHI(A, B) | COND(cond, A, B) + * next next + */ + ir_ref next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs]; + ir_insn *next; + + if (next_ref == ref) { + next_ref = ctx->use_edges[ctx->use_lists[merge_ref].refs + 1]; + } + next = &ctx->ir_base[next_ref]; + + IR_ASSERT(ctx->use_lists[start1_ref].count == 1); + IR_ASSERT(ctx->use_lists[start2_ref].count == 1); + + insn->op = IR_COND; + insn->inputs_count = 3; + insn->op1 = cond_ref; + if (start1->op == IR_IF_FALSE) { + SWAP_REFS(insn->op2, insn->op3); + } + + next->op1 = root->op1; + ir_use_list_replace_one(ctx, cond_ref, root_ref, ref); + ir_use_list_replace_one(ctx, root->op1, root_ref, next_ref); + ir_use_list_remove_all(ctx, root->op2, root_ref); + + MAKE_NOP(root); CLEAR_USES(root_ref); + MAKE_NOP(start1); CLEAR_USES(start1_ref); + MAKE_NOP(start2); CLEAR_USES(start2_ref); + MAKE_NOP(end1); CLEAR_USES(end1_ref); + MAKE_NOP(end2); CLEAR_USES(end2_ref); + MAKE_NOP(merge); CLEAR_USES(merge_ref); + + if (ctx->ir_base[next->op1].op == IR_BEGIN || ctx->ir_base[next->op1].op == IR_MERGE) { + ir_bitqueue_add(worklist, next->op1); + } + + return 1; +#endif + } + } + } + } + + return 0; +} + +static bool ir_cmp_is_true(ir_op op, ir_insn *op1, ir_insn *op2) +{ + IR_ASSERT(op1->type == op2->type); + if (IR_IS_TYPE_INT(op1->type)) { + if (op == IR_EQ) { + return op1->val.u64 == op2->val.u64; + } else if (op == IR_NE) { + return op1->val.u64 != op2->val.u64; + } else if (op == IR_LT) { + if (IR_IS_TYPE_SIGNED(op1->type)) { + return op1->val.i64 < op2->val.i64; + } else { + return op1->val.u64 < op2->val.u64; + } + } else if (op == IR_GE) { + if (IR_IS_TYPE_SIGNED(op1->type)) { + return op1->val.i64 >= op2->val.i64; + } else { + return op1->val.u64 >= op2->val.u64; + } + } else if (op == IR_LE) { + if (IR_IS_TYPE_SIGNED(op1->type)) { + return op1->val.i64 <= op2->val.i64; + } else { + return op1->val.u64 <= op2->val.u64; + } + } else if (op == IR_GT) { + if (IR_IS_TYPE_SIGNED(op1->type)) { + return op1->val.i64 > op2->val.i64; + } else { + return op1->val.u64 > op2->val.u64; + } + } else if (op == IR_ULT) { + return op1->val.u64 < op2->val.u64; + } else if (op == IR_UGE) { + return op1->val.u64 >= op2->val.u64; + } else if (op == IR_ULE) { + return op1->val.u64 <= op2->val.u64; + } else if (op == IR_UGT) { + return op1->val.u64 > op2->val.u64; + } else { + IR_ASSERT(0); + return 0; + } + } else if (op1->type == IR_DOUBLE) { + if (op == IR_EQ) { + return op1->val.d == op2->val.d; + } else if (op == IR_NE) { + return op1->val.d != op2->val.d; + } else if (op == IR_LT) { + return op1->val.d < op2->val.d; + } else if (op == IR_GE) { + return op1->val.d >= op2->val.d; + } else if (op == IR_LE) { + return op1->val.d <= op2->val.d; + } else if (op == IR_GT) { + return op1->val.d > op2->val.d; + } else if (op == IR_ULT) { + return !(op1->val.d >= op2->val.d); + } else if (op == IR_UGE) { + return !(op1->val.d < op2->val.d); + } else if (op == IR_ULE) { + return !(op1->val.d > op2->val.d); + } else if (op == IR_UGT) { + return !(op1->val.d <= op2->val.d); + } else { + IR_ASSERT(0); + return 0; + } + } else { + IR_ASSERT(op1->type == IR_FLOAT); + if (op == IR_EQ) { + return op1->val.f == op2->val.f; + } else if (op == IR_NE) { + return op1->val.f != op2->val.f; + } else if (op == IR_LT) { + return op1->val.f < op2->val.f; + } else if (op == IR_GE) { + return op1->val.f >= op2->val.f; + } else if (op == IR_LE) { + return op1->val.f <= op2->val.f; + } else if (op == IR_GT) { + return op1->val.f > op2->val.f; + } else if (op == IR_ULT) { + return !(op1->val.f >= op2->val.f); + } else if (op == IR_UGE) { + return !(op1->val.f < op2->val.f); + } else if (op == IR_ULE) { + return !(op1->val.f > op2->val.f); + } else if (op == IR_UGT) { + return !(op1->val.f <= op2->val.f); + } else { + IR_ASSERT(0); + return 0; + } + } +} + +static bool ir_try_split_if(ir_ctx *ctx, ir_ref ref, ir_insn *insn, ir_bitqueue *worklist) +{ + ir_ref cond_ref = insn->op2; + ir_insn *cond = &ctx->ir_base[cond_ref]; + + if (cond->op == IR_PHI + && cond->inputs_count == 3 + && cond->op1 == insn->op1 + && ((IR_IS_CONST_REF(cond->op2) && !IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op)) + || (IR_IS_CONST_REF(cond->op3) && !IR_IS_SYM_CONST(ctx->ir_base[cond->op3].op)))) { + ir_ref merge_ref = insn->op1; + ir_insn *merge = &ctx->ir_base[merge_ref]; + + if (ctx->use_lists[merge_ref].count == 2) { + ir_ref end1_ref = merge->op1, end2_ref = merge->op2; + ir_insn *end1 = &ctx->ir_base[end1_ref]; + ir_insn *end2 = &ctx->ir_base[end2_ref]; + + if (end1->op == IR_END && end2->op == IR_END) { + ir_ref if_true_ref, if_false_ref; + ir_insn *if_true, *if_false; + ir_op op = IR_IF_FALSE; + + ir_get_true_false_refs(ctx, ref, &if_true_ref, &if_false_ref); + + if (!IR_IS_CONST_REF(cond->op2) || IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op)) { + IR_ASSERT(IR_IS_CONST_REF(cond->op3)); + SWAP_REFS(cond->op2, cond->op3); + SWAP_REFS(merge->op1, merge->op2); + SWAP_REFS(end1_ref, end2_ref); + SWAP_INSNS(end1, end2); + } + if (ir_const_is_true(&ctx->ir_base[cond->op2])) { + SWAP_REFS(if_true_ref, if_false_ref); + op = IR_IF_TRUE; + } + if_true = &ctx->ir_base[if_true_ref]; + if_false = &ctx->ir_base[if_false_ref]; + + if (IR_IS_CONST_REF(cond->op3) && !IR_IS_SYM_CONST(ctx->ir_base[cond->op3].op)) { + if (ir_const_is_true(&ctx->ir_base[cond->op3]) ^ (op == IR_IF_TRUE)) { + /* Simple IF Split + * + * | | | | + * | END | END + * END / END \ + * | +---+ | + + * | / | | + * MERGE | | + * | \ | | + * | PHI(false, true) | | + * | / | | + * IF => | | + * | \ | | + * | +------+ | | + * | IF_TRUE | BEGIN + * IF_FALSE | BEGIN + * | | + */ + ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); + ir_use_list_replace_one(ctx, end2_ref, merge_ref, if_true_ref); + + MAKE_NOP(merge); CLEAR_USES(merge_ref); + MAKE_NOP(cond); CLEAR_USES(cond_ref); + MAKE_NOP(insn); CLEAR_USES(ref); + + if_false->optx = IR_OPTX(IR_BEGIN, IR_VOID, 1); + if_false->op1 = end1_ref; + + if_true->optx = IR_OPTX(IR_BEGIN, IR_VOID, 1); + if_true->op1 = end2_ref; + + ir_bitqueue_add(worklist, if_false_ref); + ir_bitqueue_add(worklist, if_true_ref); + + return 1; + } else { + /* Simple IF Split + * + * | | | | + * | END | END + * END / END | + * | +---+ | | + * | / | | + * MERGE | + + * | \ | / + * | PHI(false, false) | / + * | / | / + * IF => | / + * | \ | / + * | +------+ | / + * | IF_TRUE | / BEGIN(unreachable) + * IF_FALSE | MERGE + * | | + */ + ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); + ir_use_list_replace_one(ctx, end2_ref, merge_ref, if_false_ref); + + MAKE_NOP(merge); CLEAR_USES(merge_ref); + MAKE_NOP(cond); CLEAR_USES(cond_ref); + MAKE_NOP(insn); CLEAR_USES(ref); + + if_false->optx = IR_OPTX(IR_MERGE, IR_VOID, 2); + if_false->op1 = end1_ref; + if_false->op2 = end2_ref; + + if_true->optx = IR_BEGIN; + if_true->op1 = IR_UNUSED; + + ctx->flags2 &= ~IR_CFG_REACHABLE; + + ir_bitqueue_add(worklist, if_false_ref); + + return 1; + } + } + + /* Simple IF Split + * + * | | | | + * | END | IF(X) + * END / END / \ + * | +---+ | +--+ + + * | / | / | + * MERGE | IF_FALSE | + * | \ | | | + * | PHI(false, X) | | | + * | / | | | + * IF => | END | + * | \ | | | + * | +------+ | | | + * | IF_TRUE | | IF_TRUE + * IF_FALSE | MERGE + * | | + */ + ir_use_list_remove_all(ctx, merge_ref, cond_ref); + ir_use_list_remove_all(ctx, ref, if_true_ref); + if (!IR_IS_CONST_REF(cond->op3)) { + ir_use_list_replace_one(ctx, cond->op3, cond_ref, end2_ref); + } + ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); + ir_use_list_add(ctx, end2_ref, if_true_ref); + + end2->optx = IR_OPTX(IR_IF, IR_VOID, 2); + end2->op2 = cond->op3; + + merge->optx = IR_OPTX(op, IR_VOID, 1); + merge->op1 = end2_ref; + merge->op2 = IR_UNUSED; + + MAKE_NOP(cond); + CLEAR_USES(cond_ref); + + insn->optx = IR_OPTX(IR_END, IR_VOID, 1); + insn->op1 = merge_ref; + insn->op2 = IR_UNUSED; + + if_true->op1 = end2_ref; + + if_false->optx = IR_OPTX(IR_MERGE, IR_VOID, 2); + if_false->op1 = end1_ref; + if_false->op2 = ref; + + ir_bitqueue_add(worklist, if_false_ref); + if (ctx->ir_base[end2->op1].op == IR_BEGIN || ctx->ir_base[end2->op1].op == IR_MERGE) { + ir_bitqueue_add(worklist, end2->op1); + } + + return 1; + } + } + } + + return 0; +} + +static bool ir_try_split_if_cmp(ir_ctx *ctx, ir_ref ref, ir_insn *insn, ir_bitqueue *worklist) +{ + ir_ref cond_ref = insn->op2; + ir_insn *cond = &ctx->ir_base[cond_ref]; + + if (cond->op >= IR_EQ && cond->op <= IR_UGT + && IR_IS_CONST_REF(cond->op2) + && !IR_IS_SYM_CONST(ctx->ir_base[cond->op2].op) + && ctx->use_lists[insn->op2].count == 1) { + ir_ref phi_ref = cond->op1; + ir_insn *phi = &ctx->ir_base[phi_ref]; + + if (phi->op == IR_PHI + && phi->inputs_count == 3 + && phi->op1 == insn->op1 + && ctx->use_lists[phi_ref].count == 1 + && ((IR_IS_CONST_REF(phi->op2) && !IR_IS_SYM_CONST(ctx->ir_base[phi->op2].op)) + || (IR_IS_CONST_REF(phi->op3) && !IR_IS_SYM_CONST(ctx->ir_base[phi->op3].op)))) { + ir_ref merge_ref = insn->op1; + ir_insn *merge = &ctx->ir_base[merge_ref]; + + if (ctx->use_lists[merge_ref].count == 2) { + ir_ref end1_ref = merge->op1, end2_ref = merge->op2; + ir_insn *end1 = &ctx->ir_base[end1_ref]; + ir_insn *end2 = &ctx->ir_base[end2_ref]; + + if (end1->op == IR_END && end2->op == IR_END) { + ir_ref if_true_ref, if_false_ref; + ir_insn *if_true, *if_false; + ir_op op = IR_IF_FALSE; + + ir_get_true_false_refs(ctx, ref, &if_true_ref, &if_false_ref); + + if (!IR_IS_CONST_REF(phi->op2) || IR_IS_SYM_CONST(ctx->ir_base[phi->op2].op)) { + IR_ASSERT(IR_IS_CONST_REF(phi->op3)); + SWAP_REFS(phi->op2, phi->op3); + SWAP_REFS(merge->op1, merge->op2); + SWAP_REFS(end1_ref, end2_ref); + SWAP_INSNS(end1, end2); + } + if (ir_cmp_is_true(cond->op, &ctx->ir_base[phi->op2], &ctx->ir_base[cond->op2])) { + SWAP_REFS(if_true_ref, if_false_ref); + op = IR_IF_TRUE; + } + if_true = &ctx->ir_base[if_true_ref]; + if_false = &ctx->ir_base[if_false_ref]; + + if (IR_IS_CONST_REF(phi->op3) && !IR_IS_SYM_CONST(ctx->ir_base[phi->op3].op)) { + if (ir_cmp_is_true(cond->op, &ctx->ir_base[phi->op3], &ctx->ir_base[cond->op2]) ^ (op == IR_IF_TRUE)) { + /* IF Split + * + * | | | | + * | END | END + * END / END | + * | +---+ | | + * | / | | + * MERGE | | + * | \ | | + * | PHI(C1, X) | | + * | | | | + * | CMP(_, C2) | | + * | / | | + * IF => | | + * | \ | | + * | +------+ | | + * | IF_TRUE | BEGIN + * IF_FALSE | BEGIN | + * | | + */ + + ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); + ir_use_list_replace_one(ctx, end2_ref, merge_ref, if_true_ref); + + MAKE_NOP(merge); CLEAR_USES(merge_ref); + MAKE_NOP(phi); CLEAR_USES(phi_ref); + MAKE_NOP(cond); CLEAR_USES(cond_ref); + MAKE_NOP(insn); CLEAR_USES(ref); + + if_false->optx = IR_OPTX(IR_BEGIN, IR_VOID, 1); + if_false->op1 = end1_ref; + + if_true->optx = IR_OPTX(IR_BEGIN, IR_VOID, 1); + if_true->op1 = end2_ref; + + ir_bitqueue_add(worklist, if_false_ref); + ir_bitqueue_add(worklist, if_true_ref); + + return 1; + } else { + /* IF Split + * + * | | | | + * | END | END + * END / END | + * | +---+ | | + * | / | | + * MERGE | | + * | \ | | + * | PHI(C1, X) | | + * | | | + + * | CMP(_, C2) | / + * | / | / + * IF => | / + * | \ | / + * | +------+ | / + * | IF_TRUE | / BEGIN(unreachable) + * IF_FALSE | MERGE | + * | | + */ + + ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); + ir_use_list_replace_one(ctx, end2_ref, merge_ref, if_false_ref); + + MAKE_NOP(merge); CLEAR_USES(merge_ref); + MAKE_NOP(phi); CLEAR_USES(phi_ref); + MAKE_NOP(cond); CLEAR_USES(cond_ref); + MAKE_NOP(insn); CLEAR_USES(ref); + + if_false->optx = IR_OPTX(IR_MERGE, IR_VOID, 2); + if_false->op1 = end1_ref; + if_false->op2 = end2_ref; + + if_true->optx = IR_BEGIN; + if_true->op1 = IR_UNUSED; + + ctx->flags2 &= ~IR_CFG_REACHABLE; + + ir_bitqueue_add(worklist, if_false_ref); + + return 1; + } + } else { + /* IF Split + * + * | | | | + * | END | IF<----+ + * END / END / \ | + * | +---+ | +--+ + | + * | / | / | | + * MERGE | IF_FALSE | | + * | \ | | | | + * | PHI(C1, X) | | | | + * | | | | | | + * | CMP(_, C2) | | | CMP(X, C2) + * | / | | | + * IF => | END | + * | \ | | | + * | +------+ | | | + * | IF_TRUE | | IF_TRUE + * IF_FALSE | MERGE + * | | + */ + + ir_use_list_remove_all(ctx, merge_ref, phi_ref); + ir_use_list_remove_all(ctx, ref, if_true_ref); + if (!IR_IS_CONST_REF(phi->op3)) { + ir_use_list_replace_one(ctx, phi->op3, phi_ref, insn->op2); + } + ir_use_list_replace_one(ctx, end1_ref, merge_ref, if_false_ref); + ir_use_list_replace_one(ctx, cond_ref, ref, end2_ref); + ir_use_list_add(ctx, end2_ref, if_true_ref); + + end2->optx = IR_OPTX(IR_IF, IR_VOID, 2); + end2->op2 = insn->op2; + + merge->optx = IR_OPTX(op, IR_VOID, 1); + merge->op1 = end2_ref; + merge->op2 = IR_UNUSED; + + cond->op1 = phi->op3; + MAKE_NOP(phi); + CLEAR_USES(phi_ref); + + insn->optx = IR_OPTX(IR_END, IR_VOID, 1); + insn->op1 = merge_ref; + insn->op2 = IR_UNUSED; + + if_true->op1 = end2_ref; + + if_false->optx = IR_OPTX(IR_MERGE, IR_VOID, 2); + if_false->op1 = end1_ref; + if_false->op2 = ref; + + ir_bitqueue_add(worklist, if_false_ref); + if (ctx->ir_base[end2->op1].op == IR_BEGIN || ctx->ir_base[end2->op1].op == IR_MERGE) { + ir_bitqueue_add(worklist, end2->op1); + } + + return 1; + } + } + } + } + } + + return 0; +} + +static void ir_optimize_merge(ir_ctx *ctx, ir_ref merge_ref, ir_insn *merge, ir_bitqueue *worklist) +{ + ir_use_list *use_list = &ctx->use_lists[merge_ref]; + + if (use_list->count == 1) { + ir_try_remove_empty_diamond(ctx, merge_ref, merge, worklist); + } else if (use_list->count == 2) { + if (merge->inputs_count == 2) { + ir_ref phi_ref = ctx->use_edges[use_list->refs]; + ir_insn *phi = &ctx->ir_base[phi_ref]; + + ir_ref next_ref = ctx->use_edges[use_list->refs + 1]; + ir_insn *next = &ctx->ir_base[next_ref]; + IR_ASSERT(next->op != IR_PHI); + + if (phi->op == IR_PHI) { + if (next->op == IR_IF && next->op1 == merge_ref && ctx->use_lists[phi_ref].count == 1) { + if (next->op2 == phi_ref) { + if (ir_try_split_if(ctx, next_ref, next, worklist)) { + return; + } + } else { + ir_insn *cmp = &ctx->ir_base[next->op2]; + + if (cmp->op >= IR_EQ && cmp->op <= IR_UGT + && cmp->op1 == phi_ref + && IR_IS_CONST_REF(cmp->op2) + && !IR_IS_SYM_CONST(ctx->ir_base[cmp->op2].op) + && ctx->use_lists[next->op2].count == 1) { + if (ir_try_split_if_cmp(ctx, next_ref, next, worklist)) { + return; + } + } + } + } + ir_optimize_phi(ctx, merge_ref, merge, phi_ref, phi, worklist); + } + } + } +} + int ir_sccp(ir_ctx *ctx) { ir_ref i, j, n, *p, use; @@ -1126,13 +2142,13 @@ int ir_sccp(ir_ctx *ctx) insn = &ctx->ir_base[i]; flags = ir_op_flags[insn->op]; if (flags & IR_OP_FLAG_DATA) { - if (insn->op == IR_PHI) { + if (ctx->use_lists[i].count == 0) { + /* dead code */ + continue; + } else if (insn->op == IR_PHI) { if (!ir_sccp_meet_phi(ctx, _values, i, insn, &worklist)) { continue; } - } else if (ctx->use_lists[i].count == 0) { - /* dead code */ - continue; } else if (EXPECTED(IR_IS_FOLDABLE_OP(insn->op))) { bool may_benefit = 0; bool has_top = 0; @@ -1182,6 +2198,9 @@ int ir_sccp(ir_ctx *ctx) IR_MAKE_BOTTOM(i); } } else if (flags & IR_OP_FLAG_BB_START) { + if (insn->op == IR_MERGE || insn->op == IR_BEGIN) { + ir_bitqueue_add(&worklist2, i); + } if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) { ir_ref unfeasible_inputs = 0; @@ -1430,10 +2449,15 @@ int ir_sccp(ir_ctx *ctx) ir_sccp_remove_unfeasible_merge_inputs(ctx, _values, i, _values[i].op1); } + ctx->flags2 |= IR_CFG_REACHABLE; + while ((i = ir_bitqueue_pop(&worklist2)) >= 0) { insn = &ctx->ir_base[i]; if (IR_IS_FOLDABLE_OP(insn->op)) { if (ctx->use_lists[i].count == 0) { + if (insn->op == IR_PHI) { + ir_bitqueue_add(&worklist2, insn->op1); + } ir_sccp_remove_insn2(ctx, i, &worklist2); } else { insn = &ctx->ir_base[i]; @@ -1488,6 +2512,17 @@ folding: break; } } + } else if (ir_op_flags[insn->op] & IR_OP_FLAG_BB_START) { + if (!(ctx->flags & IR_OPT_CFG)) { + /* pass */ + } else if (insn->op == IR_BEGIN) { + if (ctx->ir_base[insn->op1].op == IR_END + && ctx->use_lists[i].count == 1) { + ir_merge_blocks(ctx, insn->op1, i, &worklist2); + } + } else if (insn->op == IR_MERGE) { + ir_optimize_merge(ctx, i, insn, &worklist2); + } } else if (((ir_op_flags[insn->op] & (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_MASK)) == (IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD) || insn->op == IR_ALLOCA) && ctx->use_lists[i].count == 1) { diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 66ca2ecad04..11ec4677964 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -1529,9 +1529,7 @@ static bool ir_match_try_fuse_load(ir_ctx *ctx, ir_ref ref, ir_ref root); static void ir_swap_ops(ir_insn *insn) { - ir_ref tmp = insn->op1; - insn->op1 = insn->op2; - insn->op2 = tmp; + SWAP_REFS(insn->op1, insn->op2); } static bool ir_match_try_revert_lea_to_add(ir_ctx *ctx, ir_ref ref) @@ -5744,12 +5742,9 @@ static ir_op ir_emit_cmp_fp_common(ir_ctx *ctx, ir_ref root, ir_ref cmp_ref, ir_ op2_reg = ctx->regs[cmp_ref][2]; if (op1_reg == IR_REG_NONE && op2_reg != IR_REG_NONE && (op == IR_EQ || op == IR_NE)) { - ir_ref tmp; ir_reg tmp_reg; - tmp = op1; - op1 = op2; - op2 = tmp; + SWAP_REFS(op1, op2); tmp_reg = op1_reg; op1_reg = op2_reg; op2_reg = tmp_reg;