From f016caa312ae83b15979d59d1a1bbd86248bc437 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 3 Mar 2025 23:48:08 +0300 Subject: [PATCH] Merge IR IR commit: 1a02c4819f210a1f4548b83850ed7cd5c76c13aa --- ext/opcache/jit/ir/ir.c | 36 +++++++++++++---------------- ext/opcache/jit/ir/ir_cfg.c | 22 ++++++++++++++---- ext/opcache/jit/ir/ir_check.c | 12 +++++----- ext/opcache/jit/ir/ir_dump.c | 2 +- ext/opcache/jit/ir/ir_emit.c | 4 ++-- ext/opcache/jit/ir/ir_ra.c | 11 +++++---- ext/opcache/jit/ir/ir_save.c | 2 +- ext/opcache/jit/ir/ir_sccp.c | 14 ++++++++++-- ext/opcache/jit/ir/ir_x86.dasc | 41 ++++++++++++++++++++++++++++++++-- 9 files changed, 100 insertions(+), 44 deletions(-) diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index a6bb7c993c5..2721de4a6e0 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -1294,69 +1294,65 @@ void ir_build_def_use_lists(ir_ctx *ctx) void ir_use_list_remove_all(ir_ctx *ctx, ir_ref from, ir_ref ref) { - ir_ref j, n, *p, *q, use; + ir_ref n, *p, *q, use; ir_use_list *use_list; - ir_ref skip = 0; IR_ASSERT(from > 0); use_list = &ctx->use_lists[from]; n = use_list->count; - for (j = 0, p = q = &ctx->use_edges[use_list->refs]; j < n; j++, p++) { + for (p = q = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { use = *p; - if (use == ref) { - skip++; - } else { + if (use != ref) { if (p != q) { *q = use; } q++; } } - if (skip) { - use_list->count -= skip; + if (p != q) { + use_list->count -= (p - q); do { *q = IR_UNUSED; q++; - } while (--skip); + } while (q != p); } } void ir_use_list_remove_one(ir_ctx *ctx, ir_ref from, ir_ref ref) { - ir_ref j, n, *p; + ir_ref n, *p; ir_use_list *use_list; IR_ASSERT(from > 0); use_list = &ctx->use_lists[from]; n = use_list->count; - j = 0; p = &ctx->use_edges[use_list->refs]; - while (j < n) { + while (n > 0) { if (*p == ref) { use_list->count--; - j++; - while (j < n) { + n--; + while (n > 0) { *p = *(p+1); p++; - j++; + n--; } *p = IR_UNUSED; break; } p++; - j++; + n--; } } void ir_use_list_replace_one(ir_ctx *ctx, ir_ref ref, ir_ref use, ir_ref new_use) { ir_use_list *use_list; - ir_ref i, n, *p; + ir_ref n, *p; IR_ASSERT(ref > 0); use_list = &ctx->use_lists[ref]; n = use_list->count; - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { + for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { if (*p == use) { *p = new_use; break; @@ -1367,12 +1363,12 @@ void ir_use_list_replace_one(ir_ctx *ctx, ir_ref ref, ir_ref use, ir_ref new_use void ir_use_list_replace_all(ir_ctx *ctx, ir_ref ref, ir_ref use, ir_ref new_use) { ir_use_list *use_list; - ir_ref i, n, *p; + ir_ref n, *p; IR_ASSERT(ref > 0); use_list = &ctx->use_lists[ref]; n = use_list->count; - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { + for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { if (*p == use) { *p = new_use; } diff --git a/ext/opcache/jit/ir/ir_cfg.c b/ext/opcache/jit/ir/ir_cfg.c index 3678867e46c..16facae51b1 100644 --- a/ext/opcache/jit/ir/ir_cfg.c +++ b/ext/opcache/jit/ir/ir_cfg.c @@ -328,7 +328,7 @@ static void ir_remove_predecessor(ir_ctx *ctx, ir_block *bb, uint32_t from) static void ir_remove_merge_input(ir_ctx *ctx, ir_ref merge, ir_ref from) { - ir_ref i, j, n, k, *p, use; + ir_ref i, j, n, k, *p, *q, use; ir_insn *use_insn; ir_use_list *use_list; ir_bitset life_inputs; @@ -359,7 +359,7 @@ static void ir_remove_merge_input(ir_ctx *ctx, ir_ref merge, ir_ref from) use_list = &ctx->use_lists[merge]; if (use_list->count > 1) { n++; - for (k = 0, p = &ctx->use_edges[use_list->refs]; k < use_list->count; k++, p++) { + for (k = use_list->count, p = q = &ctx->use_edges[use_list->refs]; k > 0; p++, k--) { use = *p; use_insn = &ctx->ir_base[use]; if (use_insn->op == IR_PHI) { @@ -379,8 +379,22 @@ static void ir_remove_merge_input(ir_ctx *ctx, ir_ref merge, ir_ref from) for (j = 2; j <= n; j++) { ir_insn_set_op(use_insn, j, IR_UNUSED); } - ir_use_list_remove_all(ctx, merge, use); + continue; } + + /*compact use list */ + if (p != q){ + *q = use; + } + q++; + } + + if (p != q) { + use_list->count -= (p - q); + do { + *q = IR_UNUSED; /* clenu-op the removed tail */ + q++; + } while (p != q); } } } else { @@ -389,7 +403,7 @@ static void ir_remove_merge_input(ir_ctx *ctx, ir_ref merge, ir_ref from) use_list = &ctx->use_lists[merge]; if (use_list->count > 1) { n++; - for (k = 0, p = &ctx->use_edges[use_list->refs]; k < use_list->count; k++, p++) { + for (k = use_list->count, p = &ctx->use_edges[use_list->refs]; k > 0; p++, k--) { use = *p; use_insn = &ctx->ir_base[use]; if (use_insn->op == IR_PHI) { diff --git a/ext/opcache/jit/ir/ir_check.c b/ext/opcache/jit/ir/ir_check.c index 40a475ae22f..f12b4776fa1 100644 --- a/ext/opcache/jit/ir/ir_check.c +++ b/ext/opcache/jit/ir/ir_check.c @@ -42,11 +42,11 @@ void ir_consistency_check(void) static bool ir_check_use_list(const ir_ctx *ctx, ir_ref from, ir_ref to) { - ir_ref n, j, *p; + ir_ref n, *p; ir_use_list *use_list = &ctx->use_lists[from]; n = use_list->count; - for (j = 0, p = &ctx->use_edges[use_list->refs]; j < n; j++, p++) { + for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { if (*p == to) { return 1; } @@ -304,9 +304,9 @@ bool ir_check(const ir_ctx *ctx) if (ctx->use_lists) { ir_use_list *use_list = &ctx->use_lists[i]; - ir_ref count; + ir_ref count, n = use_list->count; - for (j = 0, p = &ctx->use_edges[use_list->refs]; j < use_list->count; j++, p++) { + for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { use = *p; if (!ir_check_input_list(ctx, i, use)) { fprintf(stderr, "ir_base[%d] is in use list of ir_base[%d]\n", use, i); @@ -347,8 +347,8 @@ bool ir_check(const ir_ctx *ctx) break; default: /* skip data references */ - count = use_list->count; - for (j = 0, p = &ctx->use_edges[use_list->refs]; j < use_list->count; j++, p++) { + count = n = use_list->count; + for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { use = *p; if (!(ir_op_flags[ctx->ir_base[use].op] & IR_OP_FLAG_CONTROL)) { count--; diff --git a/ext/opcache/jit/ir/ir_dump.c b/ext/opcache/jit/ir/ir_dump.c index f9b26c4d2a1..54fddf50ac0 100644 --- a/ext/opcache/jit/ir/ir_dump.c +++ b/ext/opcache/jit/ir/ir_dump.c @@ -177,7 +177,7 @@ static void ir_dump_dessa_moves(const ir_ctx *ctx, int b, ir_block *bb, FILE *f) use_list = &ctx->use_lists[succ_bb->start]; k = ir_phi_input_number(ctx, succ_bb, b); - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < use_list->count; i++, p++) { + for (i = use_list->count, p = &ctx->use_edges[use_list->refs]; i > 0; p++, i--) { use_ref = *p; use_insn = &ctx->ir_base[use_ref]; if (use_insn->op == IR_PHI) { diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index 367dee72963..5cf44a51d0f 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -161,7 +161,7 @@ static ir_reg ir_get_param_reg(const ir_ctx *ctx, ir_ref ref) } #endif - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < use_list->count; i++, p++) { + for (i = use_list->count, p = &ctx->use_edges[use_list->refs]; i > 0; p++, i--) { use = *p; insn = &ctx->ir_base[use]; if (insn->op == IR_PARAM) { @@ -917,7 +917,7 @@ static void ir_emit_dessa_moves(ir_ctx *ctx, int b, ir_block *bb) copies = alloca(use_list->count * sizeof(ir_dessa_copy)); - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < use_list->count; i++, p++) { + for (i = use_list->count, p = &ctx->use_edges[use_list->refs]; i > 0; p++, i--) { ir_ref ref = *p; ir_insn *insn = &ctx->ir_base[ref]; diff --git a/ext/opcache/jit/ir/ir_ra.c b/ext/opcache/jit/ir/ir_ra.c index dcc67e0074f..0c0e8dec3b4 100644 --- a/ext/opcache/jit/ir/ir_ra.c +++ b/ext/opcache/jit/ir/ir_ra.c @@ -1901,7 +1901,6 @@ int ir_coalesce(ir_ctx *ctx) qsort(list, count, sizeof(ir_coalesce_block), ir_block_cmp); while (count > 0) { - uint32_t i; count--; b = list[count].b; @@ -1913,7 +1912,7 @@ int ir_coalesce(ir_ctx *ctx) k = ir_phi_input_number(ctx, succ_bb, b); use_list = &ctx->use_lists[succ_bb->start]; n = use_list->count; - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { + for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { use = *p; insn = &ctx->ir_base[use]; if (insn->op == IR_PHI) { @@ -2061,7 +2060,7 @@ int ir_coalesce(ir_ctx *ctx) int ir_compute_dessa_moves(ir_ctx *ctx) { - uint32_t b, i, n; + uint32_t b, n; ir_ref j, k, *p, use; ir_block *bb; ir_use_list *use_list; @@ -2076,7 +2075,7 @@ int ir_compute_dessa_moves(ir_ctx *ctx) if (n > 1) { IR_ASSERT(k == ctx->ir_base[bb->start].inputs_count); k++; - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < n; i++, p++) { + for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { use = *p; insn = &ctx->ir_base[use]; if (insn->op == IR_PHI) { @@ -2136,7 +2135,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy) len = ir_bitset_len(ctx->vregs_count + 1); todo = ir_bitset_malloc(ctx->vregs_count + 1); - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < use_list->count; i++, p++) { + for (i = use_list->count, p = &ctx->use_edges[use_list->refs]; i > 0; p++, i--) { ref = *p; insn = &ctx->ir_base[ref]; if (insn->op == IR_PHI) { @@ -2205,7 +2204,7 @@ int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy) ir_mem_free(loc); if (have_constants_or_addresses) { - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < use_list->count; i++, p++) { + for (i = use_list->count, p = &ctx->use_edges[use_list->refs]; i > 0; p++, i--) { ref = *p; insn = &ctx->ir_base[ref]; if (insn->op == IR_PHI) { diff --git a/ext/opcache/jit/ir/ir_save.c b/ext/opcache/jit/ir/ir_save.c index be9b142c8f3..b12cc267af6 100644 --- a/ext/opcache/jit/ir/ir_save.c +++ b/ext/opcache/jit/ir/ir_save.c @@ -53,7 +53,7 @@ static void ir_save_dessa_moves(const ir_ctx *ctx, int b, ir_block *bb, FILE *f) use_list = &ctx->use_lists[succ_bb->start]; k = ir_phi_input_number(ctx, succ_bb, b); - for (i = 0, p = &ctx->use_edges[use_list->refs]; i < use_list->count; i++, p++) { + for (i = use_list->count, p = &ctx->use_edges[use_list->refs]; i > 0; p++, i--) { use_ref = *p; use_insn = &ctx->ir_base[use_ref]; if (use_insn->op == IR_PHI) { diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 18192d7f179..4f364849390 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -667,7 +667,7 @@ static IR_NEVER_INLINE void ir_sccp_analyze(ir_ctx *ctx, ir_insn *_values, ir_bi use_list = &ctx->use_lists[i]; n = use_list->count; - for (j = 0, p = &ctx->use_edges[use_list->refs]; j < n; j++, p++) { + for (p = &ctx->use_edges[use_list->refs]; n > 0; p++, n--) { use = *p; IR_ASSERT(use > 0); use_insn = &ctx->ir_base[use]; @@ -1048,7 +1048,7 @@ static void ir_sccp_remove_unfeasible_merge_inputs(ir_ctx *ctx, ir_insn *_values n++; use_list = &ctx->use_lists[ref]; if (use_list->count > 1) { - for (k = 0, p = &ctx->use_edges[use_list->refs]; k < use_list->count; k++, p++) { + for (k = use_list->count, p = &ctx->use_edges[use_list->refs]; k > 0; p++, k--) { use = *p; use_insn = &ctx->ir_base[use]; if (use_insn->op == IR_PHI) { @@ -1715,6 +1715,7 @@ static bool ir_may_promote_i2i(ir_ctx *ctx, ir_type type, ir_ref ref) case IR_OR: case IR_AND: case IR_XOR: + case IR_SHL: return ctx->use_lists[ref].count == 1 && ir_may_promote_i2i(ctx, type, insn->op1) && ir_may_promote_i2i(ctx, type, insn->op2); @@ -1773,6 +1774,7 @@ static ir_ref ir_promote_i2i(ir_ctx *ctx, ir_type type, ir_ref ref, ir_ref use) case IR_OR: case IR_AND: case IR_XOR: + case IR_SHL: if (insn->op1 == insn->op2) { insn->op2 = insn->op1 = ir_promote_i2i(ctx, type, insn->op1, ref); } else { @@ -2952,6 +2954,14 @@ static ir_ref ir_iter_optimize_condition(ir_ctx *ctx, ir_ref control, ir_ref con { ir_insn *condition_insn = &ctx->ir_base[condition]; + while ((condition_insn->op == IR_BITCAST + || condition_insn->op == IR_ZEXT + || condition_insn->op == IR_SEXT) + && ctx->use_lists[condition].count == 1) { + condition = condition_insn->op1; + condition_insn = &ctx->ir_base[condition]; + } + if (condition_insn->opt == IR_OPT(IR_NOT, IR_BOOL)) { *swap = 1; condition = condition_insn->op1; diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index c4f0eae01c0..3b6cf156ad9 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -1010,6 +1010,7 @@ const char *ir_reg_name(int8_t reg, ir_type type) _(MOD_PWR2) \ _(SDIV_PWR2) \ _(SMOD_PWR2) \ + _(BOOL_NOT) \ _(BOOL_NOT_INT) \ _(ABS_INT) \ _(OP_INT) \ @@ -2223,10 +2224,16 @@ binop_fp: ir_match_fuse_load(ctx, insn->op2, ref); return IR_MOD_INT; case IR_BSWAP: + IR_ASSERT(IR_IS_TYPE_INT(insn->type)); + return IR_OP_INT; case IR_NOT: if (insn->type == IR_BOOL) { - IR_ASSERT(IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)); // TODO: IR_BOOL_NOT_FP - return IR_BOOL_NOT_INT; + if (ctx->ir_base[insn->op1].type == IR_BOOL) { + return IR_BOOL_NOT; + } else { + IR_ASSERT(IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)); // TODO: IR_BOOL_NOT_FP + return IR_BOOL_NOT_INT; + } } else { IR_ASSERT(IR_IS_TYPE_INT(insn->type)); return IR_OP_INT; @@ -5054,6 +5061,33 @@ static void ir_emit_abs_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } +static void ir_emit_bool_not(ir_ctx *ctx, ir_ref def, ir_insn *insn) +{ + ir_backend_data *data = ctx->data; + dasm_State **Dst = &data->dasm_state; + ir_type type = ctx->ir_base[insn->op1].type; + ir_ref op1 = insn->op1; + ir_reg def_reg = IR_REG_NUM(ctx->regs[def][0]); + ir_reg op1_reg = ctx->regs[def][1]; + + IR_ASSERT(def_reg != IR_REG_NONE); + + if (op1_reg != IR_REG_NONE && IR_REG_SPILLED(op1_reg)) { + op1_reg = IR_REG_NUM(op1_reg); + ir_emit_load(ctx, type, op1_reg, op1); + } + + if (def_reg != op1_reg) { + | mov Rb(def_reg), Rb(op1_reg) + } + + | xor Rb(def_reg), 1 + + if (IR_REG_SPILLED(ctx->regs[def][0])) { + ir_emit_store(ctx, type, def, def_reg); + } +} + static void ir_emit_bool_not_int(ir_ctx *ctx, ir_ref def, ir_insn *insn) { ir_backend_data *data = ctx->data; @@ -10602,6 +10636,9 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_ABS_INT: ir_emit_abs_int(ctx, i, insn); break; + case IR_BOOL_NOT: + ir_emit_bool_not(ctx, i, insn); + break; case IR_BOOL_NOT_INT: ir_emit_bool_not_int(ctx, i, insn); break;