From 9267ebccfc7049d6d9326b4923f2eb393fb95cf2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 21 Dec 2023 13:45:23 +0300 Subject: [PATCH] Update IR IR commit: 415f673be5116d121b934c0bdaf2a83f4d3a95fb --- ext/opcache/jit/ir/ir.h | 8 +- ext/opcache/jit/ir/ir_aarch64.dasc | 160 ++++++++++++++++++++--------- ext/opcache/jit/ir/ir_emit.c | 12 +-- ext/opcache/jit/ir/ir_sccp.c | 55 +++++++++- ext/opcache/jit/ir/ir_x86.dasc | 111 ++++++++++++++++---- 5 files changed, 263 insertions(+), 83 deletions(-) diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 666564c95fd..537daeb767a 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -757,6 +757,10 @@ int32_t ir_get_spill_slot_offset(ir_ctx *ctx, ir_ref ref); int ir_match(ir_ctx *ctx); void *ir_emit_code(ir_ctx *ctx, size_t *size); +bool ir_needs_thunk(ir_code_buffer *code_buffer, void *addr); +void *ir_emit_thunk(ir_code_buffer *code_buffer, void *addr, size_t *size_ptr); +void ir_fix_thunk(void *thunk_entry, void *addr); + /* Target address resolution (implementation in ir_emit.c) */ void *ir_resolve_sym_name(const char *name); @@ -799,11 +803,11 @@ struct _ir_loader { bool (*sym_dcl) (ir_loader *loader, const char *name, uint32_t flags, size_t size, bool has_data); bool (*sym_data) (ir_loader *loader, ir_type type, uint32_t count, const void *data); bool (*sym_data_pad) (ir_loader *loader, size_t offset); - bool (*sym_data_ref) (ir_loader *loader, ir_op op, const char *ref); + bool (*sym_data_ref) (ir_loader *loader, ir_op op, const char *ref, uintptr_t offset); bool (*sym_data_end) (ir_loader *loader); bool (*func_init) (ir_loader *loader, ir_ctx *ctx, const char *name); bool (*func_process) (ir_loader *loader, ir_ctx *ctx, const char *name); - void*(*resolve_sym_name) (ir_loader *loader, const char *name); + void*(*resolve_sym_name) (ir_loader *loader, const char *name, bool add_thunk); bool (*has_sym) (ir_loader *loader, const char *name); bool (*add_sym) (ir_loader *loader, const char *name, void *addr); }; diff --git a/ext/opcache/jit/ir/ir_aarch64.dasc b/ext/opcache/jit/ir/ir_aarch64.dasc index 6986a0358cd..5e4ce906d7a 100644 --- a/ext/opcache/jit/ir/ir_aarch64.dasc +++ b/ext/opcache/jit/ir/ir_aarch64.dasc @@ -20,44 +20,44 @@ #define ADR_IMM (1<<20) // signed imm21 #define ADRP_IMM (1LL<<32) // signed imm21 * 4096 -static bool aarch64_may_use_b(ir_ctx *ctx, const void *addr) +static bool aarch64_may_use_b(ir_code_buffer *code_buffer, const void *addr) { - if (ctx->code_buffer) { - if (addr >= ctx->code_buffer->start && (char*)addr < (char*)ctx->code_buffer->end) { - return (((char*)ctx->code_buffer->end - (char*)ctx->code_buffer->start) < B_IMM); - } else if ((char*)addr >= (char*)ctx->code_buffer->end) { - return (((char*)addr - (char*)ctx->code_buffer->start) < B_IMM); - } else if (addr < ctx->code_buffer->start) { - return (((char*)ctx->code_buffer->end - (char*)addr) < B_IMM); - } - } - return 1; //??? -} - -#if 0 -static bool aarch64_may_use_adr(ir_ctx *ctx, const void *addr) -{ - if (ctx->code_buffer) { - if (addr >= ctx->code_buffer->start && (char*)addr < (char*)ctx->code_buffer->end) { - return (((char*)ctx->code_buffer->end - (char*)ctx->code_buffer->start) < ADR_IMM); - } else if ((char*)addr >= (char*)ctx->code_buffer->end) { - return (((char*)addr - (char*)ctx->code_buffer->start) < ADR_IMM); - } else if (addr < ctx->code_buffer->start) { - return (((char*)ctx->code_buffer->end - (char*)addr) < ADR_IMM); + if (code_buffer) { + if (addr >= code_buffer->start && (char*)addr < (char*)code_buffer->end) { + return (((char*)code_buffer->end - (char*)code_buffer->start) < B_IMM); + } else if ((char*)addr >= (char*)code_buffer->end) { + return (((char*)addr - (char*)code_buffer->start) < B_IMM); + } else if (addr < code_buffer->start) { + return (((char*)code_buffer->end - (char*)addr) < B_IMM); } } return 0; } -static bool aarch64_may_use_adrp(ir_ctx *ctx, const void *addr) +#if 0 +static bool aarch64_may_use_adr(ir_code_buffer *code_buffer, const void *addr) { - if (ctx->code_buffer) { - if (addr >= ctx->code_buffer->start && (char*)addr < (char*)ctx->code_buffer->end) { - return (((char*)ctx->code_buffer->end - (char*)ctx->code_buffer->start) < ADRP_IMM); - } else if ((char*)addr >= (char*)ctx->code_buffer->end) { - return (((char*)addr - (char*)ctx->code_buffer->start) < ADRP_IMM); - } else if (addr < ctx->code_buffer->start) { - return (((char*)ctx->code_buffer->end - (char*)addr) < ADRP_IMM); + if (code_buffer) { + if (addr >= code_buffer->start && (char*)addr < (char*)code_buffer->end) { + return (((char*)code_buffer->end - (char*)code_buffer->start) < ADR_IMM); + } else if ((char*)addr >= (char*)code_buffer->end) { + return (((char*)addr - (char*)code_buffer->start) < ADR_IMM); + } else if (addr < code_buffer->start) { + return (((char*)code_buffer->end - (char*)addr) < ADR_IMM); + } + } + return 0; +} + +static bool aarch64_may_use_adrp(ir_code_buffer *code_buffer, const void *addr) +{ + if (code_buffer) { + if (addr >= code_buffer->start && (char*)addr < (char*)code_buffer->end) { + return (((char*)code_buffer->end - (char*)code_buffer->start) < ADRP_IMM); + } else if ((char*)addr >= (char*)code_buffer->end) { + return (((char*)addr - (char*)code_buffer->start) < ADRP_IMM); + } else if (addr < code_buffer->start) { + return (((char*)code_buffer->end - (char*)addr) < ADRP_IMM); } } return 0; @@ -1176,9 +1176,10 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) ir_insn *insn = &ctx->ir_base[src]; if (insn->op == IR_SYM || insn->op == IR_FUNC) { + const char *name = ir_get_str(ctx, insn->val.name); void *addr = (ctx->loader && ctx->loader->resolve_sym_name) ? - ctx->loader->resolve_sym_name(ctx->loader, ir_get_str(ctx, insn->val.name)) : - ir_resolve_sym_name(ir_get_str(ctx, insn->val.name)); + ctx->loader->resolve_sym_name(ctx->loader, name, insn->op == IR_FUNC) : + ir_resolve_sym_name(name); IR_ASSERT(addr); ir_emit_load_imm_int(ctx, type, reg, (intptr_t)addr); } else if (insn->op == IR_STR) { @@ -4360,7 +4361,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, int32_t used if (IR_IS_CONST_REF(insn->op2)) { void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]); - if (aarch64_may_use_b(ctx, addr)) { + if (aarch64_may_use_b(ctx->code_buffer, addr)) { | bl &addr } else { ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, (intptr_t)addr); @@ -4435,7 +4436,7 @@ static void ir_emit_tailcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (IR_IS_CONST_REF(insn->op2)) { void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]); - if (aarch64_may_use_b(ctx, addr)) { + if (aarch64_may_use_b(ctx->code_buffer, addr)) { | b &addr } else { ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, (intptr_t)addr); @@ -4468,7 +4469,7 @@ static void ir_emit_ijmp(ir_ctx *ctx, ir_ref def, ir_insn *insn) } else if (IR_IS_CONST_REF(insn->op2)) { void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op2]); - if (aarch64_may_use_b(ctx, addr)) { + if (aarch64_may_use_b(ctx->code_buffer, addr)) { | b &addr } else { ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, (intptr_t)addr); @@ -4494,7 +4495,7 @@ static void ir_emit_guard(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (IR_IS_CONST_REF(insn->op3)) { void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]); - if (aarch64_may_use_b(ctx, addr)) { + if (aarch64_may_use_b(ctx->code_buffer, addr)) { | b &addr } else { ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, (intptr_t)addr); @@ -4654,7 +4655,7 @@ static void ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * if (IR_IS_CONST_REF(op2) && ctx->ir_base[op2].val.u64 == 0) { if (op == IR_ULT) { /* always false */ - if (aarch64_may_use_b(ctx, addr)) { + if (aarch64_may_use_b(ctx->code_buffer, addr)) { | b &addr } else { ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, (intptr_t)addr); @@ -4807,7 +4808,7 @@ static void ir_emit_exitcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (IR_IS_CONST_REF(insn->op2)) { void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]); - if (aarch64_may_use_b(ctx, addr)) { + if (aarch64_may_use_b(ctx->code_buffer, addr)) { | bl &addr } else { ir_emit_load_imm_int(ctx, IR_ADDR, IR_REG_INT_TMP, (intptr_t)addr); @@ -5835,17 +5836,8 @@ const void *ir_emit_exitgroup(uint32_t first_exit_point, uint32_t exit_points_pe dasm_State **Dst, *dasm_state; int ret; - /* IR_ASSERT(aarch64_may_use_b(ctx, exit_addr)) */ IR_ASSERT(code_buffer); - if ((char*)exit_addr >= (char*)code_buffer->start && (char*)exit_addr < (char*)code_buffer->end) { - IR_ASSERT(((char*)code_buffer->end - (char*)code_buffer->end) < B_IMM); - } else if ((char*)exit_addr >= (char*)code_buffer->end) { - IR_ASSERT(((char*)exit_addr - (char*)code_buffer->start) < B_IMM); - } else if ((char*)exit_addr < (char*)code_buffer->start) { - IR_ASSERT(((char*)code_buffer->end - (char*)exit_addr) < B_IMM); - } else { - IR_ASSERT(0); - } + IR_ASSERT(aarch64_may_use_b(code_buffer, exit_addr)); Dst = &dasm_state; dasm_state = NULL; @@ -6010,3 +6002,73 @@ static int ir_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, ui return n; } + +bool ir_needs_thunk(ir_code_buffer *code_buffer, void *addr) +{ + return !aarch64_may_use_b(code_buffer, addr); +} + +void *ir_emit_thunk(ir_code_buffer *code_buffer, void *addr, size_t *size_ptr) +{ + void *entry; + size_t size; + dasm_State **Dst, *dasm_state; + int ret; + + Dst = &dasm_state; + dasm_state = NULL; + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, ir_lb_MAX); + dasm_setup(&dasm_state, dasm_actions); + + |.code + | movz Rx(IR_REG_INT_TMP), #((uint64_t)(addr) & 0xffff) + | movk Rx(IR_REG_INT_TMP), #(((uint64_t)(addr) >> 16) & 0xffff), lsl #16 + | movk Rx(IR_REG_INT_TMP), #(((uint64_t)(addr) >> 32) & 0xffff), lsl #32 + | movk Rx(IR_REG_INT_TMP), #(((uint64_t)(addr) >> 48) & 0xffff), lsl #48 + | br Rx(IR_REG_INT_TMP) + + ret = dasm_link(&dasm_state, &size); + if (ret != DASM_S_OK) { + IR_ASSERT(0); + dasm_free(&dasm_state); + return NULL; + } + + if (size > (size_t)((char*)code_buffer->end - (char*)code_buffer->pos)) { + dasm_free(&dasm_state); + return NULL; + } + + entry = code_buffer->pos; + ret = dasm_encode(&dasm_state, entry); + if (ret != DASM_S_OK) { + dasm_free(&dasm_state); + return NULL; + } + + *size_ptr = size; + code_buffer->pos = (char*)code_buffer->pos + size; + + dasm_free(&dasm_state); + ir_mem_flush(entry, size); + + return entry; +} + +void ir_fix_thunk(void *thunk_entry, void *addr) +{ + uint32_t *code = thunk_entry; + IR_ASSERT((code[0] & 0xffe00000) == 0xd2800000 + && (code[1] & 0xffe00000) == 0xf2a00000 + && (code[2] & 0xffe00000) == 0xf2c00000 + && (code[3] & 0xffe00000) == 0xf2e00000 + && (code[4] & 0xfffffc1f) == 0xd61f0000); + + code[0] = (code[0] & 0xffe0001f) | (uint32_t)((uint64_t)(addr) & 0xffff) << 5; + code[1] = (code[1] & 0xffe0001f) | (uint32_t)(((uint64_t)(addr) >> 16) & 0xffff) << 5; + code[2] = (code[2] & 0xffe0001f) | (uint32_t)(((uint64_t)(addr) >> 32) & 0xffff) << 5; + code[3] = (code[3] & 0xffe0001f) | (uint32_t)(((uint64_t)(addr) >> 48) & 0xffff) << 5; + + ir_mem_flush(code, sizeof(uint32_t) * 4); +} diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index 3776055c567..caf2f547876 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -302,12 +302,6 @@ void *ir_resolve_sym_name(const char *name) DWORD cbNeeded; uint32_t i = 0; - /* Quick workaraund to prevent *.irt tests failures */ - // TODO: try to find a general solution ??? - if (strcmp(name, "printf") == 0) { - return (void*)printf; - } - addr = NULL; EnumProcessModules(GetCurrentProcess(), mods, sizeof(mods), &cbNeeded); @@ -320,7 +314,6 @@ void *ir_resolve_sym_name(const char *name) i++; } #endif - IR_ASSERT(addr != NULL); return addr; } @@ -334,9 +327,10 @@ static void *ir_call_addr(ir_ctx *ctx, ir_insn *insn, ir_insn *addr_insn) IR_ASSERT(addr_insn->type == IR_ADDR); if (addr_insn->op == IR_FUNC) { + const char* name = ir_get_str(ctx, addr_insn->val.name); addr = (ctx->loader && ctx->loader->resolve_sym_name) ? - ctx->loader->resolve_sym_name(ctx->loader, ir_get_str(ctx, addr_insn->val.name)) : - ir_resolve_sym_name(ir_get_str(ctx, addr_insn->val.name)); + ctx->loader->resolve_sym_name(ctx->loader, name, 1) : + ir_resolve_sym_name(name); IR_ASSERT(addr); } else { IR_ASSERT(addr_insn->op == IR_ADDR || addr_insn->op == IR_FUNC_ADDR); diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 808b0152e5f..5ddd3722a04 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -176,6 +176,35 @@ static void ir_sccp_remove_from_use_list(ir_ctx *ctx, ir_ref from, ir_ref ref) #endif } +static void ir_sccp_remove_from_use_list_1(ir_ctx *ctx, ir_ref from, ir_ref ref) +{ + ir_ref j, n, *p; + ir_use_list *use_list = &ctx->use_lists[from]; + + n = use_list->count; + j = 0; + p = &ctx->use_edges[use_list->refs]; + while (j < n) { + if (*p == ref) { + break; + } + j++; + } + + if (j < n) { + use_list->count--; + j++; + while (j < n) { + *p = *(p+1); + p++; + j++; + } +#if IR_COMBO_COPY_PROPAGATION + *p = IR_UNUSED; +#endif + } +} + #if IR_COMBO_COPY_PROPAGATION static int ir_sccp_add_to_use_list(ir_ctx *ctx, ir_ref to, ir_ref ref) { @@ -521,7 +550,7 @@ static void ir_sccp_remove_unfeasible_merge_inputs(ir_ctx *ctx, ir_insn *_values } i++; } else if (!IR_IS_CONST_REF(input)) { - ir_sccp_remove_from_use_list(ctx, input, use); + ir_sccp_remove_from_use_list_1(ctx, input, use); } } while (i <= n) { @@ -577,7 +606,11 @@ int ir_sccp(ir_ctx *ctx) ir_ref input = ir_insn_op(insn, j + 1); if (input > 0 && IR_IS_TOP(input)) { - ir_bitqueue_add(&worklist, input); + /* do backward propagaton only once */ + if (!_values[input].op1) { + _values[input].op1 = 1; + ir_bitqueue_add(&worklist, input); + } } else if (ir_sccp_join_values(ctx, _values, i, input)) { changed = 1; } @@ -600,7 +633,11 @@ int ir_sccp(ir_ctx *ctx) if (input > 0) { if (_values[input].optx == IR_TOP) { has_top = 1; - ir_bitqueue_add(&worklist, input); + /* do backward propagaton only once */ + if (!_values[input].op1) { + _values[input].op1 = 1; + ir_bitqueue_add(&worklist, input); + } } else if (_values[input].optx != IR_BOTTOM) { /* Perform folding only if some of direct inputs * is going to be replaced by a constant or copy. @@ -660,7 +697,11 @@ int ir_sccp(ir_ctx *ctx) } if (insn->op == IR_IF) { if (IR_IS_TOP(insn->op2)) { - ir_bitqueue_add(&worklist, insn->op2); + /* do backward propagaton only once */ + if (!_values[insn->op2].op1) { + _values[insn->op2].op1 = 1; + ir_bitqueue_add(&worklist, insn->op2); + } continue; } if (!IR_IS_BOTTOM(insn->op2) @@ -693,7 +734,11 @@ int ir_sccp(ir_ctx *ctx) IR_MAKE_BOTTOM(i); } else if (insn->op == IR_SWITCH) { if (IR_IS_TOP(insn->op2)) { - ir_bitqueue_add(&worklist, insn->op2); + /* do backward propagaton only once */ + if (!_values[insn->op2].op1) { + _values[insn->op2].op1 = 1; + ir_bitqueue_add(&worklist, insn->op2); + } continue; } if (!IR_IS_BOTTOM(insn->op2)) { diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index 69fbe67a85f..8e3dd55628c 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -19,10 +19,10 @@ #define IR_IS_UNSIGNED_32BIT(val) (((uintptr_t)(val)) <= 0xffffffff) #define IR_IS_32BIT(type, val) (IR_IS_TYPE_SIGNED(type) ? IR_IS_SIGNED_32BIT((val).i64) : IR_IS_UNSIGNED_32BIT((val).u64)) #define IR_IS_FP_ZERO(insn) ((insn.type == IR_DOUBLE) ? (insn.val.u64 == 0) : (insn.val.u32 == 0)) -#define IR_MAY_USE_32BIT_ADDR(addr) \ - (ctx->code_buffer && \ - IR_IS_SIGNED_32BIT((char*)(addr) - (char*)ctx->code_buffer->start) && \ - IR_IS_SIGNED_32BIT((char*)(addr) - ((char*)ctx->code_buffer->end))) +#define IR_MAY_USE_32BIT_ADDR(code_buffer, addr) \ + ((code_buffer) && \ + IR_IS_SIGNED_32BIT((char*)(addr) - (char*)(code_buffer)->start) && \ + IR_IS_SIGNED_32BIT((char*)(addr) - ((char*)(code_buffer)->end))) #define IR_SPILL_POS_TO_OFFSET(offset) \ ((ctx->flags & IR_USE_FRAME_POINTER) ? \ @@ -2162,9 +2162,10 @@ static void ir_emit_load(ir_ctx *ctx, ir_type type, ir_reg reg, ir_ref src) ir_insn *insn = &ctx->ir_base[src]; if (insn->op == IR_SYM || insn->op == IR_FUNC) { + const char *name = ir_get_str(ctx, insn->val.name); void *addr = (ctx->loader && ctx->loader->resolve_sym_name) ? - ctx->loader->resolve_sym_name(ctx->loader, ir_get_str(ctx, insn->val.name)) : - ir_resolve_sym_name(ir_get_str(ctx, insn->val.name)); + ctx->loader->resolve_sym_name(ctx->loader, name, insn->op == IR_FUNC) : + ir_resolve_sym_name(name); IR_ASSERT(addr); ir_emit_load_imm_int(ctx, type, reg, (intptr_t)addr); } else if (insn->op == IR_STR) { @@ -2250,9 +2251,10 @@ static void ir_emit_store_mem_int_const(ir_ctx *ctx, ir_type type, ir_reg base_r int64_t val = val_insn->val.i64; if (val_insn->op == IR_FUNC || val_insn->op == IR_SYM) { + const char *name = ir_get_str(ctx, val_insn->val.name); void *addr = (ctx->loader && ctx->loader->resolve_sym_name) ? - ctx->loader->resolve_sym_name(ctx->loader, ir_get_str(ctx, val_insn->val.name)) : - ir_resolve_sym_name(ir_get_str(ctx, val_insn->val.name)); + ctx->loader->resolve_sym_name(ctx->loader, name, val_insn->op == IR_FUNC) : + ir_resolve_sym_name(name); IR_ASSERT(addr); val = (int64_t)(intptr_t)addr; } @@ -7410,7 +7412,7 @@ static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, int32_t used if (IR_IS_CONST_REF(insn->op2)) { void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]); - if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(addr)) { + if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, addr)) { | call aword &addr } else { |.if X64 @@ -7554,7 +7556,7 @@ static void ir_emit_tailcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (IR_IS_CONST_REF(insn->op2)) { void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]); - if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(addr)) { + if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, addr)) { | jmp aword &addr } else { |.if X64 @@ -7609,7 +7611,7 @@ static void ir_emit_ijmp(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (IR_IS_CONST_REF(insn->op2)) { void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op2]); - if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(addr)) { + if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, addr)) { | jmp aword &addr } else { |.if X64 @@ -7737,7 +7739,7 @@ static bool ir_emit_guard_jcc(ir_ctx *ctx, uint32_t b, ir_ref def, uint8_t op, v } else if (next_insn->op == IR_IJMP && IR_IS_CONST_REF(next_insn->op2)) { void *target_addr = ir_jmp_addr(ctx, next_insn, &ctx->ir_base[next_insn->op2]); - if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(target_addr)) { + if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, target_addr)) { if (int_cmp) { switch (op) { default: @@ -7893,7 +7895,7 @@ static bool ir_emit_guard(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) if ((insn->op == IR_GUARD && !is_true) || (insn->op == IR_GUARD_NOT && is_true)) { addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]); - if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(addr)) { + if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, addr)) { | jmp aword &addr } else { |.if X64 @@ -7931,7 +7933,7 @@ static bool ir_emit_guard(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn) } addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]); - if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(addr)) { + if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, addr)) { ir_op op; if (insn->op == IR_GUARD) { @@ -7989,7 +7991,7 @@ static bool ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn * if (IR_IS_CONST_REF(op2) && ctx->ir_base[op2].val.u64 == 0) { if (op == IR_ULT) { /* always false */ - if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(addr)) { + if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, addr)) { | jmp aword &addr } else { |.if X64 @@ -8371,7 +8373,7 @@ static void ir_emit_exitcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) if (IR_IS_CONST_REF(insn->op2)) { void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]); - if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(addr)) { + if (sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(ctx->code_buffer, addr)) { | call aword &addr } else { |.if X64 @@ -9802,8 +9804,7 @@ const void *ir_emit_exitgroup(uint32_t first_exit_point, uint32_t exit_points_pe int ret; IR_ASSERT(code_buffer); - IR_ASSERT(IR_IS_SIGNED_32BIT((char*)exit_addr - (char*)code_buffer->start)); - IR_ASSERT(IR_IS_SIGNED_32BIT((char*)exit_addr - (char*)code_buffer->end)); + IR_ASSERT(sizeof(void*) == 4 || IR_MAY_USE_32BIT_ADDR(code_buffer, exit_addr)); Dst = &dasm_state; dasm_state = NULL; @@ -9852,3 +9853,77 @@ const void *ir_emit_exitgroup(uint32_t first_exit_point, uint32_t exit_points_pe *size_ptr = size; return entry; } + +bool ir_needs_thunk(ir_code_buffer *code_buffer, void *addr) +{ + return sizeof(void*) == 8 && !IR_MAY_USE_32BIT_ADDR(code_buffer, addr); +} + +void *ir_emit_thunk(ir_code_buffer *code_buffer, void *addr, size_t *size_ptr) +{ + void *entry; + size_t size; + dasm_State **Dst, *dasm_state; + int ret; + + Dst = &dasm_state; + dasm_state = NULL; + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, ir_lb_MAX); + dasm_setup(&dasm_state, dasm_actions); + + |.code + |.if X64 + | jmp aword [>1] + |1: + | .aword &addr + |.else + | jmp &addr + |.endif + + ret = dasm_link(&dasm_state, &size); + if (ret != DASM_S_OK) { + IR_ASSERT(0); + dasm_free(&dasm_state); + return NULL; + } + + if (size > (size_t)((char*)code_buffer->end - (char*)code_buffer->pos)) { + dasm_free(&dasm_state); + return NULL; + } + + entry = code_buffer->pos; + ret = dasm_encode(&dasm_state, entry); + if (ret != DASM_S_OK) { + dasm_free(&dasm_state); + return NULL; + } + + *size_ptr = size; + code_buffer->pos = (char*)code_buffer->pos + size; + + dasm_free(&dasm_state); + ir_mem_flush(entry, size); + + return entry; +} + +void ir_fix_thunk(void *thunk_entry, void *addr) +{ + unsigned char *code = thunk_entry; + void **addr_ptr; + + if (sizeof(void*) == 8) { + int32_t *offset_ptr; + + IR_ASSERT(code[0] == 0xff && code[1] == 0x25); + offset_ptr = (int32_t*)(code + 2); + addr_ptr = (void**)(code + 6 + *offset_ptr); + *addr_ptr = addr; + } else { + IR_ASSERT(code[0] == 0xe9); + addr_ptr = (void**)(code + 1); + *addr_ptr = (void*)((unsigned char*)addr - (code + 5)); + } +}