From 711f80e8dcac83d08c9a83ce7b8c3a1e8589779c Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 15 Mar 2026 11:19:05 +0100 Subject: [PATCH 01/83] finished first ir emit pass --- src/emit.c | 1216 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1216 insertions(+) create mode 100644 src/emit.c diff --git a/src/emit.c b/src/emit.c new file mode 100644 index 000000000..faa1a1d3d --- /dev/null +++ b/src/emit.c @@ -0,0 +1,1216 @@ +/* + * Copyright (C)2015-2016 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include + +typedef enum { + LOAD, + STORE, + LEA, + TEST, + CMP, + JCOND, + JUMP, + JUMP_TABLE, + BINOP, + UNOP, + CONV, + RET, + ALLOC_STACK, + FREE_STACK, + ALLOC_GLOBAL_STACK, + NATIVE_REG, + PREFETCH, +} emit_op; + +typedef enum { + REG_RBP, +} native_reg; + +typedef enum { + I8, + I16, + I32, + I64, + F32, + F64 +} emit_mode; + +typedef struct { + int id; + hl_type *t; +} vreg; + +typedef struct { + int index; +} ereg; + +typedef struct { + emit_op op; + emit_mode mode; + short offset_param; + union { + struct { + ereg a; + ereg b; + } args; + uint64 value; + }; +} einstr; + +#define MAX_TMP_ARGS 32 +#define MAX_TRAPS 32 + +typedef struct { + hl_module *mod; + hl_function *fun; + einstr *instrs; + vreg *vregs; + int max_instrs; + int max_regs; + int emit_pos; + int op_pos; + + ereg tmp_args[MAX_TMP_ARGS]; + ereg traps[MAX_TRAPS]; + int *pos_map; + int pos_map_size; + int trap_count; + + int *jump_regs; + int jump_count; + int max_jumps; + + void *closure_list; // TODO : patch with good addresses +} emit_ctx; + +#define R(i) (ctx->vregs + (i)) + +#define emit_error(msg) _emit_error(msg,__LINE__) +#define emit_assert() emit_error("") + +#define LOAD(r) emit_load_reg(ctx, r) +#define STORE(r, v) emit_store_reg(ctx, r, v) +#define LOAD_CONST(v, t) emit_load_const(ctx, (uint64)v, t) +#define LOAD_CONST_PTR(v) LOAD_CONST(v,&hlt_bytes) +#define LOAD_MEM(v, offs, t) emit_load_mem(ctx, v, offs, t) +#define LOAD_MEM_PTR(v, offs) LOAD_MEM(v, offs, &hlt_bytes) +#define STORE_MEM(to, offs, v) emit_gen(ctx, STORE, to, v, offs) +#define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR((obj),0),HL_WSIZE*2),HL_WSIZE*(id)) +#define OFFSET(base,index,mult,offset) emit_gen(ctx, LEA, base, index, (mult) | ((offset) << 8)) + +#define CUR_REG() __current_reg(ctx) + +#define HDYN_VALUE 8 + +#define IS_FLOAT(t) (t->kind == HF64 || t->kind == HF32) + +static hl_type hlt_ui8 = { HUI8, 0 }; +static hl_type hlt_ui16 = { HUI16, 0 }; +static ereg ENULL = {-1}; + +static void _emit_error( const char *msg, int line ) { + printf("*** EMIT ERROR line %d (%s) ****\n", line, msg); + hl_debug_break(); + exit(-1); +} + +static void hl_stub_null_field_access() { emit_assert(); } +static void hl_stub_null_access() { emit_assert(); } +static void hl_stub_assert() { emit_assert(); } + +static ereg __current_reg( emit_ctx *ctx ) { + ereg r = {ctx->emit_pos-1}; + return r; +} + +static einstr *emit_instr( emit_ctx *ctx ) { + if( ctx->emit_pos == ctx->max_instrs ) { + int next_size = ctx->max_instrs ? (ctx->max_instrs * 3) >> 1 : 256; + einstr *instrs = (einstr*)malloc(sizeof(einstr) * next_size); + if( instrs == NULL ) emit_error("Out of memory"); + memcpy(instrs, ctx->instrs, ctx->emit_pos * sizeof(einstr)); + free(ctx->instrs); + ctx->instrs = instrs; + ctx->max_instrs = next_size; + } + return ctx->instrs + ctx->emit_pos++; +} + +static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { + einstr *e = emit_instr(ctx); + if( (short)mode != mode ) emit_assert(); + e->op = op; + e->mode = mode; + e->args.a = a; + e->args.b = b; + return CUR_REG(); +} + +static int emit_jump( emit_ctx *ctx, bool cond ) { + int p = ctx->emit_pos; + emit_gen(ctx, cond ? JCOND : JUMP, ENULL, ENULL, 0); + return p; +} + +static void patch_jump( emit_ctx *ctx, int jpos ) { + ctx->instrs[jpos].value = ctx->emit_pos; +} + +static void register_jump( emit_ctx *ctx, int jpos, int offs ) { + if( ctx->jump_count == ctx->max_jumps ) { + int next_size = ctx->max_jumps ? ctx->max_jumps << 1 : 64; + int *jumps = (int*)malloc(sizeof(int) * next_size); + if( jumps == NULL ) emit_error("Out of memory"); + memcpy(jumps, ctx->jump_regs, ctx->jump_count * sizeof(int)); + free(ctx->jump_regs); + ctx->jump_regs = jumps; + ctx->max_jumps = next_size; + } + ctx->jump_regs[ctx->jump_count++] = jpos; + ctx->jump_regs[ctx->jump_count++] = offs + ctx->op_pos + 1; +} + +static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { + return ENULL; +} + +static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { + return ENULL; +} + +static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) { + return ENULL; +} + +static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { + if( to->t->kind == HVOID ) return; +} + +static ereg emit_binop( emit_ctx *ctx, hl_op op, ereg a, ereg b ) { + return ENULL; +} + +static ereg emit_unop( emit_ctx *ctx, hl_op op, ereg v ) { + return ENULL; +} + +static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int nargs, hl_type *ret ) { + return ENULL; +} + +static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_type *ret ) { + return ENULL; +} + +static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t ) { + return ENULL; +} + +static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int *args ) { + emit_error("TODO"); +} + +static vclosure *alloc_static_closure( emit_ctx *ctx, int fid ) { + hl_module *m = ctx->mod; + vclosure *c = hl_malloc(&m->ctx.alloc,sizeof(vclosure)); + int fidx = m->functions_indexes[fid]; + c->hasValue = 0; + if( fidx >= m->code->nfunctions ) { + // native + c->t = m->code->natives[fidx - m->code->nfunctions].t; + c->fun = m->functions_ptrs[fid]; + c->value = NULL; + } else { + c->t = m->code->functions[fidx].type; + c->fun = (void*)(int_val)fid; + c->value = ctx->closure_list; + ctx->closure_list = c; + } + return c; +} + +static void *get_dynget( hl_type *t ) { + switch( t->kind ) { + case HF32: + return hl_dyn_getf; + case HF64: + return hl_dyn_getd; + case HI64: + case HGUID: + return hl_dyn_geti64; + case HI32: + case HUI16: + case HUI8: + case HBOOL: + return hl_dyn_geti; + default: + return hl_dyn_getp; + } +} + +static void *get_dynset( hl_type *t ) { + switch( t->kind ) { + case HF32: + return hl_dyn_setf; + case HF64: + return hl_dyn_setd; + case HI64: + case HGUID: + return hl_dyn_seti64; + case HI32: + case HUI16: + case HUI8: + case HBOOL: + return hl_dyn_seti; + default: + return hl_dyn_setp; + } +} + +static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, int src_offset, int total_size ) { + int offset = 0; + while( offset < total_size) { + int remain = total_size - offset; + hl_type *ct = remain >= HL_WSIZE ? &hlt_bytes : (remain >= 4 ? &hlt_i32 : &hlt_ui8); + STORE_MEM(dst, dst_offset+offset, LOAD_MEM(src,src_offset+offset,ct)); + offset += hl_type_size(ct); + } +} + +static ereg *get_tmp_args( emit_ctx *ctx, int count ) { + if( count > MAX_TMP_ARGS ) emit_error("Too many arguments"); + return ctx->tmp_args; +} + +static ereg emit_phy( emit_ctx *ctx, ereg v1, ereg v2 ) { + return ENULL; +} + +static ereg emit_conv( emit_ctx *ctx, ereg v, hl_op op ) { + return ENULL; +} + +static bool dyn_need_type( hl_type *t ) { + return !(IS_FLOAT(t) || t->kind == HI64 || t->kind == HGUID); +} + +static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { + vreg *dst = R(o->p1); + vreg *ra = R(o->p2); + vreg *rb = R(o->p3); + hl_module *m = ctx->mod; + switch( o->op ) { + case OMov: + case OUnsafeCast: + STORE(dst, LOAD(ra)); + break; + case OInt: + STORE(dst, LOAD_CONST(m->code->ints[o->p2], dst->t)); + break; + case OBool: + STORE(dst, LOAD_CONST(o->p2, &hlt_bool)); + break; + case ONull: + STORE(dst, LOAD_CONST(0, dst->t)); + break; + case OFloat: + { + union { + float f; + double d; + uint64 i; + } v; + if( dst->t->kind == HF32 ) + v.f = (float)m->code->floats[o->p2]; + else + v.d = m->code->floats[o->p2]; + STORE(dst, LOAD_CONST(v.i, dst->t)); + } + break; + case OString: + STORE(dst, LOAD_CONST_PTR(hl_get_ustring(m->code,o->p2))); + break; + case OBytes: + { + char *b = m->code->version >= 5 ? m->code->bytes + m->code->bytes_pos[o->p2] : m->code->strings[o->p2]; + STORE(dst,LOAD_CONST_PTR(b)); + } + break; + case OGetGlobal: + { + void *addr = m->globals_data + m->globals_indexes[o->p2]; + STORE(dst, LOAD_MEM_PTR(LOAD_CONST_PTR(addr),0)); + } + break; + case OSetGlobal: + { + void *addr = m->globals_data + m->globals_indexes[o->p1]; + STORE_MEM(LOAD_CONST_PTR(addr),0,LOAD(ra)); + } + break; + case OCall0: + emit_call_fun(ctx, dst, o->p2, 0, NULL); + break; + case OCall1: + emit_call_fun(ctx, dst, o->p2, 1, &o->p3); + break; + case OCall2: + { + int args[2] = { o->p3, (int)(int_val)o->extra }; + emit_call_fun(ctx, dst, o->p2, 2, args); + } + break; + case OCall3: + { + int args[3] = { o->p3, o->extra[0], o->extra[1] }; + emit_call_fun(ctx, dst, o->p2, 3, args); + } + break; + case OCall4: + { + int args[4] = { o->p3, o->extra[0], o->extra[1], o->extra[2] }; + emit_call_fun(ctx, dst, o->p2, 4, args); + } + break; + case OCallN: + emit_call_fun(ctx, dst, o->p2, o->p3, o->extra); + break; + case OSub: + case OAdd: + case OMul: + case OSDiv: + case OUDiv: + case OShl: + case OSShr: + case OUShr: + case OAnd: + case OOr: + case OXor: + case OSMod: + case OUMod: + { + ereg va = LOAD(ra); + ereg vb = LOAD(rb); + STORE(dst, emit_gen(ctx, BINOP, va, vb, o->op)); + } + break; + case ONeg: + case ONot: + STORE(dst, emit_gen(ctx, UNOP, LOAD(ra), ENULL, o->op)); + break; + case OJFalse: + case OJTrue: + case OJNotNull: + case OJNull: + { + emit_gen(ctx, TEST, LOAD(dst), ENULL, o->op); + int jidx = emit_jump(ctx, true); + register_jump(ctx, jidx, o->p2); + } + break; + case OJEq: + case OJNotEq: + case OJSLt: + case OJSGte: + case OJSLte: + case OJSGt: + case OJULt: + case OJUGte: + case OJNotLt: + case OJNotGte: + { + emit_gen(ctx, CMP, LOAD(dst), LOAD(ra), o->op); + int jidx = emit_jump(ctx, true); + register_jump(ctx, jidx, o->p3); + } + break; + case OJAlways: + { + int jidx = emit_jump(ctx, false); + register_jump(ctx, jidx, o->p1); + } + break; + case OToDyn: + if( ra->t->kind == HBOOL ) { + ereg arg = LOAD(ra); + STORE(dst, emit_native_call(ctx,hl_alloc_dynbool,&arg,1,&hlt_dyn)); + } else { + ereg arg = LOAD_CONST_PTR(ra->t); + ereg ret = emit_native_call(ctx,hl_alloc_dynamic,&arg,1,&hlt_dyn); + STORE_MEM(ret,HDYN_VALUE,LOAD(ra)); + } + break; + case OToSFloat: + case OToInt: + case OToUFloat: + STORE(dst, emit_conv(ctx,LOAD(ra),o->op)); + break; + case ORet: + emit_gen(ctx, RET, LOAD(dst), ENULL, 0); + break; + case OIncr: + case ODecr: + { + if( IS_FLOAT(dst->t) ) { + emit_assert(); + } else { + STORE(dst, emit_unop(ctx,o->op, LOAD(dst))); + } + } + break; + case ONew: + { + ereg arg = ENULL; + void *allocFun = NULL; + int nargs = 1; + switch( dst->t->kind ) { + case HOBJ: + case HSTRUCT: + allocFun = hl_alloc_obj; + break; + case HDYNOBJ: + allocFun = hl_alloc_dynobj; + nargs = 0; + break; + case HVIRTUAL: + allocFun = hl_alloc_virtual; + break; + default: + emit_assert(); + } + if( nargs ) arg = LOAD_CONST_PTR(dst->t); + STORE(dst, emit_native_call(ctx,allocFun,&arg,nargs,dst->t)); + } + break; + case OInstanceClosure: + { + ereg args[3]; + args[0] = LOAD_CONST_PTR(m->code->functions[m->functions_indexes[o->p2]].type); + // WRITE (emit_pos + op_count) to process later and replace address ! + args[1] = LOAD_CONST_PTR(0); + args[2] = LOAD(rb); + STORE(dst, emit_native_call(ctx,hl_alloc_closure_ptr,args,3,dst->t)); + } + break; + case OVirtualClosure: + { + hl_type *t = NULL; + hl_type *ot = ra->t; + while( t == NULL ) { + int i; + for(i=0;iobj->nproto;i++) { + hl_obj_proto *pp = ot->obj->proto + i; + if( pp->pindex == o->p3 ) { + t = m->code->functions[m->functions_indexes[pp->findex]].type; + break; + } + } + ot = ot->obj->super; + } + ereg args[3]; + ereg obj = LOAD(ra); + args[0] = LOAD_CONST_PTR(t); + args[1] = LOAD_OBJ_METHOD(obj,o->p3); + args[2] = obj; + STORE(dst, emit_native_call(ctx,hl_alloc_closure_ptr,args,3,dst->t)); + } + break; + case OCallClosure: + if( ra->t->kind == HDYN ) { + int i; + ereg st = emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, o->p3); + for(i=0;ip3;i++) { + vreg *r = R(o->extra[i]); + if( !hl_is_dynamic(r->t) ) emit_assert(); + STORE_MEM(st,i*HL_WSIZE,LOAD(r)); + } + ereg args[3]; + args[0] = LOAD(ra); + args[1] = st; + args[2] = LOAD_CONST(o->p3,&hlt_i32); + STORE(dst, emit_dyn_cast(ctx,emit_native_call(ctx,hl_dyn_call,args,3,dst->t),dst->t)); + emit_gen(ctx, FREE_STACK, ENULL, ENULL, o->p3); + } else { + ereg r = LOAD(ra); + ereg *args = get_tmp_args(ctx,o->p3+1); + // Code for if( c->hasValue ) c->fun(c->value,args) else c->fun(args) + ereg has = LOAD_MEM(r,HL_WSIZE*2,&hlt_i32); + emit_gen(ctx, TEST, has, ENULL, OJNull); + int jidx = emit_jump(ctx, true); + int i; + args[0] = LOAD_MEM_PTR(r,HL_WSIZE * 3); + for(i=0;ip3;i++) + args[i+1] = LOAD(R(o->extra[i])); + ereg v1 = emit_dyn_call(ctx,LOAD_MEM_PTR(r,HL_WSIZE),args,o->p3 + 1,dst->t); + int jend = emit_jump(ctx, false); + patch_jump(ctx, jidx); + for(i=0;ip3;i++) + args[i] = LOAD(R(o->extra[i])); + ereg v2 = emit_dyn_call(ctx,LOAD_MEM_PTR(r,HL_WSIZE),args,o->p3,dst->t); + patch_jump(ctx, jend); + STORE(dst, emit_phy(ctx,v1,v2)); + } + break; + case OStaticClosure: + { + vclosure *c = alloc_static_closure(ctx,o->p2); + STORE(dst, LOAD_CONST_PTR(c)); + } + break; + case OField: + { + switch( ra->t->kind ) { + case HOBJ: + case HSTRUCT: + { + hl_runtime_obj *rt = hl_get_obj_rt(ra->t); + ereg r = LOAD(ra); + if( dst->t->kind == HSTRUCT ) { + hl_type *ft = hl_obj_field_fetch(ra->t,o->p3)->t; + if( ft->kind == HPACKED ) { + STORE(dst,emit_gen(ctx, LEA, r, ENULL, rt->fields_indexes[o->p3])); + break; + } + } + STORE(dst, LOAD_MEM(r,rt->fields_indexes[o->p3],dst->t)); + } + break; + case HVIRTUAL: + // code for : if( hl_vfields(o)[f] ) r = *hl_vfields(o)[f]; else r = hl_dyn_get(o,hash(field),vt) + { + ereg obj = LOAD(ra); + ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p3); + emit_gen(ctx, TEST, field, ENULL, OJNull); + int jidx = emit_jump(ctx, true); + ereg v1 = LOAD_MEM(field,0,dst->t); + int jend = emit_jump(ctx, false); + patch_jump(ctx, jidx); + bool need_type = dyn_need_type(dst->t); + ereg args[3]; + args[0] = obj; + args[1] = LOAD_CONST(ra->t->virt->fields[o->p3].hashed_name,&hlt_i32); + if( need_type ) args[2] = LOAD_CONST_PTR(dst->t); + ereg v2 = emit_native_call(ctx,get_dynget(dst->t),args,need_type?3:2,dst->t); + patch_jump(ctx, jend); + STORE(dst, emit_phy(ctx, v1, v2)); + } + break; + default: + emit_assert(); + break; + } + } + break; + case OSetField: + { + switch( dst->t->kind ) { + case HOBJ: + case HSTRUCT: + { + ereg obj = LOAD(dst); + ereg val = LOAD(rb); + hl_runtime_obj *rt = hl_get_obj_rt(dst->t); + int field_pos = rt->fields_indexes[o->p2]; + if( rb->t->kind == HSTRUCT ) { + hl_type *ft = hl_obj_field_fetch(dst->t,o->p2)->t; + if( ft->kind == HPACKED ) { + emit_store_size(ctx,obj,field_pos,val,0,hl_get_obj_rt(ft->tparam)->size); + break; + } + } + STORE_MEM(obj,field_pos, val); + } + break; + case HVIRTUAL: + // code for : if( hl_vfields(o)[f] ) *hl_vfields(o)[f] = v; else hl_dyn_set(o,hash(field),vt,v) + { + ereg obj = LOAD(ra); + ereg val = LOAD(rb); + ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p3); + emit_gen(ctx, TEST, field, ENULL, OJNull); + int jidx = emit_jump(ctx, true); + STORE_MEM(field, 0, val); + int jend = emit_jump(ctx, false); + patch_jump(ctx, jidx); + bool need_type = dyn_need_type(dst->t); + ereg args[4]; + args[0] = obj; + args[1] = LOAD_CONST(ra->t->virt->fields[o->p3].hashed_name,&hlt_i32); + if( need_type ) { + args[2] = LOAD_CONST_PTR(rb->t); + args[3] = val; + } else { + args[2] = val; + } + emit_native_call(ctx,get_dynset(dst->t),args,need_type?4:3,dst->t); + patch_jump(ctx, jend); + } + break; + default: + emit_assert(); + break; + } + } + break; + case OGetThis: + { + vreg *r = R(0); + ereg obj = LOAD(r); + hl_runtime_obj *rt = hl_get_obj_rt(r->t); + int field_pos = rt->fields_indexes[o->p2]; + if( dst->t->kind == HSTRUCT ) { + hl_type *ft = hl_obj_field_fetch(r->t,o->p2)->t; + if( ft->kind == HPACKED ) { + STORE(dst, emit_gen(ctx, LEA, obj, ENULL, field_pos)); + break; + } + } + STORE(dst, LOAD_MEM(obj, field_pos, dst->t)); + } + break; + case OSetThis: + { + vreg *r = R(0); + ereg obj = LOAD(r); + ereg val = LOAD(ra); + hl_runtime_obj *rt = hl_get_obj_rt(r->t); + int field_pos = rt->fields_indexes[o->p1]; + if( ra->t->kind == HSTRUCT ) { + hl_type *ft = hl_obj_field_fetch(r->t,o->p1)->t; + if( ft->kind == HPACKED ) { + emit_store_size(ctx, obj, field_pos, val, 0, hl_get_obj_rt(ft->tparam)->size); + break; + } + } + STORE_MEM(obj,field_pos,val); + } + break; + case OCallThis: + { + int i; + int nargs = o->p3 + 1; + ereg obj = LOAD(R(0)); + ereg *args = get_tmp_args(ctx, nargs); + args[0] = obj; + for(i=1;iextra[i-1])); + ereg fun = LOAD_OBJ_METHOD(obj, o->p2); + STORE(dst, emit_dyn_call(ctx,fun,args,nargs,dst->t)); + } + break; + case OCallMethod: + { + vreg *r = R(o->extra[0]); + ereg obj = LOAD(r); + switch( r->t->kind ) { + case HOBJ: + { + int i; + int nargs = o->p3; + ereg *args = get_tmp_args(ctx, nargs); + for(i=0;iextra[i])); + ereg fun = LOAD_OBJ_METHOD(obj, o->p2); + STORE(dst, emit_dyn_call(ctx,fun,args,nargs,dst->t)); + } + break; + case HVIRTUAL: + // code for : if( hl_vfields(o)[f] ) dst = *hl_vfields(o)[f](o->value,args...); else dst = hl_dyn_call_obj(o->value,field,args,&ret) + { + vreg *_o = R(o->extra[0]); + ereg obj = LOAD(_o); + ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p2); + emit_gen(ctx, TEST, field, ENULL, OJNull); + int jidx = emit_jump(ctx, true); + + int nargs = o->p3 + 1; + ereg *args = get_tmp_args(ctx, nargs); + int i; + args[0] = LOAD_MEM_PTR(obj,HL_WSIZE); + for(i=1;iextra[i])); + ereg v1 = emit_dyn_call(ctx,LOAD_MEM_PTR(field,0),args,nargs,dst->t); + + int jend = emit_jump(ctx, false); + patch_jump(ctx, jidx); + + ereg eargs = emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, o->p3); + for(i=0;ip3;i++) + STORE_MEM(eargs,i*HL_WSIZE,LOAD(R(o->extra[i+1]))); + bool need_dyn = !hl_is_ptr(dst->t) && dst->t->kind != HVOID; + int dyn_size = sizeof(vdynamic)/HL_WSIZE; + ereg edyn = need_dyn ? emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, dyn_size) : LOAD_CONST_PTR(NULL); + + args = get_tmp_args(ctx, 4); + args[0] = LOAD_MEM_PTR(obj,HL_WSIZE); + args[1] = LOAD_CONST(_o->t->virt->fields[o->p2].hashed_name,&hlt_i32); + args[2] = eargs; + args[3] = edyn; + + ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 4, dst->t); + + emit_gen(ctx, FREE_STACK, ENULL, ENULL, o->p3 + (need_dyn ? dyn_size : 0)); + patch_jump(ctx, jend); + + STORE(dst, emit_phy(ctx, v1, v2)); + } + break; + default: + emit_assert(); + break; + } + } + break; + case OThrow: + case ORethrow: + { + ereg arg = LOAD(dst); + emit_native_call(ctx, o->op == OThrow ? hl_throw : hl_rethrow, &arg, 1, &hlt_void); + } + break; + case OLabel: + // NOP + break; + case OGetI8: + case OGetI16: + case OGetMem: + { + ereg offs = OFFSET(LOAD(ra),LOAD(rb),1,0); + ereg val = LOAD_MEM(offs, 0, dst->t); + if( o->op != OGetMem ) val = emit_conv(ctx, val, o->op); + STORE(dst, val); + } + break; + case OSetI8: + case OSetI16: + case OSetMem: + { + ereg offs = OFFSET(LOAD(dst), LOAD(ra),1,0); + ereg val = LOAD(rb); + if( o->op != OSetMem ) val = emit_conv(ctx, val, o->op); + STORE_MEM(offs, 0, val); + } + break; + case OType: + STORE(dst, LOAD_CONST_PTR(m->code->types + o->p2)); + break; + case OGetType: + { + ereg r = LOAD(ra); + emit_gen(ctx, TEST, r, ENULL, OJNotNull); + int jidx = emit_jump(ctx, true); + ereg v1 = LOAD_CONST_PTR(&hlt_void); + int jend = emit_jump(ctx, false); + patch_jump(ctx, jidx); + ereg v2 = LOAD_MEM_PTR(r,0); + patch_jump(ctx, jend); + STORE(dst, emit_phy(ctx, v1, v2)); + } + break; + case OGetArray: + { + if( ra->t->kind == HABSTRACT ) { + int osize; + bool isPtr = dst->t->kind != HOBJ && dst->t->kind != HSTRUCT; + if( isPtr ) + osize = HL_WSIZE; // a pointer into the carray + else { + hl_runtime_obj *rt = hl_get_obj_rt(dst->t); + osize = rt->size; // a mem offset into it + } + ereg pos = OFFSET(LOAD(ra), LOAD(rb), osize, 0); + ereg val = isPtr ? LOAD_MEM_PTR(pos,0) : pos; + STORE(dst, val); + } else { + ereg pos = OFFSET(LOAD(ra), LOAD(rb), hl_type_size(dst->t), sizeof(varray)); + STORE(dst, LOAD_MEM_PTR(pos,0)); + } + } + break; + case OSetArray: + { + if( dst->t->kind == HABSTRACT ) { + int osize; + bool isPtr = rb->t->kind != HOBJ && rb->t->kind != HSTRUCT; + if( isPtr) { + osize = HL_WSIZE; + } else { + hl_runtime_obj *rt = hl_get_obj_rt(rb->t); + osize = rt->size; + } + ereg pos = OFFSET(LOAD(dst), LOAD(ra), osize, 0); + emit_store_size(ctx, pos, 0, LOAD(rb), 0, osize); + } else { + ereg pos = OFFSET(LOAD(dst), LOAD(ra), hl_type_size(dst->t), sizeof(varray)); + STORE_MEM(pos, 0, LOAD(rb)); + } + } + break; + case OArraySize: + STORE(dst, LOAD_MEM_PTR(LOAD(ra),HL_WSIZE*2)); + break; + case ORef: + { + ereg addr = emit_gen(ctx, ALLOC_GLOBAL_STACK, ENULL, ENULL, hl_type_size(ra->t)); + STORE_MEM(addr, 0, LOAD(ra)); + STORE(dst, addr); + } + break; + case OUnref: + STORE(dst, LOAD_MEM(LOAD(ra),0,dst->t)); + break; + case OSetref: + STORE_MEM(LOAD(dst),0,LOAD(ra)); + break; + case ORefData: + switch( ra->t->kind ) { + case HARRAY: + STORE(dst, OFFSET(LOAD(ra),ENULL,0,sizeof(varray))); + break; + default: + emit_assert(); + } + break; + case ORefOffset: + STORE(dst, OFFSET(LOAD(ra),LOAD(rb), hl_type_size(dst->t->tparam),0)); + break; + case OToVirtual: + { + ereg args[2]; + args[0] = LOAD(ra); + args[1] = LOAD_CONST_PTR(dst->t); + STORE(dst, emit_native_call(ctx,hl_to_virtual,args,2, dst->t)); + } + break; + case OMakeEnum: + { + ereg args[2]; + args[0] = LOAD_CONST_PTR(dst->t); + args[1] = LOAD_CONST(o->p2,&hlt_i32); + ereg en = emit_native_call(ctx, hl_alloc_enum, args, 2, dst->t); + STORE(dst, en); + hl_enum_construct *c = &dst->t->tenum->constructs[o->p2]; + for(int i=0;inparams;i++) + STORE_MEM(en, c->offsets[i], LOAD(R(o->extra[i]))); + } + break; + case OEnumAlloc: + { + ereg args[2]; + args[0] = LOAD_CONST_PTR(dst->t); + args[1] = LOAD_CONST(o->p2,&hlt_i32); + STORE(dst, emit_native_call(ctx, hl_alloc_enum, args, 2, dst->t)); + } + break; + case OEnumField: + { + hl_enum_construct *c = &ra->t->tenum->constructs[o->p3]; + int slot = (int)(int_val)o->extra; + STORE(dst, LOAD_MEM(LOAD(ra),c->offsets[slot], dst->t)); + } + break; + case OEnumIndex: + STORE(dst, LOAD_MEM(LOAD(ra),HL_WSIZE,dst->t)); + break; + case OSetEnumField: + { + hl_enum_construct *c = &ra->t->tenum->constructs[0]; + STORE_MEM(LOAD(dst), c->offsets[o->p2], LOAD(rb)); + } + break; + case ONullCheck: + { + emit_gen(ctx, TEST, LOAD(dst), ENULL, OJNotNull); + int jok = emit_jump(ctx, true); + + // ----- DETECT FIELD ACCESS ---------------- + hl_function *f = ctx->fun; + hl_opcode *next = f->ops + ctx->op_pos + 1; + bool null_field_access = false; + int hashed_name = 0; + // skip const and operation between nullcheck and access + while( (next < f->ops + f->nops - 1) && (next->op >= OInt && next->op <= ODecr) ) { + next++; + } + if( (next->op == OField && next->p2 == o->p1) || (next->op == OSetField && next->p1 == o->p1) ) { + int fid = next->op == OField ? next->p3 : next->p2; + hl_obj_field *f = NULL; + if( dst->t->kind == HOBJ || dst->t->kind == HSTRUCT ) + f = hl_obj_field_fetch(dst->t, fid); + else if( dst->t->kind == HVIRTUAL ) + f = dst->t->virt->fields + fid; + if( f == NULL ) emit_assert(); + null_field_access = true; + hashed_name = f->hashed_name; + } else if( (next->op >= OCall1 && next->op <= OCallN) && next->p3 == o->p1 ) { + int fid = next->p2 < 0 ? -1 : m->functions_indexes[next->p2]; + hl_function *cf = m->code->functions + fid; + const uchar *name = fun_field_name(cf); + null_field_access = true; + hashed_name = hl_hash_gen(name, true); + } + // ----------------------------------------- + ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : ENULL; + emit_native_call(ctx, null_field_access ? hl_stub_null_field_access : hl_stub_null_access, &arg, null_field_access ? 1 : 0, &hlt_void); + patch_jump(ctx, jok); + } + break; + case OSafeCast: + STORE(dst, emit_dyn_cast(ctx, LOAD(ra), dst->t)); + break; + case ODynGet: + { + bool need_type = dyn_need_type(dst->t); + ereg args[3]; + args[0] = LOAD(ra); + args[1] = LOAD_CONST(hl_hash_utf8(m->code->strings[o->p3]),&hlt_i32); + if( need_type ) args[2] = LOAD_CONST_PTR(dst->t); + STORE(dst, emit_native_call(ctx, get_dynget(dst->t), args, need_type ? 3 : 2, dst->t)); + } + break; + case ODynSet: + { + bool need_type = dyn_need_type(dst->t); + ereg args[4]; + args[0] = LOAD(dst); + args[1] = LOAD_CONST(hl_hash_utf8(m->code->strings[o->p2]),&hlt_i32); + if( need_type ) { + args[2] = LOAD_CONST_PTR(rb->t); + args[3] = LOAD(rb); + } else + args[2] = LOAD(rb); + emit_native_call(ctx, get_dynset(rb->t), args, need_type ? 4 : 3, &hlt_void); + } + break; + case OTrap: + { + ereg st = emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, sizeof(hl_trap_ctx)); + + ereg thread, current_addr; + static hl_thread_info *tinf = NULL; + static hl_trap_ctx *trap = NULL; +# ifndef HL_THREADS + if( tinf == NULL ) tinf = hl_get_thread(); + current_addr = LOAD_CONST_PTR(&tinf->trap_current); +# else + thread = emit_native_call(ctx, hl_get_thread, NULL, 0, &hlt_bytes); + current_addr = OFFSET(thread, ENULL, 0, (int)(int_val)&tinf->trap_current); +# endif + STORE_MEM(st, (int)(int_val)&trap->prev, LOAD_MEM_PTR(current_addr,0)); + STORE_MEM(current_addr, 0, st); + + + /* + trap E,@catch + catch g + catch g2 + ... + @:catch + + // Before haxe 5 + This is a bit hackshish : we want to detect the type of exception filtered by the catch so we check the following + sequence of HL opcodes: + + trap E,@catch + ... + @catch: + global R, _ + call _, ???(R,E) + + ??? is expected to be hl.BaseType.check + */ + hl_function *f = ctx->fun; + hl_opcode *cat = f->ops + ctx->op_pos + 1; + hl_opcode *next = f->ops + ctx->op_pos + 1 + o->p2; + hl_opcode *next2 = f->ops + ctx->op_pos + 2 + o->p2; + void *addr = NULL; + if( cat->op == OCatch || (next->op == OGetGlobal && next2->op == OCall2 && next2->p3 == next->p1 && dst->id == (int)(int_val)next2->extra) ) { + int gindex = cat->op == OCatch ? cat->p1 : next->p2; + hl_type *gt = m->code->globals[gindex]; + while( gt->kind == HOBJ && gt->obj->super ) gt = gt->obj->super; + if( gt->kind == HOBJ && gt->obj->nfields && gt->obj->fields[0].t->kind == HTYPE ) + addr = m->globals_data + m->globals_indexes[gindex]; + } + STORE_MEM(st, (int)(int_val)&trap->tcheck, addr ? LOAD_MEM_PTR(LOAD_CONST_PTR(addr),0) : LOAD_CONST_PTR(NULL)); + + void *fun = setjmp; + ereg args[2]; + int nargs = 1; + args[0] = OFFSET(st, ENULL, 0, (int)(int_val)&trap->buf); +#if defined(HL_WIN) && defined(HL_64) + // On Win64 setjmp actually takes two arguments + // the jump buffer and the frame pointer (or the stack pointer if there is no FP) + nargs = 2; + args[1] = emit_gen(ctx, NATIVE_REG, ENULL, ENULL, REG_RBP); +#endif +#ifdef HL_MINGW + fun = _setjmp; +#endif + ereg ret = emit_native_call(ctx, fun, args, nargs, &hlt_i32); + emit_gen(ctx, TEST, ret, ENULL, OJNull); + int jskip = emit_jump(ctx, true); + emit_gen(ctx, FREE_STACK, ENULL, ENULL, sizeof(hl_trap_ctx)); + STORE(dst, tinf ? LOAD_CONST_PTR(&tinf->exc_value) : LOAD_MEM_PTR(thread,(int)(int_val)&tinf->exc_value)); + + int jtrap = emit_jump(ctx, false); + register_jump(ctx, jtrap, o->p2); + patch_jump(ctx, jskip); + + if( ctx->trap_count == MAX_TRAPS ) emit_error("Too many try/catch depth"); + ctx->traps[ctx->trap_count++] = st; + } + break; + case OEndTrap: + { + if( ctx->trap_count == 0 ) emit_assert(); + ereg st = ctx->traps[--ctx->trap_count]; + + ereg thread, current_addr; + static hl_thread_info *tinf = NULL; + static hl_trap_ctx *trap = NULL; +# ifndef HL_THREADS + if( tinf == NULL ) tinf = hl_get_thread(); + current_addr = LOAD_CONST_PTR(&tinf->trap_current); +# else + thread = emit_native_call(ctx, hl_get_thread, NULL, 0, &hlt_bytes); + current_addr = OFFSET(thread, ENULL, 0, (int)(int_val)&tinf->trap_current); +# endif + + STORE_MEM(current_addr, 0, LOAD_MEM_PTR(st,(int)(int_val)&trap->prev)); + +# ifdef HL_WIN + // erase eip (prevent false positive in exception stack) + { + _JUMP_BUFFER *b = NULL; +# ifdef HL_64 + int offset = (int)(int_val)&(b->Rip); +# else + int offset = (int)(int_val)&(b->Eip); +# endif + STORE_MEM(st, offset, LOAD_CONST_PTR(NULL)); + } +# endif + + emit_gen(ctx, FREE_STACK, ENULL, ENULL, sizeof(hl_trap_ctx)); + } + break; + case OSwitch: + { + ereg v = LOAD(dst); + int count = o->p2; + emit_gen(ctx, CMP, v, LOAD_CONST(count,&hlt_i32), OJUGte); + int jdefault = emit_jump(ctx, true); + emit_gen(ctx, JUMP_TABLE, v, ENULL, count); + for(int i=0; iextra[i]); + } + patch_jump(ctx, jdefault); + } + break; + case OGetTID: + STORE(dst, LOAD_MEM(LOAD(ra),0,&hlt_i32)); + break; + case OAssert: + emit_native_call(ctx, hl_stub_assert, NULL, 0, &hlt_void); + break; + case ONop: + break; + case OPrefetch: + { + ereg r = LOAD(dst); + if( o->p2 > 0 ) { + switch( dst->t->kind ) { + case HOBJ: + case HSTRUCT: + { + hl_runtime_obj *rt = hl_get_obj_rt(dst->t); + r = OFFSET(r, ENULL, 0, rt->fields_indexes[o->p2-1]); + } + break; + default: + emit_assert(); + break; + } + } + emit_gen(ctx, PREFETCH, r, ENULL, o->p3); + } + break; + case OAsm: + emit_assert(); + break; + case OCatch: + // Only used by OTrap typing + break; + default: + emit_error(hl_op_name(o->op)); + break; + } +} + +bool emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { + int i; + ctx->mod = m; + ctx->fun = f; + + if( f->nregs > ctx->max_regs ) { + free(ctx->vregs); + ctx->vregs = (vreg*)malloc(sizeof(vreg) * (f->nregs + 1)); + if( ctx->vregs == NULL ) + return false; + for(i=ctx->max_regs;inregs;i++) + R(i)->id = i; + ctx->max_regs = f->nregs; + } + + for(i=0;inregs;i++) { + vreg *r = R(i); + r->t = f->regs[i]; + } + + if( f->nops > ctx->pos_map_size ) { + free(ctx->pos_map); + ctx->pos_map = (int*)malloc(sizeof(int) * f->nops); + if( ctx->pos_map == NULL ) + return false; + ctx->pos_map_size = f->nops; + } + + for(int op_pos=0;op_posnops;op_pos++) { + ctx->op_pos = op_pos; + ctx->pos_map[op_pos] = ctx->emit_pos; + emit_opcode(ctx,f->ops + op_pos); + } + return true; +} + +emit_ctx *hl_emit_alloc() { + emit_ctx *ctx = (emit_ctx*)malloc(sizeof(emit_ctx)); + if( ctx == NULL ) return NULL; + memset(ctx,0,sizeof(emit_ctx)); + return ctx; +} + +void hl_emit_free( emit_ctx *ctx ) { + free(ctx->vregs); + free(ctx); +} From bf5620ba566f1b897a255bce255e934b16807b51 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 15 Mar 2026 14:54:53 +0100 Subject: [PATCH 02/83] call emitter instead of jit, started ir emit & dump --- src/emit.c | 366 ++++++++++++++++++++++++++++++++++++++++++------- src/hlmodule.h | 17 +-- src/module.c | 35 ++--- src/opcodes.h | 20 +-- 4 files changed, 351 insertions(+), 87 deletions(-) diff --git a/src/emit.c b/src/emit.c index faa1a1d3d..44de4f59f 100644 --- a/src/emit.c +++ b/src/emit.c @@ -22,7 +22,9 @@ #include typedef enum { - LOAD, + LOAD_ADDR, + LOAD_IMM, + LOAD_ARG, STORE, LEA, TEST, @@ -33,40 +35,77 @@ typedef enum { BINOP, UNOP, CONV, + CONV_UNSIGNED, RET, + CALL_PTR, + CALL_REG, + CALL_FUN, + PHY, ALLOC_STACK, FREE_STACK, ALLOC_GLOBAL_STACK, NATIVE_REG, PREFETCH, + DEBUG_BREAK, } emit_op; +static const char *op_names[] = { + "load-addr", + "load-imm", + "load-arg", + "store", + "lea", + "test", + "cmp", + "jcond", + "jump", + "jump-table", + "binop", + "unop", + "conv", + "conv-unsigned", + "ret", + "call-ptr", + "call-reg", + "call-fun", + "phy", + "alloc-stack", + "free-stack", + "alloc-global-stack", + "native-reg", + "prefetch", + "debug-break", +}; + typedef enum { REG_RBP, } native_reg; typedef enum { + NONE, I8, I16, I32, I64, F32, - F64 + F64, + PTR, } emit_mode; -typedef struct { - int id; - hl_type *t; -} vreg; typedef struct { int index; } ereg; +typedef struct { + int id; + hl_type *t; + ereg current; +} vreg; + typedef struct { emit_op op; emit_mode mode; - short offset_param; union { struct { ereg a; @@ -79,7 +118,7 @@ typedef struct { #define MAX_TMP_ARGS 32 #define MAX_TRAPS 32 -typedef struct { +struct _emit_ctx { hl_module *mod; hl_function *fun; einstr *instrs; @@ -100,7 +139,7 @@ typedef struct { int max_jumps; void *closure_list; // TODO : patch with good addresses -} emit_ctx; +}; #define R(i) (ctx->vregs + (i)) @@ -113,10 +152,10 @@ typedef struct { #define LOAD_CONST_PTR(v) LOAD_CONST(v,&hlt_bytes) #define LOAD_MEM(v, offs, t) emit_load_mem(ctx, v, offs, t) #define LOAD_MEM_PTR(v, offs) LOAD_MEM(v, offs, &hlt_bytes) -#define STORE_MEM(to, offs, v) emit_gen(ctx, STORE, to, v, offs) +#define STORE_MEM(to, offs, v) emit_store_mem(ctx, to, offs, v) #define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR((obj),0),HL_WSIZE*2),HL_WSIZE*(id)) #define OFFSET(base,index,mult,offset) emit_gen(ctx, LEA, base, index, (mult) | ((offset) << 8)) - +#define BREAK() emit_gen(ctx, DEBUG_BREAK, ENULL, ENULL, 0) #define CUR_REG() __current_reg(ctx) #define HDYN_VALUE 8 @@ -137,28 +176,59 @@ static void hl_stub_null_field_access() { emit_assert(); } static void hl_stub_null_access() { emit_assert(); } static void hl_stub_assert() { emit_assert(); } +static emit_mode hl_type_mode( hl_type *t ) { + if( t->kind < HBOOL ) + return (emit_mode)t->kind; + if( t->kind == HBOOL ) + return sizeof(bool) == 1 ? I8 : I32; + if( t->kind == HGUID ) + return I64; + return PTR; +} + +void hl_jit_patch_method( void*fun, void**newt ) { + emit_assert(); +} + static ereg __current_reg( emit_ctx *ctx ) { ereg r = {ctx->emit_pos-1}; return r; } -static einstr *emit_instr( emit_ctx *ctx ) { +static ereg *get_tmp_args( emit_ctx *ctx, int count ) { + if( count > MAX_TMP_ARGS ) emit_error("Too many arguments"); + return ctx->tmp_args; +} + +static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { if( ctx->emit_pos == ctx->max_instrs ) { + int pos = ctx->emit_pos; int next_size = ctx->max_instrs ? (ctx->max_instrs * 3) >> 1 : 256; einstr *instrs = (einstr*)malloc(sizeof(einstr) * next_size); if( instrs == NULL ) emit_error("Out of memory"); - memcpy(instrs, ctx->instrs, ctx->emit_pos * sizeof(einstr)); + memcpy(instrs, ctx->instrs, pos * sizeof(einstr)); + memset(instrs + pos, 0, (next_size - pos) * sizeof(einstr)); free(ctx->instrs); ctx->instrs = instrs; ctx->max_instrs = next_size; } - return ctx->instrs + ctx->emit_pos++; + einstr *e = ctx->instrs + ctx->emit_pos++; + e->op = op; + return e; +} + +static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { + einstr *e = emit_instr(ctx, STORE); + e->mode = (ctx->instrs[from.index].mode) | (offs << 8); + e->args.a = to; + e->args.b = from; +} + +static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { } static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { - einstr *e = emit_instr(ctx); - if( (short)mode != mode ) emit_assert(); - e->op = op; + einstr *e = emit_instr(ctx, op); e->mode = mode; e->args.a = a; e->args.b = b; @@ -172,7 +242,7 @@ static int emit_jump( emit_ctx *ctx, bool cond ) { } static void patch_jump( emit_ctx *ctx, int jpos ) { - ctx->instrs[jpos].value = ctx->emit_pos; + ctx->instrs[jpos].mode = ctx->emit_pos - (jpos + 1); } static void register_jump( emit_ctx *ctx, int jpos, int offs ) { @@ -190,43 +260,74 @@ static void register_jump( emit_ctx *ctx, int jpos, int offs ) { } static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { - return ENULL; + //if( r->current.index < 0 ) emit_assert(); + return r->current; } static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { - return ENULL; + einstr *e = emit_instr(ctx, LOAD_IMM); + e->mode = hl_type_mode(size_t); + e->value = value; + return CUR_REG(); } static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) { - return ENULL; + einstr *e = emit_instr(ctx, LOAD_ADDR); + e->mode = hl_type_mode(size_t); + e->args.a = v; + e->args.b.index = offset; + return CUR_REG(); } static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { if( to->t->kind == HVOID ) return; + to->current = v; } static ereg emit_binop( emit_ctx *ctx, hl_op op, ereg a, ereg b ) { - return ENULL; + return emit_gen(ctx, BINOP, a, b, op); } static ereg emit_unop( emit_ctx *ctx, hl_op op, ereg v ) { - return ENULL; + return emit_gen(ctx, UNOP, v, ENULL, op); } static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int nargs, hl_type *ret ) { - return ENULL; + einstr *e = emit_instr(ctx, CALL_PTR); + e->mode = hl_type_mode(ret); + e->value = (int64)native_ptr; + store_args(ctx, e, args, nargs); + return CUR_REG(); } static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_type *ret ) { - return ENULL; + einstr *e = emit_instr(ctx, CALL_REG); + e->mode = hl_type_mode(ret); + e->args.a = f; + store_args(ctx, e, args, nargs); + return CUR_REG(); } static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t ) { - return ENULL; + BREAK(); + return v; } -static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int *args ) { - emit_error("TODO"); +static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int *args_regs ) { + hl_module *m = ctx->mod; + int fid = m->functions_indexes[findex]; + bool isNative = fid >= m->code->nfunctions; + ereg *args = get_tmp_args(ctx, count); + for(int i=0;ifunctions_ptrs[findex], args, count, dst->t)); + else { + einstr *e = emit_instr(ctx, CALL_FUN); + e->mode = findex; + store_args(ctx, e, args, count); + STORE(dst, CUR_REG()); + } } static vclosure *alloc_static_closure( emit_ctx *ctx, int fid ) { @@ -296,17 +397,12 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, } } -static ereg *get_tmp_args( emit_ctx *ctx, int count ) { - if( count > MAX_TMP_ARGS ) emit_error("Too many arguments"); - return ctx->tmp_args; -} - static ereg emit_phy( emit_ctx *ctx, ereg v1, ereg v2 ) { - return ENULL; + return emit_gen(ctx, PHY, v1, v2, 0); } -static ereg emit_conv( emit_ctx *ctx, ereg v, hl_op op ) { - return ENULL; +static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode mode, bool _unsigned ) { + return emit_gen(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, ENULL, mode); } static bool dyn_need_type( hl_type *t ) { @@ -462,7 +558,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OToSFloat: case OToInt: case OToUFloat: - STORE(dst, emit_conv(ctx,LOAD(ra),o->op)); + STORE(dst, emit_conv(ctx,LOAD(ra),hl_type_mode(dst->t), o->op == OToUFloat)); break; case ORet: emit_gen(ctx, RET, LOAD(dst), ENULL, 0); @@ -505,7 +601,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg args[3]; args[0] = LOAD_CONST_PTR(m->code->functions[m->functions_indexes[o->p2]].type); - // WRITE (emit_pos + op_count) to process later and replace address ! + // TODO : WRITE (emit_pos + op_count) to process later and replace address ! args[1] = LOAD_CONST_PTR(0); args[2] = LOAD(rb); STORE(dst, emit_native_call(ctx,hl_alloc_closure_ptr,args,3,dst->t)); @@ -643,9 +739,9 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case HVIRTUAL: // code for : if( hl_vfields(o)[f] ) *hl_vfields(o)[f] = v; else hl_dyn_set(o,hash(field),vt,v) { - ereg obj = LOAD(ra); + ereg obj = LOAD(dst); ereg val = LOAD(rb); - ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p3); + ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p2); emit_gen(ctx, TEST, field, ENULL, OJNull); int jidx = emit_jump(ctx, true); STORE_MEM(field, 0, val); @@ -654,7 +750,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { bool need_type = dyn_need_type(dst->t); ereg args[4]; args[0] = obj; - args[1] = LOAD_CONST(ra->t->virt->fields[o->p3].hashed_name,&hlt_i32); + args[1] = LOAD_CONST(dst->t->virt->fields[o->p2].hashed_name,&hlt_i32); if( need_type ) { args[2] = LOAD_CONST_PTR(rb->t); args[3] = val; @@ -742,7 +838,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { emit_gen(ctx, TEST, field, ENULL, OJNull); int jidx = emit_jump(ctx, true); - int nargs = o->p3 + 1; + int nargs = o->p3; ereg *args = get_tmp_args(ctx, nargs); int i; args[0] = LOAD_MEM_PTR(obj,HL_WSIZE); @@ -753,8 +849,9 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); - ereg eargs = emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, o->p3); - for(i=0;ip3;i++) + nargs = o->p3 - 1; + ereg eargs = emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, nargs); + for(i=0;iextra[i+1]))); bool need_dyn = !hl_is_ptr(dst->t) && dst->t->kind != HVOID; int dyn_size = sizeof(vdynamic)/HL_WSIZE; @@ -796,7 +893,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg offs = OFFSET(LOAD(ra),LOAD(rb),1,0); ereg val = LOAD_MEM(offs, 0, dst->t); - if( o->op != OGetMem ) val = emit_conv(ctx, val, o->op); + if( o->op != OGetMem ) val = emit_conv(ctx, val, I32, false); STORE(dst, val); } break; @@ -806,7 +903,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg offs = OFFSET(LOAD(dst), LOAD(ra),1,0); ereg val = LOAD(rb); - if( o->op != OSetMem ) val = emit_conv(ctx, val, o->op); + if( o->op != OSetMem ) val = emit_conv(ctx, val, I32, false); STORE_MEM(offs, 0, val); } break; @@ -933,7 +1030,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case OSetEnumField: { - hl_enum_construct *c = &ra->t->tenum->constructs[0]; + hl_enum_construct *c = &dst->t->tenum->constructs[0]; STORE_MEM(LOAD(dst), c->offsets[o->p2], LOAD(rb)); } break; @@ -1082,7 +1179,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OEndTrap: { if( ctx->trap_count == 0 ) emit_assert(); - ereg st = ctx->traps[--ctx->trap_count]; + ereg st = ctx->traps[ctx->trap_count - 1]; ereg thread, current_addr; static hl_thread_info *tinf = NULL; @@ -1167,10 +1264,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } } -bool emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { +int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { int i; ctx->mod = m; ctx->fun = f; + ctx->emit_pos = 0; + ctx->trap_count = 0; if( f->nregs > ctx->max_regs ) { free(ctx->vregs); @@ -1185,11 +1284,16 @@ bool emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { for(i=0;inregs;i++) { vreg *r = R(i); r->t = f->regs[i]; + r->current = ENULL; + } + + for(i=0;itype->fun->nargs;i++) { + STORE(R(i), emit_gen(ctx, LOAD_ARG, ENULL, ENULL, hl_type_mode(f->type->fun->args[i]))); } - if( f->nops > ctx->pos_map_size ) { + if( f->nops >= ctx->pos_map_size ) { free(ctx->pos_map); - ctx->pos_map = (int*)malloc(sizeof(int) * f->nops); + ctx->pos_map = (int*)malloc(sizeof(int) * (f->nops+1)); if( ctx->pos_map == NULL ) return false; ctx->pos_map_size = f->nops; @@ -1200,6 +1304,18 @@ bool emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { ctx->pos_map[op_pos] = ctx->emit_pos; emit_opcode(ctx,f->ops + op_pos); } + + // patch jumps + i = 0; + while( i < ctx->jump_count ) { + int pos = ctx->jump_regs[i++]; + einstr *e = ctx->instrs + pos; + int target = ctx->jump_regs[i++]; + e->mode = ctx->pos_map[target] - (pos + 1); + } + ctx->jump_count = 0; + + ctx->pos_map[f->nops] = -1; return true; } @@ -1210,7 +1326,153 @@ emit_ctx *hl_emit_alloc() { return ctx; } -void hl_emit_free( emit_ctx *ctx ) { +void hl_emit_free( emit_ctx *ctx, h_bool can_reset ) { free(ctx->vregs); + free(ctx->instrs); + free(ctx->pos_map); + free(ctx->jump_regs); free(ctx); } + +void hl_emit_init( emit_ctx *ctx, hl_module *m ) { +} + +void hl_emit_reset( emit_ctx *ctx, hl_module *m ) { +} + +void *hl_emit_code( emit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { + printf("TODO:emit_code\n"); + exit(0); + return NULL; +} + +static void hl_dump_arg( emit_ctx *ctx, int fmt, int val, char sep ) { + if( fmt == 0 ) return; + printf("%c", sep); + switch( fmt ) { + case 1: + case 2: + printf("R%d", val); + if( val < 0 || val >= ctx->fun->nregs ) printf("?"); + break; + case 3: + printf("%d", val); + break; + case 4: + printf("[%d]", val); + break; + case 5: + case 6: + printf("@%X", val + ctx->op_pos + 1); + break; + default: + printf("?#%d", fmt); + break; + } +} + +#define OP(_,_a,_b,_c) ((_a) | (((_b)&0xFF) << 8) | (((_c)&0xFF) << 16)), +#define OP_BEGIN static int hl_op_fmt[] = { +#define OP_END }; +#undef R +#include "opcodes.h" + +static void hl_dump_op( emit_ctx *ctx, hl_opcode *op ) { + printf("%s", hl_op_name(op->op) + 1); + int fmt = hl_op_fmt[op->op]; + hl_dump_arg(ctx, fmt & 0xFF, op->p1, ' '); + if( ((fmt >> 8) & 0xFF) == 5 ) { + int count = (fmt >> 16) & 0xFF; + printf(" ["); + if( count == 4 ) { + printf("%d", op->p2); + printf(",%d", op->p3); + printf(",%d", (int)(int_val)op->extra); + } else { + if( count == 0xFF ) count = op->p3; + for(int i=0;iextra[i]); + } + } + printf("]"); + } else { + hl_dump_arg(ctx, (fmt >> 8) & 0xFF, op->p2,','); + hl_dump_arg(ctx, fmt >> 16, op->p3,','); + } +} + +static const char *emit_mode_str( emit_mode mode ) { + switch( mode ) { + case NONE: return "-void"; + case I8: return "-i8"; + case I16: return "-i16"; + case I32: return "-i32"; + case I64: return "-i64"; + case F32: return "-f32"; + case F64: return "-f64"; + case PTR: return ""; + default: + static char buf[50]; + sprintf(buf,"?%d",mode); + return buf; + } +} + +void hl_emit_dump( emit_ctx *ctx ) { + int i; + int cur_op = 0; + hl_function *f = ctx->fun; + int nargs = f->type->fun->nargs; + printf("function %X(", f->findex); + for(i=0;i 0 ) printf(","); + uprintf(USTR("R%d"), i); + } + printf(")\n"); + for(i=0;inregs;i++) + uprintf(USTR("\tR%d : %s\n"),i, hl_type_str(f->regs[i])); + for(i=0;iemit_pos;i++) { + while( ctx->pos_map[cur_op] == i ) { + printf("@%X ", cur_op); + ctx->op_pos = cur_op; + hl_dump_op(ctx, f->ops + cur_op); + printf("\n"); + cur_op++; + } + einstr *e = ctx->instrs + i; + printf("\t\t@%X %s", i, op_names[e->op]); + switch( e->op ) { + case LOAD_ADDR: + case LOAD_IMM: + case LOAD_ARG: + printf("%s", emit_mode_str(e->mode)); + break; + default: + break; + } + switch( e->op ) { + case JUMP: + case JCOND: + printf(" @%X", i + 1 + e->mode); + break; + case STORE: + { + int offs = e->mode >> 8; + printf("%s", emit_mode_str(e->mode&0xFF)); + if( offs == 0 ) + printf(" [@%X] := @%X", e->args.a.index, e->args.b.index); + else + printf(" @%X[%d] := @%X", e->args.a.index, offs, e->args.b.index); + } + break; + default: + if( e->args.a.index >= 0 ) printf(" @%X", e->args.a.index); + if( e->args.b.index >= 0 ) printf(", @%X", e->args.b.index); + break; + } + printf("\n"); + } + printf("\n\n"); + fflush(stdout); +} \ No newline at end of file diff --git a/src/hlmodule.h b/src/hlmodule.h index d8ea8c912..e82a3ff9f 100644 --- a/src/hlmodule.h +++ b/src/hlmodule.h @@ -104,7 +104,7 @@ typedef struct { bool large; } hl_debug_infos; -typedef struct _jit_ctx jit_ctx; +typedef struct _emit_ctx emit_ctx; typedef struct { @@ -131,7 +131,7 @@ typedef struct { void *jit_code; hl_code_hash *hash; hl_debug_infos *jit_debug; - jit_ctx *jit_ctx; + emit_ctx *emit_ctx; hl_module_context ctx; #ifdef WIN64_UNWIND_TABLES PRUNTIME_FUNCTION unwind_table; @@ -161,10 +161,11 @@ hl_type *hl_module_resolve_type( hl_module *m, hl_type *t, bool err ); void hl_profile_setup( int sample_count ); void hl_profile_end(); -jit_ctx *hl_jit_alloc(); -void hl_jit_free( jit_ctx *ctx, h_bool can_reset ); -void hl_jit_reset( jit_ctx *ctx, hl_module *m ); -void hl_jit_init( jit_ctx *ctx, hl_module *m ); -int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ); -void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); +emit_ctx *hl_emit_alloc(); +void hl_emit_free( emit_ctx *ctx, h_bool can_reset ); +void hl_emit_reset( emit_ctx *ctx, hl_module *m ); +void hl_emit_init( emit_ctx *ctx, hl_module *m ); +void hl_emit_dump( emit_ctx *ctx ); +int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ); +void *hl_emit_code( emit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); void hl_jit_patch_method( void *old_fun, void **new_fun_table ); diff --git a/src/module.c b/src/module.c index e668b1064..6e0b6ef8b 100644 --- a/src/module.c +++ b/src/module.c @@ -680,7 +680,7 @@ static void hl_module_add( hl_module *m ) { int hl_module_init( hl_module *m, h_bool hot_reload ) { int i; - jit_ctx *ctx; + emit_ctx *ctx; // expand globals if( hot_reload ) { int nsize = m->globals_size + HOT_RELOAD_EXTRA_GLOBALS * sizeof(void*); @@ -706,20 +706,21 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { hl_module_init_natives(m); hl_module_init_indexes(m); // JIT - ctx = hl_jit_alloc(); + ctx = hl_emit_alloc(); if( ctx == NULL ) return 0; - hl_jit_init(ctx, m); + hl_emit_init(ctx, m); for(i=0;icode->nfunctions;i++) { hl_function *f = m->code->functions + i; - int fpos = hl_jit_function(ctx, m, f); + int fpos = hl_emit_function(ctx, m, f); if( fpos < 0 ) { - hl_jit_free(ctx, false); + hl_emit_free(ctx, false); return 0; } + hl_emit_dump(ctx); m->functions_ptrs[f->findex] = (void*)(int_val)fpos; } - m->jit_code = hl_jit_code(ctx, m, &m->codesize, &m->jit_debug, NULL); + m->jit_code = hl_emit_code(ctx, m, &m->codesize, &m->jit_debug, NULL); for(i=0;icode->nfunctions;i++) { hl_function *f = m->code->functions + i; m->functions_ptrs[f->findex] = ((unsigned char*)m->jit_code) + ((int_val)m->functions_ptrs[f->findex]); @@ -736,10 +737,10 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { # ifdef HL_VTUNE hl_setup.vtune_init = modules_init_vtune; # endif - hl_jit_free(ctx, hot_reload); + hl_emit_free(ctx, hot_reload); if( hot_reload ) { hl_code_hash_finalize(m->hash); - m->jit_ctx = ctx; + m->emit_ctx = ctx; } return 1; } @@ -824,7 +825,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { int i,i1,i2; bool has_changes = false; int changes_count = 0; - jit_ctx *ctx = m1->jit_ctx; + emit_ctx *ctx = m1->emit_ctx; hl_module *m2 = hl_module_alloc(c); m2->hash = hl_code_hash_alloc(c); @@ -849,7 +850,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { hl_module_init_natives(m2); hl_module_init_indexes(m2); - hl_jit_reset(ctx, m2); + hl_emit_reset(ctx, m2); hl_code_hash_finalize(m2->hash); for(i=0;icode->nconstants;i++) { @@ -893,7 +894,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { changes_count++; m1->hash->functions_hashes[i1] = hash2; // update hash - int fpos = hl_jit_function(ctx, m2, f2); + int fpos = hl_emit_function(ctx, m2, f2); if( fpos < 0 ) return false; m2->functions_ptrs[f2->findex] = (void*)(int_val)fpos; has_changes = true; @@ -902,7 +903,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { } if( i1 == m1->code->nfunctions ) { // not found (signature changed or new method) : inject new method! - int fpos = hl_jit_function(ctx, m2, f2); + int fpos = hl_emit_function(ctx, m2, f2); if( fpos < 0 ) return false; m2->hash->functions_hashes[i2] = -1; m2->functions_ptrs[f2->findex] = (void*)(int_val)fpos; @@ -921,7 +922,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { if( !has_changes ) { printf("[HotReload] No changes found\n"); fflush(stdout); - hl_jit_free(ctx, true); + hl_emit_free(ctx, true); return false; } @@ -969,7 +970,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { } } - m2->jit_code = hl_jit_code(ctx, m2, &m2->codesize, &m2->jit_debug, m1); + m2->jit_code = hl_emit_code(ctx, m2, &m2->codesize, &m2->jit_debug, m1); // patch missing debug info int start = -1; @@ -984,7 +985,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { } } - hl_jit_free(ctx,true); + hl_emit_free(ctx,true); if( m2->jit_code == NULL ) { printf("[HotReload] Couldn't JIT result\n"); @@ -1054,7 +1055,7 @@ void hl_module_free( hl_module *m ) { free(m->jit_debug[i].offsets); free(m->jit_debug); } - if( m->jit_ctx ) - hl_jit_free(m->jit_ctx,false); + if( m->emit_ctx ) + hl_emit_free(m->emit_ctx,false); free(m); } diff --git a/src/opcodes.h b/src/opcodes.h index ab9b1fa51..9e4df7f60 100644 --- a/src/opcodes.h +++ b/src/opcodes.h @@ -67,8 +67,8 @@ OP_BEGIN OP(OIncr,R,X,X) OP(ODecr,R,X,X) - OP(OCall0,R,R,X) - OP(OCall1,R,R,R) + OP(OCall0,R,C,X) + OP(OCall1,R,C,R) OP(OCall2,R,AR,4) OP(OCall3,R,AR,5) OP(OCall4,R,AR,6) @@ -78,17 +78,17 @@ OP_BEGIN OP(OCallClosure,R,AR,VAR_ARGS) OP(OStaticClosure,R,G,X) - OP(OInstanceClosure,R,R,G) + OP(OInstanceClosure,R,C,R) OP(OVirtualClosure,R,R,G) OP(OGetGlobal,R,G,X) - OP(OSetGlobal,R_NW,G,X) - OP(OField,R,R,C) - OP(OSetField,R_NW,R,C) - OP(OGetThis,R,C,X) - OP(OSetThis,R_NW,R,X) + OP(OSetGlobal,G,R,X) + OP(OField,R,R,G) + OP(OSetField,R_NW,G,R) + OP(OGetThis,R,G,X) + OP(OSetThis,G,R,X) OP(ODynGet,R,R,C) - OP(ODynSet,R_NW,R,C) + OP(ODynSet,R_NW,C,R) OP(OJTrue,R_NW,J,X) OP(OJFalse,R_NW,J,X) @@ -134,7 +134,7 @@ OP_BEGIN OP(ONew,R,X,X) OP(OArraySize,R,R,X) - OP(OType,R,R,X) + OP(OType,R,G,X) OP(OGetType,R,R,X) OP(OGetTID,R,R,X) From d8306a942fdcf1833abcb2b086dc3c063cab78a0 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 15 Mar 2026 19:50:58 +0100 Subject: [PATCH 03/83] finalized IR and completed dump --- src/emit.c | 533 +++++++++++++++++++++++++++++++++++++++------------ src/module.c | 1 - 2 files changed, 412 insertions(+), 122 deletions(-) diff --git a/src/emit.c b/src/emit.c index 44de4f59f..8777c4f6b 100644 --- a/src/emit.c +++ b/src/emit.c @@ -40,10 +40,9 @@ typedef enum { CALL_PTR, CALL_REG, CALL_FUN, - PHY, + PHI, ALLOC_STACK, FREE_STACK, - ALLOC_GLOBAL_STACK, NATIVE_REG, PREFETCH, DEBUG_BREAK, @@ -65,13 +64,12 @@ static const char *op_names[] = { "conv", "conv-unsigned", "ret", - "call-ptr", - "call-reg", - "call-fun", - "phy", + "call", + "call", + "call", + "phi", "alloc-stack", "free-stack", - "alloc-global-stack", "native-reg", "prefetch", "debug-break", @@ -82,14 +80,15 @@ typedef enum { } native_reg; typedef enum { - NONE, - I8, - I16, - I32, - I64, - F32, - F64, - PTR, + M_NONE, + M_UI8, + M_UI16, + M_I32, + M_I64, + M_F32, + M_F64, + M_PTR, + M_VOID, } emit_mode; @@ -98,25 +97,28 @@ typedef struct { } ereg; typedef struct { - int id; hl_type *t; + int id; ereg current; } vreg; typedef struct { - emit_op op; - emit_mode mode; + unsigned char op; + unsigned char mode; + unsigned short nargs; + int size_offs; union { struct { ereg a; ereg b; - } args; + }; uint64 value; }; } einstr; #define MAX_TMP_ARGS 32 #define MAX_TRAPS 32 +#define MAX_REFS 512 // TODO : different impl struct _emit_ctx { hl_module *mod; @@ -130,9 +132,19 @@ struct _emit_ctx { ereg tmp_args[MAX_TMP_ARGS]; ereg traps[MAX_TRAPS]; + struct { + ereg r; + int reg; + } refs[MAX_REFS]; int *pos_map; int pos_map_size; int trap_count; + int ref_count; + + int *args_data; + int args_data_size; + int args_data_pos; + int *jump_regs; int jump_count; @@ -154,7 +166,7 @@ struct _emit_ctx { #define LOAD_MEM_PTR(v, offs) LOAD_MEM(v, offs, &hlt_bytes) #define STORE_MEM(to, offs, v) emit_store_mem(ctx, to, offs, v) #define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR((obj),0),HL_WSIZE*2),HL_WSIZE*(id)) -#define OFFSET(base,index,mult,offset) emit_gen(ctx, LEA, base, index, (mult) | ((offset) << 8)) +#define OFFSET(base,index,mult,offset) emit_gen_ext(ctx, LEA, base, index, 0, (mult) | ((offset) << 8)) #define BREAK() emit_gen(ctx, DEBUG_BREAK, ENULL, ENULL, 0) #define CUR_REG() __current_reg(ctx) @@ -165,10 +177,13 @@ struct _emit_ctx { static hl_type hlt_ui8 = { HUI8, 0 }; static hl_type hlt_ui16 = { HUI16, 0 }; static ereg ENULL = {-1}; +static emit_ctx *current_ctx = NULL; static void _emit_error( const char *msg, int line ) { printf("*** EMIT ERROR line %d (%s) ****\n", line, msg); + if( current_ctx ) hl_emit_dump(current_ctx); hl_debug_break(); + fflush(stdout); exit(-1); } @@ -177,13 +192,15 @@ static void hl_stub_null_access() { emit_assert(); } static void hl_stub_assert() { emit_assert(); } static emit_mode hl_type_mode( hl_type *t ) { + if( t->kind == HVOID ) + return M_VOID; if( t->kind < HBOOL ) return (emit_mode)t->kind; if( t->kind == HBOOL ) - return sizeof(bool) == 1 ? I8 : I32; + return sizeof(bool) == 1 ? M_UI8 : M_I32; if( t->kind == HGUID ) - return I64; - return PTR; + return M_I64; + return M_PTR; } void hl_jit_patch_method( void*fun, void**newt ) { @@ -200,6 +217,14 @@ static ereg *get_tmp_args( emit_ctx *ctx, int count ) { return ctx->tmp_args; } +static ereg resolve_ref( emit_ctx *ctx, int reg ) { + for(int i=0;iref_count;i++) { + if( ctx->refs[i].reg == reg ) + return ctx->refs[i].r; + } + return ENULL; +} + static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { if( ctx->emit_pos == ctx->max_instrs ) { int pos = ctx->emit_pos; @@ -219,22 +244,60 @@ static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { einstr *e = emit_instr(ctx, STORE); - e->mode = (ctx->instrs[from.index].mode) | (offs << 8); - e->args.a = to; - e->args.b = from; + e->mode = ctx->instrs[from.index].mode; + e->size_offs = offs; + e->a = to; + e->b = from; } static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { + if( count < 0 || count > 64 ) emit_assert(); + e->nargs = (unsigned short)count; + if( count == 0 ) return; + if( count == 1 ) { + e->size_offs = args[0].index; + return; + } + if( ctx->args_data_pos + count > ctx->args_data_size ) { + int next_size = ctx->args_data_size ? ctx->args_data_size << 1 : 128; + int *args = (int*)malloc(sizeof(int) * next_size); + if( args == NULL ) emit_error("Out of memory"); + memcpy(args, ctx->args_data, sizeof(int) * ctx->args_data_pos); + free(ctx->args_data); + ctx->args_data = args; + ctx->args_data_size = next_size; + } + e->size_offs = ctx->args_data_pos; + memcpy(ctx->args_data + ctx->args_data_pos, args, sizeof(int) * count); + ctx->args_data_pos += count; } -static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { +ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ) { + if( e->nargs == 0 ) + return NULL; + if( e->nargs == 1 ) + return (ereg*)&e->size_offs; + return (ereg*)(ctx->args_data + e->size_offs); +} + +static ereg emit_gen_ext( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode, int size_offs ) { einstr *e = emit_instr(ctx, op); - e->mode = mode; - e->args.a = a; - e->args.b = b; + if( (unsigned char)mode != mode ) emit_assert(); + e->mode = (unsigned char)mode; + e->size_offs = size_offs; + e->a = a; + e->b = b; return CUR_REG(); } +static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { + return emit_gen_ext(ctx,op,a,b,mode,0); +} + +static ereg emit_gen_size( emit_ctx *ctx, emit_op op, int size_offs ) { + return emit_gen_ext(ctx,op,ENULL,ENULL,0,size_offs); +} + static int emit_jump( emit_ctx *ctx, bool cond ) { int p = ctx->emit_pos; emit_gen(ctx, cond ? JCOND : JUMP, ENULL, ENULL, 0); @@ -242,7 +305,7 @@ static int emit_jump( emit_ctx *ctx, bool cond ) { } static void patch_jump( emit_ctx *ctx, int jpos ) { - ctx->instrs[jpos].mode = ctx->emit_pos - (jpos + 1); + ctx->instrs[jpos].size_offs = ctx->emit_pos - (jpos + 1); } static void register_jump( emit_ctx *ctx, int jpos, int offs ) { @@ -259,11 +322,6 @@ static void register_jump( emit_ctx *ctx, int jpos, int offs ) { ctx->jump_regs[ctx->jump_count++] = offs + ctx->op_pos + 1; } -static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { - //if( r->current.index < 0 ) emit_assert(); - return r->current; -} - static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { einstr *e = emit_instr(ctx, LOAD_IMM); e->mode = hl_type_mode(size_t); @@ -274,8 +332,8 @@ static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) { einstr *e = emit_instr(ctx, LOAD_ADDR); e->mode = hl_type_mode(size_t); - e->args.a = v; - e->args.b.index = offset; + e->a = v; + e->b.index = offset; return CUR_REG(); } @@ -284,14 +342,6 @@ static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { to->current = v; } -static ereg emit_binop( emit_ctx *ctx, hl_op op, ereg a, ereg b ) { - return emit_gen(ctx, BINOP, a, b, op); -} - -static ereg emit_unop( emit_ctx *ctx, hl_op op, ereg v ) { - return emit_gen(ctx, UNOP, v, ENULL, op); -} - static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int nargs, hl_type *ret ) { einstr *e = emit_instr(ctx, CALL_PTR); e->mode = hl_type_mode(ret); @@ -303,14 +353,23 @@ static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_type *ret ) { einstr *e = emit_instr(ctx, CALL_REG); e->mode = hl_type_mode(ret); - e->args.a = f; + e->a = f; store_args(ctx, e, args, nargs); return CUR_REG(); } -static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t ) { - BREAK(); - return v; +static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { + emit_gen_ext(ctx, TEST, v, ENULL, ctx->instrs[v.index].mode, o); +} + +static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { + if( r->current.index < 0 ) { + ereg ref = resolve_ref(ctx, r->id); + if( ref.index < 0 ) emit_assert(); + // reload from ref + return LOAD_MEM(ref,0,r->t); + } + return r->current; } static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int *args_regs ) { @@ -324,7 +383,8 @@ static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int STORE(dst, emit_native_call(ctx, m->functions_ptrs[findex], args, count, dst->t)); else { einstr *e = emit_instr(ctx, CALL_FUN); - e->mode = findex; + e->mode = hl_type_mode(dst->t); + e->a.index = findex; store_args(ctx, e, args, count); STORE(dst, CUR_REG()); } @@ -387,6 +447,25 @@ static void *get_dynset( hl_type *t ) { } } +static void *get_dyncast( hl_type *t ) { + switch( t->kind ) { + case HF32: + return hl_dyn_castf; + case HF64: + return hl_dyn_castd; + case HI64: + case HGUID: + return hl_dyn_casti64; + case HI32: + case HUI16: + case HUI8: + case HBOOL: + return hl_dyn_casti; + default: + return hl_dyn_castp; + } +} + static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, int src_offset, int total_size ) { int offset = 0; while( offset < total_size) { @@ -397,8 +476,8 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, } } -static ereg emit_phy( emit_ctx *ctx, ereg v1, ereg v2 ) { - return emit_gen(ctx, PHY, v1, v2, 0); +static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { + return emit_gen(ctx, PHI, v1, v2, 0); } static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode mode, bool _unsigned ) { @@ -409,11 +488,37 @@ static bool dyn_need_type( hl_type *t ) { return !(IS_FLOAT(t) || t->kind == HI64 || t->kind == HGUID); } +static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { + if( t->kind == HNULL && t->tparam->kind == dt->kind ) { + emit_test(ctx, v, OJNotNull); + int jnot = emit_jump(ctx, false); + ereg v1 = LOAD_CONST(0,dt); + int jend = emit_jump(ctx, true); + patch_jump(ctx, jnot); + ereg v2 = LOAD_MEM(v,0,dt); + patch_jump(ctx, jend); + return emit_phi(ctx, v1, v2); + } + bool need_dyn = dyn_need_type(dt); + ereg st = emit_gen_size(ctx, ALLOC_STACK, 1); + STORE_MEM(st, 0, v); + ereg args[3]; + args[0] = st; + args[1] = LOAD_CONST_PTR(t); + if( need_dyn ) args[2] = LOAD_CONST_PTR(dt); + ereg r = emit_native_call(ctx, get_dyncast(dt), args, need_dyn ? 3 : 2, dt); + emit_gen_size(ctx, FREE_STACK, 1); + return r; +} + static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { vreg *dst = R(o->p1); vreg *ra = R(o->p2); vreg *rb = R(o->p3); hl_module *m = ctx->mod; +#ifdef HL_DEBUG + int uid = (ctx->fun->findex << 16) | ctx->op_pos; +#endif switch( o->op ) { case OMov: case OUnsafeCast: @@ -506,19 +611,19 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg va = LOAD(ra); ereg vb = LOAD(rb); - STORE(dst, emit_gen(ctx, BINOP, va, vb, o->op)); + STORE(dst, emit_gen_ext(ctx, BINOP, va, vb, hl_type_mode(dst->t), o->op)); } break; case ONeg: case ONot: - STORE(dst, emit_gen(ctx, UNOP, LOAD(ra), ENULL, o->op)); + STORE(dst, emit_gen_ext(ctx, UNOP, LOAD(ra), ENULL, hl_type_mode(dst->t), o->op)); break; case OJFalse: case OJTrue: case OJNotNull: case OJNull: { - emit_gen(ctx, TEST, LOAD(dst), ENULL, o->op); + emit_test(ctx, LOAD(dst), o->op); int jidx = emit_jump(ctx, true); register_jump(ctx, jidx, o->p2); } @@ -534,7 +639,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OJNotLt: case OJNotGte: { - emit_gen(ctx, CMP, LOAD(dst), LOAD(ra), o->op); + emit_gen_ext(ctx, CMP, LOAD(dst), LOAD(ra), hl_type_mode(dst->t), o->op); int jidx = emit_jump(ctx, true); register_jump(ctx, jidx, o->p3); } @@ -553,6 +658,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg arg = LOAD_CONST_PTR(ra->t); ereg ret = emit_native_call(ctx,hl_alloc_dynamic,&arg,1,&hlt_dyn); STORE_MEM(ret,HDYN_VALUE,LOAD(ra)); + STORE(dst, ret); } break; case OToSFloat: @@ -561,7 +667,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, emit_conv(ctx,LOAD(ra),hl_type_mode(dst->t), o->op == OToUFloat)); break; case ORet: - emit_gen(ctx, RET, LOAD(dst), ENULL, 0); + if( dst->t->kind == HVOID ) + emit_gen(ctx,RET,ENULL,ENULL,M_VOID); + else + emit_gen(ctx, RET, LOAD(dst), ENULL, hl_type_mode(dst->t)); break; case OIncr: case ODecr: @@ -569,7 +678,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( IS_FLOAT(dst->t) ) { emit_assert(); } else { - STORE(dst, emit_unop(ctx,o->op, LOAD(dst))); + STORE(dst, emit_gen_ext(ctx,UNOP,LOAD(dst),ENULL,hl_type_mode(dst->t),o->op)); } } break; @@ -633,7 +742,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OCallClosure: if( ra->t->kind == HDYN ) { int i; - ereg st = emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, o->p3); + ereg st = emit_gen_size(ctx, ALLOC_STACK, o->p3); for(i=0;ip3;i++) { vreg *r = R(o->extra[i]); if( !hl_is_dynamic(r->t) ) emit_assert(); @@ -643,14 +752,14 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[0] = LOAD(ra); args[1] = st; args[2] = LOAD_CONST(o->p3,&hlt_i32); - STORE(dst, emit_dyn_cast(ctx,emit_native_call(ctx,hl_dyn_call,args,3,dst->t),dst->t)); - emit_gen(ctx, FREE_STACK, ENULL, ENULL, o->p3); + STORE(dst, emit_dyn_cast(ctx,emit_native_call(ctx,hl_dyn_call,args,3,dst->t),ra->t,dst->t)); + emit_gen_size(ctx, FREE_STACK, o->p3); } else { ereg r = LOAD(ra); ereg *args = get_tmp_args(ctx,o->p3+1); // Code for if( c->hasValue ) c->fun(c->value,args) else c->fun(args) ereg has = LOAD_MEM(r,HL_WSIZE*2,&hlt_i32); - emit_gen(ctx, TEST, has, ENULL, OJNull); + emit_test(ctx, has, OJNull); int jidx = emit_jump(ctx, true); int i; args[0] = LOAD_MEM_PTR(r,HL_WSIZE * 3); @@ -663,7 +772,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[i] = LOAD(R(o->extra[i])); ereg v2 = emit_dyn_call(ctx,LOAD_MEM_PTR(r,HL_WSIZE),args,o->p3,dst->t); patch_jump(ctx, jend); - STORE(dst, emit_phy(ctx,v1,v2)); + STORE(dst, emit_phi(ctx,v1,v2)); } break; case OStaticClosure: @@ -683,7 +792,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( dst->t->kind == HSTRUCT ) { hl_type *ft = hl_obj_field_fetch(ra->t,o->p3)->t; if( ft->kind == HPACKED ) { - STORE(dst,emit_gen(ctx, LEA, r, ENULL, rt->fields_indexes[o->p3])); + STORE(dst,OFFSET(r, ENULL, 0, rt->fields_indexes[o->p3])); break; } } @@ -695,7 +804,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg obj = LOAD(ra); ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p3); - emit_gen(ctx, TEST, field, ENULL, OJNull); + emit_test(ctx, field, OJNull); int jidx = emit_jump(ctx, true); ereg v1 = LOAD_MEM(field,0,dst->t); int jend = emit_jump(ctx, false); @@ -707,7 +816,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( need_type ) args[2] = LOAD_CONST_PTR(dst->t); ereg v2 = emit_native_call(ctx,get_dynget(dst->t),args,need_type?3:2,dst->t); patch_jump(ctx, jend); - STORE(dst, emit_phy(ctx, v1, v2)); + STORE(dst, emit_phi(ctx, v1, v2)); } break; default: @@ -742,7 +851,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg obj = LOAD(dst); ereg val = LOAD(rb); ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p2); - emit_gen(ctx, TEST, field, ENULL, OJNull); + emit_test(ctx, field, OJNull); int jidx = emit_jump(ctx, true); STORE_MEM(field, 0, val); int jend = emit_jump(ctx, false); @@ -776,7 +885,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( dst->t->kind == HSTRUCT ) { hl_type *ft = hl_obj_field_fetch(r->t,o->p2)->t; if( ft->kind == HPACKED ) { - STORE(dst, emit_gen(ctx, LEA, obj, ENULL, field_pos)); + STORE(dst, OFFSET(obj, ENULL, 0, field_pos)); break; } } @@ -835,7 +944,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { vreg *_o = R(o->extra[0]); ereg obj = LOAD(_o); ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p2); - emit_gen(ctx, TEST, field, ENULL, OJNull); + emit_test(ctx, field, OJNull); int jidx = emit_jump(ctx, true); int nargs = o->p3; @@ -850,12 +959,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { patch_jump(ctx, jidx); nargs = o->p3 - 1; - ereg eargs = emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, nargs); + ereg eargs = emit_gen_size(ctx, ALLOC_STACK, nargs); for(i=0;iextra[i+1]))); bool need_dyn = !hl_is_ptr(dst->t) && dst->t->kind != HVOID; int dyn_size = sizeof(vdynamic)/HL_WSIZE; - ereg edyn = need_dyn ? emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, dyn_size) : LOAD_CONST_PTR(NULL); + ereg edyn = need_dyn ? emit_gen_size(ctx, ALLOC_STACK, dyn_size) : LOAD_CONST_PTR(NULL); args = get_tmp_args(ctx, 4); args[0] = LOAD_MEM_PTR(obj,HL_WSIZE); @@ -865,10 +974,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 4, dst->t); - emit_gen(ctx, FREE_STACK, ENULL, ENULL, o->p3 + (need_dyn ? dyn_size : 0)); + emit_gen_size(ctx, FREE_STACK, o->p3 + (need_dyn ? dyn_size : 0)); patch_jump(ctx, jend); - STORE(dst, emit_phy(ctx, v1, v2)); + STORE(dst, emit_phi(ctx, v1, v2)); } break; default: @@ -893,7 +1002,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg offs = OFFSET(LOAD(ra),LOAD(rb),1,0); ereg val = LOAD_MEM(offs, 0, dst->t); - if( o->op != OGetMem ) val = emit_conv(ctx, val, I32, false); + if( o->op != OGetMem ) val = emit_conv(ctx, val, M_I32, false); STORE(dst, val); } break; @@ -903,7 +1012,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg offs = OFFSET(LOAD(dst), LOAD(ra),1,0); ereg val = LOAD(rb); - if( o->op != OSetMem ) val = emit_conv(ctx, val, I32, false); + if( o->op != OSetMem ) val = emit_conv(ctx, val, M_I32, false); STORE_MEM(offs, 0, val); } break; @@ -913,14 +1022,14 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OGetType: { ereg r = LOAD(ra); - emit_gen(ctx, TEST, r, ENULL, OJNotNull); + emit_test(ctx, r, OJNotNull); int jidx = emit_jump(ctx, true); ereg v1 = LOAD_CONST_PTR(&hlt_void); int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); ereg v2 = LOAD_MEM_PTR(r,0); patch_jump(ctx, jend); - STORE(dst, emit_phy(ctx, v1, v2)); + STORE(dst, emit_phi(ctx, v1, v2)); } break; case OGetArray: @@ -967,9 +1076,11 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case ORef: { - ereg addr = emit_gen(ctx, ALLOC_GLOBAL_STACK, ENULL, ENULL, hl_type_size(ra->t)); - STORE_MEM(addr, 0, LOAD(ra)); - STORE(dst, addr); + ereg ref = resolve_ref(ctx, ra->id); + if( ref.index < 0 ) emit_assert(); + if( ra->current.index >= 0 ) STORE_MEM(ref, 0, LOAD(ra)); + ra->current.index = -1; // ref will be modified + STORE(dst, ref); } break; case OUnref: @@ -1036,7 +1147,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case ONullCheck: { - emit_gen(ctx, TEST, LOAD(dst), ENULL, OJNotNull); + emit_test(ctx, LOAD(dst), OJNotNull); int jok = emit_jump(ctx, true); // ----- DETECT FIELD ACCESS ---------------- @@ -1072,7 +1183,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case OSafeCast: - STORE(dst, emit_dyn_cast(ctx, LOAD(ra), dst->t)); + STORE(dst, emit_dyn_cast(ctx, LOAD(ra), ra->t, dst->t)); break; case ODynGet: { @@ -1100,7 +1211,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case OTrap: { - ereg st = emit_gen(ctx, ALLOC_STACK, ENULL, ENULL, sizeof(hl_trap_ctx)); + ereg st = emit_gen_size(ctx, ALLOC_STACK, sizeof(hl_trap_ctx)); ereg thread, current_addr; static hl_thread_info *tinf = NULL; @@ -1163,9 +1274,9 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { fun = _setjmp; #endif ereg ret = emit_native_call(ctx, fun, args, nargs, &hlt_i32); - emit_gen(ctx, TEST, ret, ENULL, OJNull); + emit_test(ctx, ret, OJNull); int jskip = emit_jump(ctx, true); - emit_gen(ctx, FREE_STACK, ENULL, ENULL, sizeof(hl_trap_ctx)); + emit_gen_size(ctx, FREE_STACK, sizeof(hl_trap_ctx)); STORE(dst, tinf ? LOAD_CONST_PTR(&tinf->exc_value) : LOAD_MEM_PTR(thread,(int)(int_val)&tinf->exc_value)); int jtrap = emit_jump(ctx, false); @@ -1207,16 +1318,16 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } # endif - emit_gen(ctx, FREE_STACK, ENULL, ENULL, sizeof(hl_trap_ctx)); + emit_gen_size(ctx, FREE_STACK, sizeof(hl_trap_ctx)); } break; case OSwitch: { ereg v = LOAD(dst); int count = o->p2; - emit_gen(ctx, CMP, v, LOAD_CONST(count,&hlt_i32), OJUGte); + emit_gen_ext(ctx, CMP, v, LOAD_CONST(count,&hlt_i32), M_I32, OJUGte); int jdefault = emit_jump(ctx, true); - emit_gen(ctx, JUMP_TABLE, v, ENULL, count); + emit_gen_ext(ctx, JUMP_TABLE, v, ENULL, 0, count); for(int i=0; iextra[i]); @@ -1264,19 +1375,33 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } } +static void hl_emit_flush( emit_ctx *ctx ) { + int i = 0; + while( i < ctx->jump_count ) { + int pos = ctx->jump_regs[i++]; + einstr *e = ctx->instrs + pos; + int target = ctx->jump_regs[i++]; + e->size_offs = ctx->pos_map[target] - (pos + 1); + } + ctx->jump_count = 0; + ctx->pos_map[ctx->fun->nops] = -1; +} + int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { int i; ctx->mod = m; ctx->fun = f; ctx->emit_pos = 0; ctx->trap_count = 0; + ctx->ref_count = 0; + current_ctx = ctx; if( f->nregs > ctx->max_regs ) { free(ctx->vregs); ctx->vregs = (vreg*)malloc(sizeof(vreg) * (f->nregs + 1)); if( ctx->vregs == NULL ) return false; - for(i=ctx->max_regs;inregs;i++) + for(i=0;inregs;i++) R(i)->id = i; ctx->max_regs = f->nregs; } @@ -1291,6 +1416,18 @@ int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { STORE(R(i), emit_gen(ctx, LOAD_ARG, ENULL, ENULL, hl_type_mode(f->type->fun->args[i]))); } + for(i=f->nops-1;i>=0;i--) { + hl_opcode *o = f->ops + i; + if( o->op == ORef ) { + ereg ref = resolve_ref(ctx, o->p2); + if( ref.index >= 0 ) continue; + if( ctx->ref_count == MAX_REFS ) emit_error("Too many refs"); + ctx->refs[ctx->ref_count].r = emit_gen_size(ctx, ALLOC_STACK, hl_type_size(R(o->p2)->t)); + ctx->refs[ctx->ref_count].reg = o->p2; + ctx->ref_count++; + } + } + if( f->nops >= ctx->pos_map_size ) { free(ctx->pos_map); ctx->pos_map = (int*)malloc(sizeof(int) * (f->nops+1)); @@ -1305,17 +1442,8 @@ int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { emit_opcode(ctx,f->ops + op_pos); } - // patch jumps - i = 0; - while( i < ctx->jump_count ) { - int pos = ctx->jump_regs[i++]; - einstr *e = ctx->instrs + pos; - int target = ctx->jump_regs[i++]; - e->mode = ctx->pos_map[target] - (pos + 1); - } - ctx->jump_count = 0; - - ctx->pos_map[f->nops] = -1; + hl_emit_flush(ctx); + current_ctx = NULL; return true; } @@ -1323,6 +1451,7 @@ emit_ctx *hl_emit_alloc() { emit_ctx *ctx = (emit_ctx*)malloc(sizeof(emit_ctx)); if( ctx == NULL ) return NULL; memset(ctx,0,sizeof(emit_ctx)); + if( sizeof(einstr) != 16 ) emit_assert(); return ctx; } @@ -1331,6 +1460,7 @@ void hl_emit_free( emit_ctx *ctx, h_bool can_reset ) { free(ctx->instrs); free(ctx->pos_map); free(ctx->jump_regs); + free(ctx->args_data); free(ctx); } @@ -1388,6 +1518,12 @@ static void hl_dump_op( emit_ctx *ctx, hl_opcode *op ) { printf("%d", op->p2); printf(",%d", op->p3); printf(",%d", (int)(int_val)op->extra); + } else if( op->op == OSwitch ) { + for(int i=0;ip2;i++) { + if( i != 0 ) printf(","); + printf("@%X", (op->extra[i] + ctx->op_pos + 1)); + } + printf(",def=@%X", op->p3 + ctx->op_pos + 1); } else { if( count == 0xFF ) count = op->p3; for(int i=0;i= -0x10000 && (int)value <= 0x10000 ) + printf("%d",(int)value); + else + printf("0x%X",(int)value); + break; + case M_F32: + tmp.v = value; + printf("%f",tmp.f); + break; + case M_F64: + tmp.v = value; + printf("%g",tmp.d); + break; + default: + if( value == 0 ) + printf("NULL"); + else + printf("0x%llX",value); + break; + } +} + +static void hl_dump_fun_name( hl_function *f ) { + if( f->obj ) + uprintf(USTR("%s.%s"),f->obj->name,f->field.name); + else if( f->field.ref ) + uprintf(USTR("%s.~%s.%d"),f->field.ref->obj->name, f->field.ref->field.name, f->ref); + printf("[%X]", f->findex); +} + +static void hl_dump_args( emit_ctx *ctx, einstr *e ) { + ereg *v = hl_emit_get_args(ctx, e); + printf("("); + for(int i=0;inargs;i++) { + if( i != 0 ) printf(","); + printf("@%X", v[i].index); + } + printf(")"); +} + + +typedef struct { const char *name; void *ptr; } named_ptr; +static void hl_dump_ptr_name( emit_ctx *ctx, void *ptr ) { +# define N(v) ptr_names[i].name = #v; ptr_names[i].ptr = v; i++ +# define N2(n,v) ptr_names[i].name = n; ptr_names[i].ptr = v; i++ + static named_ptr ptr_names[256] = { NULL }; + int i = 0; + if( !ptr_names[0].ptr ) { + i = 0; + N(hl_alloc_dynbool); + N(hl_alloc_dynamic); + N(hl_alloc_obj); + N(hl_alloc_dynobj); + N(hl_alloc_virtual); + N(hl_alloc_closure_ptr); + N(hl_dyn_call); + N(hl_dyn_call_obj); + N(hl_throw); + N(hl_rethrow); + N(hl_to_virtual); + N(hl_alloc_enum); + N(hl_dyn_castf); + N(hl_dyn_castd); + N(hl_dyn_casti64); + N(hl_dyn_casti); + N(hl_dyn_castp); + N2("null_field",hl_stub_null_field_access); + N2("null_access",hl_stub_null_access); + N(hl_get_thread); + N(setjmp); + N(_setjmp); + N2("assert",hl_stub_assert); + } +# undef N +# undef N2 + while( true ) { + named_ptr p = ptr_names[i++]; + if( !p.ptr ) break; + if( p.ptr == ptr ) { + printf("<%s>",p.name); + return; + } + } + for(i=0;imod->code->nnatives;i++) { + hl_native *n = ctx->mod->code->natives + i; + if( ctx->mod->functions_ptrs[n->findex] == ptr ) { + printf("<%s.%s>",n->lib,n->name); + return; + } + } + printf("",(uint64)ptr); +} + void hl_emit_dump( emit_ctx *ctx ) { int i; int cur_op = 0; hl_function *f = ctx->fun; int nargs = f->type->fun->nargs; - printf("function %X(", f->findex); + hl_emit_flush(ctx); // if it not was not before (in case of dump during emit) + printf("function "); + hl_dump_fun_name(f); + printf("("); for(i=0;i 0 ) printf(","); uprintf(USTR("R%d"), i); @@ -1443,36 +1688,82 @@ void hl_emit_dump( emit_ctx *ctx ) { einstr *e = ctx->instrs + i; printf("\t\t@%X %s", i, op_names[e->op]); switch( e->op ) { - case LOAD_ADDR: - case LOAD_IMM: - case LOAD_ARG: - printf("%s", emit_mode_str(e->mode)); + case TEST: + case CMP: + case BINOP: + case UNOP: + printf("-%s", hl_op_name(e->size_offs)+2); break; default: break; } + if( e->mode ) + printf("%s", emit_mode_str(e->mode)); switch( e->op ) { + case CALL_FUN: + printf(" "); + { + int fid = ctx->mod->functions_indexes[e->a.index]; + hl_code *code = ctx->mod->code; + if( fid < code->nfunctions ) { + hl_dump_fun_name(&code->functions[fid]); + } else { + printf("???"); + } + } + hl_dump_args(ctx,e); + break; + case CALL_REG: + printf(" @%X", e->a.index); + hl_dump_args(ctx,e); + break; + case CALL_PTR: + printf(" "); + hl_dump_ptr_name(ctx, (void*)e->value); + hl_dump_args(ctx,e); + break; case JUMP: case JCOND: - printf(" @%X", i + 1 + e->mode); + printf(" @%X", i + 1 + e->size_offs); + break; + case LOAD_IMM: + printf(" "); + dump_value(e->value, e->mode); + break; + case ALLOC_STACK: + printf(" %d", e->size_offs); + break; + case LOAD_ADDR: + if( (e->b.index>>8) ) + printf(" @%X[%Xh]", e->a.index, e->b.index); + else + printf(" @%X[%d]", e->a.index, e->b.index); break; case STORE: { - int offs = e->mode >> 8; - printf("%s", emit_mode_str(e->mode&0xFF)); + int offs = e->size_offs; if( offs == 0 ) - printf(" [@%X] := @%X", e->args.a.index, e->args.b.index); + printf(" [@%X] = @%X", e->a.index, e->b.index); else - printf(" @%X[%d] := @%X", e->args.a.index, offs, e->args.b.index); + printf(" @%X[%d] = @%X", e->a.index, offs, e->b.index); } break; default: - if( e->args.a.index >= 0 ) printf(" @%X", e->args.a.index); - if( e->args.b.index >= 0 ) printf(", @%X", e->args.b.index); + if( e->a.index >= 0 ) { + printf(" @%X", e->a.index); + if( e->b.index >= 0 ) printf(", @%X", e->b.index); + } break; } printf("\n"); } + // interrupted + if( cur_op < f->nops ) { + printf("@%X ", cur_op); + ctx->op_pos = cur_op; + hl_dump_op(ctx, f->ops + cur_op); + printf("\n\t\t...\n"); + } printf("\n\n"); fflush(stdout); } \ No newline at end of file diff --git a/src/module.c b/src/module.c index 6e0b6ef8b..1dbd9f2cf 100644 --- a/src/module.c +++ b/src/module.c @@ -717,7 +717,6 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { hl_emit_free(ctx, false); return 0; } - hl_emit_dump(ctx); m->functions_ptrs[f->findex] = (void*)(int_val)fpos; } m->jit_code = hl_emit_code(ctx, m, &m->codesize, &m->jit_debug, NULL); From 42995f1a8612c09a3840b8438c756402c846855d Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 15 Mar 2026 22:27:00 +0100 Subject: [PATCH 04/83] minor --- src/emit.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/emit.c b/src/emit.c index 8777c4f6b..6f58733a7 100644 --- a/src/emit.c +++ b/src/emit.c @@ -295,7 +295,7 @@ static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { } static ereg emit_gen_size( emit_ctx *ctx, emit_op op, int size_offs ) { - return emit_gen_ext(ctx,op,ENULL,ENULL,0,size_offs); + return emit_gen_ext(ctx,op,ENULL,ENULL,op==ALLOC_STACK ? M_PTR : 0,size_offs); } static int emit_jump( emit_ctx *ctx, bool cond ) { @@ -477,7 +477,9 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, } static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { - return emit_gen(ctx, PHI, v1, v2, 0); + int mode = ctx->instrs[v1.index].mode; + if( mode != ctx->instrs[v2.index].mode ) emit_assert(); + return emit_gen(ctx, PHI, v1, v2, mode); } static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode mode, bool _unsigned ) { @@ -1613,6 +1615,7 @@ typedef struct { const char *name; void *ptr; } named_ptr; static void hl_dump_ptr_name( emit_ctx *ctx, void *ptr ) { # define N(v) ptr_names[i].name = #v; ptr_names[i].ptr = v; i++ # define N2(n,v) ptr_names[i].name = n; ptr_names[i].ptr = v; i++ +# define DYN(p) N2("dyn_get" #p, hl_dyn_get##p); N2("dyn_set" #p, hl_dyn_set##p); N2("dyn_cast" #p, hl_dyn_cast##p) static named_ptr ptr_names[256] = { NULL }; int i = 0; if( !ptr_names[0].ptr ) { @@ -1629,11 +1632,11 @@ static void hl_dump_ptr_name( emit_ctx *ctx, void *ptr ) { N(hl_rethrow); N(hl_to_virtual); N(hl_alloc_enum); - N(hl_dyn_castf); - N(hl_dyn_castd); - N(hl_dyn_casti64); - N(hl_dyn_casti); - N(hl_dyn_castp); + DYN(f); + DYN(d); + DYN(i64); + DYN(i); + DYN(p); N2("null_field",hl_stub_null_field_access); N2("null_access",hl_stub_null_access); N(hl_get_thread); @@ -1746,6 +1749,8 @@ void hl_emit_dump( emit_ctx *ctx ) { printf(" [@%X] = @%X", e->a.index, e->b.index); else printf(" @%X[%d] = @%X", e->a.index, offs, e->b.index); + if( e->mode == 0 || e->mode != ctx->instrs[e->b.index].mode ) + printf(" ???"); } break; default: From fabdc4756f8c873ba7e143cd26e42b4b127f1aa1 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Tue, 24 Mar 2026 22:32:01 +0100 Subject: [PATCH 05/83] more work on phi emit and loop managements (ongoing) --- src/emit.c | 637 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 479 insertions(+), 158 deletions(-) diff --git a/src/emit.c b/src/emit.c index 6f58733a7..47ace840c 100644 --- a/src/emit.c +++ b/src/emit.c @@ -89,6 +89,7 @@ typedef enum { M_F64, M_PTR, M_VOID, + M_NORET, } emit_mode; @@ -100,12 +101,19 @@ typedef struct { hl_type *t; int id; ereg current; + bool written; } vreg; typedef struct { - unsigned char op; - unsigned char mode; - unsigned short nargs; + union { + struct { + unsigned char op; + unsigned char mode; + unsigned char nargs; + unsigned char _unused; + }; + int header; + }; int size_offs; union { struct { @@ -120,6 +128,30 @@ typedef struct { #define MAX_TRAPS 32 #define MAX_REFS 512 // TODO : different impl +typedef struct _linked_inf linked_inf; +typedef struct _emit_block emit_block; + +struct _linked_inf { + int id; + void *ptr; + linked_inf *next; +}; + +struct _emit_block { + int id; + int start_pos; + int wait_nexts; + linked_inf *preds; + linked_inf *written_vars; + linked_inf *incomplete_phis; +}; + +typedef struct { + int *data; + int max; + int cur; +} int_alloc; + struct _emit_ctx { hl_module *mod; hl_function *fun; @@ -141,15 +173,14 @@ struct _emit_ctx { int trap_count; int ref_count; - int *args_data; - int args_data_size; - int args_data_pos; - + int_alloc args_data; + int_alloc jump_regs; + int_alloc block_writes; + int_alloc phi_gather; - int *jump_regs; - int jump_count; - int max_jumps; - + hl_alloc falloc; + emit_block *current_block; + linked_inf *arrival_points; void *closure_list; // TODO : patch with good addresses }; @@ -160,25 +191,27 @@ struct _emit_ctx { #define LOAD(r) emit_load_reg(ctx, r) #define STORE(r, v) emit_store_reg(ctx, r, v) -#define LOAD_CONST(v, t) emit_load_const(ctx, (uint64)v, t) +#define LOAD_CONST(v, t) emit_load_const(ctx, (uint64)(v), t) #define LOAD_CONST_PTR(v) LOAD_CONST(v,&hlt_bytes) #define LOAD_MEM(v, offs, t) emit_load_mem(ctx, v, offs, t) #define LOAD_MEM_PTR(v, offs) LOAD_MEM(v, offs, &hlt_bytes) #define STORE_MEM(to, offs, v) emit_store_mem(ctx, to, offs, v) -#define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR((obj),0),HL_WSIZE*2),HL_WSIZE*(id)) -#define OFFSET(base,index,mult,offset) emit_gen_ext(ctx, LEA, base, index, 0, (mult) | ((offset) << 8)) +#define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR(obj,0),HL_WSIZE*2),HL_WSIZE*(id)) +#define OFFSET(base,index,mult,offset) emit_gen_ext(ctx, LEA, base, index, M_PTR, (mult) | ((offset) << 8)) #define BREAK() emit_gen(ctx, DEBUG_BREAK, ENULL, ENULL, 0) #define CUR_REG() __current_reg(ctx) #define HDYN_VALUE 8 -#define IS_FLOAT(t) (t->kind == HF64 || t->kind == HF32) +#define IS_FLOAT(t) ((t)->kind == HF64 || (t)->kind == HF32) static hl_type hlt_ui8 = { HUI8, 0 }; static hl_type hlt_ui16 = { HUI16, 0 }; static ereg ENULL = {-1}; static emit_ctx *current_ctx = NULL; +//#define BLOCK_DEBUG + static void _emit_error( const char *msg, int line ) { printf("*** EMIT ERROR line %d (%s) ****\n", line, msg); if( current_ctx ) hl_emit_dump(current_ctx); @@ -191,6 +224,88 @@ static void hl_stub_null_field_access() { emit_assert(); } static void hl_stub_null_access() { emit_assert(); } static void hl_stub_assert() { emit_assert(); } +static void int_alloc_reset( int_alloc *a ) { + a->cur = 0; +} + +static void int_alloc_free( int_alloc *a ) { + free(a->data); + a->cur = 0; + a->max = 0; + a->data = NULL; +} + +static int *int_alloc_get( int_alloc *a, int count ) { + while( a->cur + count > a->max ) { + int next_size = a->max ? a->max << 1 : 128; + int *new_data = (int*)malloc(sizeof(int) * next_size); + if( new_data == NULL ) emit_error("Out of memory"); + memcpy(new_data, a->data, sizeof(int) * a->cur); + free(a->data); + a->data = new_data; + a->max = next_size; + } + int *ptr = a->data + a->cur; + a->cur += count; + return ptr; +} + +static void int_alloc_store( int_alloc *a, int v ) { + *int_alloc_get(a,1) = v; +} + +static void int_alloc_store_unique( int_alloc *a, int v ) { + int i = 0; + while( i < a->cur ) { + if( a->data[i] == v ) return; + i++; + } + *int_alloc_get(a,1) = v; +} + +static linked_inf *link_add( emit_ctx *ctx, int id, void *ptr, linked_inf *head ) { + linked_inf *l = hl_malloc(&ctx->falloc,sizeof(linked_inf)); + l->id = id; + l->ptr = ptr; + l->next = head; + return l; +} + +static linked_inf *link_add_sort_unique( emit_ctx *ctx, int id, void *ptr, linked_inf *head ) { + linked_inf *prev = NULL; + linked_inf *cur = head; + while( cur && cur->id < id ) { + prev = cur; + cur = cur->next; + } + // check duplicate + while( cur && cur->id == id ) { + if( cur->ptr == ptr ) + return head; + cur = cur->next; + } + // insert + linked_inf *l = hl_malloc(&ctx->falloc,sizeof(linked_inf)); + l->id = id; + l->ptr = ptr; + if( !prev ) { + l->next = head; + return l; + } else { + l->next = prev->next; + prev->next = l; + return head; + } +} + +static void *link_sort_lookup( linked_inf *head, int id ) { + while( head && head->id < id ) + head = head->next; + if( head && head->id == id ) + return head->ptr; + return NULL; +} + static emit_mode hl_type_mode( hl_type *t ) { if( t->kind == HVOID ) return M_VOID; @@ -203,6 +318,10 @@ static emit_mode hl_type_mode( hl_type *t ) { return M_PTR; } +static void mark_no_return( emit_ctx *ctx ) { + ctx->instrs[ctx->emit_pos-1].mode = M_NORET; +} + void hl_jit_patch_method( void*fun, void**newt ) { emit_assert(); } @@ -228,7 +347,7 @@ static ereg resolve_ref( emit_ctx *ctx, int reg ) { static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { if( ctx->emit_pos == ctx->max_instrs ) { int pos = ctx->emit_pos; - int next_size = ctx->max_instrs ? (ctx->max_instrs * 3) >> 1 : 256; + int next_size = ctx->max_instrs ? (ctx->max_instrs << 1) : 256; einstr *instrs = (einstr*)malloc(sizeof(einstr) * next_size); if( instrs == NULL ) emit_error("Out of memory"); memcpy(instrs, ctx->instrs, pos * sizeof(einstr)); @@ -236,7 +355,8 @@ static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { free(ctx->instrs); ctx->instrs = instrs; ctx->max_instrs = next_size; - } + } else if( (ctx->emit_pos & 0xFF) == 0 ) + memset(ctx->instrs + ctx->emit_pos, 0, 256 * sizeof(einstr)); einstr *e = ctx->instrs + ctx->emit_pos++; e->op = op; return e; @@ -252,24 +372,19 @@ static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { if( count < 0 || count > 64 ) emit_assert(); - e->nargs = (unsigned short)count; + e->nargs = (unsigned char)count; if( count == 0 ) return; if( count == 1 ) { e->size_offs = args[0].index; return; } - if( ctx->args_data_pos + count > ctx->args_data_size ) { - int next_size = ctx->args_data_size ? ctx->args_data_size << 1 : 128; - int *args = (int*)malloc(sizeof(int) * next_size); - if( args == NULL ) emit_error("Out of memory"); - memcpy(args, ctx->args_data, sizeof(int) * ctx->args_data_pos); - free(ctx->args_data); - ctx->args_data = args; - ctx->args_data_size = next_size; + if( e->op == PHI && count <= 3 ) { + memcpy(&e->size_offs, args, sizeof(int) * count); + return; } - e->size_offs = ctx->args_data_pos; - memcpy(ctx->args_data + ctx->args_data_pos, args, sizeof(int) * count); - ctx->args_data_pos += count; + int *args_data = int_alloc_get(&ctx->args_data, count); + e->size_offs = (int)(args_data - ctx->args_data.data); + memcpy(args_data, args, sizeof(int) * count); } ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ) { @@ -277,7 +392,9 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ) { return NULL; if( e->nargs == 1 ) return (ereg*)&e->size_offs; - return (ereg*)(ctx->args_data + e->size_offs); + if( e->op == PHI && e->nargs <= 3 ) + return (ereg*)&e->size_offs; + return (ereg*)(ctx->args_data.data + e->size_offs); } static ereg emit_gen_ext( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode, int size_offs ) { @@ -298,6 +415,18 @@ static ereg emit_gen_size( emit_ctx *ctx, emit_op op, int size_offs ) { return emit_gen_ext(ctx,op,ENULL,ENULL,op==ALLOC_STACK ? M_PTR : 0,size_offs); } +static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { + unsigned char mode = ctx->instrs[v1.index].mode; + if( mode != ctx->instrs[v2.index].mode ) emit_assert(); + einstr *e = emit_instr(ctx, PHI); + e->mode = mode; + ereg args[2]; + args[0] = v1; + args[1] = v2; + store_args(ctx, e, args, 2); + return CUR_REG(); +} + static int emit_jump( emit_ctx *ctx, bool cond ) { int p = ctx->emit_pos; emit_gen(ctx, cond ? JCOND : JUMP, ENULL, ENULL, 0); @@ -308,18 +437,57 @@ static void patch_jump( emit_ctx *ctx, int jpos ) { ctx->instrs[jpos].size_offs = ctx->emit_pos - (jpos + 1); } +static emit_block *alloc_block( emit_ctx *ctx ) { + return hl_zalloc(&ctx->falloc, sizeof(emit_block)); +} + +static void block_add_prec( emit_ctx *ctx, emit_block *b, emit_block *p ) { + b->preds = link_add(ctx,0,p,b->preds); +# ifdef BLOCK_DEBUG + printf(" PRED %d\n",p->id); +# endif +} + +static void split_block( emit_ctx *ctx ) { + // flush previous block + int i = 0; + emit_block *cur = ctx->current_block; + while( i < ctx->block_writes.cur ) { + vreg *r = R(ctx->block_writes.data[i++]); + cur->written_vars = link_add_sort_unique(ctx,r->id,(void*)(int_val)(r->current.index+1),cur->written_vars); +# ifdef BLOCK_DEBUG + printf(" WRITES R%d @%X\n", r->id, r->current.index); +# endif + r->current.index = -1; + r->written = false; + } + int_alloc_reset(&ctx->block_writes); + // split + emit_block *b = alloc_block(ctx); + b->id = ctx->current_block->id + 1; + b->start_pos = ctx->emit_pos; +# ifdef BLOCK_DEBUG + printf("BLOCK #%d@%X\n",b->id,ctx->op_pos); +# endif + einstr *eprev = &ctx->instrs[ctx->emit_pos-1]; + if( (eprev->op != JUMP && eprev->op != RET) || ctx->fun->ops[ctx->op_pos].op == OTrap ) + block_add_prec(ctx, b, ctx->current_block); + while( ctx->arrival_points && ctx->arrival_points->id == ctx->op_pos ) { + block_add_prec(ctx, b, (emit_block*)ctx->arrival_points->ptr); + ctx->arrival_points = ctx->arrival_points->next; + } + ctx->current_block = b; +} + static void register_jump( emit_ctx *ctx, int jpos, int offs ) { - if( ctx->jump_count == ctx->max_jumps ) { - int next_size = ctx->max_jumps ? ctx->max_jumps << 1 : 64; - int *jumps = (int*)malloc(sizeof(int) * next_size); - if( jumps == NULL ) emit_error("Out of memory"); - memcpy(jumps, ctx->jump_regs, ctx->jump_count * sizeof(int)); - free(ctx->jump_regs); - ctx->jump_regs = jumps; - ctx->max_jumps = next_size; + int target = offs + ctx->op_pos + 1; + int_alloc_store(&ctx->jump_regs, jpos); + int_alloc_store(&ctx->jump_regs, target); + if( offs > 0 ) { + ctx->arrival_points = link_add_sort_unique(ctx, target, ctx->current_block, ctx->arrival_points); + if( ctx->arrival_points->id != ctx->op_pos + 1 && ctx->fun->ops[ctx->op_pos].op != OSwitch ) + split_block(ctx); } - ctx->jump_regs[ctx->jump_count++] = jpos; - ctx->jump_regs[ctx->jump_count++] = offs + ctx->op_pos + 1; } static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { @@ -339,13 +507,18 @@ static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { if( to->t->kind == HVOID ) return; + if( v.index < 0 ) emit_assert(); + if( !to->written ) { + to->written = true; + int_alloc_store(&ctx->block_writes,to->id); + } to->current = v; } static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int nargs, hl_type *ret ) { einstr *e = emit_instr(ctx, CALL_PTR); e->mode = hl_type_mode(ret); - e->value = (int64)native_ptr; + e->value = (int_val)native_ptr; store_args(ctx, e, args, nargs); return CUR_REG(); } @@ -362,12 +535,47 @@ static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { emit_gen_ext(ctx, TEST, v, ENULL, ctx->instrs[v.index].mode, o); } +static void emit_gather_phi_rec( emit_ctx *ctx, emit_block *b, vreg *r ) { + if( b->wait_nexts > 0 ) { + if( ctx->instrs[ctx->emit_pos-1].op != PHI ) emit_assert(); + int_alloc_store_unique(&ctx->phi_gather, -1); + b->incomplete_phis = link_add_sort_unique(ctx, ctx->emit_pos - 1, r, b->incomplete_phis); + return; + } + int eid = (int)(int_val)link_sort_lookup(b->written_vars, r->id) - 1; + if( eid >= 0 ) { + if( eid != ctx->emit_pos - 1 ) + int_alloc_store_unique(&ctx->phi_gather, eid); + } else { + b->written_vars = link_add_sort_unique(ctx, r->id, (void*)(int_val)ctx->emit_pos, b->written_vars); + linked_inf *l = b->preds; + while( l ) { + emit_gather_phi_rec(ctx, (emit_block*)l->ptr, r); + l = l->next; + } + } +} + static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { if( r->current.index < 0 ) { ereg ref = resolve_ref(ctx, r->id); - if( ref.index < 0 ) emit_assert(); - // reload from ref - return LOAD_MEM(ref,0,r->t); + if( ref.index >= 0 ) + return LOAD_MEM(ref,0,r->t); + einstr *p = emit_instr(ctx, PHI); + int_alloc_reset(&ctx->phi_gather); + emit_gather_phi_rec(ctx, ctx->current_block, r); + if( ctx->phi_gather.cur == 1 && ctx->phi_gather.data[0] != -1 ) { +# ifdef BLOCK_DEBUG + printf(" SKIP-PHI @%X\n",ctx->emit_pos-1); +# endif + // remove phi + ctx->emit_pos--; + r->current.index = ctx->phi_gather.data[0]; + } else { + if( ctx->phi_gather.cur == 0 ) emit_assert(); + store_args(ctx, p, (ereg*)ctx->phi_gather.data, ctx->phi_gather.cur); + r->current = CUR_REG(); + } } return r->current; } @@ -476,12 +684,6 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, } } -static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { - int mode = ctx->instrs[v1.index].mode; - if( mode != ctx->instrs[v2.index].mode ) emit_assert(); - return emit_gen(ctx, PHI, v1, v2, mode); -} - static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode mode, bool _unsigned ) { return emit_gen(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, ENULL, mode); } @@ -513,6 +715,165 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { return r; } +static void emit_opcode( emit_ctx *ctx, hl_opcode *o ); + +static void hl_emit_flush( emit_ctx *ctx ) { + int i = 0; + while( i < ctx->jump_regs.cur ) { + int pos = ctx->jump_regs.data[i++]; + einstr *e = ctx->instrs + pos; + int target = ctx->jump_regs.data[i++]; + e->size_offs = ctx->pos_map[target] - (pos + 1); + } + ctx->pos_map[ctx->fun->nops] = -1; +} + +int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { + int i; + ctx->mod = m; + ctx->fun = f; + ctx->emit_pos = 0; + ctx->trap_count = 0; + ctx->ref_count = 0; + int_alloc_reset(&ctx->args_data); + int_alloc_reset(&ctx->jump_regs); + int_alloc_reset(&ctx->block_writes); + current_ctx = ctx; + hl_free(&ctx->falloc); + ctx->current_block = alloc_block(ctx); + ctx->arrival_points = NULL; +# ifdef BLOCK_DEBUG + printf("---- begin ----\n"); +# endif + if( f->nregs > ctx->max_regs ) { + free(ctx->vregs); + ctx->vregs = (vreg*)malloc(sizeof(vreg) * (f->nregs + 1)); + if( ctx->vregs == NULL ) + return false; + for(i=0;inregs;i++) + R(i)->id = i; + ctx->max_regs = f->nregs; + } + + for(i=0;inregs;i++) { + vreg *r = R(i); + r->t = f->regs[i]; + r->current = ENULL; + r->written = false; + } + + for(i=0;itype->fun->nargs;i++) { + STORE(R(i), emit_gen(ctx, LOAD_ARG, ENULL, ENULL, hl_type_mode(f->type->fun->args[i]))); + } + + for(i=f->nops-1;i>=0;i--) { + hl_opcode *o = f->ops + i; + if( o->op == ORef ) { + ereg ref = resolve_ref(ctx, o->p2); + if( ref.index >= 0 ) continue; + if( ctx->ref_count == MAX_REFS ) emit_error("Too many refs"); + ctx->refs[ctx->ref_count].r = emit_gen_size(ctx, ALLOC_STACK, hl_type_size(R(o->p2)->t)); + ctx->refs[ctx->ref_count].reg = o->p2; + ctx->ref_count++; + } + } + + if( f->nops >= ctx->pos_map_size ) { + free(ctx->pos_map); + ctx->pos_map = (int*)malloc(sizeof(int) * (f->nops+1)); + if( ctx->pos_map == NULL ) + return false; + ctx->pos_map_size = f->nops; + } + + for(int op_pos=0;op_posnops;op_pos++) { + ctx->op_pos = op_pos; + ctx->pos_map[op_pos] = ctx->emit_pos; + if( ctx->arrival_points && ctx->arrival_points->id == op_pos ) + split_block(ctx); + emit_opcode(ctx,f->ops + op_pos); + } + + hl_emit_flush(ctx); + current_ctx = NULL; + return true; +} + +emit_ctx *hl_emit_alloc() { + emit_ctx *ctx = (emit_ctx*)malloc(sizeof(emit_ctx)); + if( ctx == NULL ) return NULL; + memset(ctx,0,sizeof(emit_ctx)); + if( sizeof(einstr) != 16 ) emit_assert(); + hl_alloc_init(&ctx->falloc); + return ctx; +} + +void hl_emit_free( emit_ctx *ctx, h_bool can_reset ) { + hl_free(&ctx->falloc); + free(ctx->vregs); + free(ctx->instrs); + free(ctx->pos_map); + int_alloc_free(&ctx->jump_regs); + int_alloc_free(&ctx->args_data); + int_alloc_free(&ctx->block_writes); + free(ctx); +} + +void hl_emit_init( emit_ctx *ctx, hl_module *m ) { +} + +void hl_emit_reset( emit_ctx *ctx, hl_module *m ) { +} + +static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { + if( b->start_pos < target ) + return false; + if( b->start_pos == target ) { + b->wait_nexts--; + b->preds = link_add(ctx,0,ctx->current_block,b->preds); + split_block(ctx); + if( b->wait_nexts == 0 ) { +# ifdef BLOCK_DEBUG + printf(" SEAL #%d\n",b->id); +# endif + // seal block + linked_inf *l = b->incomplete_phis; + while( l ) { + einstr *e = ctx->instrs + l->id; + vreg *r = (vreg*)l->ptr; + if( e->op != PHI ) emit_assert(); + int prev_emit = ctx->emit_pos; + ctx->emit_pos = l->id + 1; + int_alloc_reset(&ctx->phi_gather); + emit_gather_phi_rec(ctx, b, r); + ctx->emit_pos = prev_emit; + store_args(ctx, e, (ereg*)ctx->phi_gather.data, ctx->phi_gather.cur); + l = l->next; + } + b->incomplete_phis = NULL; + } + return true; + } + linked_inf *l = b->preds; + while( l ) { + emit_block *n = (emit_block*)l->ptr; + if( n->start_pos < b->start_pos && seal_block_rec(ctx,n,target) ) + return true; + l = l->next; + } + return false; +} + +static void register_block_jump( emit_ctx *ctx, int offs, bool cond ) { + int jidx = emit_jump(ctx, cond); + register_jump(ctx, jidx, offs); + if( offs < 0 ) { + int target = ctx->pos_map[ctx->op_pos + 1 + offs]; + emit_block *b = ctx->current_block; + if( !seal_block_rec(ctx, b, target) ) emit_assert(); + } +} + static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { vreg *dst = R(o->p1); vreg *ra = R(o->p2); @@ -626,8 +987,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OJNull: { emit_test(ctx, LOAD(dst), o->op); - int jidx = emit_jump(ctx, true); - register_jump(ctx, jidx, o->p2); + register_block_jump(ctx, o->p2, true); } break; case OJEq: @@ -642,15 +1002,11 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OJNotGte: { emit_gen_ext(ctx, CMP, LOAD(dst), LOAD(ra), hl_type_mode(dst->t), o->op); - int jidx = emit_jump(ctx, true); - register_jump(ctx, jidx, o->p3); + register_block_jump(ctx, o->p3, true); } break; case OJAlways: - { - int jidx = emit_jump(ctx, false); - register_jump(ctx, jidx, o->p1); - } + register_block_jump(ctx, o->p1, false); break; case OToDyn: if( ra->t->kind == HBOOL ) { @@ -993,10 +1349,51 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg arg = LOAD(dst); emit_native_call(ctx, o->op == OThrow ? hl_throw : hl_rethrow, &arg, 1, &hlt_void); + mark_no_return(ctx); } break; case OLabel: // NOP + { + if( ctx->current_block->start_pos != ctx->emit_pos ) + split_block(ctx); + int i; + for(i=ctx->op_pos+1;ifun->nops;i++) { + hl_opcode *op = &ctx->fun->ops[i]; + int offs = 0; + switch( op->op ) { + case OJFalse: + case OJTrue: + case OJNotNull: + case OJNull: + offs = op->p2; + break; + case OJAlways: + offs = op->p1; + break; + case OJEq: + case OJNotEq: + case OJSLt: + case OJSGte: + case OJSLte: + case OJSGt: + case OJULt: + case OJUGte: + case OJNotLt: + case OJNotGte: + offs = op->p3; + break; + default: + break; + } + if( offs < 0 && i + 1 + offs == ctx->op_pos ) { +# ifdef BLOCK_DEBUG + printf(" WAIT @%X\n",i); +# endif + ctx->current_block->wait_nexts++; + } + } + } break; case OGetI8: case OGetI16: @@ -1181,6 +1578,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { // ----------------------------------------- ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : ENULL; emit_native_call(ctx, null_field_access ? hl_stub_null_field_access : hl_stub_null_access, &arg, null_field_access ? 1 : 0, &hlt_void); + mark_no_return(ctx); patch_jump(ctx, jok); } break; @@ -1377,100 +1775,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } } -static void hl_emit_flush( emit_ctx *ctx ) { - int i = 0; - while( i < ctx->jump_count ) { - int pos = ctx->jump_regs[i++]; - einstr *e = ctx->instrs + pos; - int target = ctx->jump_regs[i++]; - e->size_offs = ctx->pos_map[target] - (pos + 1); - } - ctx->jump_count = 0; - ctx->pos_map[ctx->fun->nops] = -1; -} - -int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { - int i; - ctx->mod = m; - ctx->fun = f; - ctx->emit_pos = 0; - ctx->trap_count = 0; - ctx->ref_count = 0; - current_ctx = ctx; - - if( f->nregs > ctx->max_regs ) { - free(ctx->vregs); - ctx->vregs = (vreg*)malloc(sizeof(vreg) * (f->nregs + 1)); - if( ctx->vregs == NULL ) - return false; - for(i=0;inregs;i++) - R(i)->id = i; - ctx->max_regs = f->nregs; - } - - for(i=0;inregs;i++) { - vreg *r = R(i); - r->t = f->regs[i]; - r->current = ENULL; - } - - for(i=0;itype->fun->nargs;i++) { - STORE(R(i), emit_gen(ctx, LOAD_ARG, ENULL, ENULL, hl_type_mode(f->type->fun->args[i]))); - } - - for(i=f->nops-1;i>=0;i--) { - hl_opcode *o = f->ops + i; - if( o->op == ORef ) { - ereg ref = resolve_ref(ctx, o->p2); - if( ref.index >= 0 ) continue; - if( ctx->ref_count == MAX_REFS ) emit_error("Too many refs"); - ctx->refs[ctx->ref_count].r = emit_gen_size(ctx, ALLOC_STACK, hl_type_size(R(o->p2)->t)); - ctx->refs[ctx->ref_count].reg = o->p2; - ctx->ref_count++; - } - } - - if( f->nops >= ctx->pos_map_size ) { - free(ctx->pos_map); - ctx->pos_map = (int*)malloc(sizeof(int) * (f->nops+1)); - if( ctx->pos_map == NULL ) - return false; - ctx->pos_map_size = f->nops; - } - - for(int op_pos=0;op_posnops;op_pos++) { - ctx->op_pos = op_pos; - ctx->pos_map[op_pos] = ctx->emit_pos; - emit_opcode(ctx,f->ops + op_pos); - } - - hl_emit_flush(ctx); - current_ctx = NULL; - return true; -} - -emit_ctx *hl_emit_alloc() { - emit_ctx *ctx = (emit_ctx*)malloc(sizeof(emit_ctx)); - if( ctx == NULL ) return NULL; - memset(ctx,0,sizeof(emit_ctx)); - if( sizeof(einstr) != 16 ) emit_assert(); - return ctx; -} - -void hl_emit_free( emit_ctx *ctx, h_bool can_reset ) { - free(ctx->vregs); - free(ctx->instrs); - free(ctx->pos_map); - free(ctx->jump_regs); - free(ctx->args_data); - free(ctx); -} - -void hl_emit_init( emit_ctx *ctx, hl_module *m ) { -} - -void hl_emit_reset( emit_ctx *ctx, hl_module *m ) { -} +// -------------------- CODE ---------------------------------------------------------- void *hl_emit_code( emit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { printf("TODO:emit_code\n"); @@ -1478,6 +1783,8 @@ void *hl_emit_code( emit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos * return NULL; } +// -------------------- DUMP ---------------------------------------------------------- + static void hl_dump_arg( emit_ctx *ctx, int fmt, int val, char sep ) { if( fmt == 0 ) return; printf("%c", sep); @@ -1550,6 +1857,7 @@ static const char *emit_mode_str( emit_mode mode ) { case M_F64: return "-f64"; case M_PTR: return ""; case M_VOID: return "-void"; + case M_NORET: return "-noret"; default: static char buf[50]; sprintf(buf,"?%d",mode); @@ -1557,12 +1865,14 @@ static const char *emit_mode_str( emit_mode mode ) { } } -static void dump_value( uint64 value, emit_mode mode ) { +static void dump_value( emit_ctx *ctx, uint64 value, emit_mode mode ) { union { uint64 v; double d; float f; } tmp; + hl_module *mod = ctx->mod; + hl_code *code = ctx->mod->code; switch( mode ) { case M_NONE: printf("?0x%llX",value); @@ -1586,6 +1896,12 @@ static void dump_value( uint64 value, emit_mode mode ) { default: if( value == 0 ) printf("NULL"); + else if( mode == M_PTR && value >= (uint64)code->types && value < (uint64)(code->types + code->ntypes) ) + uprintf(USTR("<%s>"),hl_type_str((hl_type*)value)); + else if( mode == M_PTR && value >= (uint64)mod->globals_data && value < (uint64)(mod->globals_data + mod->globals_size) ) + printf("",(int)(value - (uint64)mod->globals_data)); + else if( value == (uint64)&hlt_void ) + printf(""); else printf("0x%llX",value); break; @@ -1619,7 +1935,6 @@ static void hl_dump_ptr_name( emit_ctx *ctx, void *ptr ) { static named_ptr ptr_names[256] = { NULL }; int i = 0; if( !ptr_names[0].ptr ) { - i = 0; N(hl_alloc_dynbool); N(hl_alloc_dynamic); N(hl_alloc_obj); @@ -1643,6 +1958,7 @@ static void hl_dump_ptr_name( emit_ctx *ctx, void *ptr ) { N(setjmp); N(_setjmp); N2("assert",hl_stub_assert); + i = 0; } # undef N # undef N2 @@ -1657,7 +1973,7 @@ static void hl_dump_ptr_name( emit_ctx *ctx, void *ptr ) { for(i=0;imod->code->nnatives;i++) { hl_native *n = ctx->mod->code->natives + i; if( ctx->mod->functions_ptrs[n->findex] == ptr ) { - printf("<%s.%s>",n->lib,n->name); + printf("<%s.%s>",n->lib[0] == '?' ? n->lib + 1 : n->lib,n->name); return; } } @@ -1693,9 +2009,11 @@ void hl_emit_dump( emit_ctx *ctx ) { switch( e->op ) { case TEST: case CMP: + printf("-%s", hl_op_name(e->size_offs)+2); + break; case BINOP: case UNOP: - printf("-%s", hl_op_name(e->size_offs)+2); + printf("-%s", hl_op_name(e->size_offs)+1); break; default: break; @@ -1725,13 +2043,16 @@ void hl_emit_dump( emit_ctx *ctx ) { hl_dump_ptr_name(ctx, (void*)e->value); hl_dump_args(ctx,e); break; + case PHI: + hl_dump_args(ctx,e); + break; case JUMP: case JCOND: printf(" @%X", i + 1 + e->size_offs); break; case LOAD_IMM: printf(" "); - dump_value(e->value, e->mode); + dump_value(ctx, e->value, e->mode); break; case ALLOC_STACK: printf(" %d", e->size_offs); @@ -1771,4 +2092,4 @@ void hl_emit_dump( emit_ctx *ctx ) { } printf("\n\n"); fflush(stdout); -} \ No newline at end of file +} From 78df97e0b11ba00bf4caadc0bcebe7bdb9b87201 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Tue, 24 Mar 2026 23:16:02 +0100 Subject: [PATCH 06/83] small fixes and invalid phi detect in dump --- src/emit.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/emit.c b/src/emit.c index 47ace840c..bdb610524 100644 --- a/src/emit.c +++ b/src/emit.c @@ -371,7 +371,12 @@ static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { } static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { - if( count < 0 || count > 64 ) emit_assert(); + if( count < 0 ) emit_assert(); + if( e->op == PHI ) { + if( count > 0x200 ) emit_error("Too many branches"); + } else { + if( count > 64 ) emit_error("Too many arguments"); + } e->nargs = (unsigned char)count; if( count == 0 ) return; if( count == 1 ) { @@ -562,6 +567,7 @@ static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { if( ref.index >= 0 ) return LOAD_MEM(ref,0,r->t); einstr *p = emit_instr(ctx, PHI); + p->mode = hl_type_mode(r->t); int_alloc_reset(&ctx->phi_gather); emit_gather_phi_rec(ctx, ctx->current_block, r); if( ctx->phi_gather.cur == 1 && ctx->phi_gather.data[0] != -1 ) { @@ -1471,7 +1477,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case OArraySize: - STORE(dst, LOAD_MEM_PTR(LOAD(ra),HL_WSIZE*2)); + STORE(dst, LOAD_MEM(LOAD(ra),HL_WSIZE*2,&hlt_i32)); break; case ORef: { @@ -2018,7 +2024,7 @@ void hl_emit_dump( emit_ctx *ctx ) { default: break; } - if( e->mode ) + if( e->mode && e->op != PHI ) printf("%s", emit_mode_str(e->mode)); switch( e->op ) { case CALL_FUN: @@ -2045,6 +2051,17 @@ void hl_emit_dump( emit_ctx *ctx ) { break; case PHI: hl_dump_args(ctx,e); + { + int i; + ereg *args = hl_emit_get_args(ctx, e); + for(i=0;inargs;i++) { + einstr *a = ctx->instrs + args[i].index; + if( a->mode != e->mode ) { + printf(" ???"); + break; + } + } + } break; case JUMP: case JCOND: From 82710786afb1c7553fdb7e595bf297dfee136991 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 25 Mar 2026 07:28:20 +0100 Subject: [PATCH 07/83] fixes emit phi --- src/emit.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/emit.c b/src/emit.c index bdb610524..53377355b 100644 --- a/src/emit.c +++ b/src/emit.c @@ -141,6 +141,7 @@ struct _emit_block { int id; int start_pos; int wait_nexts; + int mark; linked_inf *preds; linked_inf *written_vars; linked_inf *incomplete_phis; @@ -161,6 +162,7 @@ struct _emit_ctx { int max_regs; int emit_pos; int op_pos; + int uid; ereg tmp_args[MAX_TMP_ARGS]; ereg traps[MAX_TRAPS]; @@ -373,7 +375,7 @@ static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { if( count < 0 ) emit_assert(); if( e->op == PHI ) { - if( count > 0x200 ) emit_error("Too many branches"); + if( count > 256 ) emit_error("Too many branches"); } else { if( count > 64 ) emit_error("Too many arguments"); } @@ -540,22 +542,24 @@ static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { emit_gen_ext(ctx, TEST, v, ENULL, ctx->instrs[v.index].mode, o); } -static void emit_gather_phi_rec( emit_ctx *ctx, emit_block *b, vreg *r ) { +static void emit_gather_phi_rec( emit_ctx *ctx, emit_block *b, vreg *r, int uid ) { if( b->wait_nexts > 0 ) { if( ctx->instrs[ctx->emit_pos-1].op != PHI ) emit_assert(); int_alloc_store_unique(&ctx->phi_gather, -1); b->incomplete_phis = link_add_sort_unique(ctx, ctx->emit_pos - 1, r, b->incomplete_phis); return; } + if( b->mark == uid ) + return; + b->mark = uid; int eid = (int)(int_val)link_sort_lookup(b->written_vars, r->id) - 1; if( eid >= 0 ) { if( eid != ctx->emit_pos - 1 ) int_alloc_store_unique(&ctx->phi_gather, eid); } else { - b->written_vars = link_add_sort_unique(ctx, r->id, (void*)(int_val)ctx->emit_pos, b->written_vars); linked_inf *l = b->preds; while( l ) { - emit_gather_phi_rec(ctx, (emit_block*)l->ptr, r); + emit_gather_phi_rec(ctx, (emit_block*)l->ptr, r, uid); l = l->next; } } @@ -569,7 +573,7 @@ static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { einstr *p = emit_instr(ctx, PHI); p->mode = hl_type_mode(r->t); int_alloc_reset(&ctx->phi_gather); - emit_gather_phi_rec(ctx, ctx->current_block, r); + emit_gather_phi_rec(ctx, ctx->current_block, r, ++ctx->uid); if( ctx->phi_gather.cur == 1 && ctx->phi_gather.data[0] != -1 ) { # ifdef BLOCK_DEBUG printf(" SKIP-PHI @%X\n",ctx->emit_pos-1); @@ -749,7 +753,7 @@ int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { ctx->current_block = alloc_block(ctx); ctx->arrival_points = NULL; # ifdef BLOCK_DEBUG - printf("---- begin ----\n"); + printf("---- begin [%X] ----\n",f->findex); # endif if( f->nregs > ctx->max_regs ) { free(ctx->vregs); @@ -851,9 +855,10 @@ static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { int prev_emit = ctx->emit_pos; ctx->emit_pos = l->id + 1; int_alloc_reset(&ctx->phi_gather); - emit_gather_phi_rec(ctx, b, r); + emit_gather_phi_rec(ctx, b, r, ++ctx->uid); ctx->emit_pos = prev_emit; store_args(ctx, e, (ereg*)ctx->phi_gather.data, ctx->phi_gather.cur); + b->written_vars = link_add_sort_unique(ctx,r->id,(void*)(int_val)(l->id + 1), b->written_vars); l = l->next; } b->incomplete_phis = NULL; From f2bfaa4134dbf0f4ae2b1bb20413370a02b2fd70 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 27 Mar 2026 00:24:12 +0100 Subject: [PATCH 08/83] new jit files split --- hl.vcxproj | 9 +- hl.vcxproj.filters | 5 +- src/hlmodule.h | 19 +- src/jit.h | 155 +++++++++ src/jit_dump.c | 374 ++++++++++++++++++++++ src/{emit.c => jit_emit.c} | 632 ++++--------------------------------- src/{jit.c => jit_old.c} | 0 src/module.c | 38 ++- 8 files changed, 630 insertions(+), 602 deletions(-) create mode 100644 src/jit.h create mode 100644 src/jit_dump.c rename src/{emit.c => jit_emit.c} (73%) rename src/{jit.c => jit_old.c} (100%) diff --git a/hl.vcxproj b/hl.vcxproj index 88e95b28b..a2c72f702 100644 --- a/hl.vcxproj +++ b/hl.vcxproj @@ -79,21 +79,21 @@ false true Unicode - v142 + v143 Application false true Unicode - v142 + v143 Application false true Unicode - v120 + v143 @@ -361,6 +361,8 @@ + + @@ -369,6 +371,7 @@ + diff --git a/hl.vcxproj.filters b/hl.vcxproj.filters index f86723996..83b94a469 100644 --- a/hl.vcxproj.filters +++ b/hl.vcxproj.filters @@ -4,14 +4,17 @@ - + + + + \ No newline at end of file diff --git a/src/hlmodule.h b/src/hlmodule.h index e82a3ff9f..01ab8be2e 100644 --- a/src/hlmodule.h +++ b/src/hlmodule.h @@ -19,6 +19,9 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ +#ifndef HL_MODULE_H +#define HL_MODULE_H + #include #include #include "opcodes.h" @@ -104,9 +107,6 @@ typedef struct { bool large; } hl_debug_infos; -typedef struct _emit_ctx emit_ctx; - - typedef struct { hl_code *code; int *types_hashes; @@ -120,6 +120,8 @@ typedef struct { #define WIN64_UNWIND_TABLES #endif +typedef struct _jit_ctx jit_ctx; + typedef struct { hl_code *code; int codesize; @@ -131,7 +133,7 @@ typedef struct { void *jit_code; hl_code_hash *hash; hl_debug_infos *jit_debug; - emit_ctx *emit_ctx; + jit_ctx *jit_ctx; hl_module_context ctx; #ifdef WIN64_UNWIND_TABLES PRUNTIME_FUNCTION unwind_table; @@ -161,11 +163,4 @@ hl_type *hl_module_resolve_type( hl_module *m, hl_type *t, bool err ); void hl_profile_setup( int sample_count ); void hl_profile_end(); -emit_ctx *hl_emit_alloc(); -void hl_emit_free( emit_ctx *ctx, h_bool can_reset ); -void hl_emit_reset( emit_ctx *ctx, hl_module *m ); -void hl_emit_init( emit_ctx *ctx, hl_module *m ); -void hl_emit_dump( emit_ctx *ctx ); -int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ); -void *hl_emit_code( emit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); -void hl_jit_patch_method( void *old_fun, void **new_fun_table ); +#endif diff --git a/src/jit.h b/src/jit.h new file mode 100644 index 000000000..19fc09b77 --- /dev/null +++ b/src/jit.h @@ -0,0 +1,155 @@ +/* + * Copyright (C)2005-2016 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef JIT_H +#define JIT_H + +#include + +typedef enum { + LOAD_ADDR, + LOAD_IMM, + LOAD_ARG, + STORE, + LEA, + TEST, + CMP, + JCOND, + JUMP, + JUMP_TABLE, + BINOP, + UNOP, + CONV, + CONV_UNSIGNED, + RET, + CALL_PTR, + CALL_REG, + CALL_FUN, + PHI, + ALLOC_STACK, + FREE_STACK, + NATIVE_REG, + PREFETCH, + DEBUG_BREAK, +} emit_op; + +typedef enum { + REG_RBP, +} native_reg; + +typedef enum { + M_NONE, + M_UI8, + M_UI16, + M_I32, + M_I64, + M_F32, + M_F64, + M_PTR, + M_VOID, + M_NORET, +} emit_mode; + +typedef struct { + int index; +} ereg; + +typedef struct { + union { + struct { + unsigned char op; + unsigned char mode; + unsigned char nargs; + unsigned char _unused; + }; + int header; + }; + int size_offs; + union { + struct { + ereg a; + ereg b; + }; + uint64 value; + }; +} einstr; + +typedef struct _emit_ctx emit_ctx; + +typedef struct { + int *data; + int max; + int cur; +} int_alloc; + +typedef struct _jit_ctx jit_ctx; + +struct _jit_ctx { + hl_module *mod; + hl_function *fun; + hl_alloc falloc; + emit_ctx *emit; + // emit output + einstr *instrs; + int instr_count; + int *emit_pos_map; +}; + +jit_ctx *hl_jit_alloc(); +void hl_jit_free( jit_ctx *ctx, h_bool can_reset ); +void hl_jit_reset( jit_ctx *ctx, hl_module *m ); +void hl_jit_init( jit_ctx *ctx, hl_module *m ); +int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ); + +void hl_jit_null_field_access(); +void hl_jit_null_access(); +void hl_jit_assert(); + +void int_alloc_reset( int_alloc *a ); +void int_alloc_free( int_alloc *a ); +int *int_alloc_get( int_alloc *a, int count ); +void int_alloc_store( int_alloc *a, int v ); +void int_alloc_store_unique( int_alloc *a, int v ); + + +void hl_emit_dump( jit_ctx *ctx ); +ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); + +#ifdef HL_DEBUG +# define JIT_DEBUG +#endif + +#define jit_error(msg) hl_jit_error(msg,__LINE__) +#define jit_assert() jit_error("") + +#ifdef JIT_DEBUG +# define jit_debug(...) printf(__VA_ARGS__) +#else +# define jit_debug(...) +#endif + +void hl_jit_error( const char *msg, int line ); + + +void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); +void hl_jit_patch_method( void *old_fun, void **new_fun_table ); + +#endif diff --git a/src/jit_dump.c b/src/jit_dump.c new file mode 100644 index 000000000..1e524cdd2 --- /dev/null +++ b/src/jit_dump.c @@ -0,0 +1,374 @@ +/* + * Copyright (C)2015-2016 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include + +static const char *op_names[] = { + "load-addr", + "load-imm", + "load-arg", + "store", + "lea", + "test", + "cmp", + "jcond", + "jump", + "jump-table", + "binop", + "unop", + "conv", + "conv-unsigned", + "ret", + "call", + "call", + "call", + "phi", + "alloc-stack", + "free-stack", + "native-reg", + "prefetch", + "debug-break", +}; + +static void hl_dump_arg( hl_function *fun, int fmt, int val, char sep, int pos ) { + if( fmt == 0 ) return; + printf("%c", sep); + switch( fmt ) { + case 1: + case 2: + printf("R%d", val); + if( val < 0 || val >= fun->nregs ) printf("?"); + break; + case 3: + printf("%d", val); + break; + case 4: + printf("[%d]", val); + break; + case 5: + case 6: + printf("@%X", val + pos + 1); + break; + default: + printf("?#%d", fmt); + break; + } +} + +#define OP(_,_a,_b,_c) ((_a) | (((_b)&0xFF) << 8) | (((_c)&0xFF) << 16)), +#define OP_BEGIN static int hl_op_fmt[] = { +#define OP_END }; +#undef R +#include "opcodes.h" + +static void hl_dump_op( hl_function *fun, hl_opcode *op ) { + printf("%s", hl_op_name(op->op) + 1); + int fmt = hl_op_fmt[op->op]; + int pos = (int)(op - fun->ops); + hl_dump_arg(fun, fmt & 0xFF, op->p1, ' ', pos); + if( ((fmt >> 8) & 0xFF) == 5 ) { + int count = (fmt >> 16) & 0xFF; + printf(" ["); + if( count == 4 ) { + printf("%d", op->p2); + printf(",%d", op->p3); + printf(",%d", (int)(int_val)op->extra); + } else if( op->op == OSwitch ) { + for(int i=0;ip2;i++) { + if( i != 0 ) printf(","); + printf("@%X", (op->extra[i] + pos + 1)); + } + printf(",def=@%X", op->p3 + pos + 1); + } else { + if( count == 0xFF ) + count = op->p3; + else { + printf("%d,%d,",op->p2,op->p3); + count -= 3; + } + for(int i=0;iextra[i]); + } + } + printf("]"); + } else { + hl_dump_arg(fun, (fmt >> 8) & 0xFF, op->p2,',', pos); + hl_dump_arg(fun, fmt >> 16, op->p3,',', pos); + } +} + +static const char *emit_mode_str( emit_mode mode ) { + switch( mode ) { + case M_UI8: return "-ui8"; + case M_UI16: return "-ui16"; + case M_I32: return "-i32"; + case M_I64: return "-i64"; + case M_F32: return "-f32"; + case M_F64: return "-f64"; + case M_PTR: return ""; + case M_VOID: return "-void"; + case M_NORET: return "-noret"; + default: + static char buf[50]; + sprintf(buf,"?%d",mode); + return buf; + } +} + +static void dump_value( jit_ctx *ctx, uint64 value, emit_mode mode ) { + union { + uint64 v; + double d; + float f; + } tmp; + hl_module *mod = ctx->mod; + hl_code *code = ctx->mod->code; + switch( mode ) { + case M_NONE: + printf("?0x%llX",value); + break; + case M_UI8: + case M_UI16: + case M_I32: + if( (int)value >= -0x10000 && (int)value <= 0x10000 ) + printf("%d",(int)value); + else + printf("0x%X",(int)value); + break; + case M_F32: + tmp.v = value; + printf("%f",tmp.f); + break; + case M_F64: + tmp.v = value; + printf("%g",tmp.d); + break; + default: + if( value == 0 ) + printf("NULL"); + else if( mode == M_PTR && value >= (uint64)code->types && value < (uint64)(code->types + code->ntypes) ) + uprintf(USTR("<%s>"),hl_type_str((hl_type*)value)); + else if( mode == M_PTR && value >= (uint64)mod->globals_data && value < (uint64)(mod->globals_data + mod->globals_size) ) + printf("",(int)(value - (uint64)mod->globals_data)); + else if( value == (uint64)&hlt_void ) + printf(""); + else + printf("0x%llX",value); + break; + } +} + +static void hl_dump_fun_name( hl_function *f ) { + if( f->obj ) + uprintf(USTR("%s.%s"),f->obj->name,f->field.name); + else if( f->field.ref ) + uprintf(USTR("%s.~%s.%d"),f->field.ref->obj->name, f->field.ref->field.name, f->ref); + printf("[%X]", f->findex); +} + +static void hl_dump_args( jit_ctx *ctx, einstr *e ) { + ereg *v = hl_emit_get_args(ctx->emit, e); + printf("("); + for(int i=0;inargs;i++) { + if( i != 0 ) printf(","); + printf("@%X", v[i].index); + } + printf(")"); +} + +typedef struct { const char *name; void *ptr; } named_ptr; +static void hl_dump_ptr_name( jit_ctx *ctx, void *ptr ) { +# define N(v) ptr_names[i].name = #v; ptr_names[i].ptr = v; i++ +# define N2(n,v) ptr_names[i].name = n; ptr_names[i].ptr = v; i++ +# define DYN(p) N2("dyn_get" #p, hl_dyn_get##p); N2("dyn_set" #p, hl_dyn_set##p); N2("dyn_cast" #p, hl_dyn_cast##p) + static named_ptr ptr_names[256] = { NULL }; + int i = 0; + if( !ptr_names[0].ptr ) { + N(hl_alloc_dynbool); + N(hl_alloc_dynamic); + N(hl_alloc_obj); + N(hl_alloc_dynobj); + N(hl_alloc_virtual); + N(hl_alloc_closure_ptr); + N(hl_dyn_call); + N(hl_dyn_call_obj); + N(hl_throw); + N(hl_rethrow); + N(hl_to_virtual); + N(hl_alloc_enum); + DYN(f); + DYN(d); + DYN(i64); + DYN(i); + DYN(p); + N2("null_field",hl_jit_null_field_access); + N2("null_access",hl_jit_null_access); + N(hl_get_thread); + N(setjmp); + N(_setjmp); + N2("assert",hl_jit_assert); + i = 0; + } +# undef N +# undef N2 + while( true ) { + named_ptr p = ptr_names[i++]; + if( !p.ptr ) break; + if( p.ptr == ptr ) { + printf("<%s>",p.name); + return; + } + } + for(i=0;imod->code->nnatives;i++) { + hl_native *n = ctx->mod->code->natives + i; + if( ctx->mod->functions_ptrs[n->findex] == ptr ) { + printf("<%s.%s>",n->lib[0] == '?' ? n->lib + 1 : n->lib,n->name); + return; + } + } + printf("",(uint64)ptr); +} + +void hl_emit_flush( jit_ctx *ctx ); + +void hl_emit_dump( jit_ctx *ctx ) { + int i; + int cur_op = 0; + hl_function *f = ctx->fun; + int nargs = f->type->fun->nargs; + hl_emit_flush(ctx); // if it not was not before (in case of dump during emit) + printf("function "); + hl_dump_fun_name(f); + printf("("); + for(i=0;i 0 ) printf(","); + uprintf(USTR("R%d"), i); + } + printf(")\n"); + for(i=0;inregs;i++) + uprintf(USTR("\tR%d : %s\n"),i, hl_type_str(f->regs[i])); + for(i=0;iinstr_count;i++) { + while( ctx->emit_pos_map[cur_op] == i ) { + printf("@%X ", cur_op); + hl_dump_op(ctx->fun, f->ops + cur_op); + printf("\n"); + cur_op++; + } + einstr *e = ctx->instrs + i; + printf("\t\t@%X %s", i, op_names[e->op]); + switch( e->op ) { + case TEST: + case CMP: + printf("-%s", hl_op_name(e->size_offs)+2); + break; + case BINOP: + case UNOP: + printf("-%s", hl_op_name(e->size_offs)+1); + break; + default: + break; + } + if( e->mode && e->op != PHI ) + printf("%s", emit_mode_str(e->mode)); + switch( e->op ) { + case CALL_FUN: + printf(" "); + { + int fid = ctx->mod->functions_indexes[e->a.index]; + hl_code *code = ctx->mod->code; + if( fid < code->nfunctions ) { + hl_dump_fun_name(&code->functions[fid]); + } else { + printf("???"); + } + } + hl_dump_args(ctx,e); + break; + case CALL_REG: + printf(" @%X", e->a.index); + hl_dump_args(ctx,e); + break; + case CALL_PTR: + printf(" "); + hl_dump_ptr_name(ctx, (void*)e->value); + hl_dump_args(ctx,e); + break; + case PHI: + hl_dump_args(ctx,e); + { + int i; + ereg *args = hl_emit_get_args(ctx->emit, e); + for(i=0;inargs;i++) { + einstr *a = ctx->instrs + args[i].index; + if( a->mode != e->mode ) { + printf(" ???"); + break; + } + } + } + break; + case JUMP: + case JCOND: + printf(" @%X", i + 1 + e->size_offs); + break; + case LOAD_IMM: + printf(" "); + dump_value(ctx, e->value, e->mode); + break; + case ALLOC_STACK: + printf(" %d", e->size_offs); + break; + case LOAD_ADDR: + if( (e->b.index>>8) ) + printf(" @%X[%Xh]", e->a.index, e->b.index); + else + printf(" @%X[%d]", e->a.index, e->b.index); + break; + case STORE: + { + int offs = e->size_offs; + if( offs == 0 ) + printf(" [@%X] = @%X", e->a.index, e->b.index); + else + printf(" @%X[%d] = @%X", e->a.index, offs, e->b.index); + if( e->mode == 0 || e->mode != ctx->instrs[e->b.index].mode ) + printf(" ???"); + } + break; + default: + if( e->a.index >= 0 ) { + printf(" @%X", e->a.index); + if( e->b.index >= 0 ) printf(", @%X", e->b.index); + } + break; + } + printf("\n"); + } + // interrupted + if( cur_op < f->nops ) { + printf("@%X ", cur_op); + hl_dump_op(ctx->fun, f->ops + cur_op); + printf("\n\t\t...\n"); + } + printf("\n\n"); + fflush(stdout); +} diff --git a/src/emit.c b/src/jit_emit.c similarity index 73% rename from src/emit.c rename to src/jit_emit.c index 53377355b..868c6dd02 100644 --- a/src/emit.c +++ b/src/jit_emit.c @@ -20,82 +20,7 @@ * DEALINGS IN THE SOFTWARE. */ #include - -typedef enum { - LOAD_ADDR, - LOAD_IMM, - LOAD_ARG, - STORE, - LEA, - TEST, - CMP, - JCOND, - JUMP, - JUMP_TABLE, - BINOP, - UNOP, - CONV, - CONV_UNSIGNED, - RET, - CALL_PTR, - CALL_REG, - CALL_FUN, - PHI, - ALLOC_STACK, - FREE_STACK, - NATIVE_REG, - PREFETCH, - DEBUG_BREAK, -} emit_op; - -static const char *op_names[] = { - "load-addr", - "load-imm", - "load-arg", - "store", - "lea", - "test", - "cmp", - "jcond", - "jump", - "jump-table", - "binop", - "unop", - "conv", - "conv-unsigned", - "ret", - "call", - "call", - "call", - "phi", - "alloc-stack", - "free-stack", - "native-reg", - "prefetch", - "debug-break", -}; - -typedef enum { - REG_RBP, -} native_reg; - -typedef enum { - M_NONE, - M_UI8, - M_UI16, - M_I32, - M_I64, - M_F32, - M_F64, - M_PTR, - M_VOID, - M_NORET, -} emit_mode; - - -typedef struct { - int index; -} ereg; +#include typedef struct { hl_type *t; @@ -104,26 +29,6 @@ typedef struct { bool written; } vreg; -typedef struct { - union { - struct { - unsigned char op; - unsigned char mode; - unsigned char nargs; - unsigned char _unused; - }; - int header; - }; - int size_offs; - union { - struct { - ereg a; - ereg b; - }; - uint64 value; - }; -} einstr; - #define MAX_TMP_ARGS 32 #define MAX_TRAPS 32 #define MAX_REFS 512 // TODO : different impl @@ -147,15 +52,11 @@ struct _emit_block { linked_inf *incomplete_phis; }; -typedef struct { - int *data; - int max; - int cur; -} int_alloc; - struct _emit_ctx { hl_module *mod; hl_function *fun; + jit_ctx *jit; + einstr *instrs; vreg *vregs; int max_instrs; @@ -180,7 +81,6 @@ struct _emit_ctx { int_alloc block_writes; int_alloc phi_gather; - hl_alloc falloc; emit_block *current_block; linked_inf *arrival_points; void *closure_list; // TODO : patch with good addresses @@ -188,9 +88,6 @@ struct _emit_ctx { #define R(i) (ctx->vregs + (i)) -#define emit_error(msg) _emit_error(msg,__LINE__) -#define emit_assert() emit_error("") - #define LOAD(r) emit_load_reg(ctx, r) #define STORE(r, v) emit_store_reg(ctx, r, v) #define LOAD_CONST(v, t) emit_load_const(ctx, (uint64)(v), t) @@ -210,63 +107,9 @@ struct _emit_ctx { static hl_type hlt_ui8 = { HUI8, 0 }; static hl_type hlt_ui16 = { HUI16, 0 }; static ereg ENULL = {-1}; -static emit_ctx *current_ctx = NULL; - -//#define BLOCK_DEBUG - -static void _emit_error( const char *msg, int line ) { - printf("*** EMIT ERROR line %d (%s) ****\n", line, msg); - if( current_ctx ) hl_emit_dump(current_ctx); - hl_debug_break(); - fflush(stdout); - exit(-1); -} - -static void hl_stub_null_field_access() { emit_assert(); } -static void hl_stub_null_access() { emit_assert(); } -static void hl_stub_assert() { emit_assert(); } - -static void int_alloc_reset( int_alloc *a ) { - a->cur = 0; -} - -static void int_alloc_free( int_alloc *a ) { - free(a->data); - a->cur = 0; - a->max = 0; - a->data = NULL; -} - -static int *int_alloc_get( int_alloc *a, int count ) { - while( a->cur + count > a->max ) { - int next_size = a->max ? a->max << 1 : 128; - int *new_data = (int*)malloc(sizeof(int) * next_size); - if( new_data == NULL ) emit_error("Out of memory"); - memcpy(new_data, a->data, sizeof(int) * a->cur); - free(a->data); - a->data = new_data; - a->max = next_size; - } - int *ptr = a->data + a->cur; - a->cur += count; - return ptr; -} - -static void int_alloc_store( int_alloc *a, int v ) { - *int_alloc_get(a,1) = v; -} - -static void int_alloc_store_unique( int_alloc *a, int v ) { - int i = 0; - while( i < a->cur ) { - if( a->data[i] == v ) return; - i++; - } - *int_alloc_get(a,1) = v; -} static linked_inf *link_add( emit_ctx *ctx, int id, void *ptr, linked_inf *head ) { - linked_inf *l = hl_malloc(&ctx->falloc,sizeof(linked_inf)); + linked_inf *l = hl_malloc(&ctx->jit->falloc,sizeof(linked_inf)); l->id = id; l->ptr = ptr; l->next = head; @@ -287,7 +130,7 @@ static linked_inf *link_add_sort_unique( emit_ctx *ctx, int id, void *ptr, linke cur = cur->next; } // insert - linked_inf *l = hl_malloc(&ctx->falloc,sizeof(linked_inf)); + linked_inf *l = hl_malloc(&ctx->jit->falloc,sizeof(linked_inf)); l->id = id; l->ptr = ptr; if( !prev ) { @@ -325,7 +168,7 @@ static void mark_no_return( emit_ctx *ctx ) { } void hl_jit_patch_method( void*fun, void**newt ) { - emit_assert(); + jit_assert(); } static ereg __current_reg( emit_ctx *ctx ) { @@ -334,7 +177,7 @@ static ereg __current_reg( emit_ctx *ctx ) { } static ereg *get_tmp_args( emit_ctx *ctx, int count ) { - if( count > MAX_TMP_ARGS ) emit_error("Too many arguments"); + if( count > MAX_TMP_ARGS ) jit_error("Too many arguments"); return ctx->tmp_args; } @@ -351,7 +194,7 @@ static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { int pos = ctx->emit_pos; int next_size = ctx->max_instrs ? (ctx->max_instrs << 1) : 256; einstr *instrs = (einstr*)malloc(sizeof(einstr) * next_size); - if( instrs == NULL ) emit_error("Out of memory"); + if( instrs == NULL ) jit_error("Out of memory"); memcpy(instrs, ctx->instrs, pos * sizeof(einstr)); memset(instrs + pos, 0, (next_size - pos) * sizeof(einstr)); free(ctx->instrs); @@ -373,11 +216,11 @@ static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { } static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { - if( count < 0 ) emit_assert(); + if( count < 0 ) jit_assert(); if( e->op == PHI ) { - if( count > 256 ) emit_error("Too many branches"); + if( count > 256 ) jit_error("Too many branches"); } else { - if( count > 64 ) emit_error("Too many arguments"); + if( count > 64 ) jit_error("Too many arguments"); } e->nargs = (unsigned char)count; if( count == 0 ) return; @@ -406,7 +249,7 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ) { static ereg emit_gen_ext( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode, int size_offs ) { einstr *e = emit_instr(ctx, op); - if( (unsigned char)mode != mode ) emit_assert(); + if( (unsigned char)mode != mode ) jit_assert(); e->mode = (unsigned char)mode; e->size_offs = size_offs; e->a = a; @@ -424,7 +267,7 @@ static ereg emit_gen_size( emit_ctx *ctx, emit_op op, int size_offs ) { static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { unsigned char mode = ctx->instrs[v1.index].mode; - if( mode != ctx->instrs[v2.index].mode ) emit_assert(); + if( mode != ctx->instrs[v2.index].mode ) jit_assert(); einstr *e = emit_instr(ctx, PHI); e->mode = mode; ereg args[2]; @@ -445,14 +288,12 @@ static void patch_jump( emit_ctx *ctx, int jpos ) { } static emit_block *alloc_block( emit_ctx *ctx ) { - return hl_zalloc(&ctx->falloc, sizeof(emit_block)); + return hl_zalloc(&ctx->jit->falloc, sizeof(emit_block)); } static void block_add_prec( emit_ctx *ctx, emit_block *b, emit_block *p ) { b->preds = link_add(ctx,0,p,b->preds); -# ifdef BLOCK_DEBUG - printf(" PRED %d\n",p->id); -# endif + jit_debug(" PRED %d\n",p->id); } static void split_block( emit_ctx *ctx ) { @@ -462,9 +303,7 @@ static void split_block( emit_ctx *ctx ) { while( i < ctx->block_writes.cur ) { vreg *r = R(ctx->block_writes.data[i++]); cur->written_vars = link_add_sort_unique(ctx,r->id,(void*)(int_val)(r->current.index+1),cur->written_vars); -# ifdef BLOCK_DEBUG - printf(" WRITES R%d @%X\n", r->id, r->current.index); -# endif + jit_debug(" WRITES R%d @%X\n", r->id, r->current.index); r->current.index = -1; r->written = false; } @@ -473,9 +312,7 @@ static void split_block( emit_ctx *ctx ) { emit_block *b = alloc_block(ctx); b->id = ctx->current_block->id + 1; b->start_pos = ctx->emit_pos; -# ifdef BLOCK_DEBUG - printf("BLOCK #%d@%X\n",b->id,ctx->op_pos); -# endif + jit_debug("BLOCK #%d@%X\n",b->id,ctx->op_pos); einstr *eprev = &ctx->instrs[ctx->emit_pos-1]; if( (eprev->op != JUMP && eprev->op != RET) || ctx->fun->ops[ctx->op_pos].op == OTrap ) block_add_prec(ctx, b, ctx->current_block); @@ -514,7 +351,7 @@ static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { if( to->t->kind == HVOID ) return; - if( v.index < 0 ) emit_assert(); + if( v.index < 0 ) jit_assert(); if( !to->written ) { to->written = true; int_alloc_store(&ctx->block_writes,to->id); @@ -544,7 +381,7 @@ static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { static void emit_gather_phi_rec( emit_ctx *ctx, emit_block *b, vreg *r, int uid ) { if( b->wait_nexts > 0 ) { - if( ctx->instrs[ctx->emit_pos-1].op != PHI ) emit_assert(); + if( ctx->instrs[ctx->emit_pos-1].op != PHI ) jit_assert(); int_alloc_store_unique(&ctx->phi_gather, -1); b->incomplete_phis = link_add_sort_unique(ctx, ctx->emit_pos - 1, r, b->incomplete_phis); return; @@ -575,14 +412,12 @@ static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { int_alloc_reset(&ctx->phi_gather); emit_gather_phi_rec(ctx, ctx->current_block, r, ++ctx->uid); if( ctx->phi_gather.cur == 1 && ctx->phi_gather.data[0] != -1 ) { -# ifdef BLOCK_DEBUG - printf(" SKIP-PHI @%X\n",ctx->emit_pos-1); -# endif + jit_debug(" SKIP-PHI @%X\n",ctx->emit_pos-1); // remove phi ctx->emit_pos--; r->current.index = ctx->phi_gather.data[0]; } else { - if( ctx->phi_gather.cur == 0 ) emit_assert(); + if( ctx->phi_gather.cur == 0 ) jit_assert(); store_args(ctx, p, (ereg*)ctx->phi_gather.data, ctx->phi_gather.cur); r->current = CUR_REG(); } @@ -727,7 +562,8 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { static void emit_opcode( emit_ctx *ctx, hl_opcode *o ); -static void hl_emit_flush( emit_ctx *ctx ) { +void hl_emit_flush( jit_ctx *jit ) { + emit_ctx *ctx = jit->emit; int i = 0; while( i < ctx->jump_regs.cur ) { int pos = ctx->jump_regs.data[i++]; @@ -736,11 +572,16 @@ static void hl_emit_flush( emit_ctx *ctx ) { e->size_offs = ctx->pos_map[target] - (pos + 1); } ctx->pos_map[ctx->fun->nops] = -1; + jit->instrs = ctx->instrs; + jit->instr_count = ctx->emit_pos; + jit->emit_pos_map = ctx->pos_map; } -int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { +void hl_emit_function( jit_ctx *jit ) { + emit_ctx *ctx = jit->emit; + hl_function *f = jit->fun; int i; - ctx->mod = m; + ctx->mod = jit->mod; ctx->fun = f; ctx->emit_pos = 0; ctx->trap_count = 0; @@ -748,18 +589,13 @@ int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { int_alloc_reset(&ctx->args_data); int_alloc_reset(&ctx->jump_regs); int_alloc_reset(&ctx->block_writes); - current_ctx = ctx; - hl_free(&ctx->falloc); ctx->current_block = alloc_block(ctx); ctx->arrival_points = NULL; -# ifdef BLOCK_DEBUG - printf("---- begin [%X] ----\n",f->findex); -# endif + jit_debug("---- begin [%X] ----\n",f->findex); if( f->nregs > ctx->max_regs ) { free(ctx->vregs); ctx->vregs = (vreg*)malloc(sizeof(vreg) * (f->nregs + 1)); - if( ctx->vregs == NULL ) - return false; + if( ctx->vregs == NULL ) jit_assert(); for(i=0;inregs;i++) R(i)->id = i; ctx->max_regs = f->nregs; @@ -781,7 +617,7 @@ int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { if( o->op == ORef ) { ereg ref = resolve_ref(ctx, o->p2); if( ref.index >= 0 ) continue; - if( ctx->ref_count == MAX_REFS ) emit_error("Too many refs"); + if( ctx->ref_count == MAX_REFS ) jit_error("Too many refs"); ctx->refs[ctx->ref_count].r = emit_gen_size(ctx, ALLOC_STACK, hl_type_size(R(o->p2)->t)); ctx->refs[ctx->ref_count].reg = o->p2; ctx->ref_count++; @@ -791,8 +627,7 @@ int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { if( f->nops >= ctx->pos_map_size ) { free(ctx->pos_map); ctx->pos_map = (int*)malloc(sizeof(int) * (f->nops+1)); - if( ctx->pos_map == NULL ) - return false; + if( ctx->vregs == NULL ) jit_assert(); ctx->pos_map_size = f->nops; } @@ -804,22 +639,20 @@ int hl_emit_function( emit_ctx *ctx, hl_module *m, hl_function *f ) { emit_opcode(ctx,f->ops + op_pos); } - hl_emit_flush(ctx); - current_ctx = NULL; - return true; + hl_emit_flush(ctx->jit); } -emit_ctx *hl_emit_alloc() { +void hl_emit_alloc( jit_ctx *jit ) { emit_ctx *ctx = (emit_ctx*)malloc(sizeof(emit_ctx)); - if( ctx == NULL ) return NULL; + if( ctx == NULL ) jit_assert(); memset(ctx,0,sizeof(emit_ctx)); - if( sizeof(einstr) != 16 ) emit_assert(); - hl_alloc_init(&ctx->falloc); - return ctx; + ctx->jit = jit; + jit->emit = ctx; + if( sizeof(einstr) != 16 ) jit_assert(); } -void hl_emit_free( emit_ctx *ctx, h_bool can_reset ) { - hl_free(&ctx->falloc); +void hl_emit_free( jit_ctx *jit ) { + emit_ctx *ctx = jit->emit; free(ctx->vregs); free(ctx->instrs); free(ctx->pos_map); @@ -827,12 +660,7 @@ void hl_emit_free( emit_ctx *ctx, h_bool can_reset ) { int_alloc_free(&ctx->args_data); int_alloc_free(&ctx->block_writes); free(ctx); -} - -void hl_emit_init( emit_ctx *ctx, hl_module *m ) { -} - -void hl_emit_reset( emit_ctx *ctx, hl_module *m ) { + jit->emit = NULL; } static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { @@ -843,15 +671,13 @@ static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { b->preds = link_add(ctx,0,ctx->current_block,b->preds); split_block(ctx); if( b->wait_nexts == 0 ) { -# ifdef BLOCK_DEBUG - printf(" SEAL #%d\n",b->id); -# endif + jit_debug(" SEAL #%d\n",b->id); // seal block linked_inf *l = b->incomplete_phis; while( l ) { einstr *e = ctx->instrs + l->id; vreg *r = (vreg*)l->ptr; - if( e->op != PHI ) emit_assert(); + if( e->op != PHI ) jit_assert(); int prev_emit = ctx->emit_pos; ctx->emit_pos = l->id + 1; int_alloc_reset(&ctx->phi_gather); @@ -881,7 +707,7 @@ static void register_block_jump( emit_ctx *ctx, int offs, bool cond ) { if( offs < 0 ) { int target = ctx->pos_map[ctx->op_pos + 1 + offs]; emit_block *b = ctx->current_block; - if( !seal_block_rec(ctx, b, target) ) emit_assert(); + if( !seal_block_rec(ctx, b, target) ) jit_assert(); } } @@ -1045,7 +871,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case ODecr: { if( IS_FLOAT(dst->t) ) { - emit_assert(); + jit_assert(); } else { STORE(dst, emit_gen_ext(ctx,UNOP,LOAD(dst),ENULL,hl_type_mode(dst->t),o->op)); } @@ -1069,7 +895,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { allocFun = hl_alloc_virtual; break; default: - emit_assert(); + jit_assert(); } if( nargs ) arg = LOAD_CONST_PTR(dst->t); STORE(dst, emit_native_call(ctx,allocFun,&arg,nargs,dst->t)); @@ -1114,7 +940,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg st = emit_gen_size(ctx, ALLOC_STACK, o->p3); for(i=0;ip3;i++) { vreg *r = R(o->extra[i]); - if( !hl_is_dynamic(r->t) ) emit_assert(); + if( !hl_is_dynamic(r->t) ) jit_assert(); STORE_MEM(st,i*HL_WSIZE,LOAD(r)); } ereg args[3]; @@ -1189,7 +1015,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; default: - emit_assert(); + jit_assert(); break; } } @@ -1240,7 +1066,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; default: - emit_assert(); + jit_assert(); break; } } @@ -1350,7 +1176,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; default: - emit_assert(); + jit_assert(); break; } } @@ -1398,9 +1224,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; } if( offs < 0 && i + 1 + offs == ctx->op_pos ) { -# ifdef BLOCK_DEBUG - printf(" WAIT @%X\n",i); -# endif + jit_debug(" WAIT @%X\n",i); ctx->current_block->wait_nexts++; } } @@ -1487,7 +1311,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case ORef: { ereg ref = resolve_ref(ctx, ra->id); - if( ref.index < 0 ) emit_assert(); + if( ref.index < 0 ) jit_assert(); if( ra->current.index >= 0 ) STORE_MEM(ref, 0, LOAD(ra)); ra->current.index = -1; // ref will be modified STORE(dst, ref); @@ -1505,7 +1329,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, OFFSET(LOAD(ra),ENULL,0,sizeof(varray))); break; default: - emit_assert(); + jit_assert(); } break; case ORefOffset: @@ -1576,7 +1400,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { f = hl_obj_field_fetch(dst->t, fid); else if( dst->t->kind == HVIRTUAL ) f = dst->t->virt->fields + fid; - if( f == NULL ) emit_assert(); + if( f == NULL ) jit_assert(); null_field_access = true; hashed_name = f->hashed_name; } else if( (next->op >= OCall1 && next->op <= OCallN) && next->p3 == o->p1 ) { @@ -1588,7 +1412,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } // ----------------------------------------- ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : ENULL; - emit_native_call(ctx, null_field_access ? hl_stub_null_field_access : hl_stub_null_access, &arg, null_field_access ? 1 : 0, &hlt_void); + emit_native_call(ctx, null_field_access ? hl_jit_null_field_access : hl_jit_null_access, &arg, null_field_access ? 1 : 0, &hlt_void); mark_no_return(ctx); patch_jump(ctx, jok); } @@ -1694,13 +1518,13 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { register_jump(ctx, jtrap, o->p2); patch_jump(ctx, jskip); - if( ctx->trap_count == MAX_TRAPS ) emit_error("Too many try/catch depth"); + if( ctx->trap_count == MAX_TRAPS ) jit_error("Too many try/catch depth"); ctx->traps[ctx->trap_count++] = st; } break; case OEndTrap: { - if( ctx->trap_count == 0 ) emit_assert(); + if( ctx->trap_count == 0 ) jit_assert(); ereg st = ctx->traps[ctx->trap_count - 1]; ereg thread, current_addr; @@ -1750,7 +1574,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, LOAD_MEM(LOAD(ra),0,&hlt_i32)); break; case OAssert: - emit_native_call(ctx, hl_stub_assert, NULL, 0, &hlt_void); + emit_native_call(ctx, hl_jit_assert, NULL, 0, &hlt_void); break; case ONop: break; @@ -1767,7 +1591,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; default: - emit_assert(); + jit_assert(); break; } } @@ -1775,343 +1599,13 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case OAsm: - emit_assert(); + jit_assert(); break; case OCatch: // Only used by OTrap typing break; default: - emit_error(hl_op_name(o->op)); - break; - } -} - -// -------------------- CODE ---------------------------------------------------------- - -void *hl_emit_code( emit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { - printf("TODO:emit_code\n"); - exit(0); - return NULL; -} - -// -------------------- DUMP ---------------------------------------------------------- - -static void hl_dump_arg( emit_ctx *ctx, int fmt, int val, char sep ) { - if( fmt == 0 ) return; - printf("%c", sep); - switch( fmt ) { - case 1: - case 2: - printf("R%d", val); - if( val < 0 || val >= ctx->fun->nregs ) printf("?"); - break; - case 3: - printf("%d", val); - break; - case 4: - printf("[%d]", val); - break; - case 5: - case 6: - printf("@%X", val + ctx->op_pos + 1); - break; - default: - printf("?#%d", fmt); - break; - } -} - -#define OP(_,_a,_b,_c) ((_a) | (((_b)&0xFF) << 8) | (((_c)&0xFF) << 16)), -#define OP_BEGIN static int hl_op_fmt[] = { -#define OP_END }; -#undef R -#include "opcodes.h" - -static void hl_dump_op( emit_ctx *ctx, hl_opcode *op ) { - printf("%s", hl_op_name(op->op) + 1); - int fmt = hl_op_fmt[op->op]; - hl_dump_arg(ctx, fmt & 0xFF, op->p1, ' '); - if( ((fmt >> 8) & 0xFF) == 5 ) { - int count = (fmt >> 16) & 0xFF; - printf(" ["); - if( count == 4 ) { - printf("%d", op->p2); - printf(",%d", op->p3); - printf(",%d", (int)(int_val)op->extra); - } else if( op->op == OSwitch ) { - for(int i=0;ip2;i++) { - if( i != 0 ) printf(","); - printf("@%X", (op->extra[i] + ctx->op_pos + 1)); - } - printf(",def=@%X", op->p3 + ctx->op_pos + 1); - } else { - if( count == 0xFF ) count = op->p3; - for(int i=0;iextra[i]); - } - } - printf("]"); - } else { - hl_dump_arg(ctx, (fmt >> 8) & 0xFF, op->p2,','); - hl_dump_arg(ctx, fmt >> 16, op->p3,','); - } -} - -static const char *emit_mode_str( emit_mode mode ) { - switch( mode ) { - case M_UI8: return "-ui8"; - case M_UI16: return "-ui16"; - case M_I32: return "-i32"; - case M_I64: return "-i64"; - case M_F32: return "-f32"; - case M_F64: return "-f64"; - case M_PTR: return ""; - case M_VOID: return "-void"; - case M_NORET: return "-noret"; - default: - static char buf[50]; - sprintf(buf,"?%d",mode); - return buf; - } -} - -static void dump_value( emit_ctx *ctx, uint64 value, emit_mode mode ) { - union { - uint64 v; - double d; - float f; - } tmp; - hl_module *mod = ctx->mod; - hl_code *code = ctx->mod->code; - switch( mode ) { - case M_NONE: - printf("?0x%llX",value); - break; - case M_UI8: - case M_UI16: - case M_I32: - if( (int)value >= -0x10000 && (int)value <= 0x10000 ) - printf("%d",(int)value); - else - printf("0x%X",(int)value); - break; - case M_F32: - tmp.v = value; - printf("%f",tmp.f); - break; - case M_F64: - tmp.v = value; - printf("%g",tmp.d); - break; - default: - if( value == 0 ) - printf("NULL"); - else if( mode == M_PTR && value >= (uint64)code->types && value < (uint64)(code->types + code->ntypes) ) - uprintf(USTR("<%s>"),hl_type_str((hl_type*)value)); - else if( mode == M_PTR && value >= (uint64)mod->globals_data && value < (uint64)(mod->globals_data + mod->globals_size) ) - printf("",(int)(value - (uint64)mod->globals_data)); - else if( value == (uint64)&hlt_void ) - printf(""); - else - printf("0x%llX",value); + jit_error(hl_op_name(o->op)); break; } } - -static void hl_dump_fun_name( hl_function *f ) { - if( f->obj ) - uprintf(USTR("%s.%s"),f->obj->name,f->field.name); - else if( f->field.ref ) - uprintf(USTR("%s.~%s.%d"),f->field.ref->obj->name, f->field.ref->field.name, f->ref); - printf("[%X]", f->findex); -} - -static void hl_dump_args( emit_ctx *ctx, einstr *e ) { - ereg *v = hl_emit_get_args(ctx, e); - printf("("); - for(int i=0;inargs;i++) { - if( i != 0 ) printf(","); - printf("@%X", v[i].index); - } - printf(")"); -} - - -typedef struct { const char *name; void *ptr; } named_ptr; -static void hl_dump_ptr_name( emit_ctx *ctx, void *ptr ) { -# define N(v) ptr_names[i].name = #v; ptr_names[i].ptr = v; i++ -# define N2(n,v) ptr_names[i].name = n; ptr_names[i].ptr = v; i++ -# define DYN(p) N2("dyn_get" #p, hl_dyn_get##p); N2("dyn_set" #p, hl_dyn_set##p); N2("dyn_cast" #p, hl_dyn_cast##p) - static named_ptr ptr_names[256] = { NULL }; - int i = 0; - if( !ptr_names[0].ptr ) { - N(hl_alloc_dynbool); - N(hl_alloc_dynamic); - N(hl_alloc_obj); - N(hl_alloc_dynobj); - N(hl_alloc_virtual); - N(hl_alloc_closure_ptr); - N(hl_dyn_call); - N(hl_dyn_call_obj); - N(hl_throw); - N(hl_rethrow); - N(hl_to_virtual); - N(hl_alloc_enum); - DYN(f); - DYN(d); - DYN(i64); - DYN(i); - DYN(p); - N2("null_field",hl_stub_null_field_access); - N2("null_access",hl_stub_null_access); - N(hl_get_thread); - N(setjmp); - N(_setjmp); - N2("assert",hl_stub_assert); - i = 0; - } -# undef N -# undef N2 - while( true ) { - named_ptr p = ptr_names[i++]; - if( !p.ptr ) break; - if( p.ptr == ptr ) { - printf("<%s>",p.name); - return; - } - } - for(i=0;imod->code->nnatives;i++) { - hl_native *n = ctx->mod->code->natives + i; - if( ctx->mod->functions_ptrs[n->findex] == ptr ) { - printf("<%s.%s>",n->lib[0] == '?' ? n->lib + 1 : n->lib,n->name); - return; - } - } - printf("",(uint64)ptr); -} - -void hl_emit_dump( emit_ctx *ctx ) { - int i; - int cur_op = 0; - hl_function *f = ctx->fun; - int nargs = f->type->fun->nargs; - hl_emit_flush(ctx); // if it not was not before (in case of dump during emit) - printf("function "); - hl_dump_fun_name(f); - printf("("); - for(i=0;i 0 ) printf(","); - uprintf(USTR("R%d"), i); - } - printf(")\n"); - for(i=0;inregs;i++) - uprintf(USTR("\tR%d : %s\n"),i, hl_type_str(f->regs[i])); - for(i=0;iemit_pos;i++) { - while( ctx->pos_map[cur_op] == i ) { - printf("@%X ", cur_op); - ctx->op_pos = cur_op; - hl_dump_op(ctx, f->ops + cur_op); - printf("\n"); - cur_op++; - } - einstr *e = ctx->instrs + i; - printf("\t\t@%X %s", i, op_names[e->op]); - switch( e->op ) { - case TEST: - case CMP: - printf("-%s", hl_op_name(e->size_offs)+2); - break; - case BINOP: - case UNOP: - printf("-%s", hl_op_name(e->size_offs)+1); - break; - default: - break; - } - if( e->mode && e->op != PHI ) - printf("%s", emit_mode_str(e->mode)); - switch( e->op ) { - case CALL_FUN: - printf(" "); - { - int fid = ctx->mod->functions_indexes[e->a.index]; - hl_code *code = ctx->mod->code; - if( fid < code->nfunctions ) { - hl_dump_fun_name(&code->functions[fid]); - } else { - printf("???"); - } - } - hl_dump_args(ctx,e); - break; - case CALL_REG: - printf(" @%X", e->a.index); - hl_dump_args(ctx,e); - break; - case CALL_PTR: - printf(" "); - hl_dump_ptr_name(ctx, (void*)e->value); - hl_dump_args(ctx,e); - break; - case PHI: - hl_dump_args(ctx,e); - { - int i; - ereg *args = hl_emit_get_args(ctx, e); - for(i=0;inargs;i++) { - einstr *a = ctx->instrs + args[i].index; - if( a->mode != e->mode ) { - printf(" ???"); - break; - } - } - } - break; - case JUMP: - case JCOND: - printf(" @%X", i + 1 + e->size_offs); - break; - case LOAD_IMM: - printf(" "); - dump_value(ctx, e->value, e->mode); - break; - case ALLOC_STACK: - printf(" %d", e->size_offs); - break; - case LOAD_ADDR: - if( (e->b.index>>8) ) - printf(" @%X[%Xh]", e->a.index, e->b.index); - else - printf(" @%X[%d]", e->a.index, e->b.index); - break; - case STORE: - { - int offs = e->size_offs; - if( offs == 0 ) - printf(" [@%X] = @%X", e->a.index, e->b.index); - else - printf(" @%X[%d] = @%X", e->a.index, offs, e->b.index); - if( e->mode == 0 || e->mode != ctx->instrs[e->b.index].mode ) - printf(" ???"); - } - break; - default: - if( e->a.index >= 0 ) { - printf(" @%X", e->a.index); - if( e->b.index >= 0 ) printf(", @%X", e->b.index); - } - break; - } - printf("\n"); - } - // interrupted - if( cur_op < f->nops ) { - printf("@%X ", cur_op); - ctx->op_pos = cur_op; - hl_dump_op(ctx, f->ops + cur_op); - printf("\n\t\t...\n"); - } - printf("\n\n"); - fflush(stdout); -} diff --git a/src/jit.c b/src/jit_old.c similarity index 100% rename from src/jit.c rename to src/jit_old.c diff --git a/src/module.c b/src/module.c index 1dbd9f2cf..e46f73f13 100644 --- a/src/module.c +++ b/src/module.c @@ -21,6 +21,7 @@ */ #include #include +#include #ifdef HL_WIN # undef _GUID @@ -680,7 +681,7 @@ static void hl_module_add( hl_module *m ) { int hl_module_init( hl_module *m, h_bool hot_reload ) { int i; - emit_ctx *ctx; + jit_ctx *ctx; // expand globals if( hot_reload ) { int nsize = m->globals_size + HOT_RELOAD_EXTRA_GLOBALS * sizeof(void*); @@ -706,20 +707,23 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { hl_module_init_natives(m); hl_module_init_indexes(m); // JIT - ctx = hl_emit_alloc(); + ctx = hl_jit_alloc(); if( ctx == NULL ) return 0; - hl_emit_init(ctx, m); + hl_jit_init(ctx, m); for(i=0;icode->nfunctions;i++) { hl_function *f = m->code->functions + i; - int fpos = hl_emit_function(ctx, m, f); + int fpos = hl_jit_function(ctx, m, f); if( fpos < 0 ) { - hl_emit_free(ctx, false); + hl_jit_free(ctx, false); return 0; } m->functions_ptrs[f->findex] = (void*)(int_val)fpos; +# ifdef HL_DEBUG + if( hl_setup.sys_nargs > 0 && ucmp(hl_setup.sys_args[0],USTR("--dump")) == 0 ) hl_emit_dump(ctx); +# endif } - m->jit_code = hl_emit_code(ctx, m, &m->codesize, &m->jit_debug, NULL); + m->jit_code = hl_jit_code(ctx, m, &m->codesize, &m->jit_debug, NULL); for(i=0;icode->nfunctions;i++) { hl_function *f = m->code->functions + i; m->functions_ptrs[f->findex] = ((unsigned char*)m->jit_code) + ((int_val)m->functions_ptrs[f->findex]); @@ -736,10 +740,10 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { # ifdef HL_VTUNE hl_setup.vtune_init = modules_init_vtune; # endif - hl_emit_free(ctx, hot_reload); + hl_jit_free(ctx, hot_reload); if( hot_reload ) { hl_code_hash_finalize(m->hash); - m->emit_ctx = ctx; + m->jit_ctx = ctx; } return 1; } @@ -824,7 +828,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { int i,i1,i2; bool has_changes = false; int changes_count = 0; - emit_ctx *ctx = m1->emit_ctx; + jit_ctx *ctx = m1->jit_ctx; hl_module *m2 = hl_module_alloc(c); m2->hash = hl_code_hash_alloc(c); @@ -849,7 +853,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { hl_module_init_natives(m2); hl_module_init_indexes(m2); - hl_emit_reset(ctx, m2); + hl_jit_reset(ctx, m2); hl_code_hash_finalize(m2->hash); for(i=0;icode->nconstants;i++) { @@ -893,7 +897,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { changes_count++; m1->hash->functions_hashes[i1] = hash2; // update hash - int fpos = hl_emit_function(ctx, m2, f2); + int fpos = hl_jit_function(ctx, m2, f2); if( fpos < 0 ) return false; m2->functions_ptrs[f2->findex] = (void*)(int_val)fpos; has_changes = true; @@ -902,7 +906,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { } if( i1 == m1->code->nfunctions ) { // not found (signature changed or new method) : inject new method! - int fpos = hl_emit_function(ctx, m2, f2); + int fpos = hl_jit_function(ctx, m2, f2); if( fpos < 0 ) return false; m2->hash->functions_hashes[i2] = -1; m2->functions_ptrs[f2->findex] = (void*)(int_val)fpos; @@ -921,7 +925,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { if( !has_changes ) { printf("[HotReload] No changes found\n"); fflush(stdout); - hl_emit_free(ctx, true); + hl_jit_free(ctx, true); return false; } @@ -969,7 +973,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { } } - m2->jit_code = hl_emit_code(ctx, m2, &m2->codesize, &m2->jit_debug, m1); + m2->jit_code = hl_jit_code(ctx, m2, &m2->codesize, &m2->jit_debug, m1); // patch missing debug info int start = -1; @@ -984,7 +988,7 @@ h_bool hl_module_patch( hl_module *m1, hl_code *c ) { } } - hl_emit_free(ctx,true); + hl_jit_free(ctx,true); if( m2->jit_code == NULL ) { printf("[HotReload] Couldn't JIT result\n"); @@ -1054,7 +1058,7 @@ void hl_module_free( hl_module *m ) { free(m->jit_debug[i].offsets); free(m->jit_debug); } - if( m->emit_ctx ) - hl_emit_free(m->emit_ctx,false); + if( m->jit_ctx ) + hl_jit_free(m->jit_ctx,false); free(m); } From 5da805fc6294c57e0e8e09ef6ac077c1fc3da70f Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 27 Mar 2026 00:24:29 +0100 Subject: [PATCH 09/83] added --- src/jit.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 src/jit.c diff --git a/src/jit.c b/src/jit.c new file mode 100644 index 000000000..739ab44d9 --- /dev/null +++ b/src/jit.c @@ -0,0 +1,115 @@ +/* + * Copyright (C)2015-2016 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include + +static jit_ctx *current_ctx = NULL; + +void hl_jit_error( const char *msg, int line ) { + printf("*** EMIT ERROR line %d (%s) ****\n", line, msg); + if( current_ctx ) hl_emit_dump(current_ctx); + hl_debug_break(); + fflush(stdout); + exit(-1); +} + +void hl_jit_null_field_access() { jit_assert(); } +void hl_jit_null_access() { jit_assert(); } +void hl_jit_assert() { jit_assert(); } + +void int_alloc_reset( int_alloc *a ) { + a->cur = 0; +} + +void int_alloc_free( int_alloc *a ) { + free(a->data); + a->cur = 0; + a->max = 0; + a->data = NULL; +} + +int *int_alloc_get( int_alloc *a, int count ) { + while( a->cur + count > a->max ) { + int next_size = a->max ? a->max << 1 : 128; + int *new_data = (int*)malloc(sizeof(int) * next_size); + if( new_data == NULL ) jit_error("Out of memory"); + memcpy(new_data, a->data, sizeof(int) * a->cur); + free(a->data); + a->data = new_data; + a->max = next_size; + } + int *ptr = a->data + a->cur; + a->cur += count; + return ptr; +} + +void int_alloc_store( int_alloc *a, int v ) { + *int_alloc_get(a,1) = v; +} + +void int_alloc_store_unique( int_alloc *a, int v ) { + int i = 0; + while( i < a->cur ) { + if( a->data[i] == v ) return; + i++; + } + *int_alloc_get(a,1) = v; +} + +void hl_emit_alloc( jit_ctx *jit ); +void hl_emit_free( jit_ctx *jit ); +void hl_emit_function( jit_ctx *jit ); + +jit_ctx *hl_jit_alloc() { + jit_ctx *ctx = (jit_ctx*)malloc(sizeof(jit_ctx)); + memset(ctx,0,sizeof(jit_ctx)); + hl_alloc_init(&ctx->falloc); + hl_emit_alloc(ctx); + return ctx; +} + +void hl_jit_init( jit_ctx *ctx, hl_module *m ) { +} + +void hl_jit_free( jit_ctx *ctx, h_bool can_reset ) { + hl_emit_free(ctx); + hl_free(&ctx->falloc); + free(ctx); +} + +void hl_jit_reset( jit_ctx *ctx, hl_module *m ) { +} + +int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { + hl_free(&ctx->falloc); + ctx->mod = m; + ctx->fun = f; + current_ctx = ctx; + hl_emit_function(ctx); + current_ctx = NULL; + return 0; +} + +void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { + printf("TODO:emit_code\n"); + exit(0); + return NULL; +} From f1248a6bb9c6b636ff6fef52dcf0d1ab760d933f Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 27 Mar 2026 00:26:34 +0100 Subject: [PATCH 10/83] minor --- src/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.c b/src/main.c index a25b673cc..60c4db77e 100644 --- a/src/main.c +++ b/src/main.c @@ -20,7 +20,7 @@ * DEALINGS IN THE SOFTWARE. */ #include -#include +#include #include "hlsystem.h" #ifdef HL_WIN From c8ecc31b09e34f54815b518159a58967c8c81d3c Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 27 Mar 2026 07:47:05 +0100 Subject: [PATCH 11/83] emit block output & dump check --- src/jit.c | 6 +++++- src/jit.h | 21 +++++++++++++++++---- src/jit_dump.c | 11 +++++++++++ src/jit_emit.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 79 insertions(+), 6 deletions(-) diff --git a/src/jit.c b/src/jit.c index 739ab44d9..e4cb0eae1 100644 --- a/src/jit.c +++ b/src/jit.c @@ -25,7 +25,11 @@ static jit_ctx *current_ctx = NULL; void hl_jit_error( const char *msg, int line ) { printf("*** EMIT ERROR line %d (%s) ****\n", line, msg); - if( current_ctx ) hl_emit_dump(current_ctx); + if( current_ctx ) { + jit_ctx *ctx = current_ctx; + current_ctx = NULL; + hl_emit_dump(ctx); + } hl_debug_break(); fflush(stdout); exit(-1); diff --git a/src/jit.h b/src/jit.h index 19fc09b77..55f3323ab 100644 --- a/src/jit.h +++ b/src/jit.h @@ -92,7 +92,6 @@ typedef struct { }; } einstr; -typedef struct _emit_ctx emit_ctx; typedef struct { int *data; @@ -100,7 +99,19 @@ typedef struct { int cur; } int_alloc; -typedef struct _jit_ctx jit_ctx; +typedef struct _eblock { + int id; + int start_pos; + int end_pos; + int next_count; + int pred_count; + int *preds; + int *nexts; +} eblock; + +typedef struct _emit_ctx emit_ctx; + +typedef struct _jit_ctx ji_ctx; struct _jit_ctx { hl_module *mod; @@ -108,8 +119,10 @@ struct _jit_ctx { hl_alloc falloc; emit_ctx *emit; // emit output - einstr *instrs; int instr_count; + int block_count; + einstr *instrs; + eblock *blocks; int *emit_pos_map; }; @@ -146,7 +159,7 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); # define jit_debug(...) #endif -void hl_jit_error( const char *msg, int line ); +HL_NO_RETURN( void hl_jit_error( const char *msg, int line ) ); void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); diff --git a/src/jit_dump.c b/src/jit_dump.c index 1e524cdd2..169d6c678 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -266,6 +266,17 @@ void hl_emit_dump( jit_ctx *ctx ) { printf(")\n"); for(i=0;inregs;i++) uprintf(USTR("\tR%d : %s\n"),i, hl_type_str(f->regs[i])); + // check blocks intervals + int cur = 0; + for(i=0;iblock_count;i++) { + eblock *b = ctx->blocks + i; + if( b->id < 0 || b->id >= ctx->block_count ) jit_assert(); + if( b->start_pos != cur ) printf(" ??? BLOCK %d START AT %X != %X\n", i, b->start_pos, cur); + cur = b->end_pos + 1; + } + if( cur != ctx->instr_count ) + printf(" ??? MISSING BLOCK FOR RANGE %X-%X\n", cur, ctx->instr_count); + // print instrs for(i=0;iinstr_count;i++) { while( ctx->emit_pos_map[cur_op] == i ) { printf("@%X ", cur_op); diff --git a/src/jit_emit.c b/src/jit_emit.c index 868c6dd02..44c22ab80 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -45,8 +45,10 @@ struct _linked_inf { struct _emit_block { int id; int start_pos; + int end_pos; int wait_nexts; int mark; + linked_inf *nexts; linked_inf *preds; linked_inf *written_vars; linked_inf *incomplete_phis; @@ -81,6 +83,7 @@ struct _emit_ctx { int_alloc block_writes; int_alloc phi_gather; + emit_block *root_block; emit_block *current_block; linked_inf *arrival_points; void *closure_list; // TODO : patch with good addresses @@ -293,6 +296,7 @@ static emit_block *alloc_block( emit_ctx *ctx ) { static void block_add_prec( emit_ctx *ctx, emit_block *b, emit_block *p ) { b->preds = link_add(ctx,0,p,b->preds); + p->nexts = link_add(ctx,0,b,p->nexts); jit_debug(" PRED %d\n",p->id); } @@ -320,6 +324,7 @@ static void split_block( emit_ctx *ctx ) { block_add_prec(ctx, b, (emit_block*)ctx->arrival_points->ptr); ctx->arrival_points = ctx->arrival_points->next; } + ctx->current_block->end_pos = b->start_pos - 1; ctx->current_block = b; } @@ -562,6 +567,42 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { static void emit_opcode( emit_ctx *ctx, hl_opcode *o ); +static void emit_flush_blocks( jit_ctx *jit, emit_block *b ) { + eblock *bl = jit->blocks + b->id; + if( bl->id ) return; // already set + bl->id = b->id; + bl->start_pos = b->start_pos; + bl->end_pos = b->end_pos; + linked_inf *tmp; + tmp = b->preds; + while( tmp ) { + bl->pred_count++; + tmp = tmp->next; + } + tmp = b->nexts; + while( tmp ) { + bl->next_count++; + tmp = tmp->next; + } + bl->preds = (int*)hl_malloc(&jit->falloc,sizeof(int)*bl->pred_count); + bl->nexts = (int*)hl_malloc(&jit->falloc,sizeof(int)*bl->next_count); + int i; + i = 0; + tmp = b->preds; + while( tmp ) { + bl->preds[i++] = ((emit_block*)tmp->ptr)->id; + tmp = tmp->next; + } + i = 0; + tmp = b->nexts; + while( tmp ) { + emit_block *n = (emit_block*)tmp->ptr; + bl->nexts[i++] = n->id; + if( n->start_pos > b->start_pos ) emit_flush_blocks(jit, n); + tmp = tmp->next; + } +} + void hl_emit_flush( jit_ctx *jit ) { emit_ctx *ctx = jit->emit; int i = 0; @@ -572,9 +613,13 @@ void hl_emit_flush( jit_ctx *jit ) { e->size_offs = ctx->pos_map[target] - (pos + 1); } ctx->pos_map[ctx->fun->nops] = -1; + ctx->current_block->end_pos = ctx->emit_pos - 1; jit->instrs = ctx->instrs; jit->instr_count = ctx->emit_pos; jit->emit_pos_map = ctx->pos_map; + jit->block_count = ctx->current_block->id + 1; + jit->blocks = hl_zalloc(&jit->falloc,sizeof(eblock) * jit->block_count); + emit_flush_blocks(jit, ctx->root_block); } void hl_emit_function( jit_ctx *jit ) { @@ -589,7 +634,7 @@ void hl_emit_function( jit_ctx *jit ) { int_alloc_reset(&ctx->args_data); int_alloc_reset(&ctx->jump_regs); int_alloc_reset(&ctx->block_writes); - ctx->current_block = alloc_block(ctx); + ctx->root_block = ctx->current_block = alloc_block(ctx); ctx->arrival_points = NULL; jit_debug("---- begin [%X] ----\n",f->findex); if( f->nregs > ctx->max_regs ) { From b7b7f29fdf807ec0eb881f27e8b99e031f342ca4 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 28 Mar 2026 09:38:17 +0100 Subject: [PATCH 12/83] finished block emit --- src/jit.c | 4 ++-- src/jit.h | 5 +++-- src/jit_dump.c | 4 +++- src/jit_emit.c | 25 +++++++++++++++---------- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/jit.c b/src/jit.c index e4cb0eae1..bb1d2c78b 100644 --- a/src/jit.c +++ b/src/jit.c @@ -23,8 +23,8 @@ static jit_ctx *current_ctx = NULL; -void hl_jit_error( const char *msg, int line ) { - printf("*** EMIT ERROR line %d (%s) ****\n", line, msg); +void hl_jit_error( const char *msg, const char *func, int line ) { + printf("*** JIT ERROR %s:%d (%s)****\n", func, line, msg); if( current_ctx ) { jit_ctx *ctx = current_ctx; current_ctx = NULL; diff --git a/src/jit.h b/src/jit.h index 55f3323ab..6e73f84ab 100644 --- a/src/jit.h +++ b/src/jit.h @@ -43,6 +43,7 @@ typedef enum { CALL_PTR, CALL_REG, CALL_FUN, + MOV, PHI, ALLOC_STACK, FREE_STACK, @@ -150,7 +151,7 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); # define JIT_DEBUG #endif -#define jit_error(msg) hl_jit_error(msg,__LINE__) +#define jit_error(msg) hl_jit_error(msg,__func__,__LINE__) #define jit_assert() jit_error("") #ifdef JIT_DEBUG @@ -159,7 +160,7 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); # define jit_debug(...) #endif -HL_NO_RETURN( void hl_jit_error( const char *msg, int line ) ); +HL_NO_RETURN( void hl_jit_error( const char *msg, const char *func, int line ) ); void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); diff --git a/src/jit_dump.c b/src/jit_dump.c index 169d6c678..5817432b3 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -40,6 +40,7 @@ static const char *op_names[] = { "call", "call", "call", + "mov", "phi", "alloc-stack", "free-stack", @@ -270,8 +271,9 @@ void hl_emit_dump( jit_ctx *ctx ) { int cur = 0; for(i=0;iblock_count;i++) { eblock *b = ctx->blocks + i; - if( b->id < 0 || b->id >= ctx->block_count ) jit_assert(); + if( b->id != i ) printf(" ??? BLOCK @%d ID is %d\n",i,b->id); if( b->start_pos != cur ) printf(" ??? BLOCK %d START AT %X != %X\n", i, b->start_pos, cur); + if( b->end_pos < b->start_pos ) printf(" ??? BLOCK %d RANGE [%X,%X]\n", i, b->start_pos, b->end_pos); cur = b->end_pos + 1; } if( cur != ctx->instr_count ) diff --git a/src/jit_emit.c b/src/jit_emit.c index 44c22ab80..e160ad2c0 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -300,8 +300,7 @@ static void block_add_prec( emit_ctx *ctx, emit_block *b, emit_block *p ) { jit_debug(" PRED %d\n",p->id); } -static void split_block( emit_ctx *ctx ) { - // flush previous block +static void flush_block( emit_ctx *ctx ) { int i = 0; emit_block *cur = ctx->current_block; while( i < ctx->block_writes.cur ) { @@ -312,19 +311,25 @@ static void split_block( emit_ctx *ctx ) { r->written = false; } int_alloc_reset(&ctx->block_writes); +} + +static void split_block( emit_ctx *ctx ) { + flush_block(ctx); + // split emit_block *b = alloc_block(ctx); b->id = ctx->current_block->id + 1; b->start_pos = ctx->emit_pos; jit_debug("BLOCK #%d@%X\n",b->id,ctx->op_pos); einstr *eprev = &ctx->instrs[ctx->emit_pos-1]; - if( (eprev->op != JUMP && eprev->op != RET) || ctx->fun->ops[ctx->op_pos].op == OTrap ) - block_add_prec(ctx, b, ctx->current_block); while( ctx->arrival_points && ctx->arrival_points->id == ctx->op_pos ) { block_add_prec(ctx, b, (emit_block*)ctx->arrival_points->ptr); ctx->arrival_points = ctx->arrival_points->next; } - ctx->current_block->end_pos = b->start_pos - 1; + bool dead_code = b->preds == NULL; // if we have no reach, force previous block dependency, this is rare dead code emit by compiler + if( (eprev->op != JUMP && eprev->op != RET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) + block_add_prec(ctx, b, ctx->current_block); + ctx->current_block->end_pos = ctx->emit_pos - 1; ctx->current_block = b; } @@ -567,7 +572,7 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { static void emit_opcode( emit_ctx *ctx, hl_opcode *o ); -static void emit_flush_blocks( jit_ctx *jit, emit_block *b ) { +static void emit_write_blocks( jit_ctx *jit, emit_block *b ) { eblock *bl = jit->blocks + b->id; if( bl->id ) return; // already set bl->id = b->id; @@ -598,7 +603,7 @@ static void emit_flush_blocks( jit_ctx *jit, emit_block *b ) { while( tmp ) { emit_block *n = (emit_block*)tmp->ptr; bl->nexts[i++] = n->id; - if( n->start_pos > b->start_pos ) emit_flush_blocks(jit, n); + if( n->start_pos > b->start_pos ) emit_write_blocks(jit, n); tmp = tmp->next; } } @@ -619,7 +624,7 @@ void hl_emit_flush( jit_ctx *jit ) { jit->emit_pos_map = ctx->pos_map; jit->block_count = ctx->current_block->id + 1; jit->blocks = hl_zalloc(&jit->falloc,sizeof(eblock) * jit->block_count); - emit_flush_blocks(jit, ctx->root_block); + emit_write_blocks(jit, ctx->root_block); } void hl_emit_function( jit_ctx *jit ) { @@ -714,7 +719,7 @@ static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { if( b->start_pos == target ) { b->wait_nexts--; b->preds = link_add(ctx,0,ctx->current_block,b->preds); - split_block(ctx); + flush_block(ctx); if( b->wait_nexts == 0 ) { jit_debug(" SEAL #%d\n",b->id); // seal block @@ -767,7 +772,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { switch( o->op ) { case OMov: case OUnsafeCast: - STORE(dst, LOAD(ra)); + STORE(dst, emit_gen(ctx,MOV,LOAD(ra),ENULL,hl_type_mode(ra->t))); break; case OInt: STORE(dst, LOAD_CONST(m->code->ints[o->p2], dst->t)); From 5c739b65ce822a70c3a95443509a13931fa1ad4f Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 28 Mar 2026 11:27:34 +0100 Subject: [PATCH 13/83] change writes to be values instead of opcode indexes --- src/jit.h | 2 ++ src/jit_dump.c | 37 +++++++++++++-------- src/jit_emit.c | 88 +++++++++++++++++++++++++++----------------------- 3 files changed, 72 insertions(+), 55 deletions(-) diff --git a/src/jit.h b/src/jit.h index 6e73f84ab..a457ec835 100644 --- a/src/jit.h +++ b/src/jit.h @@ -122,8 +122,10 @@ struct _jit_ctx { // emit output int instr_count; int block_count; + int value_count; einstr *instrs; eblock *blocks; + int *values_writes; int *emit_pos_map; }; diff --git a/src/jit_dump.c b/src/jit_dump.c index 5817432b3..b17b8fac2 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -191,7 +191,7 @@ static void hl_dump_args( jit_ctx *ctx, einstr *e ) { printf("("); for(int i=0;inargs;i++) { if( i != 0 ) printf(","); - printf("@%X", v[i].index); + printf("V%d", v[i].index); } printf(")"); } @@ -279,6 +279,7 @@ void hl_emit_dump( jit_ctx *ctx ) { if( cur != ctx->instr_count ) printf(" ??? MISSING BLOCK FOR RANGE %X-%X\n", cur, ctx->instr_count); // print instrs + int vpos = 0; for(i=0;iinstr_count;i++) { while( ctx->emit_pos_map[cur_op] == i ) { printf("@%X ", cur_op); @@ -287,15 +288,21 @@ void hl_emit_dump( jit_ctx *ctx ) { cur_op++; } einstr *e = ctx->instrs + i; - printf("\t\t@%X %s", i, op_names[e->op]); + printf("\t\t@%X ", i); + if( vpos < ctx->value_count && ctx->values_writes[vpos] == i ) + printf("V%d = ", vpos++); + printf("%s", op_names[e->op]); + bool show_size = true; switch( e->op ) { case TEST: case CMP: printf("-%s", hl_op_name(e->size_offs)+2); + show_size = false; break; case BINOP: case UNOP: printf("-%s", hl_op_name(e->size_offs)+1); + show_size = false; break; default: break; @@ -317,7 +324,7 @@ void hl_emit_dump( jit_ctx *ctx ) { hl_dump_args(ctx,e); break; case CALL_REG: - printf(" @%X", e->a.index); + printf(" V%d", e->a.index); hl_dump_args(ctx,e); break; case CALL_PTR: @@ -331,7 +338,7 @@ void hl_emit_dump( jit_ctx *ctx ) { int i; ereg *args = hl_emit_get_args(ctx->emit, e); for(i=0;inargs;i++) { - einstr *a = ctx->instrs + args[i].index; + einstr *a = ctx->instrs + ctx->values_writes[args[i].index]; if( a->mode != e->mode ) { printf(" ???"); break; @@ -347,35 +354,37 @@ void hl_emit_dump( jit_ctx *ctx ) { printf(" "); dump_value(ctx, e->value, e->mode); break; - case ALLOC_STACK: - printf(" %d", e->size_offs); - break; case LOAD_ADDR: if( (e->b.index>>8) ) - printf(" @%X[%Xh]", e->a.index, e->b.index); + printf(" V%d[%Xh]", e->a.index, e->b.index); else - printf(" @%X[%d]", e->a.index, e->b.index); + printf(" V%d[%d]", e->a.index, e->b.index); break; case STORE: { int offs = e->size_offs; if( offs == 0 ) - printf(" [@%X] = @%X", e->a.index, e->b.index); + printf(" [V%d] = V%d", e->a.index, e->b.index); else - printf(" @%X[%d] = @%X", e->a.index, offs, e->b.index); - if( e->mode == 0 || e->mode != ctx->instrs[e->b.index].mode ) + printf(" V%d[%d] = V%d", e->a.index, offs, e->b.index); + if( e->mode == 0 || e->mode != ctx->instrs[ctx->values_writes[e->b.index]].mode ) printf(" ???"); } break; default: if( e->a.index >= 0 ) { - printf(" @%X", e->a.index); - if( e->b.index >= 0 ) printf(", @%X", e->b.index); + printf(" V%d", e->a.index); + if( e->b.index >= 0 ) printf(", V%d", e->b.index); } + if( show_size && e->size_offs != 0 ) + printf(" %d", e->size_offs); break; } printf("\n"); } + // invalid ? + while( vpos < ctx->value_count ) + printf(" ??? UNWRITTEN VALUE V%d @%X\n", vpos, ctx->values_writes[vpos++]); // interrupted if( cur_op < f->nops ) { printf("@%X ", cur_op); diff --git a/src/jit_emit.c b/src/jit_emit.c index e160ad2c0..f688b46d9 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -82,6 +82,7 @@ struct _emit_ctx { int_alloc jump_regs; int_alloc block_writes; int_alloc phi_gather; + int_alloc values; emit_block *root_block; emit_block *current_block; @@ -101,7 +102,7 @@ struct _emit_ctx { #define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR(obj,0),HL_WSIZE*2),HL_WSIZE*(id)) #define OFFSET(base,index,mult,offset) emit_gen_ext(ctx, LEA, base, index, M_PTR, (mult) | ((offset) << 8)) #define BREAK() emit_gen(ctx, DEBUG_BREAK, ENULL, ENULL, 0) -#define CUR_REG() __current_reg(ctx) +#define GET_WRITE(r) ctx->instrs[ctx->values.data[(r).index]] #define HDYN_VALUE 8 @@ -166,16 +167,13 @@ static emit_mode hl_type_mode( hl_type *t ) { return M_PTR; } -static void mark_no_return( emit_ctx *ctx ) { - ctx->instrs[ctx->emit_pos-1].mode = M_NORET; -} - void hl_jit_patch_method( void*fun, void**newt ) { jit_assert(); } -static ereg __current_reg( emit_ctx *ctx ) { - ereg r = {ctx->emit_pos-1}; +static ereg new_value( emit_ctx *ctx ) { + ereg r = {ctx->values.cur}; + int_alloc_store(&ctx->values, ctx->emit_pos-1); return r; } @@ -212,7 +210,7 @@ static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { einstr *e = emit_instr(ctx, STORE); - e->mode = ctx->instrs[from.index].mode; + e->mode = GET_WRITE(from).mode; e->size_offs = offs; e->a = to; e->b = from; @@ -257,7 +255,7 @@ static ereg emit_gen_ext( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode, i e->size_offs = size_offs; e->a = a; e->b = b; - return CUR_REG(); + return mode == 0 || mode == M_NORET ? ENULL : new_value(ctx); } static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { @@ -269,15 +267,15 @@ static ereg emit_gen_size( emit_ctx *ctx, emit_op op, int size_offs ) { } static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { - unsigned char mode = ctx->instrs[v1.index].mode; - if( mode != ctx->instrs[v2.index].mode ) jit_assert(); + unsigned char mode = GET_WRITE(v1).mode; + if( mode != GET_WRITE(v2).mode ) jit_assert(); einstr *e = emit_instr(ctx, PHI); e->mode = mode; ereg args[2]; args[0] = v1; args[1] = v2; store_args(ctx, e, args, 2); - return CUR_REG(); + return new_value(ctx); } static int emit_jump( emit_ctx *ctx, bool cond ) { @@ -321,12 +319,12 @@ static void split_block( emit_ctx *ctx ) { b->id = ctx->current_block->id + 1; b->start_pos = ctx->emit_pos; jit_debug("BLOCK #%d@%X\n",b->id,ctx->op_pos); - einstr *eprev = &ctx->instrs[ctx->emit_pos-1]; while( ctx->arrival_points && ctx->arrival_points->id == ctx->op_pos ) { block_add_prec(ctx, b, (emit_block*)ctx->arrival_points->ptr); ctx->arrival_points = ctx->arrival_points->next; } bool dead_code = b->preds == NULL; // if we have no reach, force previous block dependency, this is rare dead code emit by compiler + einstr *eprev = &ctx->instrs[ctx->emit_pos-1]; if( (eprev->op != JUMP && eprev->op != RET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) block_add_prec(ctx, b, ctx->current_block); ctx->current_block->end_pos = ctx->emit_pos - 1; @@ -348,7 +346,7 @@ static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { einstr *e = emit_instr(ctx, LOAD_IMM); e->mode = hl_type_mode(size_t); e->value = value; - return CUR_REG(); + return new_value(ctx); } static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) { @@ -356,7 +354,7 @@ static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) e->mode = hl_type_mode(size_t); e->a = v; e->b.index = offset; - return CUR_REG(); + return new_value(ctx); } static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { @@ -371,10 +369,10 @@ static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int nargs, hl_type *ret ) { einstr *e = emit_instr(ctx, CALL_PTR); - e->mode = hl_type_mode(ret); + e->mode = (unsigned char)(ret ? hl_type_mode(ret) : M_NORET); e->value = (int_val)native_ptr; store_args(ctx, e, args, nargs); - return CUR_REG(); + return ret == NULL || e->mode == M_VOID ? ENULL : new_value(ctx); } static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_type *ret ) { @@ -382,11 +380,16 @@ static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_typ e->mode = hl_type_mode(ret); e->a = f; store_args(ctx, e, args, nargs); - return CUR_REG(); + return e->mode == M_VOID ? ENULL : new_value(ctx); +} + +static void patch_instr_mode( emit_ctx *ctx, int mode ) { + ctx->instrs[ctx->emit_pos-1].mode = (unsigned char)mode; } static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { - emit_gen_ext(ctx, TEST, v, ENULL, ctx->instrs[v.index].mode, o); + emit_gen_ext(ctx, TEST, v, ENULL, 0, o); + patch_instr_mode(ctx, GET_WRITE(v).mode); } static void emit_gather_phi_rec( emit_ctx *ctx, emit_block *b, vreg *r, int uid ) { @@ -429,7 +432,7 @@ static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { } else { if( ctx->phi_gather.cur == 0 ) jit_assert(); store_args(ctx, p, (ereg*)ctx->phi_gather.data, ctx->phi_gather.cur); - r->current = CUR_REG(); + r->current = new_value(ctx); } } return r->current; @@ -449,7 +452,7 @@ static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int e->mode = hl_type_mode(dst->t); e->a.index = findex; store_args(ctx, e, args, count); - STORE(dst, CUR_REG()); + STORE(dst, e->mode == M_VOID ? ENULL : new_value(ctx)); } } @@ -624,6 +627,8 @@ void hl_emit_flush( jit_ctx *jit ) { jit->emit_pos_map = ctx->pos_map; jit->block_count = ctx->current_block->id + 1; jit->blocks = hl_zalloc(&jit->falloc,sizeof(eblock) * jit->block_count); + jit->value_count = ctx->values.cur; + jit->values_writes = ctx->values.data; emit_write_blocks(jit, ctx->root_block); } @@ -639,6 +644,7 @@ void hl_emit_function( jit_ctx *jit ) { int_alloc_reset(&ctx->args_data); int_alloc_reset(&ctx->jump_regs); int_alloc_reset(&ctx->block_writes); + int_alloc_reset(&ctx->values); ctx->root_block = ctx->current_block = alloc_block(ctx); ctx->arrival_points = NULL; jit_debug("---- begin [%X] ----\n",f->findex); @@ -651,6 +657,13 @@ void hl_emit_function( jit_ctx *jit ) { ctx->max_regs = f->nregs; } + if( f->nops >= ctx->pos_map_size ) { + free(ctx->pos_map); + ctx->pos_map = (int*)malloc(sizeof(int) * (f->nops+1)); + if( ctx->pos_map == NULL ) jit_assert(); + ctx->pos_map_size = f->nops; + } + for(i=0;inregs;i++) { vreg *r = R(i); r->t = f->regs[i]; @@ -659,7 +672,9 @@ void hl_emit_function( jit_ctx *jit ) { } for(i=0;itype->fun->nargs;i++) { - STORE(R(i), emit_gen(ctx, LOAD_ARG, ENULL, ENULL, hl_type_mode(f->type->fun->args[i]))); + hl_type *t = f->type->fun->args[i]; + if( t->kind == HVOID ) continue; + STORE(R(i), emit_gen(ctx, LOAD_ARG, ENULL, ENULL, hl_type_mode(t))); } for(i=f->nops-1;i>=0;i--) { @@ -674,13 +689,6 @@ void hl_emit_function( jit_ctx *jit ) { } } - if( f->nops >= ctx->pos_map_size ) { - free(ctx->pos_map); - ctx->pos_map = (int*)malloc(sizeof(int) * (f->nops+1)); - if( ctx->vregs == NULL ) jit_assert(); - ctx->pos_map_size = f->nops; - } - for(int op_pos=0;op_posnops;op_pos++) { ctx->op_pos = op_pos; ctx->pos_map[op_pos] = ctx->emit_pos; @@ -888,7 +896,8 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OJNotLt: case OJNotGte: { - emit_gen_ext(ctx, CMP, LOAD(dst), LOAD(ra), hl_type_mode(dst->t), o->op); + emit_gen_ext(ctx, CMP, LOAD(dst), LOAD(ra), 0, o->op); + patch_instr_mode(ctx, hl_type_mode(dst->t)); register_block_jump(ctx, o->p3, true); } break; @@ -912,10 +921,8 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, emit_conv(ctx,LOAD(ra),hl_type_mode(dst->t), o->op == OToUFloat)); break; case ORet: - if( dst->t->kind == HVOID ) - emit_gen(ctx,RET,ENULL,ENULL,M_VOID); - else - emit_gen(ctx, RET, LOAD(dst), ENULL, hl_type_mode(dst->t)); + emit_gen(ctx, RET, dst->t->kind == HVOID ? ENULL : LOAD(dst), ENULL, M_NORET); + patch_instr_mode(ctx, hl_type_mode(dst->t)); break; case OIncr: case ODecr: @@ -1017,7 +1024,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[i] = LOAD(R(o->extra[i])); ereg v2 = emit_dyn_call(ctx,LOAD_MEM_PTR(r,HL_WSIZE),args,o->p3,dst->t); patch_jump(ctx, jend); - STORE(dst, emit_phi(ctx,v1,v2)); + if( dst->t->kind != HVOID ) STORE(dst, emit_phi(ctx,v1,v2)); } break; case OStaticClosure: @@ -1222,7 +1229,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { emit_gen_size(ctx, FREE_STACK, o->p3 + (need_dyn ? dyn_size : 0)); patch_jump(ctx, jend); - STORE(dst, emit_phi(ctx, v1, v2)); + if( dst->t->kind != HVOID ) STORE(dst, emit_phi(ctx, v1, v2)); } break; default: @@ -1235,8 +1242,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case ORethrow: { ereg arg = LOAD(dst); - emit_native_call(ctx, o->op == OThrow ? hl_throw : hl_rethrow, &arg, 1, &hlt_void); - mark_no_return(ctx); + emit_native_call(ctx, o->op == OThrow ? hl_throw : hl_rethrow, &arg, 1, NULL); } break; case OLabel: @@ -1462,8 +1468,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } // ----------------------------------------- ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : ENULL; - emit_native_call(ctx, null_field_access ? hl_jit_null_field_access : hl_jit_null_access, &arg, null_field_access ? 1 : 0, &hlt_void); - mark_no_return(ctx); + emit_native_call(ctx, null_field_access ? hl_jit_null_field_access : hl_jit_null_access, &arg, null_field_access ? 1 : 0, NULL); patch_jump(ctx, jok); } break; @@ -1610,7 +1615,8 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg v = LOAD(dst); int count = o->p2; - emit_gen_ext(ctx, CMP, v, LOAD_CONST(count,&hlt_i32), M_I32, OJUGte); + emit_gen_ext(ctx, CMP, v, LOAD_CONST(count,&hlt_i32), 0, OJUGte); + patch_instr_mode(ctx, M_I32); int jdefault = emit_jump(ctx, true); emit_gen_ext(ctx, JUMP_TABLE, v, ENULL, 0, count); for(int i=0; i Date: Sun, 29 Mar 2026 14:24:37 +0200 Subject: [PATCH 14/83] rewrote phi management using separate values --- src/jit.c | 13 +- src/jit.h | 21 ++- src/jit_dump.c | 73 +++++---- src/jit_emit.c | 432 ++++++++++++++++++++++++++++++------------------- src/module.c | 1 + 5 files changed, 333 insertions(+), 207 deletions(-) diff --git a/src/jit.c b/src/jit.c index bb1d2c78b..626cfde03 100644 --- a/src/jit.c +++ b/src/jit.c @@ -69,15 +69,6 @@ void int_alloc_store( int_alloc *a, int v ) { *int_alloc_get(a,1) = v; } -void int_alloc_store_unique( int_alloc *a, int v ) { - int i = 0; - while( i < a->cur ) { - if( a->data[i] == v ) return; - i++; - } - *int_alloc_get(a,1) = v; -} - void hl_emit_alloc( jit_ctx *jit ); void hl_emit_free( jit_ctx *jit ); void hl_emit_function( jit_ctx *jit ); @@ -117,3 +108,7 @@ void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **d exit(0); return NULL; } + +void hl_jit_patch_method( void*fun, void**newt ) { + jit_assert(); +} diff --git a/src/jit.h b/src/jit.h index a457ec835..a1e080ec8 100644 --- a/src/jit.h +++ b/src/jit.h @@ -44,7 +44,6 @@ typedef enum { CALL_REG, CALL_FUN, MOV, - PHI, ALLOC_STACK, FREE_STACK, NATIVE_REG, @@ -93,6 +92,8 @@ typedef struct { }; } einstr; +#define VAL_NULL 0x80000000 +#define IS_NULL(e) ((e).index == VAL_NULL) typedef struct { int *data; @@ -100,14 +101,24 @@ typedef struct { int cur; } int_alloc; +typedef struct _ephi ephi; + +struct _ephi { + ereg value; + int nvalues; + ereg *values; +}; + typedef struct _eblock { int id; int start_pos; int end_pos; int next_count; int pred_count; - int *preds; + int phi_count; int *nexts; + int *preds; + ephi *phis; } eblock; typedef struct _emit_ctx emit_ctx; @@ -143,12 +154,14 @@ void int_alloc_reset( int_alloc *a ); void int_alloc_free( int_alloc *a ); int *int_alloc_get( int_alloc *a, int count ); void int_alloc_store( int_alloc *a, int v ); -void int_alloc_store_unique( int_alloc *a, int v ); - void hl_emit_dump( jit_ctx *ctx ); +const char *hl_emit_regstr( ereg v ); ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); +#define val_str(v) hl_emit_regstr(v) + + #ifdef HL_DEBUG # define JIT_DEBUG #endif diff --git a/src/jit_dump.c b/src/jit_dump.c index b17b8fac2..2ef167285 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -41,7 +41,6 @@ static const char *op_names[] = { "call", "call", "mov", - "phi", "alloc-stack", "free-stack", "native-reg", @@ -49,6 +48,20 @@ static const char *op_names[] = { "debug-break", }; +const char *hl_emit_regstr( ereg v ) { + static char fmts[4][10]; + static int flip = 0; + // allow up to four concurrent val_str + char *fmt = fmts[flip++&3]; + if( IS_NULL(v) ) + sprintf(fmt,"NULL???"); + else if( v.index < 0 ) + sprintf(fmt,"P%d",-v.index); + else + sprintf(fmt,"V%d",v.index); + return fmt; +} + static void hl_dump_arg( hl_function *fun, int fmt, int val, char sep, int pos ) { if( fmt == 0 ) return; printf("%c", sep); @@ -191,7 +204,7 @@ static void hl_dump_args( jit_ctx *ctx, einstr *e ) { printf("("); for(int i=0;inargs;i++) { if( i != 0 ) printf(","); - printf("V%d", v[i].index); + printf("%s", val_str(v[i])); } printf(")"); } @@ -280,7 +293,25 @@ void hl_emit_dump( jit_ctx *ctx ) { printf(" ??? MISSING BLOCK FOR RANGE %X-%X\n", cur, ctx->instr_count); // print instrs int vpos = 0; + cur = 0; for(i=0;iinstr_count;i++) { + while( ctx->blocks[cur].start_pos == i ) { + eblock *b = &ctx->blocks[cur]; + printf("--- BLOCK #%d ---\n", cur); + for(int k=0;kphi_count;k++) { + ephi *p = b->phis + k; + printf("\t\t@%X %s = phi(",i,val_str(p->value)); + for(int n=0;nnvalues;n++) { + if( n > 0 ) printf(","); + printf("%s",val_str(p->values[n])); + } + printf(")"); + if( p->nvalues <= 1 ) + printf(" ???"); + printf("\n"); + } + cur++; + } while( ctx->emit_pos_map[cur_op] == i ) { printf("@%X ", cur_op); hl_dump_op(ctx->fun, f->ops + cur_op); @@ -307,7 +338,7 @@ void hl_emit_dump( jit_ctx *ctx ) { default: break; } - if( e->mode && e->op != PHI ) + if( e->mode ) printf("%s", emit_mode_str(e->mode)); switch( e->op ) { case CALL_FUN: @@ -324,7 +355,7 @@ void hl_emit_dump( jit_ctx *ctx ) { hl_dump_args(ctx,e); break; case CALL_REG: - printf(" V%d", e->a.index); + printf(" %s", val_str(e->a)); hl_dump_args(ctx,e); break; case CALL_PTR: @@ -332,20 +363,6 @@ void hl_emit_dump( jit_ctx *ctx ) { hl_dump_ptr_name(ctx, (void*)e->value); hl_dump_args(ctx,e); break; - case PHI: - hl_dump_args(ctx,e); - { - int i; - ereg *args = hl_emit_get_args(ctx->emit, e); - for(i=0;inargs;i++) { - einstr *a = ctx->instrs + ctx->values_writes[args[i].index]; - if( a->mode != e->mode ) { - printf(" ???"); - break; - } - } - } - break; case JUMP: case JCOND: printf(" @%X", i + 1 + e->size_offs); @@ -356,25 +373,27 @@ void hl_emit_dump( jit_ctx *ctx ) { break; case LOAD_ADDR: if( (e->b.index>>8) ) - printf(" V%d[%Xh]", e->a.index, e->b.index); + printf(" %s[%Xh]", val_str(e->a), e->b.index); else - printf(" V%d[%d]", e->a.index, e->b.index); + printf(" %s[%d]", val_str(e->a), e->b.index); break; case STORE: { int offs = e->size_offs; if( offs == 0 ) - printf(" [V%d] = V%d", e->a.index, e->b.index); + printf(" [%s]", val_str(e->a)); else - printf(" V%d[%d] = V%d", e->a.index, offs, e->b.index); - if( e->mode == 0 || e->mode != ctx->instrs[ctx->values_writes[e->b.index]].mode ) - printf(" ???"); + printf(" %s[%d]", val_str(e->a), offs); + printf(" = %s", val_str(e->b)); + //if( e->mode == 0 || e->mode != ctx->instrs[ctx->values_writes[e->b.index]].mode ) + // printf(" ???"); } break; default: - if( e->a.index >= 0 ) { - printf(" V%d", e->a.index); - if( e->b.index >= 0 ) printf(", V%d", e->b.index); + if( !IS_NULL(e->a) ) { + printf(" %s", val_str(e->a)); + if( !IS_NULL(e->b) ) printf(", %s", val_str(e->b)); + if( e->a.index >= vpos || e->b.index >= vpos ) printf(" ???"); } if( show_size && e->size_offs != 0 ) printf(" %d", e->size_offs); diff --git a/src/jit_emit.c b/src/jit_emit.c index f688b46d9..485030fce 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -25,8 +25,6 @@ typedef struct { hl_type *t; int id; - ereg current; - bool written; } vreg; #define MAX_TMP_ARGS 32 @@ -35,6 +33,7 @@ typedef struct { typedef struct _linked_inf linked_inf; typedef struct _emit_block emit_block; +typedef struct _tmp_phi tmp_phi; struct _linked_inf { int id; @@ -47,11 +46,19 @@ struct _emit_block { int start_pos; int end_pos; int wait_nexts; - int mark; + bool sealed; linked_inf *nexts; linked_inf *preds; linked_inf *written_vars; - linked_inf *incomplete_phis; + linked_inf *phis; + emit_block *wait_seal_next; +}; + +struct _tmp_phi { + ereg value; + vreg *r; + emit_block *b; + linked_inf *vals; }; struct _emit_ctx { @@ -65,7 +72,8 @@ struct _emit_ctx { int max_regs; int emit_pos; int op_pos; - int uid; + int phi_count; + bool flushed; ereg tmp_args[MAX_TMP_ARGS]; ereg traps[MAX_TRAPS]; @@ -80,15 +88,14 @@ struct _emit_ctx { int_alloc args_data; int_alloc jump_regs; - int_alloc block_writes; - int_alloc phi_gather; int_alloc values; emit_block *root_block; emit_block *current_block; + emit_block *wait_seal; linked_inf *arrival_points; void *closure_list; // TODO : patch with good addresses -}; +}; #define R(i) (ctx->vregs + (i)) @@ -110,7 +117,7 @@ struct _emit_ctx { static hl_type hlt_ui8 = { HUI8, 0 }; static hl_type hlt_ui16 = { HUI16, 0 }; -static ereg ENULL = {-1}; +static ereg ENULL = {VAL_NULL}; static linked_inf *link_add( emit_ctx *ctx, int id, void *ptr, linked_inf *head ) { linked_inf *l = hl_malloc(&ctx->jit->falloc,sizeof(linked_inf)); @@ -147,6 +154,32 @@ static linked_inf *link_add_sort_unique( emit_ctx *ctx, int id, void *ptr, linke } } +static linked_inf *link_add_sort_replace( emit_ctx *ctx, int id, void *ptr, linked_inf *head ) { + linked_inf *prev = NULL; + linked_inf *cur = head; + while( cur && cur->id < id ) { + prev = cur; + cur = cur->next; + } + // replace duplicate + if( cur && cur->id == id ) { + cur->ptr = ptr; + return head; + } + // insert + linked_inf *l = hl_malloc(&ctx->jit->falloc,sizeof(linked_inf)); + l->id = id; + l->ptr = ptr; + if( !prev ) { + l->next = head; + return l; + } else { + l->next = prev->next; + prev->next = l; + return head; + } +} + static void *link_sort_lookup( linked_inf *head, int id ) { while( head && head->id < id ) head = head->next; @@ -155,6 +188,22 @@ static void *link_sort_lookup( linked_inf *head, int id ) { return NULL; } +static void *link_sort_remove( linked_inf *head, int id ) { + linked_inf *prev = NULL; + linked_inf *cur = head; + while( cur && cur->id < id ) { + prev = cur; + cur = cur->next; + } + if( cur && cur->id == id ) { + if( !prev ) + return head->ptr; + prev->next = cur->next; + return head; + } + return head; +} + static emit_mode hl_type_mode( hl_type *t ) { if( t->kind == HVOID ) return M_VOID; @@ -167,10 +216,6 @@ static emit_mode hl_type_mode( hl_type *t ) { return M_PTR; } -void hl_jit_patch_method( void*fun, void**newt ) { - jit_assert(); -} - static ereg new_value( emit_ctx *ctx ) { ereg r = {ctx->values.cur}; int_alloc_store(&ctx->values, ctx->emit_pos-1); @@ -218,21 +263,13 @@ static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { if( count < 0 ) jit_assert(); - if( e->op == PHI ) { - if( count > 256 ) jit_error("Too many branches"); - } else { - if( count > 64 ) jit_error("Too many arguments"); - } + if( count > 64 ) jit_error("Too many arguments"); e->nargs = (unsigned char)count; if( count == 0 ) return; if( count == 1 ) { e->size_offs = args[0].index; return; } - if( e->op == PHI && count <= 3 ) { - memcpy(&e->size_offs, args, sizeof(int) * count); - return; - } int *args_data = int_alloc_get(&ctx->args_data, count); e->size_offs = (int)(args_data - ctx->args_data.data); memcpy(args_data, args, sizeof(int) * count); @@ -243,8 +280,6 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ) { return NULL; if( e->nargs == 1 ) return (ereg*)&e->size_offs; - if( e->op == PHI && e->nargs <= 3 ) - return (ereg*)&e->size_offs; return (ereg*)(ctx->args_data.data + e->size_offs); } @@ -266,16 +301,17 @@ static ereg emit_gen_size( emit_ctx *ctx, emit_op op, int size_offs ) { return emit_gen_ext(ctx,op,ENULL,ENULL,op==ALLOC_STACK ? M_PTR : 0,size_offs); } -static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { - unsigned char mode = GET_WRITE(v1).mode; - if( mode != GET_WRITE(v2).mode ) jit_assert(); - einstr *e = emit_instr(ctx, PHI); - e->mode = mode; - ereg args[2]; - args[0] = v1; - args[1] = v2; - store_args(ctx, e, args, 2); - return new_value(ctx); +static void patch_instr_mode( emit_ctx *ctx, int mode ) { + ctx->instrs[ctx->emit_pos-1].mode = (unsigned char)mode; +} + +static tmp_phi *alloc_phi( emit_ctx *ctx, emit_block *b, vreg *r ) { + tmp_phi *p = (tmp_phi*)hl_zalloc(&ctx->jit->falloc, sizeof(tmp_phi)); + p->b = b; + p->r = r; + p->value.index = -(++ctx->phi_count); + b->phis = link_add(ctx, r->id, p, b->phis); + return p; } static int emit_jump( emit_ctx *ctx, bool cond ) { @@ -292,41 +328,44 @@ static emit_block *alloc_block( emit_ctx *ctx ) { return hl_zalloc(&ctx->jit->falloc, sizeof(emit_block)); } -static void block_add_prec( emit_ctx *ctx, emit_block *b, emit_block *p ) { +static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { b->preds = link_add(ctx,0,p,b->preds); p->nexts = link_add(ctx,0,b,p->nexts); - jit_debug(" PRED %d\n",p->id); + jit_debug(" PRED #%d\n",p->id); } -static void flush_block( emit_ctx *ctx ) { - int i = 0; - emit_block *cur = ctx->current_block; - while( i < ctx->block_writes.cur ) { - vreg *r = R(ctx->block_writes.data[i++]); - cur->written_vars = link_add_sort_unique(ctx,r->id,(void*)(int_val)(r->current.index+1),cur->written_vars); - jit_debug(" WRITES R%d @%X\n", r->id, r->current.index); - r->current.index = -1; - r->written = false; - } - int_alloc_reset(&ctx->block_writes); +static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { + if( IS_NULL(v) ) jit_assert(); + b->written_vars = link_add_sort_replace(ctx,r->id,(void*)(int_val)(v.index < 0 ? v.index : v.index + 1),b->written_vars); } -static void split_block( emit_ctx *ctx ) { - flush_block(ctx); +static ereg lookup_block_var( emit_block *b, vreg *r ) { + void *p = link_sort_lookup(b->written_vars,r->id); + if( !p ) return ENULL; + int e = (int)(int_val)p; + ereg v; + v.index = e < 0 ? e : e-1; + return v; +} + +static void remove_block_var( emit_block *b, vreg *r ) { + b->written_vars = link_sort_remove(b->written_vars, r->id); +} - // split +static void split_block( emit_ctx *ctx ) { emit_block *b = alloc_block(ctx); + b->sealed = true; b->id = ctx->current_block->id + 1; b->start_pos = ctx->emit_pos; - jit_debug("BLOCK #%d@%X\n",b->id,ctx->op_pos); + jit_debug("BLOCK #%d@%X[%X]\n",b->id,b->start_pos,ctx->op_pos); while( ctx->arrival_points && ctx->arrival_points->id == ctx->op_pos ) { - block_add_prec(ctx, b, (emit_block*)ctx->arrival_points->ptr); + block_add_pred(ctx, b, (emit_block*)ctx->arrival_points->ptr); ctx->arrival_points = ctx->arrival_points->next; } bool dead_code = b->preds == NULL; // if we have no reach, force previous block dependency, this is rare dead code emit by compiler einstr *eprev = &ctx->instrs[ctx->emit_pos-1]; - if( (eprev->op != JUMP && eprev->op != RET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) - block_add_prec(ctx, b, ctx->current_block); + if( (eprev->op != JUMP && eprev->op != RET && eprev->mode != M_NORET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) + block_add_pred(ctx, b, ctx->current_block); ctx->current_block->end_pos = ctx->emit_pos - 1; ctx->current_block = b; } @@ -359,12 +398,8 @@ static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { if( to->t->kind == HVOID ) return; - if( v.index < 0 ) jit_assert(); - if( !to->written ) { - to->written = true; - int_alloc_store(&ctx->block_writes,to->id); - } - to->current = v; + if( IS_NULL(v) ) jit_assert(); + store_block_var(ctx,ctx->current_block,to,v); } static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int nargs, hl_type *ret ) { @@ -383,59 +418,94 @@ static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_typ return e->mode == M_VOID ? ENULL : new_value(ctx); } -static void patch_instr_mode( emit_ctx *ctx, int mode ) { - ctx->instrs[ctx->emit_pos-1].mode = (unsigned char)mode; -} - static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { emit_gen_ext(ctx, TEST, v, ENULL, 0, o); patch_instr_mode(ctx, GET_WRITE(v).mode); } -static void emit_gather_phi_rec( emit_ctx *ctx, emit_block *b, vreg *r, int uid ) { - if( b->wait_nexts > 0 ) { - if( ctx->instrs[ctx->emit_pos-1].op != PHI ) jit_assert(); - int_alloc_store_unique(&ctx->phi_gather, -1); - b->incomplete_phis = link_add_sort_unique(ctx, ctx->emit_pos - 1, r, b->incomplete_phis); - return; - } - if( b->mark == uid ) +static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { + if( !p->b ) jit_assert(); + if( IS_NULL(v) ) jit_assert(); + if( link_sort_lookup(p->vals,v.index) ) return; - b->mark = uid; - int eid = (int)(int_val)link_sort_lookup(b->written_vars, r->id) - 1; - if( eid >= 0 ) { - if( eid != ctx->emit_pos - 1 ) - int_alloc_store_unique(&ctx->phi_gather, eid); - } else { - linked_inf *l = b->preds; - while( l ) { - emit_gather_phi_rec(ctx, (emit_block*)l->ptr, r, uid); + jit_debug(" PHI-DEP %s = %s\n", val_str(p->value), val_str(v)); + p->vals = link_add_sort_unique(ctx,v.index,p,p->vals); +} + +static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { + ereg same = ENULL; + linked_inf *l = p->vals; + while( l ) { + if( l->id == same.index || l->id == p->value.index ) { l = l->next; + continue; } + if( !IS_NULL(same) ) + return p->value; + same.index = l->id; + l = l->next; } + if( IS_NULL(same) ) jit_assert(); // unwritten var access ? + // TODO + jit_debug(" PHI-OPT %s = %s\n", val_str(p->value), val_str(same)); + return same; +} + +static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ); + +static ereg gather_phis( emit_ctx *ctx, tmp_phi *p ) { + linked_inf *l = p->b->preds; + while( l ) { + ereg r = emit_load_reg_block(ctx, (emit_block*)l->ptr, p->r); + phi_add_val(ctx, p, r); + l = l->next; + } + return optimize_phi_rec(ctx, p); +} + +static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { + ereg v = lookup_block_var(b,r); + if( !IS_NULL(v) ) + return v; + if( !b->sealed ) { + tmp_phi *p = alloc_phi(ctx,b,r); + v = p->value; + } else if( b->preds && !b->preds->next ) + v = emit_load_reg_block(ctx, (emit_block*)b->preds->ptr, r); + else { + tmp_phi *p = alloc_phi(ctx,b,r); + store_block_var(ctx,b,r,p->value); + v = gather_phis(ctx, p); + } + store_block_var(ctx,b,r,v); + return v; } static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { - if( r->current.index < 0 ) { - ereg ref = resolve_ref(ctx, r->id); - if( ref.index >= 0 ) - return LOAD_MEM(ref,0,r->t); - einstr *p = emit_instr(ctx, PHI); - p->mode = hl_type_mode(r->t); - int_alloc_reset(&ctx->phi_gather); - emit_gather_phi_rec(ctx, ctx->current_block, r, ++ctx->uid); - if( ctx->phi_gather.cur == 1 && ctx->phi_gather.data[0] != -1 ) { - jit_debug(" SKIP-PHI @%X\n",ctx->emit_pos-1); - // remove phi - ctx->emit_pos--; - r->current.index = ctx->phi_gather.data[0]; - } else { - if( ctx->phi_gather.cur == 0 ) jit_assert(); - store_args(ctx, p, (ereg*)ctx->phi_gather.data, ctx->phi_gather.cur); - r->current = new_value(ctx); - } + ereg ref = resolve_ref(ctx, r->id); + if( ref.index >= 0 ) + return LOAD_MEM(ref,0,r->t); + return emit_load_reg_block(ctx, ctx->current_block, r); +} + +static void seal_block( emit_ctx *ctx, emit_block *b ) { + jit_debug(" SEAL #%d\n",b->id); + linked_inf *l = b->phis; + while( l ) { + tmp_phi *p = (tmp_phi*)l->ptr; + gather_phis(ctx, p); + l = l->next; } - return r->current; + b->sealed = true; +} + +static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { + unsigned char mode = GET_WRITE(v1).mode; + if( mode != GET_WRITE(v2).mode ) jit_assert(); + tmp_phi *p = alloc_phi(ctx, ctx->current_block, NULL); + phi_add_val(ctx, p, v1); + phi_add_val(ctx, p, v2); + return p->value; } static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int *args_regs ) { @@ -577,7 +647,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ); static void emit_write_blocks( jit_ctx *jit, emit_block *b ) { eblock *bl = jit->blocks + b->id; - if( bl->id ) return; // already set + if( bl->id >= 0 ) return; // already set bl->id = b->id; bl->start_pos = b->start_pos; bl->end_pos = b->end_pos; @@ -609,11 +679,40 @@ static void emit_write_blocks( jit_ctx *jit, emit_block *b ) { if( n->start_pos > b->start_pos ) emit_write_blocks(jit, n); tmp = tmp->next; } + // write phis + tmp = b->phis; + while( tmp ) { + bl->phi_count++; + tmp = tmp->next; + } + bl->phis = (ephi*)hl_zalloc(&jit->falloc,sizeof(ephi)*bl->phi_count); + tmp = b->phis; + i = 0; + while( tmp ) { + tmp_phi *p = (tmp_phi*)tmp->ptr; + ephi *p2 = bl->phis + i++; + p2->value = p->value; + linked_inf *l = p->vals; + while( l ) { + p2->nvalues++; + l = l->next; + } + p2->values = (ereg*)hl_malloc(&jit->falloc,sizeof(ereg)*p2->nvalues); + l = p->vals; + int k = 0; + while( l ) { + p2->values[k++].index = l->id; + l = l->next; + } + tmp = tmp->next; + } } void hl_emit_flush( jit_ctx *jit ) { emit_ctx *ctx = jit->emit; int i = 0; + if( ctx->flushed ) return; + ctx->flushed = true; while( i < ctx->jump_regs.cur ) { int pos = ctx->jump_regs.data[i++]; einstr *e = ctx->instrs + pos; @@ -627,6 +726,8 @@ void hl_emit_flush( jit_ctx *jit ) { jit->emit_pos_map = ctx->pos_map; jit->block_count = ctx->current_block->id + 1; jit->blocks = hl_zalloc(&jit->falloc,sizeof(eblock) * jit->block_count); + for(i=0;iblock_count;i++) + jit->blocks[i].id = -1; jit->value_count = ctx->values.cur; jit->values_writes = ctx->values.data; emit_write_blocks(jit, ctx->root_block); @@ -641,11 +742,13 @@ void hl_emit_function( jit_ctx *jit ) { ctx->emit_pos = 0; ctx->trap_count = 0; ctx->ref_count = 0; + ctx->phi_count = 0; + ctx->flushed = false; int_alloc_reset(&ctx->args_data); int_alloc_reset(&ctx->jump_regs); - int_alloc_reset(&ctx->block_writes); int_alloc_reset(&ctx->values); ctx->root_block = ctx->current_block = alloc_block(ctx); + ctx->current_block->sealed = true; ctx->arrival_points = NULL; jit_debug("---- begin [%X] ----\n",f->findex); if( f->nregs > ctx->max_regs ) { @@ -667,8 +770,6 @@ void hl_emit_function( jit_ctx *jit ) { for(i=0;inregs;i++) { vreg *r = R(i); r->t = f->regs[i]; - r->current = ENULL; - r->written = false; } for(i=0;itype->fun->nargs;i++) { @@ -716,7 +817,7 @@ void hl_emit_free( jit_ctx *jit ) { free(ctx->pos_map); int_alloc_free(&ctx->jump_regs); int_alloc_free(&ctx->args_data); - int_alloc_free(&ctx->block_writes); + int_alloc_free(&ctx->values); free(ctx); jit->emit = NULL; } @@ -726,26 +827,11 @@ static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { return false; if( b->start_pos == target ) { b->wait_nexts--; - b->preds = link_add(ctx,0,ctx->current_block,b->preds); - flush_block(ctx); - if( b->wait_nexts == 0 ) { - jit_debug(" SEAL #%d\n",b->id); - // seal block - linked_inf *l = b->incomplete_phis; - while( l ) { - einstr *e = ctx->instrs + l->id; - vreg *r = (vreg*)l->ptr; - if( e->op != PHI ) jit_assert(); - int prev_emit = ctx->emit_pos; - ctx->emit_pos = l->id + 1; - int_alloc_reset(&ctx->phi_gather); - emit_gather_phi_rec(ctx, b, r, ++ctx->uid); - ctx->emit_pos = prev_emit; - store_args(ctx, e, (ereg*)ctx->phi_gather.data, ctx->phi_gather.cur); - b->written_vars = link_add_sort_unique(ctx,r->id,(void*)(int_val)(l->id + 1), b->written_vars); - l = l->next; - } - b->incomplete_phis = NULL; + block_add_pred(ctx, b, ctx->current_block); + while( b && b->wait_nexts == 0 && ctx->wait_seal == b ) { + seal_block(ctx,b); + b = b->wait_seal_next; + ctx->wait_seal = b; } return true; } @@ -769,6 +855,51 @@ static void register_block_jump( emit_ctx *ctx, int offs, bool cond ) { } } +static void prepare_loop_block( emit_ctx *ctx ) { + int i, last_jump = -1; + emit_block *b = ctx->current_block; + // gather all backward jumps to know when the block will be finished + for(i=ctx->op_pos+1;ifun->nops;i++) { + hl_opcode *op = &ctx->fun->ops[i]; + int offs = 0; + switch( op->op ) { + case OJFalse: + case OJTrue: + case OJNotNull: + case OJNull: + offs = op->p2; + break; + case OJAlways: + offs = op->p1; + break; + case OJEq: + case OJNotEq: + case OJSLt: + case OJSGte: + case OJSLte: + case OJSGt: + case OJULt: + case OJUGte: + case OJNotLt: + case OJNotGte: + offs = op->p3; + break; + default: + break; + } + if( offs < 0 && i + 1 + offs == ctx->op_pos ) { + jit_debug(" WAIT @%X\n",i); + b->wait_nexts++; + if( b->sealed ) { + b->sealed = false; + b->wait_seal_next = ctx->wait_seal; + ctx->wait_seal = b; + } + last_jump = i; + } + } +} + static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { vreg *dst = R(o->p1); vreg *ra = R(o->p2); @@ -1246,45 +1377,9 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case OLabel: - // NOP - { - if( ctx->current_block->start_pos != ctx->emit_pos ) - split_block(ctx); - int i; - for(i=ctx->op_pos+1;ifun->nops;i++) { - hl_opcode *op = &ctx->fun->ops[i]; - int offs = 0; - switch( op->op ) { - case OJFalse: - case OJTrue: - case OJNotNull: - case OJNull: - offs = op->p2; - break; - case OJAlways: - offs = op->p1; - break; - case OJEq: - case OJNotEq: - case OJSLt: - case OJSGte: - case OJSLte: - case OJSGt: - case OJULt: - case OJUGte: - case OJNotLt: - case OJNotGte: - offs = op->p3; - break; - default: - break; - } - if( offs < 0 && i + 1 + offs == ctx->op_pos ) { - jit_debug(" WAIT @%X\n",i); - ctx->current_block->wait_nexts++; - } - } - } + if( ctx->current_block->start_pos != ctx->emit_pos ) + split_block(ctx); + prepare_loop_block(ctx); break; case OGetI8: case OGetI16: @@ -1367,9 +1462,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case ORef: { ereg ref = resolve_ref(ctx, ra->id); - if( ref.index < 0 ) jit_assert(); - if( ra->current.index >= 0 ) STORE_MEM(ref, 0, LOAD(ra)); - ra->current.index = -1; // ref will be modified + if( IS_NULL(ref) ) jit_assert(); + ereg r = lookup_block_var(ctx->current_block, ra); + if( !IS_NULL(r) ) { + STORE_MEM(ref, 0, LOAD(ra)); + remove_block_var(ctx->current_block, ra); + } STORE(dst, ref); } break; diff --git a/src/module.c b/src/module.c index e46f73f13..4ba7eeabf 100644 --- a/src/module.c +++ b/src/module.c @@ -713,6 +713,7 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { hl_jit_init(ctx, m); for(i=0;icode->nfunctions;i++) { hl_function *f = m->code->functions + i; + if( f->findex != 0x16E ) continue; int fpos = hl_jit_function(ctx, m, f); if( fpos < 0 ) { hl_jit_free(ctx, false); From e4c778a5a5dff749f9543dcf5e3d8b8fede67409 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 29 Mar 2026 17:08:19 +0200 Subject: [PATCH 15/83] minor fixes --- src/jit.c | 2 -- src/jit.h | 5 ++--- src/jit_dump.c | 2 +- src/module.c | 1 - 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/jit.c b/src/jit.c index 626cfde03..19e8a233c 100644 --- a/src/jit.c +++ b/src/jit.c @@ -30,9 +30,7 @@ void hl_jit_error( const char *msg, const char *func, int line ) { current_ctx = NULL; hl_emit_dump(ctx); } - hl_debug_break(); fflush(stdout); - exit(-1); } void hl_jit_null_field_access() { jit_assert(); } diff --git a/src/jit.h b/src/jit.h index a1e080ec8..820e79bd5 100644 --- a/src/jit.h +++ b/src/jit.h @@ -166,7 +166,7 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); # define JIT_DEBUG #endif -#define jit_error(msg) hl_jit_error(msg,__func__,__LINE__) +#define jit_error(msg) { hl_jit_error(msg,__func__,__LINE__); hl_debug_break(); exit(-1); } #define jit_assert() jit_error("") #ifdef JIT_DEBUG @@ -175,8 +175,7 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); # define jit_debug(...) #endif -HL_NO_RETURN( void hl_jit_error( const char *msg, const char *func, int line ) ); - +void hl_jit_error( const char *msg, const char *func, int line ); void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); void hl_jit_patch_method( void *old_fun, void **new_fun_table ); diff --git a/src/jit_dump.c b/src/jit_dump.c index 2ef167285..50d8eff6d 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -295,7 +295,7 @@ void hl_emit_dump( jit_ctx *ctx ) { int vpos = 0; cur = 0; for(i=0;iinstr_count;i++) { - while( ctx->blocks[cur].start_pos == i ) { + while( cur < ctx->block_count && ctx->blocks[cur].start_pos == i ) { eblock *b = &ctx->blocks[cur]; printf("--- BLOCK #%d ---\n", cur); for(int k=0;kphi_count;k++) { diff --git a/src/module.c b/src/module.c index 4ba7eeabf..e46f73f13 100644 --- a/src/module.c +++ b/src/module.c @@ -713,7 +713,6 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { hl_jit_init(ctx, m); for(i=0;icode->nfunctions;i++) { hl_function *f = m->code->functions + i; - if( f->findex != 0x16E ) continue; int fpos = hl_jit_function(ctx, m, f); if( fpos < 0 ) { hl_jit_free(ctx, false); From b3569f58aaee5e120386703fffd226f46cabfc3c Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 29 Mar 2026 17:10:00 +0200 Subject: [PATCH 16/83] added phi optimization (still needs cleanup post emit) --- src/jit_emit.c | 97 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 14 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 485030fce..1dabde220 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -57,8 +57,11 @@ struct _emit_block { struct _tmp_phi { ereg value; vreg *r; + bool locked; + unsigned char mode; emit_block *b; linked_inf *vals; + linked_inf *uses; }; struct _emit_ctx { @@ -68,11 +71,14 @@ struct _emit_ctx { einstr *instrs; vreg *vregs; + tmp_phi **phis; int max_instrs; int max_regs; + int max_phis; int emit_pos; int op_pos; int phi_count; + int phi_depth; bool flushed; ereg tmp_args[MAX_TMP_ARGS]; @@ -109,8 +115,8 @@ struct _emit_ctx { #define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR(obj,0),HL_WSIZE*2),HL_WSIZE*(id)) #define OFFSET(base,index,mult,offset) emit_gen_ext(ctx, LEA, base, index, M_PTR, (mult) | ((offset) << 8)) #define BREAK() emit_gen(ctx, DEBUG_BREAK, ENULL, ENULL, 0) -#define GET_WRITE(r) ctx->instrs[ctx->values.data[(r).index]] - +#define GET_MODE(r) emit_get_mode(ctx,r) +#define GET_PHI(r) ctx->phis[-(r).index-1] #define HDYN_VALUE 8 #define IS_FLOAT(t) ((t)->kind == HF64 || (t)->kind == HF32) @@ -188,7 +194,7 @@ static void *link_sort_lookup( linked_inf *head, int id ) { return NULL; } -static void *link_sort_remove( linked_inf *head, int id ) { +static linked_inf *link_sort_remove( linked_inf *head, int id ) { linked_inf *prev = NULL; linked_inf *cur = head; while( cur && cur->id < id ) { @@ -197,7 +203,7 @@ static void *link_sort_remove( linked_inf *head, int id ) { } if( cur && cur->id == id ) { if( !prev ) - return head->ptr; + return cur->next; prev->next = cur->next; return head; } @@ -235,6 +241,22 @@ static ereg resolve_ref( emit_ctx *ctx, int reg ) { return ENULL; } +static unsigned char emit_get_mode( emit_ctx *ctx, ereg v ) { + if( IS_NULL(v) ) jit_assert(); + if( v.index < 0 ) + return GET_PHI(v)->mode; + return ctx->instrs[ctx->values.data[v.index]].mode; +} + +static const char *phi_prefix( emit_ctx *ctx ) { + static char tmp[20]; + int sp = 3 + ctx->phi_depth * 2; + if( sp > 19 ) sp = 19; + memset(tmp,0x20,sp); + tmp[sp] = 0; + return tmp; +} + static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { if( ctx->emit_pos == ctx->max_instrs ) { int pos = ctx->emit_pos; @@ -255,7 +277,7 @@ static einstr *emit_instr( emit_ctx *ctx, emit_op op ) { static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { einstr *e = emit_instr(ctx, STORE); - e->mode = GET_WRITE(from).mode; + e->mode = GET_MODE(from); e->size_offs = offs; e->a = to; e->b = from; @@ -306,11 +328,22 @@ static void patch_instr_mode( emit_ctx *ctx, int mode ) { } static tmp_phi *alloc_phi( emit_ctx *ctx, emit_block *b, vreg *r ) { + if( ctx->phi_count == ctx->max_phis ) { + int new_size = ctx->max_phis ? ctx->max_phis << 1 : 64; + tmp_phi **phis = (tmp_phi**)malloc(sizeof(tmp_phi*) * new_size); + if( phis == NULL ) jit_error("Out of memory"); + memcpy(phis, ctx->phis, sizeof(tmp_phi*) * ctx->phi_count); + free(ctx->phis); + ctx->phis = phis; + ctx->max_phis = new_size; + } tmp_phi *p = (tmp_phi*)hl_zalloc(&ctx->jit->falloc, sizeof(tmp_phi)); p->b = b; p->r = r; + if( r ) p->mode = hl_type_mode(r->t); p->value.index = -(++ctx->phi_count); - b->phis = link_add(ctx, r->id, p, b->phis); + b->phis = link_add(ctx, 0, p, b->phis); + GET_PHI(p->value) = p; return p; } @@ -420,19 +453,31 @@ static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_typ static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { emit_gen_ext(ctx, TEST, v, ENULL, 0, o); - patch_instr_mode(ctx, GET_WRITE(v).mode); + patch_instr_mode(ctx, GET_MODE(v)); +} + +static void phi_remove_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { + p->vals = link_sort_remove(p->vals,v.index); + jit_debug("%sPHI-REM-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); } static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { if( !p->b ) jit_assert(); if( IS_NULL(v) ) jit_assert(); + if( p->value.index == v.index ) + return; if( link_sort_lookup(p->vals,v.index) ) return; - jit_debug(" PHI-DEP %s = %s\n", val_str(p->value), val_str(v)); + jit_debug("%sPHI-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); p->vals = link_add_sort_unique(ctx,v.index,p,p->vals); + if( v.index < 0 ) { + tmp_phi *p2 = GET_PHI(v); + p2->uses = link_add(ctx,0,p,p2->uses); + } } static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { + if( p->locked ) jit_assert(); ereg same = ENULL; linked_inf *l = p->vals; while( l ) { @@ -445,9 +490,30 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { same.index = l->id; l = l->next; } - if( IS_NULL(same) ) jit_assert(); // unwritten var access ? - // TODO - jit_debug(" PHI-OPT %s = %s\n", val_str(p->value), val_str(same)); + if( IS_NULL(same) ) + same = p->value; // unwritten var access ? + if( !p->uses ) + return same; + + jit_debug("%sPHI-OPT %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); + ctx->phi_depth++; + l = p->uses; + while( l ) { + tmp_phi *p2 = (tmp_phi*)l->ptr; + phi_remove_val(ctx,p2,p->value); + phi_add_val(ctx,p2,same); + l = l->next; + } + l = p->uses; + p->uses = NULL; + while( l ) { + tmp_phi *p2 = (tmp_phi*)l->ptr; + optimize_phi_rec(ctx, p2); + l = l->next; + } + ctx->phi_depth--; + same = optimize_phi_rec(ctx,p); // check again + jit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); return same; } @@ -455,11 +521,13 @@ static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ); static ereg gather_phis( emit_ctx *ctx, tmp_phi *p ) { linked_inf *l = p->b->preds; + p->locked = true; while( l ) { - ereg r = emit_load_reg_block(ctx, (emit_block*)l->ptr, p->r); + ereg r = p->r ? emit_load_reg_block(ctx, (emit_block*)l->ptr, p->r) : p->value; phi_add_val(ctx, p, r); l = l->next; } + p->locked = false; return optimize_phi_rec(ctx, p); } @@ -500,9 +568,10 @@ static void seal_block( emit_ctx *ctx, emit_block *b ) { } static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { - unsigned char mode = GET_WRITE(v1).mode; - if( mode != GET_WRITE(v2).mode ) jit_assert(); + unsigned char mode = GET_MODE(v1); + if( mode != GET_MODE(v2) ) jit_assert(); tmp_phi *p = alloc_phi(ctx, ctx->current_block, NULL); + p->mode = mode; phi_add_val(ctx, p, v1); phi_add_val(ctx, p, v2); return p->value; From eb1b86acfe4b4196de941b5ca3f304b2f7068346 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Tue, 31 Mar 2026 19:42:31 +0200 Subject: [PATCH 17/83] improved phi optimize and added phi cleanup --- src/jit_dump.c | 6 +- src/jit_emit.c | 154 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 139 insertions(+), 21 deletions(-) diff --git a/src/jit_dump.c b/src/jit_dump.c index 50d8eff6d..f9c8ce348 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -372,10 +372,10 @@ void hl_emit_dump( jit_ctx *ctx ) { dump_value(ctx, e->value, e->mode); break; case LOAD_ADDR: - if( (e->b.index>>8) ) - printf(" %s[%Xh]", val_str(e->a), e->b.index); + if( (e->size_offs>>8) ) + printf(" %s[%Xh]", val_str(e->a), e->size_offs); else - printf(" %s[%d]", val_str(e->a), e->b.index); + printf(" %s[%d]", val_str(e->a), e->size_offs); break; case STORE: { diff --git a/src/jit_emit.c b/src/jit_emit.c index 1dabde220..caf83690f 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -46,6 +46,7 @@ struct _emit_block { int start_pos; int end_pos; int wait_nexts; + int mark; bool sealed; linked_inf *nexts; linked_inf *preds; @@ -57,11 +58,15 @@ struct _emit_block { struct _tmp_phi { ereg value; vreg *r; + ereg target; + int final_id; bool locked; + bool opt; unsigned char mode; emit_block *b; linked_inf *vals; - linked_inf *uses; + linked_inf *ref_phis; + linked_inf *ref_blocks; }; struct _emit_ctx { @@ -370,6 +375,10 @@ static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { if( IS_NULL(v) ) jit_assert(); b->written_vars = link_add_sort_replace(ctx,r->id,(void*)(int_val)(v.index < 0 ? v.index : v.index + 1),b->written_vars); + if( v.index < 0 ) { + tmp_phi *p = GET_PHI(v); + p->ref_blocks = link_add_sort_unique(ctx,b->id,b,p->ref_blocks); + } } static ereg lookup_block_var( emit_block *b, vreg *r ) { @@ -425,7 +434,7 @@ static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) einstr *e = emit_instr(ctx, LOAD_ADDR); e->mode = hl_type_mode(size_t); e->a = v; - e->b.index = offset; + e->size_offs = offset; return new_value(ctx); } @@ -472,11 +481,12 @@ static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { p->vals = link_add_sort_unique(ctx,v.index,p,p->vals); if( v.index < 0 ) { tmp_phi *p2 = GET_PHI(v); - p2->uses = link_add(ctx,0,p,p2->uses); + p2->ref_phis = link_add(ctx,0,p,p2->ref_phis); } } static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { + if( p->locked ) jit_assert(); ereg same = ENULL; linked_inf *l = p->vals; @@ -490,31 +500,42 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { same.index = l->id; l = l->next; } - if( IS_NULL(same) ) - same = p->value; // unwritten var access ? - if( !p->uses ) + if( IS_NULL(same) ) + return p->value; // sealed (no dep yet) + + if( !p->ref_phis && !p->ref_blocks ) return same; + if( p->locked || p->opt ) jit_assert(); + jit_debug("%sPHI-OPT %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); + p->opt = true; ctx->phi_depth++; - l = p->uses; + l = p->ref_blocks; + while( l ) { + emit_block *b = (emit_block*)l->ptr; + if( lookup_block_var(b,p->r).index == p->value.index ) + store_block_var(ctx,b,p->r,same); + l = l->next; + } + l = p->ref_phis; while( l ) { tmp_phi *p2 = (tmp_phi*)l->ptr; phi_remove_val(ctx,p2,p->value); phi_add_val(ctx,p2,same); l = l->next; } - l = p->uses; - p->uses = NULL; + l = p->ref_phis; + p->ref_phis = NULL; + p->ref_blocks = NULL; while( l ) { tmp_phi *p2 = (tmp_phi*)l->ptr; optimize_phi_rec(ctx, p2); l = l->next; } ctx->phi_depth--; - same = optimize_phi_rec(ctx,p); // check again jit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); - return same; + return optimize_phi_rec(ctx,p); } static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ); @@ -537,6 +558,7 @@ static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { return v; if( !b->sealed ) { tmp_phi *p = alloc_phi(ctx,b,r); + jit_debug("%sPHI-SEALED %s = R%d\n",phi_prefix(ctx),val_str(p->value),r->id); v = p->value; } else if( b->preds && !b->preds->next ) v = emit_load_reg_block(ctx, (emit_block*)b->preds->ptr, r); @@ -549,6 +571,24 @@ static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { return v; } +static void emit_walk_blocks_rec( emit_ctx *ctx, emit_block *b, int mark, void (*fun)(emit_ctx*,emit_block*) ) { + if( b->mark == mark ) return; + b->mark = mark; + fun(ctx, b); + linked_inf *tmp = b->nexts; + while( tmp ) { + emit_walk_blocks_rec(ctx,(emit_block*)tmp->ptr,mark,fun); + tmp = tmp->next; + } +} + +static void emit_walk_blocks( emit_ctx *ctx, void (*fun)(emit_ctx*,emit_block*) ) { + static int MARK_UID = 0; + int mark = ++MARK_UID; + if( mark == 0 ) mark = ++MARK_UID; + emit_walk_blocks_rec(ctx, ctx->root_block, mark, fun); +} + static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { ereg ref = resolve_ref(ctx, r->id); if( ref.index >= 0 ) @@ -714,9 +754,25 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { static void emit_opcode( emit_ctx *ctx, hl_opcode *o ); -static void emit_write_blocks( jit_ctx *jit, emit_block *b ) { +static void remap_phi_reg( emit_ctx *ctx, ereg *r ) { + if( r->index >= 0 || IS_NULL(*r) ) + return; + tmp_phi *p = GET_PHI(*r); + while( p->final_id < 0 ) { + if( p->target.index >= 0 ) { + r->index = p->target.index; + return; + } + p = GET_PHI(p->target); + } + if( p->final_id == 0 ) + return; + r->index = -p->final_id; // new phis +} + +static void emit_write_block( emit_ctx *ctx, emit_block *b ) { + jit_ctx *jit = ctx->jit; eblock *bl = jit->blocks + b->id; - if( bl->id >= 0 ) return; // already set bl->id = b->id; bl->start_pos = b->start_pos; bl->end_pos = b->end_pos; @@ -745,13 +801,14 @@ static void emit_write_blocks( jit_ctx *jit, emit_block *b ) { while( tmp ) { emit_block *n = (emit_block*)tmp->ptr; bl->nexts[i++] = n->id; - if( n->start_pos > b->start_pos ) emit_write_blocks(jit, n); tmp = tmp->next; } // write phis tmp = b->phis; while( tmp ) { - bl->phi_count++; + tmp_phi *p = (tmp_phi*)tmp->ptr; + if( p->final_id >= 0 ) + bl->phi_count++; tmp = tmp->next; } bl->phis = (ephi*)hl_zalloc(&jit->falloc,sizeof(ephi)*bl->phi_count); @@ -759,8 +816,15 @@ static void emit_write_blocks( jit_ctx *jit, emit_block *b ) { i = 0; while( tmp ) { tmp_phi *p = (tmp_phi*)tmp->ptr; + if( p->final_id < 0 ) { + tmp = tmp->next; + continue; + } ephi *p2 = bl->phis + i++; - p2->value = p->value; + if( p->final_id == 0 ) + p2->value = p->value; + else + p2->value.index = -p->final_id; linked_inf *l = p->vals; while( l ) { p2->nvalues++; @@ -770,7 +834,9 @@ static void emit_write_blocks( jit_ctx *jit, emit_block *b ) { l = p->vals; int k = 0; while( l ) { - p2->values[k++].index = l->id; + ereg v = {l->id}; + remap_phi_reg(ctx, &v); + p2->values[k++] = v; l = l->next; } tmp = tmp->next; @@ -799,7 +865,58 @@ void hl_emit_flush( jit_ctx *jit ) { jit->blocks[i].id = -1; jit->value_count = ctx->values.cur; jit->values_writes = ctx->values.data; - emit_write_blocks(jit, ctx->root_block); + emit_walk_blocks(ctx,emit_write_block); +} + +static void hl_iter_instr_reg( einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ) { + switch( e->op ) { + case CALL_REG: + iter_reg(ctx,&e->a); + case CALL_FUN: + case CALL_PTR: + { + int i; + ereg *args = hl_emit_get_args(ctx, e); + for(i=0;inargs;i++) + iter_reg(ctx, args + i); + } + break; + case LOAD_IMM: + // skip + break; + default: + if( !IS_NULL(e->a) ) { + iter_reg(ctx,&e->a); + if( !IS_NULL(e->b) ) + iter_reg(ctx,&e->b); + } + break; + } +} + +static void hl_emit_clean_phis( emit_ctx *ctx ) { + for(int i=0;iphi_count;i++) { + tmp_phi *p = ctx->phis[i]; + tmp_phi *cur = p; + ereg r; + while( true ) { + cur->opt = false; + r = optimize_phi_rec(ctx,cur); + if( r.index >= 0 || r.index == cur->value.index ) break; + cur = GET_PHI(r); + } + p->target = r; + } + int new_phis = 0; + for(int i=0;iphi_count;i++) { + tmp_phi *p = ctx->phis[i]; + if( p->target.index == p->value.index ) + p->final_id = ++new_phis; + else + p->final_id = -1; + } + for(int i=0;iemit_pos;i++) + hl_iter_instr_reg(ctx->instrs + i, ctx, remap_phi_reg); } void hl_emit_function( jit_ctx *jit ) { @@ -867,6 +984,7 @@ void hl_emit_function( jit_ctx *jit ) { emit_opcode(ctx,f->ops + op_pos); } + hl_emit_clean_phis(ctx); hl_emit_flush(ctx->jit); } From ee4f24267b7008cbfe100e597be50c63c7bc6296 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 1 Apr 2026 00:01:22 +0200 Subject: [PATCH 18/83] added new data structs for emit --- src/data_struct.c | 211 ++++++++++++++++++++++++++++++++++++++++++++++ src/data_struct.h | 71 ++++++++++++++++ src/jit_emit.c | 95 ++++++++------------- 3 files changed, 318 insertions(+), 59 deletions(-) create mode 100644 src/data_struct.c create mode 100644 src/data_struct.h diff --git a/src/data_struct.c b/src/data_struct.c new file mode 100644 index 000000000..733a9350e --- /dev/null +++ b/src/data_struct.c @@ -0,0 +1,211 @@ +/* + * Copyright (C)2015-2026 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifdef S_TYPE + +// is included by data_struct.h + +#ifdef S_MAP +# define S_ARGS S_KEY k, S_VALUE v +#else +# define S_ARGS S_VALUE k +# define S_KEY S_VALUE +#endif + +typedef struct { + int cur; + int max; + S_KEY *keys; +# ifdef S_MAP + S_VALUE *values; +# endif +} S_TYPE; + +typedef S_VALUE S_NAME(_value); + +static void S_NAME(check_size)( hl_alloc *alloc, S_TYPE *st ) { + if( st->cur == st->max ) { + int n = st->max ? (st->max << 1) : STRUCT_DEF_SIZE; + S_KEY *keys = (S_KEY*)hl_malloc(alloc,sizeof(S_KEY) * n); + memcpy(keys,st->keys,sizeof(S_KEY) * st->cur); + st->keys = keys; +# ifdef S_MAP + S_VALUE *vals = (S_VALUE*)hl_malloc(alloc,sizeof(S_VALUE) * n); + memcpy(vals,st->values,sizeof(S_VALUE) * st->cur); + st->values = vals; +# endif + st->max = n; + } +} + +#ifndef S_SORTED + +static void S_NAME(add_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { + S_NAME(check_size)(alloc,st); + st->keys[st->cur] = k; +# ifdef S_MAP + st->values[st->cur] = v; +# endif + st->cur++; +} + +static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { + for(int i=0;icur; + int pos; + while( min < max ) { + int mid = (min + max) >> 1; + S_KEY k2 = st->keys[mid]; + if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else return; + } + S_NAME(check_size)(alloc,st); + pos = (min + max) >> 1; + memmove(st->keys + pos + 1, st->keys + pos, (st->cur - pos) * sizeof(S_KEY)); +# ifdef S_MAP + memmove(st->values + pos + 1, st->values + pos, (st->cur - pos) * sizeof(S_VALUE)); +# endif + st->keys[pos] = k; +# ifdef S_MAP + st->values[pos] = v; +# endif + st->cur++; +} + +#ifdef S_MAP +static void S_NAME(replace_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { + int min = 0; + int max = st->cur; + int pos; + while( min < max ) { + int mid = (min + max) >> 1; + S_KEY k2 = st->keys[mid]; + if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else { + st->values[mid] = v; + return; + } + } + S_NAME(check_size)(alloc,st); + pos = (min + max) >> 1; + memmove(st->keys + pos + 1, st->keys + pos, (st->cur - pos) * sizeof(S_KEY)); + memmove(st->values + pos + 1, st->values + pos, (st->cur - pos) * sizeof(S_VALUE)); + st->keys[pos] = k; + st->values[pos] = v; + st->cur++; +} +#endif + +static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { + int min = 0; + int max = st.cur; + while( min < max ) { + int mid = (min + max) >> 1; + S_KEY k2 = st.keys[mid]; + if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else return true; + } + return false; +} + +#ifdef S_MAP +static S_VALUE S_NAME(find)( S_TYPE st, S_KEY k ) { + int min = 0; + int max = st.cur; + while( min < max ) { + int mid = (min + max) >> 1; + S_KEY k2 = st.keys[mid]; + if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else return st.values[mid]; + } + return (S_VALUE)0; +} +#endif + +static bool S_NAME(remove)( S_TYPE *st, S_KEY k ) { + int min = 0; + int max = st->cur; + while( min < max ) { + int mid = (min + max) >> 1; + S_KEY k2 = st->keys[mid]; + if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else { + int pos = mid; + memmove(st->keys + pos, st->keys + pos + 1, (st->cur - pos - 1) * sizeof(S_KEY)); +# ifdef S_MAP + memmove(st->values + pos, st->values + pos + 1, (st->cur - pos - 1) * sizeof(S_VALUE)); +# endif + st->cur--; + return true; + } + } + return false; +} + +#endif + +static void S_NAME(reset)( S_TYPE *st ) { + st->cur = 0; +} + +static int S_NAME(count)( S_TYPE st ) { + return st.cur; +} + +static S_VALUE S_NAME(get)( S_TYPE st, int idx ) { +# ifdef S_MAP + return st.values[idx]; +# else + return st.keys[idx]; +# endif +} + +static S_VALUE S_NAME(first)( S_TYPE st ) { + return S_NAME(get)(st,0); +} + +static bool S_NAME(iter_next)( S_TYPE st, S_VALUE *val, int idx ) { + if( idx < st.cur ) *val = S_NAME(get)(st,idx); + return idx < st.cur; +} + +#undef S_NAME +#undef S_TYPE +#undef S_VALUE +#undef S_KEY +#undef S_ARGS +#undef STRUCT_NAME + +#endif diff --git a/src/data_struct.h b/src/data_struct.h new file mode 100644 index 000000000..cc57ca0ca --- /dev/null +++ b/src/data_struct.h @@ -0,0 +1,71 @@ +/* + * Copyright (C)2015-2026 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef HL_DATA_STRUCT_H +#define HL_DATA_STRUCT_H + +#include + +#define STRUCT_DEF_SIZE 2 +#define for_iter(name,var,set) name##__value var; for(int __idx=0;name##_iter_next(set,&var,__idx);__idx++) + +#define S_TYPE ptr_set +#define S_NAME(name) ptr_set_##name +#define S_VALUE void* +#include "data_struct.c" +#define ptr_set_add(set,v) ptr_set_add_impl(DEF_ALLOC,&(set),v) + +#define S_TYPE int_arr +#define S_NAME(name) int_arr_##name +#define S_VALUE int +#include "data_struct.c" +#define int_arr_add(set,v) int_arr_add_impl(DEF_ALLOC,&(set),v) + +#define S_SORTED + +#define S_TYPE int_set +#define S_NAME(name) int_set_##name +#define S_VALUE int +#include "data_struct.c" +#define int_set_add(set,v) int_set_add_impl(DEF_ALLOC,&(set),v) + +#define S_MAP + +#define S_TYPE int_map +#define S_NAME(name) int_map_##name +#define S_KEY int +#define S_VALUE int +#include "data_struct.c" +#define int_map_add(map,k,v) int_map_add_impl(DEF_ALLOC,&(map),k,v) +#define int_map_replace(map,k,v) int_map_replace_impl(DEF_ALLOC,&(map),k,v) + +#define S_TYPE ptr_map +#define S_NAME(name) ptr_map_##name +#define S_KEY int +#define S_VALUE void* +#include "data_struct.c" +#define ptr_map_add(map,k,v) ptr_map_add_impl(DEF_ALLOC,&(map),k,v) +#define ptr_map_replace(map,k,v) ptr_map_replace_impl(DEF_ALLOC,&(map),k,v) + +#undef S_MAP +#undef S_SORTED + +#endif diff --git a/src/jit_emit.c b/src/jit_emit.c index caf83690f..e0b5ab75a 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -21,6 +21,7 @@ */ #include #include +#include "data_struct.h" typedef struct { hl_type *t; @@ -35,6 +36,14 @@ typedef struct _linked_inf linked_inf; typedef struct _emit_block emit_block; typedef struct _tmp_phi tmp_phi; +#define DEF_ALLOC &ctx->jit->falloc + +#define S_TYPE blocks +#define S_NAME(name) blocks_##name +#define S_VALUE emit_block* +#include "data_struct.c" +#define blocks_add(set,v) blocks_add_impl(DEF_ALLOC,&(set),v) + struct _linked_inf { int id; void *ptr; @@ -48,9 +57,9 @@ struct _emit_block { int wait_nexts; int mark; bool sealed; - linked_inf *nexts; - linked_inf *preds; - linked_inf *written_vars; + blocks nexts; + blocks preds; + int_map written_vars; linked_inf *phis; emit_block *wait_seal_next; }; @@ -367,14 +376,14 @@ static emit_block *alloc_block( emit_ctx *ctx ) { } static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { - b->preds = link_add(ctx,0,p,b->preds); - p->nexts = link_add(ctx,0,b,p->nexts); + blocks_add(b->preds,p); + blocks_add(p->nexts,b); jit_debug(" PRED #%d\n",p->id); } static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { if( IS_NULL(v) ) jit_assert(); - b->written_vars = link_add_sort_replace(ctx,r->id,(void*)(int_val)(v.index < 0 ? v.index : v.index + 1),b->written_vars); + int_map_replace(b->written_vars,r->id,v.index < 0 ? v.index : v.index + 1); if( v.index < 0 ) { tmp_phi *p = GET_PHI(v); p->ref_blocks = link_add_sort_unique(ctx,b->id,b,p->ref_blocks); @@ -382,18 +391,13 @@ static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { } static ereg lookup_block_var( emit_block *b, vreg *r ) { - void *p = link_sort_lookup(b->written_vars,r->id); - if( !p ) return ENULL; - int e = (int)(int_val)p; + int e = int_map_find(b->written_vars,r->id); + if( !e ) return ENULL; ereg v; v.index = e < 0 ? e : e-1; return v; } -static void remove_block_var( emit_block *b, vreg *r ) { - b->written_vars = link_sort_remove(b->written_vars, r->id); -} - static void split_block( emit_ctx *ctx ) { emit_block *b = alloc_block(ctx); b->sealed = true; @@ -404,7 +408,7 @@ static void split_block( emit_ctx *ctx ) { block_add_pred(ctx, b, (emit_block*)ctx->arrival_points->ptr); ctx->arrival_points = ctx->arrival_points->next; } - bool dead_code = b->preds == NULL; // if we have no reach, force previous block dependency, this is rare dead code emit by compiler + bool dead_code = blocks_count(b->preds) == 0; // if we have no reach, force previous block dependency, this is rare dead code emit by compiler einstr *eprev = &ctx->instrs[ctx->emit_pos-1]; if( (eprev->op != JUMP && eprev->op != RET && eprev->mode != M_NORET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) block_add_pred(ctx, b, ctx->current_block); @@ -434,6 +438,7 @@ static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) einstr *e = emit_instr(ctx, LOAD_ADDR); e->mode = hl_type_mode(size_t); e->a = v; + e->b = ENULL; e->size_offs = offset; return new_value(ctx); } @@ -541,12 +546,10 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ); static ereg gather_phis( emit_ctx *ctx, tmp_phi *p ) { - linked_inf *l = p->b->preds; p->locked = true; - while( l ) { - ereg r = p->r ? emit_load_reg_block(ctx, (emit_block*)l->ptr, p->r) : p->value; + for_iter(blocks,b,p->b->preds) { + ereg r = p->r ? emit_load_reg_block(ctx, b, p->r) : p->value; phi_add_val(ctx, p, r); - l = l->next; } p->locked = false; return optimize_phi_rec(ctx, p); @@ -560,8 +563,8 @@ static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { tmp_phi *p = alloc_phi(ctx,b,r); jit_debug("%sPHI-SEALED %s = R%d\n",phi_prefix(ctx),val_str(p->value),r->id); v = p->value; - } else if( b->preds && !b->preds->next ) - v = emit_load_reg_block(ctx, (emit_block*)b->preds->ptr, r); + } else if( blocks_count(b->preds) == 1 ) + v = emit_load_reg_block(ctx, blocks_get(b->preds,0), r); else { tmp_phi *p = alloc_phi(ctx,b,r); store_block_var(ctx,b,r,p->value); @@ -575,11 +578,8 @@ static void emit_walk_blocks_rec( emit_ctx *ctx, emit_block *b, int mark, void ( if( b->mark == mark ) return; b->mark = mark; fun(ctx, b); - linked_inf *tmp = b->nexts; - while( tmp ) { - emit_walk_blocks_rec(ctx,(emit_block*)tmp->ptr,mark,fun); - tmp = tmp->next; - } + for_iter(blocks,n,b->nexts) + emit_walk_blocks_rec(ctx,n,mark,fun); } static void emit_walk_blocks( emit_ctx *ctx, void (*fun)(emit_ctx*,emit_block*) ) { @@ -776,35 +776,16 @@ static void emit_write_block( emit_ctx *ctx, emit_block *b ) { bl->id = b->id; bl->start_pos = b->start_pos; bl->end_pos = b->end_pos; - linked_inf *tmp; - tmp = b->preds; - while( tmp ) { - bl->pred_count++; - tmp = tmp->next; - } - tmp = b->nexts; - while( tmp ) { - bl->next_count++; - tmp = tmp->next; - } + bl->pred_count = blocks_count(b->preds); + bl->next_count = blocks_count(b->nexts); bl->preds = (int*)hl_malloc(&jit->falloc,sizeof(int)*bl->pred_count); bl->nexts = (int*)hl_malloc(&jit->falloc,sizeof(int)*bl->next_count); - int i; - i = 0; - tmp = b->preds; - while( tmp ) { - bl->preds[i++] = ((emit_block*)tmp->ptr)->id; - tmp = tmp->next; - } - i = 0; - tmp = b->nexts; - while( tmp ) { - emit_block *n = (emit_block*)tmp->ptr; - bl->nexts[i++] = n->id; - tmp = tmp->next; - } + for(int i=0;ipred_count;i++) + bl->preds[i++] = blocks_get(b->preds,i)->id; + for(int i=0;inext_count;i++) + bl->nexts[i++] = blocks_get(b->nexts,i)->id; // write phis - tmp = b->phis; + linked_inf *tmp = b->phis; while( tmp ) { tmp_phi *p = (tmp_phi*)tmp->ptr; if( p->final_id >= 0 ) @@ -813,7 +794,7 @@ static void emit_write_block( emit_ctx *ctx, emit_block *b ) { } bl->phis = (ephi*)hl_zalloc(&jit->falloc,sizeof(ephi)*bl->phi_count); tmp = b->phis; - i = 0; + int i = 0; while( tmp ) { tmp_phi *p = (tmp_phi*)tmp->ptr; if( p->final_id < 0 ) { @@ -1022,13 +1003,9 @@ static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { } return true; } - linked_inf *l = b->preds; - while( l ) { - emit_block *n = (emit_block*)l->ptr; - if( n->start_pos < b->start_pos && seal_block_rec(ctx,n,target) ) + for_iter(blocks,p,b->preds) + if( p->start_pos < b->start_pos && seal_block_rec(ctx,p,target) ) return true; - l = l->next; - } return false; } @@ -1653,7 +1630,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg r = lookup_block_var(ctx->current_block, ra); if( !IS_NULL(r) ) { STORE_MEM(ref, 0, LOAD(ra)); - remove_block_var(ctx->current_block, ra); + int_map_remove(&ctx->current_block->written_vars, ra->id); } STORE(dst, ref); } From 3efb299fc643240de68e98404b99226ad2ac3bfe Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 1 Apr 2026 08:36:24 +0200 Subject: [PATCH 19/83] upgrade vs projects (partial, temporary) --- hl.vcxproj | 11 ++++++----- hl.vcxproj.filters | 1 + libhl.vcxproj | 12 ++++++------ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/hl.vcxproj b/hl.vcxproj index a2c72f702..791c08677 100644 --- a/hl.vcxproj +++ b/hl.vcxproj @@ -45,34 +45,34 @@ Application true Unicode - v142 + v143 Application true Unicode - v142 + v143 Application false true Unicode - v142 + v143 Application false true Unicode - v142 + v143 Application false true Unicode - v120 + v143 Application @@ -368,6 +368,7 @@ + diff --git a/hl.vcxproj.filters b/hl.vcxproj.filters index 83b94a469..9e66b8869 100644 --- a/hl.vcxproj.filters +++ b/hl.vcxproj.filters @@ -16,5 +16,6 @@ + \ No newline at end of file diff --git a/libhl.vcxproj b/libhl.vcxproj index 40f1a2eff..1f86fe1a7 100644 --- a/libhl.vcxproj +++ b/libhl.vcxproj @@ -36,40 +36,40 @@ DynamicLibrary true - v142 + v143 Unicode DynamicLibrary false - v142 + v143 true Unicode DynamicLibrary false - v120 + v143 true Unicode DynamicLibrary true - v142 + v143 Unicode DynamicLibrary false - v142 + v143 true Unicode DynamicLibrary false - v120 + v143 true Unicode From 60ca2b232c22df5d2cfd4bfc76396baef58d016b Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 1 Apr 2026 19:48:07 +0200 Subject: [PATCH 20/83] more usage of data structs in emit --- src/data_struct.c | 40 ++++++++++++++----- src/jit_emit.c | 99 ++++++++++++++++++++--------------------------- 2 files changed, 73 insertions(+), 66 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index 733a9350e..32456ea4c 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -28,6 +28,15 @@ #else # define S_ARGS S_VALUE k # define S_KEY S_VALUE +# define keys values +#endif + +#ifndef S_DEFVAL +# define S_DEFVAL (S_VALUE)0 +#endif + +#ifndef S_CMP +# define S_CMP(a,b) a > b #endif typedef struct { @@ -86,14 +95,14 @@ static S_VALUE S_NAME(find)( S_TYPE st, S_KEY k ) { #else -static void S_NAME(add_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { +static bool S_NAME(add_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { int min = 0; int max = st->cur; int pos; while( min < max ) { int mid = (min + max) >> 1; S_KEY k2 = st->keys[mid]; - if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else return; + if( S_CMP(k,k2) ) min = mid + 1; else if( S_CMP(k2,k) ) max = mid; else return false; } S_NAME(check_size)(alloc,st); pos = (min + max) >> 1; @@ -106,6 +115,7 @@ static void S_NAME(add_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { st->values[pos] = v; # endif st->cur++; + return true; } #ifdef S_MAP @@ -137,7 +147,7 @@ static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { while( min < max ) { int mid = (min + max) >> 1; S_KEY k2 = st.keys[mid]; - if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else return true; + if( S_CMP(k,k2) ) min = mid + 1; else if( S_CMP(k2,k) ) max = mid; else return true; } return false; } @@ -161,7 +171,7 @@ static bool S_NAME(remove)( S_TYPE *st, S_KEY k ) { while( min < max ) { int mid = (min + max) >> 1; S_KEY k2 = st->keys[mid]; - if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else { + if( S_CMP(k,k2) ) min = mid + 1; else if( S_CMP(k2,k) ) max = mid; else { int pos = mid; memmove(st->keys + pos, st->keys + pos + 1, (st->cur - pos - 1) * sizeof(S_KEY)); # ifdef S_MAP @@ -180,24 +190,31 @@ static void S_NAME(reset)( S_TYPE *st ) { st->cur = 0; } +static S_VALUE *S_NAME(free)( S_TYPE *st ) { + st->cur = 0; + st->max = 0; + S_VALUE *vals = st->values; +# ifdef S_MAP + st->keys = NULL; +# endif + st->values = NULL; + return vals; +} + static int S_NAME(count)( S_TYPE st ) { return st.cur; } static S_VALUE S_NAME(get)( S_TYPE st, int idx ) { -# ifdef S_MAP return st.values[idx]; -# else - return st.keys[idx]; -# endif } static S_VALUE S_NAME(first)( S_TYPE st ) { - return S_NAME(get)(st,0); + return st.cur == 0 ? S_DEFVAL : st.values[0]; } static bool S_NAME(iter_next)( S_TYPE st, S_VALUE *val, int idx ) { - if( idx < st.cur ) *val = S_NAME(get)(st,idx); + if( idx < st.cur ) *val = st.values[idx]; return idx < st.cur; } @@ -207,5 +224,8 @@ static bool S_NAME(iter_next)( S_TYPE st, S_VALUE *val, int idx ) { #undef S_KEY #undef S_ARGS #undef STRUCT_NAME +#undef S_CMP +#undef S_DEFVAL +#undef keys #endif diff --git a/src/jit_emit.c b/src/jit_emit.c index e0b5ab75a..f57ad13e5 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -23,6 +23,8 @@ #include #include "data_struct.h" +static ereg ENULL = {VAL_NULL}; + typedef struct { hl_type *t; int id; @@ -44,6 +46,21 @@ typedef struct _tmp_phi tmp_phi; #include "data_struct.c" #define blocks_add(set,v) blocks_add_impl(DEF_ALLOC,&(set),v) +#define S_SORTED +#define S_DEFVAL ENULL +#define S_CMP(a,b) a.index > b.index +#define S_TYPE ereg_map +#define S_NAME(name) ereg_##name +#define S_VALUE ereg +#include "data_struct.c" +#define ereg_add(set,v) ereg_add_impl(DEF_ALLOC,&(set),v) + +#define S_TYPE phi_arr +#define S_NAME(name) phi_##name +#define S_VALUE tmp_phi* +#include "data_struct.c" +#define phi_add(set,v) phi_add_impl(DEF_ALLOC,&(set),v) + struct _linked_inf { int id; void *ptr; @@ -60,7 +77,7 @@ struct _emit_block { blocks nexts; blocks preds; int_map written_vars; - linked_inf *phis; + phi_arr phis; emit_block *wait_seal_next; }; @@ -73,8 +90,8 @@ struct _tmp_phi { bool opt; unsigned char mode; emit_block *b; - linked_inf *vals; - linked_inf *ref_phis; + ereg_map vals; + phi_arr ref_phis; linked_inf *ref_blocks; }; @@ -137,7 +154,6 @@ struct _emit_ctx { static hl_type hlt_ui8 = { HUI8, 0 }; static hl_type hlt_ui16 = { HUI16, 0 }; -static ereg ENULL = {VAL_NULL}; static linked_inf *link_add( emit_ctx *ctx, int id, void *ptr, linked_inf *head ) { linked_inf *l = hl_malloc(&ctx->jit->falloc,sizeof(linked_inf)); @@ -356,7 +372,7 @@ static tmp_phi *alloc_phi( emit_ctx *ctx, emit_block *b, vreg *r ) { p->r = r; if( r ) p->mode = hl_type_mode(r->t); p->value.index = -(++ctx->phi_count); - b->phis = link_add(ctx, 0, p, b->phis); + phi_add(b->phis,p); GET_PHI(p->value) = p; return p; } @@ -471,7 +487,7 @@ static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { } static void phi_remove_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { - p->vals = link_sort_remove(p->vals,v.index); + ereg_remove(&p->vals,v); jit_debug("%sPHI-REM-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); } @@ -480,13 +496,12 @@ static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { if( IS_NULL(v) ) jit_assert(); if( p->value.index == v.index ) return; - if( link_sort_lookup(p->vals,v.index) ) + if( !ereg_add(p->vals,v) ) return; jit_debug("%sPHI-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); - p->vals = link_add_sort_unique(ctx,v.index,p,p->vals); if( v.index < 0 ) { tmp_phi *p2 = GET_PHI(v); - p2->ref_phis = link_add(ctx,0,p,p2->ref_phis); + phi_add(p2->ref_phis,p); } } @@ -494,21 +509,17 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { if( p->locked ) jit_assert(); ereg same = ENULL; - linked_inf *l = p->vals; - while( l ) { - if( l->id == same.index || l->id == p->value.index ) { - l = l->next; + for_iter(ereg,v,p->vals) { + if( v.index == same.index || v.index == p->value.index ) continue; - } if( !IS_NULL(same) ) return p->value; - same.index = l->id; - l = l->next; + same = v; } if( IS_NULL(same) ) return p->value; // sealed (no dep yet) - if( !p->ref_phis && !p->ref_blocks ) + if( !phi_count(p->ref_phis) && !p->ref_blocks ) return same; if( p->locked || p->opt ) jit_assert(); @@ -516,28 +527,22 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { jit_debug("%sPHI-OPT %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); p->opt = true; ctx->phi_depth++; - l = p->ref_blocks; + linked_inf *l = p->ref_blocks; while( l ) { emit_block *b = (emit_block*)l->ptr; if( lookup_block_var(b,p->r).index == p->value.index ) store_block_var(ctx,b,p->r,same); l = l->next; } - l = p->ref_phis; - while( l ) { - tmp_phi *p2 = (tmp_phi*)l->ptr; + for_iter(phi,p2,p->ref_phis) { phi_remove_val(ctx,p2,p->value); phi_add_val(ctx,p2,same); - l = l->next; } - l = p->ref_phis; - p->ref_phis = NULL; p->ref_blocks = NULL; - while( l ) { - tmp_phi *p2 = (tmp_phi*)l->ptr; - optimize_phi_rec(ctx, p2); - l = l->next; - } + int count = phi_count(p->ref_phis); + tmp_phi **phis = phi_free(&p->ref_phis); + for(int i=0;iphi_depth--; jit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); return optimize_phi_rec(ctx,p); @@ -598,12 +603,8 @@ static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { static void seal_block( emit_ctx *ctx, emit_block *b ) { jit_debug(" SEAL #%d\n",b->id); - linked_inf *l = b->phis; - while( l ) { - tmp_phi *p = (tmp_phi*)l->ptr; + for_iter(phi,p,b->phis) gather_phis(ctx, p); - l = l->next; - } b->sealed = true; } @@ -785,42 +786,28 @@ static void emit_write_block( emit_ctx *ctx, emit_block *b ) { for(int i=0;inext_count;i++) bl->nexts[i++] = blocks_get(b->nexts,i)->id; // write phis - linked_inf *tmp = b->phis; - while( tmp ) { - tmp_phi *p = (tmp_phi*)tmp->ptr; - if( p->final_id >= 0 ) - bl->phi_count++; - tmp = tmp->next; + { + for_iter(phi,p,b->phis) + if( p->final_id >= 0 ) + bl->phi_count++; } bl->phis = (ephi*)hl_zalloc(&jit->falloc,sizeof(ephi)*bl->phi_count); - tmp = b->phis; int i = 0; - while( tmp ) { - tmp_phi *p = (tmp_phi*)tmp->ptr; - if( p->final_id < 0 ) { - tmp = tmp->next; + for_iter(phi,p,b->phis) { + if( p->final_id < 0 ) continue; - } ephi *p2 = bl->phis + i++; if( p->final_id == 0 ) p2->value = p->value; else p2->value.index = -p->final_id; - linked_inf *l = p->vals; - while( l ) { - p2->nvalues++; - l = l->next; - } + p2->nvalues = ereg_count(p->vals); p2->values = (ereg*)hl_malloc(&jit->falloc,sizeof(ereg)*p2->nvalues); - l = p->vals; int k = 0; - while( l ) { - ereg v = {l->id}; + for_iter(ereg,v,p->vals) { remap_phi_reg(ctx, &v); p2->values[k++] = v; - l = l->next; } - tmp = tmp->next; } } From 0e51b224a2071aafa423934c6ce4591d490515f0 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Thu, 2 Apr 2026 00:02:47 +0200 Subject: [PATCH 21/83] minor changes and force inline --- src/data_struct.c | 30 +++++++++++++++--------------- src/data_struct.h | 8 ++++++++ src/jit_emit.c | 41 +++++++++++++++++++++++------------------ 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index 32456ea4c..12a466216 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -50,7 +50,7 @@ typedef struct { typedef S_VALUE S_NAME(_value); -static void S_NAME(check_size)( hl_alloc *alloc, S_TYPE *st ) { +INLINE static void S_NAME(check_size)( hl_alloc *alloc, S_TYPE *st ) { if( st->cur == st->max ) { int n = st->max ? (st->max << 1) : STRUCT_DEF_SIZE; S_KEY *keys = (S_KEY*)hl_malloc(alloc,sizeof(S_KEY) * n); @@ -67,7 +67,7 @@ static void S_NAME(check_size)( hl_alloc *alloc, S_TYPE *st ) { #ifndef S_SORTED -static void S_NAME(add_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { +INLINE static void S_NAME(add_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { S_NAME(check_size)(alloc,st); st->keys[st->cur] = k; # ifdef S_MAP @@ -76,7 +76,7 @@ static void S_NAME(add_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { st->cur++; } -static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { +INLINE static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { for(int i=0;icur; int pos; @@ -119,7 +119,7 @@ static bool S_NAME(add_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { } #ifdef S_MAP -static void S_NAME(replace_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { +INLINE static void S_NAME(replace_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { int min = 0; int max = st->cur; int pos; @@ -141,7 +141,7 @@ static void S_NAME(replace_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { } #endif -static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { +INLINE static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { int min = 0; int max = st.cur; while( min < max ) { @@ -153,7 +153,7 @@ static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { } #ifdef S_MAP -static S_VALUE S_NAME(find)( S_TYPE st, S_KEY k ) { +INLINE static S_VALUE S_NAME(find)( S_TYPE st, S_KEY k ) { int min = 0; int max = st.cur; while( min < max ) { @@ -161,11 +161,11 @@ static S_VALUE S_NAME(find)( S_TYPE st, S_KEY k ) { S_KEY k2 = st.keys[mid]; if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else return st.values[mid]; } - return (S_VALUE)0; + return S_DEFVAL; } #endif -static bool S_NAME(remove)( S_TYPE *st, S_KEY k ) { +INLINE static bool S_NAME(remove)( S_TYPE *st, S_KEY k ) { int min = 0; int max = st->cur; while( min < max ) { @@ -186,11 +186,11 @@ static bool S_NAME(remove)( S_TYPE *st, S_KEY k ) { #endif -static void S_NAME(reset)( S_TYPE *st ) { +INLINE static void S_NAME(reset)( S_TYPE *st ) { st->cur = 0; } -static S_VALUE *S_NAME(free)( S_TYPE *st ) { +INLINE static S_VALUE *S_NAME(free)( S_TYPE *st ) { st->cur = 0; st->max = 0; S_VALUE *vals = st->values; @@ -201,19 +201,19 @@ static S_VALUE *S_NAME(free)( S_TYPE *st ) { return vals; } -static int S_NAME(count)( S_TYPE st ) { +INLINE static int S_NAME(count)( S_TYPE st ) { return st.cur; } -static S_VALUE S_NAME(get)( S_TYPE st, int idx ) { +INLINE static S_VALUE S_NAME(get)( S_TYPE st, int idx ) { return st.values[idx]; } -static S_VALUE S_NAME(first)( S_TYPE st ) { +INLINE static S_VALUE S_NAME(first)( S_TYPE st ) { return st.cur == 0 ? S_DEFVAL : st.values[0]; } -static bool S_NAME(iter_next)( S_TYPE st, S_VALUE *val, int idx ) { +INLINE static bool S_NAME(iter_next)( S_TYPE st, S_VALUE *val, int idx ) { if( idx < st.cur ) *val = st.values[idx]; return idx < st.cur; } diff --git a/src/data_struct.h b/src/data_struct.h index cc57ca0ca..1bd52c652 100644 --- a/src/data_struct.h +++ b/src/data_struct.h @@ -24,6 +24,14 @@ #include +#if defined(__GNUC__) || defined(__clang__) +#define INLINE inline __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define INLINE __forceinline +#else +#define INLINE inline +#endif + #define STRUCT_DEF_SIZE 2 #define for_iter(name,var,set) name##__value var; for(int __idx=0;name##_iter_next(set,&var,__idx);__idx++) diff --git a/src/jit_emit.c b/src/jit_emit.c index f57ad13e5..5ebf22ec2 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -24,6 +24,7 @@ #include "data_struct.h" static ereg ENULL = {VAL_NULL}; +static void __ignore( void *value ) {} typedef struct { hl_type *t; @@ -46,7 +47,14 @@ typedef struct _tmp_phi tmp_phi; #include "data_struct.c" #define blocks_add(set,v) blocks_add_impl(DEF_ALLOC,&(set),v) +#define S_TYPE phi_arr +#define S_NAME(name) phi_##name +#define S_VALUE tmp_phi* +#include "data_struct.c" +#define phi_add(set,v) phi_add_impl(DEF_ALLOC,&(set),v) + #define S_SORTED + #define S_DEFVAL ENULL #define S_CMP(a,b) a.index > b.index #define S_TYPE ereg_map @@ -55,11 +63,15 @@ typedef struct _tmp_phi tmp_phi; #include "data_struct.c" #define ereg_add(set,v) ereg_add_impl(DEF_ALLOC,&(set),v) -#define S_TYPE phi_arr -#define S_NAME(name) phi_##name -#define S_VALUE tmp_phi* +#define S_MAP + +#define S_DEFVAL ENULL +#define S_TYPE vreg_map +#define S_NAME(name) vreg_##name +#define S_KEY int +#define S_VALUE ereg #include "data_struct.c" -#define phi_add(set,v) phi_add_impl(DEF_ALLOC,&(set),v) +#define vreg_replace(set,k,v) vreg_replace_impl(DEF_ALLOC,&(set),k,v) struct _linked_inf { int id; @@ -76,7 +88,7 @@ struct _emit_block { bool sealed; blocks nexts; blocks preds; - int_map written_vars; + vreg_map written_vars; phi_arr phis; emit_block *wait_seal_next; }; @@ -399,21 +411,13 @@ static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { if( IS_NULL(v) ) jit_assert(); - int_map_replace(b->written_vars,r->id,v.index < 0 ? v.index : v.index + 1); + vreg_replace(b->written_vars,r->id,v); if( v.index < 0 ) { tmp_phi *p = GET_PHI(v); p->ref_blocks = link_add_sort_unique(ctx,b->id,b,p->ref_blocks); } } -static ereg lookup_block_var( emit_block *b, vreg *r ) { - int e = int_map_find(b->written_vars,r->id); - if( !e ) return ENULL; - ereg v; - v.index = e < 0 ? e : e-1; - return v; -} - static void split_block( emit_ctx *ctx ) { emit_block *b = alloc_block(ctx); b->sealed = true; @@ -530,7 +534,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { linked_inf *l = p->ref_blocks; while( l ) { emit_block *b = (emit_block*)l->ptr; - if( lookup_block_var(b,p->r).index == p->value.index ) + if( vreg_find(b->written_vars,p->r->id).index == p->value.index ) store_block_var(ctx,b,p->r,same); l = l->next; } @@ -561,7 +565,7 @@ static ereg gather_phis( emit_ctx *ctx, tmp_phi *p ) { } static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { - ereg v = lookup_block_var(b,r); + ereg v = vreg_find(b->written_vars,r->id); if( !IS_NULL(v) ) return v; if( !b->sealed ) { @@ -1058,6 +1062,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hl_module *m = ctx->mod; #ifdef HL_DEBUG int uid = (ctx->fun->findex << 16) | ctx->op_pos; + __ignore(&uid); #endif switch( o->op ) { case OMov: @@ -1614,10 +1619,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg ref = resolve_ref(ctx, ra->id); if( IS_NULL(ref) ) jit_assert(); - ereg r = lookup_block_var(ctx->current_block, ra); + ereg r = vreg_find(ctx->current_block->written_vars, ra->id); if( !IS_NULL(r) ) { STORE_MEM(ref, 0, LOAD(ra)); - int_map_remove(&ctx->current_block->written_vars, ra->id); + vreg_remove(&ctx->current_block->written_vars, ra->id); } STORE(dst, ref); } From a7b2349d7868987b9f7da314921c3a80386090ee Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 12 Apr 2026 17:17:26 +0200 Subject: [PATCH 22/83] started regs alloc (wip) --- hl.vcxproj | 4 +- hl.vcxproj.filters | 2 + src/data_struct.c | 14 ++ src/jit.c | 9 ++ src/jit.h | 53 ++++++-- src/jit_dump.c | 196 ++++++++++++++++------------ src/jit_emit.c | 146 ++++++++++++--------- src/jit_regs.c | 314 +++++++++++++++++++++++++++++++++++++++++++++ src/jit_x86_64.c | 129 +++++++++++++++++++ 9 files changed, 714 insertions(+), 153 deletions(-) create mode 100644 src/jit_regs.c create mode 100644 src/jit_x86_64.c diff --git a/hl.vcxproj b/hl.vcxproj index 791c08677..df1736b29 100644 --- a/hl.vcxproj +++ b/hl.vcxproj @@ -186,7 +186,7 @@ EnableAllWarnings Disabled WIN32;_DEBUG;_CONSOLE;HL_VTUNE;%(PreprocessorDefinitions) - /wd4456 /wd4100 /wd4204 /wd4702 /wd4457 %(AdditionalOptions) + /wd4456 /wd4100 /wd4204 /wd4702 /wd4457 /we4013 %(AdditionalOptions) true stdc11 @@ -363,6 +363,8 @@ + + diff --git a/hl.vcxproj.filters b/hl.vcxproj.filters index 9e66b8869..8a8395f72 100644 --- a/hl.vcxproj.filters +++ b/hl.vcxproj.filters @@ -9,6 +9,8 @@ + + diff --git a/src/data_struct.c b/src/data_struct.c index 12a466216..8f38a58ec 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -83,6 +83,20 @@ INLINE static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { return false; } +INLINE static bool S_NAME(remove)( S_TYPE *st, S_KEY k ) { + for(int i=0;icur;i++) + if( st->keys[i] == k ) { + int pos = i; + memmove(st->keys + pos, st->keys + pos + 1, (st->cur - pos - 1) * sizeof(S_KEY)); +# ifdef S_MAP + memmove(st->values + pos, st->values + pos + 1, (st->cur - pos - 1) * sizeof(S_VALUE)); +# endif + st->cur--; + return true; + } + return false; +} + #ifdef S_MAP static S_VALUE S_NAME(find)( S_TYPE st, S_KEY k ) { for(int i=0;ifalloc); hl_emit_alloc(ctx); + hl_regs_alloc(ctx); return ctx; } @@ -83,6 +90,7 @@ void hl_jit_init( jit_ctx *ctx, hl_module *m ) { } void hl_jit_free( jit_ctx *ctx, h_bool can_reset ) { + hl_regs_free(ctx); hl_emit_free(ctx); hl_free(&ctx->falloc); free(ctx); @@ -97,6 +105,7 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { ctx->fun = f; current_ctx = ctx; hl_emit_function(ctx); + hl_regs_function(ctx); current_ctx = NULL; return 0; } diff --git a/src/jit.h b/src/jit.h index 820e79bd5..a085fe508 100644 --- a/src/jit.h +++ b/src/jit.h @@ -26,7 +26,7 @@ typedef enum { LOAD_ADDR, - LOAD_IMM, + LOAD_CONST, LOAD_ARG, STORE, LEA, @@ -44,6 +44,9 @@ typedef enum { CALL_REG, CALL_FUN, MOV, + PUSH_CONST, + PUSH, + POP, ALLOC_STACK, FREE_STACK, NATIVE_REG, @@ -68,9 +71,7 @@ typedef enum { M_NORET, } emit_mode; -typedef struct { - int index; -} ereg; +typedef int ereg; typedef struct { union { @@ -92,8 +93,14 @@ typedef struct { }; } einstr; -#define VAL_NULL 0x80000000 -#define IS_NULL(e) ((e).index == VAL_NULL) +#define VAL_NULL 0x80000000 +#define FL_NATREG 0x40000000 +#define FL_STACKREG (FL_NATREG | 0x20000000) +#define IS_NULL(e) ((e) == VAL_NULL) +#define IS_NATREG(e) ((e) & FL_NATREG) +#define MK_STACK_REG(v) (((v)&0xFFFFFFF) | FL_STACKREG) +#define GET_STACK_OFFS(v) (((v) & 0x8000000) ? ((v) | 0xF0000000) : ((v)&0xFFFFFFF)); +#define IS_CALL(op) ((op) == CALL_PTR || (op) == CALL_REG || (op) == CALL_FUN) typedef struct { int *data; @@ -106,6 +113,7 @@ typedef struct _ephi ephi; struct _ephi { ereg value; int nvalues; + emit_mode mode; ereg *values; }; @@ -122,22 +130,41 @@ typedef struct _eblock { } eblock; typedef struct _emit_ctx emit_ctx; - +typedef struct _regs_ctx regs_ctx; typedef struct _jit_ctx ji_ctx; +typedef struct { + int nscratchs; + int npersists; + int nargs; + ereg ret; + ereg *scratch; + ereg *persist; + ereg *arg; +} reg_config; + +typedef reg_config regs_config[2]; + struct _jit_ctx { hl_module *mod; hl_function *fun; hl_alloc falloc; emit_ctx *emit; + regs_ctx *regs; // emit output int instr_count; int block_count; int value_count; + int phi_count; einstr *instrs; eblock *blocks; int *values_writes; int *emit_pos_map; + // regs output + int reg_instr_count; + einstr *reg_instrs; + ereg *reg_writes; + int *reg_pos_map; }; jit_ctx *hl_jit_alloc(); @@ -155,13 +182,15 @@ void int_alloc_free( int_alloc *a ); int *int_alloc_get( int_alloc *a, int count ); void int_alloc_store( int_alloc *a, int v ); +// emit & dump void hl_emit_dump( jit_ctx *ctx ); const char *hl_emit_regstr( ereg v ); ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); - +ereg **hl_emit_get_regs( einstr *e, int *count ); +void hl_emit_reg_iter( jit_ctx *jit, einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ); +extern int hl_emit_mode_sizes[]; #define val_str(v) hl_emit_regstr(v) - #ifdef HL_DEBUG # define JIT_DEBUG #endif @@ -169,12 +198,16 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); #define jit_error(msg) { hl_jit_error(msg,__func__,__LINE__); hl_debug_break(); exit(-1); } #define jit_assert() jit_error("") -#ifdef JIT_DEBUG +#if defined(JIT_DEBUG) # define jit_debug(...) printf(__VA_ARGS__) #else # define jit_debug(...) #endif +#define DEF_ALLOC &ctx->jit->falloc + +#define jit_pad_size(size,k) ((k == 0) ? 0 : ((-(size)) & (k - 1))) + void hl_jit_error( const char *msg, const char *func, int line ); void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); diff --git a/src/jit_dump.c b/src/jit_dump.c index f9c8ce348..081c86540 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -23,7 +23,7 @@ static const char *op_names[] = { "load-addr", - "load-imm", + "load-const", "load-arg", "store", "lea", @@ -41,6 +41,9 @@ static const char *op_names[] = { "call", "call", "mov", + "push-const", + "push", + "pop", "alloc-stack", "free-stack", "native-reg", @@ -48,6 +51,8 @@ static const char *op_names[] = { "debug-break", }; +const char *hl_natreg_str( int reg ); + const char *hl_emit_regstr( ereg v ) { static char fmts[4][10]; static int flip = 0; @@ -55,10 +60,18 @@ const char *hl_emit_regstr( ereg v ) { char *fmt = fmts[flip++&3]; if( IS_NULL(v) ) sprintf(fmt,"NULL???"); - else if( v.index < 0 ) - sprintf(fmt,"P%d",-v.index); + else if( v < 0 ) + sprintf(fmt,"P%d",-v); + else if( (v&FL_STACKREG) == FL_STACKREG ) { + int index = GET_STACK_OFFS(v); + if( index < 0 ) + sprintf(fmt,"[ST%d]", index); + else + sprintf(fmt,"[ST+%d]", index); + } else if( IS_NATREG(v) ) + sprintf(fmt,"%s", hl_natreg_str(v&(FL_NATREG-1))); else - sprintf(fmt,"V%d",v.index); + sprintf(fmt,"V%d",v); return fmt; } @@ -200,6 +213,8 @@ static void hl_dump_fun_name( hl_function *f ) { } static void hl_dump_args( jit_ctx *ctx, einstr *e ) { + if( e->nargs == 0xFF ) + return; ereg *v = hl_emit_get_args(ctx->emit, e); printf("("); for(int i=0;inargs;i++) { @@ -263,13 +278,97 @@ static void hl_dump_ptr_name( jit_ctx *ctx, void *ptr ) { } void hl_emit_flush( jit_ctx *ctx ); +void hl_regs_flush( jit_ctx *ctx ); + + +static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { + printf("%s", op_names[e->op]); + bool show_size = true; + switch( e->op ) { + case TEST: + case CMP: + printf("-%s", hl_op_name(e->size_offs)+2); + show_size = false; + break; + case BINOP: + case UNOP: + printf("-%s", hl_op_name(e->size_offs)+1); + show_size = false; + break; + default: + break; + } + if( e->mode ) + printf("%s", emit_mode_str(e->mode)); + switch( e->op ) { + case CALL_FUN: + printf(" "); + { + int fid = ctx->mod->functions_indexes[e->a]; + hl_code *code = ctx->mod->code; + if( fid < code->nfunctions ) { + hl_dump_fun_name(&code->functions[fid]); + } else { + printf("???"); + } + } + hl_dump_args(ctx,e); + break; + case CALL_REG: + printf(" %s", val_str(e->a)); + hl_dump_args(ctx,e); + break; + case CALL_PTR: + printf(" "); + hl_dump_ptr_name(ctx, (void*)e->value); + hl_dump_args(ctx,e); + break; + case JUMP: + case JCOND: + printf(" @%X", cur_pos + 1 + e->size_offs); + break; + case LOAD_CONST: + case PUSH_CONST: + printf(" "); + dump_value(ctx, e->value, e->mode); + break; + case LOAD_ADDR: + if( (e->size_offs>>8) ) + printf(" %s[%Xh]", val_str(e->a), e->size_offs); + else + printf(" %s[%d]", val_str(e->a), e->size_offs); + break; + case STORE: + { + int offs = e->size_offs; + if( offs == 0 ) + printf(" [%s]", val_str(e->a)); + else + printf(" %s[%d]", val_str(e->a), offs); + printf(" = %s", val_str(e->b)); + //if( e->mode == 0 || e->mode != ctx->instrs[ctx->values_writes[e->b.index]].mode ) + // printf(" ???"); + } + break; + default: + if( !IS_NULL(e->a) ) { + printf(" %s", val_str(e->a)); + if( !IS_NULL(e->b) ) printf(", %s", val_str(e->b)); + } + if( show_size && e->size_offs != 0 ) + printf(" %d", e->size_offs); + break; + } +} void hl_emit_dump( jit_ctx *ctx ) { int i; int cur_op = 0; hl_function *f = ctx->fun; int nargs = f->type->fun->nargs; - hl_emit_flush(ctx); // if it not was not before (in case of dump during emit) + // if it not was not before (in case of dump during process) + hl_emit_flush(ctx); + hl_regs_flush(ctx); printf("function "); hl_dump_fun_name(f); printf("("); @@ -293,6 +392,7 @@ void hl_emit_dump( jit_ctx *ctx ) { printf(" ??? MISSING BLOCK FOR RANGE %X-%X\n", cur, ctx->instr_count); // print instrs int vpos = 0; + int rpos = 0; cur = 0; for(i=0;iinstr_count;i++) { while( cur < ctx->block_count && ctx->blocks[cur].start_pos == i ) { @@ -300,7 +400,7 @@ void hl_emit_dump( jit_ctx *ctx ) { printf("--- BLOCK #%d ---\n", cur); for(int k=0;kphi_count;k++) { ephi *p = b->phis + k; - printf("\t\t@%X %s = phi(",i,val_str(p->value)); + printf("\t\t@%X %s = phi%s(",i,val_str(p->value),emit_mode_str(p->mode)); for(int n=0;nnvalues;n++) { if( n > 0 ) printf(","); printf("%s",val_str(p->values[n])); @@ -322,82 +422,14 @@ void hl_emit_dump( jit_ctx *ctx ) { printf("\t\t@%X ", i); if( vpos < ctx->value_count && ctx->values_writes[vpos] == i ) printf("V%d = ", vpos++); - printf("%s", op_names[e->op]); - bool show_size = true; - switch( e->op ) { - case TEST: - case CMP: - printf("-%s", hl_op_name(e->size_offs)+2); - show_size = false; - break; - case BINOP: - case UNOP: - printf("-%s", hl_op_name(e->size_offs)+1); - show_size = false; - break; - default: - break; - } - if( e->mode ) - printf("%s", emit_mode_str(e->mode)); - switch( e->op ) { - case CALL_FUN: - printf(" "); - { - int fid = ctx->mod->functions_indexes[e->a.index]; - hl_code *code = ctx->mod->code; - if( fid < code->nfunctions ) { - hl_dump_fun_name(&code->functions[fid]); - } else { - printf("???"); - } - } - hl_dump_args(ctx,e); - break; - case CALL_REG: - printf(" %s", val_str(e->a)); - hl_dump_args(ctx,e); - break; - case CALL_PTR: - printf(" "); - hl_dump_ptr_name(ctx, (void*)e->value); - hl_dump_args(ctx,e); - break; - case JUMP: - case JCOND: - printf(" @%X", i + 1 + e->size_offs); - break; - case LOAD_IMM: - printf(" "); - dump_value(ctx, e->value, e->mode); - break; - case LOAD_ADDR: - if( (e->size_offs>>8) ) - printf(" %s[%Xh]", val_str(e->a), e->size_offs); - else - printf(" %s[%d]", val_str(e->a), e->size_offs); - break; - case STORE: - { - int offs = e->size_offs; - if( offs == 0 ) - printf(" [%s]", val_str(e->a)); - else - printf(" %s[%d]", val_str(e->a), offs); - printf(" = %s", val_str(e->b)); - //if( e->mode == 0 || e->mode != ctx->instrs[ctx->values_writes[e->b.index]].mode ) - // printf(" ???"); - } - break; - default: - if( !IS_NULL(e->a) ) { - printf(" %s", val_str(e->a)); - if( !IS_NULL(e->b) ) printf(", %s", val_str(e->b)); - if( e->a.index >= vpos || e->b.index >= vpos ) printf(" ???"); - } - if( show_size && e->size_offs != 0 ) - printf(" %d", e->size_offs); - break; + dump_instr(ctx, e, i); + while( rpos < ctx->reg_instr_count && rpos < ctx->reg_pos_map[i+1] ) { + ereg out = ctx->reg_writes[rpos]; + einstr *er = ctx->reg_instrs + rpos; + printf("\n\t\t\t\t@%X ",rpos); + if( !IS_NULL(out) ) printf("%s = ",val_str(out)); + dump_instr(ctx,er,rpos); + rpos++; } printf("\n"); } diff --git a/src/jit_emit.c b/src/jit_emit.c index 5ebf22ec2..2f384ab05 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -23,8 +23,8 @@ #include #include "data_struct.h" -static ereg ENULL = {VAL_NULL}; static void __ignore( void *value ) {} +int hl_emit_mode_sizes[] = {0,1,2,4,8,4,8,sizeof(void*),0,0}; typedef struct { hl_type *t; @@ -39,8 +39,6 @@ typedef struct _linked_inf linked_inf; typedef struct _emit_block emit_block; typedef struct _tmp_phi tmp_phi; -#define DEF_ALLOC &ctx->jit->falloc - #define S_TYPE blocks #define S_NAME(name) blocks_##name #define S_VALUE emit_block* @@ -55,8 +53,7 @@ typedef struct _tmp_phi tmp_phi; #define S_SORTED -#define S_DEFVAL ENULL -#define S_CMP(a,b) a.index > b.index +#define S_DEFVAL VAL_NULL #define S_TYPE ereg_map #define S_NAME(name) ereg_##name #define S_VALUE ereg @@ -65,7 +62,7 @@ typedef struct _tmp_phi tmp_phi; #define S_MAP -#define S_DEFVAL ENULL +#define S_DEFVAL VAL_NULL #define S_TYPE vreg_map #define S_NAME(name) vreg_##name #define S_KEY int @@ -157,9 +154,9 @@ struct _emit_ctx { #define STORE_MEM(to, offs, v) emit_store_mem(ctx, to, offs, v) #define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR(obj,0),HL_WSIZE*2),HL_WSIZE*(id)) #define OFFSET(base,index,mult,offset) emit_gen_ext(ctx, LEA, base, index, M_PTR, (mult) | ((offset) << 8)) -#define BREAK() emit_gen(ctx, DEBUG_BREAK, ENULL, ENULL, 0) +#define BREAK() emit_gen(ctx, DEBUG_BREAK, VAL_NULL, VAL_NULL, 0) #define GET_MODE(r) emit_get_mode(ctx,r) -#define GET_PHI(r) ctx->phis[-(r).index-1] +#define GET_PHI(r) ctx->phis[-(r)-1] #define HDYN_VALUE 8 #define IS_FLOAT(t) ((t)->kind == HF64 || (t)->kind == HF32) @@ -280,14 +277,14 @@ static ereg resolve_ref( emit_ctx *ctx, int reg ) { if( ctx->refs[i].reg == reg ) return ctx->refs[i].r; } - return ENULL; + return VAL_NULL; } static unsigned char emit_get_mode( emit_ctx *ctx, ereg v ) { if( IS_NULL(v) ) jit_assert(); - if( v.index < 0 ) + if( v < 0 ) return GET_PHI(v)->mode; - return ctx->instrs[ctx->values.data[v.index]].mode; + return ctx->instrs[ctx->values.data[v]].mode; } static const char *phi_prefix( emit_ctx *ctx ) { @@ -331,7 +328,7 @@ static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { e->nargs = (unsigned char)count; if( count == 0 ) return; if( count == 1 ) { - e->size_offs = args[0].index; + e->size_offs = args[0]; return; } int *args_data = int_alloc_get(&ctx->args_data, count); @@ -354,7 +351,7 @@ static ereg emit_gen_ext( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode, i e->size_offs = size_offs; e->a = a; e->b = b; - return mode == 0 || mode == M_NORET ? ENULL : new_value(ctx); + return mode == 0 || mode == M_NORET ? VAL_NULL : new_value(ctx); } static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { @@ -362,7 +359,7 @@ static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { } static ereg emit_gen_size( emit_ctx *ctx, emit_op op, int size_offs ) { - return emit_gen_ext(ctx,op,ENULL,ENULL,op==ALLOC_STACK ? M_PTR : 0,size_offs); + return emit_gen_ext(ctx,op,VAL_NULL,VAL_NULL,op==ALLOC_STACK ? M_PTR : 0,size_offs); } static void patch_instr_mode( emit_ctx *ctx, int mode ) { @@ -383,7 +380,7 @@ static tmp_phi *alloc_phi( emit_ctx *ctx, emit_block *b, vreg *r ) { p->b = b; p->r = r; if( r ) p->mode = hl_type_mode(r->t); - p->value.index = -(++ctx->phi_count); + p->value = -(++ctx->phi_count); phi_add(b->phis,p); GET_PHI(p->value) = p; return p; @@ -391,7 +388,7 @@ static tmp_phi *alloc_phi( emit_ctx *ctx, emit_block *b, vreg *r ) { static int emit_jump( emit_ctx *ctx, bool cond ) { int p = ctx->emit_pos; - emit_gen(ctx, cond ? JCOND : JUMP, ENULL, ENULL, 0); + emit_gen(ctx, cond ? JCOND : JUMP, VAL_NULL, VAL_NULL, 0); return p; } @@ -412,7 +409,7 @@ static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { if( IS_NULL(v) ) jit_assert(); vreg_replace(b->written_vars,r->id,v); - if( v.index < 0 ) { + if( v < 0 ) { tmp_phi *p = GET_PHI(v); p->ref_blocks = link_add_sort_unique(ctx,b->id,b,p->ref_blocks); } @@ -448,7 +445,7 @@ static void register_jump( emit_ctx *ctx, int jpos, int offs ) { } static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { - einstr *e = emit_instr(ctx, LOAD_IMM); + einstr *e = emit_instr(ctx, LOAD_CONST); e->mode = hl_type_mode(size_t); e->value = value; return new_value(ctx); @@ -458,7 +455,7 @@ static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) einstr *e = emit_instr(ctx, LOAD_ADDR); e->mode = hl_type_mode(size_t); e->a = v; - e->b = ENULL; + e->b = VAL_NULL; e->size_offs = offset; return new_value(ctx); } @@ -474,7 +471,7 @@ static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int e->mode = (unsigned char)(ret ? hl_type_mode(ret) : M_NORET); e->value = (int_val)native_ptr; store_args(ctx, e, args, nargs); - return ret == NULL || e->mode == M_VOID ? ENULL : new_value(ctx); + return ret == NULL || e->mode == M_VOID ? VAL_NULL : new_value(ctx); } static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_type *ret ) { @@ -482,11 +479,11 @@ static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_typ e->mode = hl_type_mode(ret); e->a = f; store_args(ctx, e, args, nargs); - return e->mode == M_VOID ? ENULL : new_value(ctx); + return e->mode == M_VOID ? VAL_NULL : new_value(ctx); } static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { - emit_gen_ext(ctx, TEST, v, ENULL, 0, o); + emit_gen_ext(ctx, TEST, v, VAL_NULL, 0, o); patch_instr_mode(ctx, GET_MODE(v)); } @@ -498,12 +495,12 @@ static void phi_remove_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { if( !p->b ) jit_assert(); if( IS_NULL(v) ) jit_assert(); - if( p->value.index == v.index ) + if( p->value == v ) return; if( !ereg_add(p->vals,v) ) return; jit_debug("%sPHI-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); - if( v.index < 0 ) { + if( v < 0 ) { tmp_phi *p2 = GET_PHI(v); phi_add(p2->ref_phis,p); } @@ -512,9 +509,9 @@ static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { if( p->locked ) jit_assert(); - ereg same = ENULL; + ereg same = VAL_NULL; for_iter(ereg,v,p->vals) { - if( v.index == same.index || v.index == p->value.index ) + if( v == same || v == p->value ) continue; if( !IS_NULL(same) ) return p->value; @@ -534,7 +531,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { linked_inf *l = p->ref_blocks; while( l ) { emit_block *b = (emit_block*)l->ptr; - if( vreg_find(b->written_vars,p->r->id).index == p->value.index ) + if( vreg_find(b->written_vars,p->r->id) == p->value ) store_block_var(ctx,b,p->r,same); l = l->next; } @@ -600,7 +597,7 @@ static void emit_walk_blocks( emit_ctx *ctx, void (*fun)(emit_ctx*,emit_block*) static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { ereg ref = resolve_ref(ctx, r->id); - if( ref.index >= 0 ) + if( ref >= 0 ) return LOAD_MEM(ref,0,r->t); return emit_load_reg_block(ctx, ctx->current_block, r); } @@ -634,9 +631,9 @@ static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int else { einstr *e = emit_instr(ctx, CALL_FUN); e->mode = hl_type_mode(dst->t); - e->a.index = findex; + e->a = findex; store_args(ctx, e, args, count); - STORE(dst, e->mode == M_VOID ? ENULL : new_value(ctx)); + STORE(dst, e->mode == M_VOID ? VAL_NULL : new_value(ctx)); } } @@ -727,7 +724,7 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, } static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode mode, bool _unsigned ) { - return emit_gen(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, ENULL, mode); + return emit_gen(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, VAL_NULL, mode); } static bool dyn_need_type( hl_type *t ) { @@ -760,19 +757,19 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { static void emit_opcode( emit_ctx *ctx, hl_opcode *o ); static void remap_phi_reg( emit_ctx *ctx, ereg *r ) { - if( r->index >= 0 || IS_NULL(*r) ) + if( *r >= 0 || IS_NULL(*r) ) return; tmp_phi *p = GET_PHI(*r); while( p->final_id < 0 ) { - if( p->target.index >= 0 ) { - r->index = p->target.index; + if( p->target >= 0 ) { + *r = p->target; return; } p = GET_PHI(p->target); } if( p->final_id == 0 ) return; - r->index = -p->final_id; // new phis + *r = -p->final_id; // new phis } static void emit_write_block( emit_ctx *ctx, emit_block *b ) { @@ -796,6 +793,7 @@ static void emit_write_block( emit_ctx *ctx, emit_block *b ) { bl->phi_count++; } bl->phis = (ephi*)hl_zalloc(&jit->falloc,sizeof(ephi)*bl->phi_count); + jit->phi_count += bl->phi_count; int i = 0; for_iter(phi,p,b->phis) { if( p->final_id < 0 ) @@ -804,7 +802,8 @@ static void emit_write_block( emit_ctx *ctx, emit_block *b ) { if( p->final_id == 0 ) p2->value = p->value; else - p2->value.index = -p->final_id; + p2->value = -p->final_id; + p2->mode = p->mode; p2->nvalues = ereg_count(p->vals); p2->values = (ereg*)hl_malloc(&jit->falloc,sizeof(ereg)*p2->nvalues); int k = 0; @@ -831,6 +830,7 @@ void hl_emit_flush( jit_ctx *jit ) { jit->instrs = ctx->instrs; jit->instr_count = ctx->emit_pos; jit->emit_pos_map = ctx->pos_map; + jit->phi_count = 0; jit->block_count = ctx->current_block->id + 1; jit->blocks = hl_zalloc(&jit->falloc,sizeof(eblock) * jit->block_count); for(i=0;iblock_count;i++) @@ -840,7 +840,7 @@ void hl_emit_flush( jit_ctx *jit ) { emit_walk_blocks(ctx,emit_write_block); } -static void hl_iter_instr_reg( einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ) { +void hl_emit_reg_iter( jit_ctx *jit, einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ) { switch( e->op ) { case CALL_REG: iter_reg(ctx,&e->a); @@ -848,12 +848,13 @@ static void hl_iter_instr_reg( einstr *e, void *ctx, void (*iter_reg)( void *, e case CALL_PTR: { int i; - ereg *args = hl_emit_get_args(ctx, e); + ereg *args = hl_emit_get_args(jit->emit, e); for(i=0;inargs;i++) iter_reg(ctx, args + i); } break; - case LOAD_IMM: + case LOAD_CONST: + case PUSH_CONST: // skip break; default: @@ -866,6 +867,31 @@ static void hl_iter_instr_reg( einstr *e, void *ctx, void (*iter_reg)( void *, e } } +ereg **hl_emit_get_regs( einstr *e, int *count ) { + static ereg *tmp[2]; + int k = 0; + switch( e->op ) { + case CALL_REG: + case CALL_FUN: + case CALL_PTR: + jit_assert(); + break; + case LOAD_CONST: + case PUSH_CONST: + // skip + break; + default: + if( !IS_NULL(e->a) ) { + tmp[k++] = &e->a; + if( !IS_NULL(e->b) ) + tmp[k++] = &e->b; + } + break; + } + *count = k; + return tmp; +} + static void hl_emit_clean_phis( emit_ctx *ctx ) { for(int i=0;iphi_count;i++) { tmp_phi *p = ctx->phis[i]; @@ -874,7 +900,7 @@ static void hl_emit_clean_phis( emit_ctx *ctx ) { while( true ) { cur->opt = false; r = optimize_phi_rec(ctx,cur); - if( r.index >= 0 || r.index == cur->value.index ) break; + if( r >= 0 || r == cur->value ) break; cur = GET_PHI(r); } p->target = r; @@ -882,13 +908,13 @@ static void hl_emit_clean_phis( emit_ctx *ctx ) { int new_phis = 0; for(int i=0;iphi_count;i++) { tmp_phi *p = ctx->phis[i]; - if( p->target.index == p->value.index ) + if( p->target == p->value ) p->final_id = ++new_phis; else p->final_id = -1; } for(int i=0;iemit_pos;i++) - hl_iter_instr_reg(ctx->instrs + i, ctx, remap_phi_reg); + hl_emit_reg_iter(ctx->jit, ctx->instrs + i, ctx, remap_phi_reg); } void hl_emit_function( jit_ctx *jit ) { @@ -933,14 +959,14 @@ void hl_emit_function( jit_ctx *jit ) { for(i=0;itype->fun->nargs;i++) { hl_type *t = f->type->fun->args[i]; if( t->kind == HVOID ) continue; - STORE(R(i), emit_gen(ctx, LOAD_ARG, ENULL, ENULL, hl_type_mode(t))); + STORE(R(i), emit_gen(ctx, LOAD_ARG, VAL_NULL, VAL_NULL, hl_type_mode(t))); } for(i=f->nops-1;i>=0;i--) { hl_opcode *o = f->ops + i; if( o->op == ORef ) { ereg ref = resolve_ref(ctx, o->p2); - if( ref.index >= 0 ) continue; + if( ref >= 0 ) continue; if( ctx->ref_count == MAX_REFS ) jit_error("Too many refs"); ctx->refs[ctx->ref_count].r = emit_gen_size(ctx, ALLOC_STACK, hl_type_size(R(o->p2)->t)); ctx->refs[ctx->ref_count].reg = o->p2; @@ -1067,7 +1093,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { switch( o->op ) { case OMov: case OUnsafeCast: - STORE(dst, emit_gen(ctx,MOV,LOAD(ra),ENULL,hl_type_mode(ra->t))); + STORE(dst, emit_gen(ctx,MOV,LOAD(ra),VAL_NULL,hl_type_mode(ra->t))); break; case OInt: STORE(dst, LOAD_CONST(m->code->ints[o->p2], dst->t)); @@ -1161,7 +1187,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case ONeg: case ONot: - STORE(dst, emit_gen_ext(ctx, UNOP, LOAD(ra), ENULL, hl_type_mode(dst->t), o->op)); + STORE(dst, emit_gen_ext(ctx, UNOP, LOAD(ra), VAL_NULL, hl_type_mode(dst->t), o->op)); break; case OJFalse: case OJTrue: @@ -1208,7 +1234,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, emit_conv(ctx,LOAD(ra),hl_type_mode(dst->t), o->op == OToUFloat)); break; case ORet: - emit_gen(ctx, RET, dst->t->kind == HVOID ? ENULL : LOAD(dst), ENULL, M_NORET); + emit_gen(ctx, RET, dst->t->kind == HVOID ? VAL_NULL : LOAD(dst), VAL_NULL, M_NORET); patch_instr_mode(ctx, hl_type_mode(dst->t)); break; case OIncr: @@ -1217,13 +1243,13 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( IS_FLOAT(dst->t) ) { jit_assert(); } else { - STORE(dst, emit_gen_ext(ctx,UNOP,LOAD(dst),ENULL,hl_type_mode(dst->t),o->op)); + STORE(dst, emit_gen_ext(ctx,UNOP,LOAD(dst),VAL_NULL,hl_type_mode(dst->t),o->op)); } } break; case ONew: { - ereg arg = ENULL; + ereg arg = VAL_NULL; void *allocFun = NULL; int nargs = 1; switch( dst->t->kind ) { @@ -1331,7 +1357,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( dst->t->kind == HSTRUCT ) { hl_type *ft = hl_obj_field_fetch(ra->t,o->p3)->t; if( ft->kind == HPACKED ) { - STORE(dst,OFFSET(r, ENULL, 0, rt->fields_indexes[o->p3])); + STORE(dst,OFFSET(r, VAL_NULL, 0, rt->fields_indexes[o->p3])); break; } } @@ -1424,7 +1450,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( dst->t->kind == HSTRUCT ) { hl_type *ft = hl_obj_field_fetch(r->t,o->p2)->t; if( ft->kind == HPACKED ) { - STORE(dst, OFFSET(obj, ENULL, 0, field_pos)); + STORE(dst, OFFSET(obj, VAL_NULL, 0, field_pos)); break; } } @@ -1636,7 +1662,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case ORefData: switch( ra->t->kind ) { case HARRAY: - STORE(dst, OFFSET(LOAD(ra),ENULL,0,sizeof(varray))); + STORE(dst, OFFSET(LOAD(ra),VAL_NULL,0,sizeof(varray))); break; default: jit_assert(); @@ -1721,7 +1747,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hashed_name = hl_hash_gen(name, true); } // ----------------------------------------- - ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : ENULL; + ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : VAL_NULL; emit_native_call(ctx, null_field_access ? hl_jit_null_field_access : hl_jit_null_access, &arg, null_field_access ? 1 : 0, NULL); patch_jump(ctx, jok); } @@ -1765,7 +1791,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { current_addr = LOAD_CONST_PTR(&tinf->trap_current); # else thread = emit_native_call(ctx, hl_get_thread, NULL, 0, &hlt_bytes); - current_addr = OFFSET(thread, ENULL, 0, (int)(int_val)&tinf->trap_current); + current_addr = OFFSET(thread, VAL_NULL, 0, (int)(int_val)&tinf->trap_current); # endif STORE_MEM(st, (int)(int_val)&trap->prev, LOAD_MEM_PTR(current_addr,0)); STORE_MEM(current_addr, 0, st); @@ -1807,12 +1833,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { void *fun = setjmp; ereg args[2]; int nargs = 1; - args[0] = OFFSET(st, ENULL, 0, (int)(int_val)&trap->buf); + args[0] = OFFSET(st, VAL_NULL, 0, (int)(int_val)&trap->buf); #if defined(HL_WIN) && defined(HL_64) // On Win64 setjmp actually takes two arguments // the jump buffer and the frame pointer (or the stack pointer if there is no FP) nargs = 2; - args[1] = emit_gen(ctx, NATIVE_REG, ENULL, ENULL, REG_RBP); + args[1] = emit_gen(ctx, NATIVE_REG, VAL_NULL, VAL_NULL, REG_RBP); #endif #ifdef HL_MINGW fun = _setjmp; @@ -1844,7 +1870,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { current_addr = LOAD_CONST_PTR(&tinf->trap_current); # else thread = emit_native_call(ctx, hl_get_thread, NULL, 0, &hlt_bytes); - current_addr = OFFSET(thread, ENULL, 0, (int)(int_val)&tinf->trap_current); + current_addr = OFFSET(thread, VAL_NULL, 0, (int)(int_val)&tinf->trap_current); # endif STORE_MEM(current_addr, 0, LOAD_MEM_PTR(st,(int)(int_val)&trap->prev)); @@ -1872,7 +1898,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { emit_gen_ext(ctx, CMP, v, LOAD_CONST(count,&hlt_i32), 0, OJUGte); patch_instr_mode(ctx, M_I32); int jdefault = emit_jump(ctx, true); - emit_gen_ext(ctx, JUMP_TABLE, v, ENULL, 0, count); + emit_gen_ext(ctx, JUMP_TABLE, v, VAL_NULL, 0, count); for(int i=0; iextra[i]); @@ -1897,7 +1923,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case HSTRUCT: { hl_runtime_obj *rt = hl_get_obj_rt(dst->t); - r = OFFSET(r, ENULL, 0, rt->fields_indexes[o->p2-1]); + r = OFFSET(r, VAL_NULL, 0, rt->fields_indexes[o->p2-1]); } break; default: @@ -1905,7 +1931,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; } } - emit_gen(ctx, PREFETCH, r, ENULL, o->p3); + emit_gen(ctx, PREFETCH, r, VAL_NULL, o->p3); } break; case OAsm: diff --git a/src/jit_regs.c b/src/jit_regs.c new file mode 100644 index 000000000..0949e2ec5 --- /dev/null +++ b/src/jit_regs.c @@ -0,0 +1,314 @@ +/* + * Copyright (C)2015-2016 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include "data_struct.h" + +#define VAL(k) (ctx->values + (k)) + +#if defined(HL_WIN_CALL) && defined(HL_64) +# define IS_WINCALL64 1 +#else +# define IS_WINCALL64 0 +#endif + +#define VIDX(e) (((e) < 0) ? ctx->jit->value_count + (-(e)-1) : (e)) +#define REG_MODE(m) (((m) == M_F32 || (m) == M_F64) ? 1 :0) + +typedef struct { + int stack_pos; + emit_mode mode; + ereg current; + bool saved; +} value_info; + +#define S_TYPE values +#define S_NAME(name) values_##name +#define S_VALUE value_info* +#include "data_struct.c" +#define values_add(set,v) values_add_impl(DEF_ALLOC,&(set),v) + +struct _regs_ctx { + jit_ctx *jit; + regs_config cfg; + value_info *values; + values scratch; + int max_instrs; + int cur_op; + int emit_pos; + int persists_uses[2]; + einstr *instrs; + ereg *out_write; + int *pos_map; +}; + +typedef int call_regs[2]; + +void hl_jit_init_regs( regs_config cfg ); + +static ereg get_call_reg( regs_ctx *ctx, call_regs regs, emit_mode m ) { + ereg r; + int mode = REG_MODE(m); + reg_config *cfg = &ctx->cfg[mode]; + if( regs[mode] < cfg->nargs ) + r = cfg->arg[regs[mode]++]; + else + r = VAL_NULL; + return r; +} + +static void regs_write_instr( regs_ctx *ctx, einstr *e, ereg out ) { + if( ctx->emit_pos == ctx->max_instrs ) { + int pos = ctx->emit_pos; + int next_size = ctx->max_instrs ? (ctx->max_instrs << 1) : 256; + einstr *instrs = (einstr*)malloc(sizeof(einstr) * next_size); + ereg *out = (ereg*)malloc(sizeof(ereg) * next_size); + if( instrs == NULL || out == NULL ) jit_error("Out of memory"); + memcpy(instrs, ctx->instrs, pos * sizeof(einstr)); + memcpy(out, ctx->out_write, pos * sizeof(ereg)); + memset(instrs + pos, 0, (next_size - pos) * sizeof(einstr)); + free(ctx->instrs); + free(ctx->out_write); + ctx->instrs = instrs; + ctx->out_write = out; + ctx->max_instrs = next_size; + } else if( (ctx->emit_pos & 0xFF) == 0 ) + memset(ctx->instrs + ctx->emit_pos, 0, 256 * sizeof(einstr)); + ctx->out_write[ctx->emit_pos] = out; + ctx->instrs[ctx->emit_pos++] = *e; +} + +static void regs_emit_mov( regs_ctx *ctx, ereg to, ereg from, emit_mode m ) { + if( to == from ) return; + einstr e; + e.header = MOV; + e.mode = m; + e.size_offs = 0; + e.a = from; + e.b = VAL_NULL; + regs_write_instr(ctx, &e, to); +} + +#define regs_todo() regs_emit_todo_impl(ctx,__LINE__) +static void regs_emit_todo_impl( regs_ctx *ctx, int line ) { + einstr e; + e.header = PUSH_CONST; + e.mode = M_I32; + e.value = line; + regs_write_instr(ctx, &e, VAL_NULL); + einstr e2; + e2.header = DEBUG_BREAK; + e2.size_offs = 0; + e2.a = VAL_NULL; + e2.b = VAL_NULL; + regs_write_instr(ctx, &e2, VAL_NULL); +} + + +static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { + if( forced ) { + if( v->current == *forced ) + return; + // erase previous value + for_iter(values,v2,ctx->scratch) { + if( v2->current == *forced ) { + if( !v2->saved ) jit_assert(); + v2->current = VAL_NULL; + values_remove(&ctx->scratch,v2); + break; + } + } + v->current = *forced; + values_add(ctx->scratch,v); + } else { + // lookup available reg + int mode = REG_MODE(v->mode); + reg_config *cfg = &ctx->cfg[mode]; + for(int i=0;inscratchs;i++) { + ereg r = cfg->scratch[i]; + for_iter(values,v2,ctx->scratch) { + if( v2->current == r ) { + r = VAL_NULL; + break; + } + } + if( !IS_NULL(r) ) { + v->current = r; + values_add(ctx->scratch,v); + return; + } + } + if( ctx->persists_uses[mode] < cfg->npersists ) { + v->current = cfg->persist[ctx->persists_uses[mode]++]; + return; + } + // free the oldest scratch reg + value_info *v2 = values_first(ctx->scratch); + if( !v2->saved ) { + regs_emit_mov(ctx, MK_STACK_REG(v2->stack_pos), v2->current, v2->mode); + v2->saved = true; + } + v->current = v2->current; + values_remove(&ctx->scratch, v2); + values_add(ctx->scratch, v); + } +} + +static void regs_assign_stack_regs( regs_ctx *ctx ) { + // dummy regs assign for testing : set all values on stack + int stack_pos = 0; + int nargs = ctx->jit->fun->type->fun->nargs; + call_regs regs = {0}; + int args_size = 0; + for(int i=0;ijit->value_count;i++) { + value_info *v = VAL(i); + einstr *e = ctx->jit->instrs + ctx->jit->values_writes[i]; + int size = hl_emit_mode_sizes[e->mode]; + if( size <= 0 ) hl_jit_assert(); + if( i < nargs ) { + ereg r = get_call_reg(ctx,regs,e->mode); + if( !IS_NULL(r) ) { + v->current = r; + values_add(ctx->scratch,v); + } + if( IS_NULL(r) || IS_WINCALL64 ) { + // use existing stack storage + v->stack_pos = args_size + HL_WSIZE*2; + args_size += size < 4 ? 4 : size; + continue; + } + } + stack_pos += size; + stack_pos += jit_pad_size(stack_pos,size); + v->stack_pos = MK_STACK_REG(-stack_pos); + } + for(int i=0;ijit->block_count;i++) { + eblock *b = ctx->jit->blocks + i; + for(int k=0;kphi_count;k++) { + ephi *p = b->phis + k; + int size = hl_emit_mode_sizes[p->mode]; + value_info *v = VAL(VIDX(p->value)); + stack_pos += size; + stack_pos += jit_pad_size(stack_pos,size); + v->stack_pos = MK_STACK_REG(-stack_pos); + } + } +} + +void hl_regs_alloc( jit_ctx *jit ) { + regs_ctx *ctx = malloc(sizeof(regs_ctx)); + memset(ctx,0,sizeof(regs_ctx)); + ctx->jit = jit; + jit->regs = ctx; + hl_jit_init_regs(ctx->cfg); +} + +void hl_regs_flush( jit_ctx *jit ) { + regs_ctx *ctx = jit->regs; + jit->reg_instr_count = ctx->emit_pos; + jit->reg_instrs = ctx->instrs; + jit->reg_writes = ctx->out_write; + jit->reg_pos_map = ctx->pos_map; + ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; +} + +void hl_regs_function( jit_ctx *jit ) { + regs_ctx *ctx = jit->regs; + int nvalues = jit->value_count + jit->phi_count; + memset(ctx->persists_uses,0,sizeof(ctx->persists_uses)); + free(ctx->pos_map); + ctx->pos_map = (int*)malloc((jit->instr_count + 1) * sizeof(int)); + ctx->emit_pos = 0; + ctx->cur_op = 0; + ctx->pos_map[0] = 0; + values_reset(&ctx->scratch); + ctx->values = malloc(sizeof(value_info) * nvalues); + memset(ctx->values, 0, sizeof(value_info) * nvalues); + for(int i=0;icurrent = VAL_NULL; + } + regs_assign_stack_regs(ctx); + int write_index = 0; + for(int i=0;iinstr_count;i++) { + einstr e = jit->instrs[i]; + ereg *ret_val = NULL; + int nread; + ctx->cur_op = i; + if( i > 0 ) + ctx->pos_map[i] = ctx->emit_pos; + if( IS_CALL(e.op) ) { + ereg *args = hl_emit_get_args(ctx->jit->emit,&e); + for_iter(values,v,ctx->scratch) { + if( !v->saved ) { + v->saved = true; + regs_emit_mov(ctx, MK_STACK_REG(v->stack_pos), v->current, v->mode); + } + } + call_regs regs = {0}; + for(int k=0;kmode); + if( IS_NULL(r) ) { + regs_todo(); + } else if( IS_NULL(v->current) ) + regs_emit_mov(ctx, r, MK_STACK_REG(v->stack_pos), v->mode); + else + regs_emit_mov(ctx, r, v->current, v->mode); + } + for_iter(values,v2,ctx->scratch) + v2->current = VAL_NULL; + values_reset(&ctx->scratch); + e.nargs = 0xFF; + ret_val = &ctx->cfg[REG_MODE(e.mode)].ret; + } else { + ereg **regs = hl_emit_get_regs(&e,&nread); + for(int k=0;kcurrent) ) { + regs_assign(ctx,v,NULL); + regs_emit_mov(ctx, v->current, MK_STACK_REG(v->stack_pos), v->mode); + } + *r = v->current; + } + } + ereg out; + if( write_index < jit->value_count && jit->values_writes[write_index] == i ) { + value_info *v = VAL(write_index++); + regs_assign(ctx,v,ret_val); + out = v->current; + } else + out = VAL_NULL; + regs_write_instr(ctx, &e, out); + } + free(ctx->values); + hl_regs_flush(ctx->jit); +} + +void hl_regs_free( regs_ctx *ctx ) { + free(ctx->pos_map); + free(ctx->instrs); + free(ctx->out_write); + free(ctx); +} + diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c new file mode 100644 index 000000000..9a2e5b1bd --- /dev/null +++ b/src/jit_x86_64.c @@ -0,0 +1,129 @@ +/* + * Copyright (C)2015-2016 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include "data_struct.h" + +typedef enum { + RAX = 0, + RCX = 1, + RDX = 2, + RBX = 3, + RSP = 4, + RBP = 5, + RSI = 6, + RDI = 7, +#ifdef HL_64 + R8 = 8, + R9 = 9, + R10 = 10, + R11 = 11, + R12 = 12, + R13 = 13, + R14 = 14, + R15 = 15, +#endif + _UNUSED = 0xFF +} Reg; + +typedef enum { + K64 = 0, + K32 = 1, + K16 = 2, + K8 = 3, + KF32 = 4, + KF64 = 5 +} RegKind; + +#define R(id,k) ((id) | ((k) << 8) | FL_NATREG) +#define X(id) R(id,K64) + +const char *hl_natreg_str( int reg ) { + static char out[8]; + static const char *regs_str[] = { "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI" }; + RegKind k = (reg >> 8) & 0xFF; + Reg r = (reg & 0xFF); + switch( k ) { + case K64: + if( r < 8 ) + sprintf(out,"R%s",regs_str[r]); + else + sprintf(out,"R%d",r); + break; + case K32: + if( r < 8 ) + sprintf(out,"E%s",regs_str[r]); + else + sprintf(out,"R%dD",r); + break; + case K16: + if( r < 8 ) + sprintf(out,"%s",regs_str[r]); + else + sprintf(out,"R%dW",r); + break; + case K8: + if( r < 8 ) + sprintf(out,"%s",regs_str[r]); + else + sprintf(out,"R%dB",r); + break; + default: + sprintf(out,"XMM%d",r); + break; + } + return out; +} + +void hl_jit_init_regs( regs_config cfg ) { +# ifdef HL_WIN_CALL + static int scratch_regs[] = { X(RAX), X(RCX), X(RDX), X(R8), X(R9), X(R10), X(R11) }; + static int free_regs[] = { X(RSI), X(RDI), X(RBX), X(R12), X(R13), X(R14), X(R15) }; + static int call_regs[] = { X(RCX), X(RDX), X(R8), X(R9) }; + cfg[1].nargs = 4; + cfg[1].nscratchs = 6; +# else + static int scratch_regs[] = { X(RAX), X(RCX), X(RDX), X(RSI), X(RDI), X(R8), X(R9), X(R10), X(R11) }; + static int free_regs[] = { X(RBX), X(R12), X(R13), X(R14), X(R15) }; + static int call_regs[] = { X(RDI), X(RSI), X(RDX), X(RCX), X(R8), X(R9) }; + cfg[1].nargs = 8; + cfg[1].nscratchs = 16; +# endif + static int floats[] = { + R(0,KF64), R(1,KF64), R(2,KF64), R(3,KF64), + R(4,KF64), R(5,KF64), R(6,KF64), R(7,KF64), + R(8,KF64), R(9,KF64), R(10,KF64), R(11,KF64), + R(12,KF64), R(13,KF64), R(14,KF64), R(15,KF64) + }; + cfg[0].ret = scratch_regs[0]; + cfg[0].nscratchs = sizeof(scratch_regs) / sizeof(int); + cfg[0].npersists = sizeof(free_regs) / sizeof(int); + cfg[0].nargs = sizeof(call_regs) / sizeof(int); + cfg[0].scratch = (ereg*)scratch_regs; + cfg[0].persist = (ereg*)free_regs; + cfg[0].arg = (ereg*)call_regs; + cfg[1].ret = floats[0]; + cfg[1].scratch = (ereg*)floats; + cfg[1].arg = (ereg*)floats; + cfg[1].persist = (ereg*)floats+cfg[1].nscratchs; + cfg[1].npersists = 16 - cfg[1].nscratchs; +} From f748a027aa911b7ffe6b62fdc70897804f163d63 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Mon, 13 Apr 2026 07:03:32 +0200 Subject: [PATCH 23/83] more regs alloc, cleanup alloc_stack & refs handling --- src/data_struct.c | 5 +++ src/data_struct.h | 1 + src/jit.h | 4 ++- src/jit_dump.c | 20 +++++++---- src/jit_emit.c | 90 +++++++++++++++++------------------------------ src/jit_regs.c | 84 ++++++++++++++++++++++++++++++++++--------- src/jit_x86_64.c | 59 +++++++++++++++---------------- src/module.c | 26 +++++++++++++- 8 files changed, 176 insertions(+), 113 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index 8f38a58ec..a3410a3ed 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -232,6 +232,11 @@ INLINE static bool S_NAME(iter_next)( S_TYPE st, S_VALUE *val, int idx ) { return idx < st.cur; } +INLINE static bool S_NAME(iter_prev)( S_TYPE st, S_VALUE *val, int idx ) { + if( idx >= 0 ) *val = st.values[idx]; + return idx >= 0; +} + #undef S_NAME #undef S_TYPE #undef S_VALUE diff --git a/src/data_struct.h b/src/data_struct.h index 1bd52c652..0dfe08a6d 100644 --- a/src/data_struct.h +++ b/src/data_struct.h @@ -34,6 +34,7 @@ #define STRUCT_DEF_SIZE 2 #define for_iter(name,var,set) name##__value var; for(int __idx=0;name##_iter_next(set,&var,__idx);__idx++) +#define for_iter_back(name,var,set) name##__value var; for(int __idx=(set).cur-1;name##_iter_prev(set,&var,__idx);__idx--) #define S_TYPE ptr_set #define S_NAME(name) ptr_set_##name diff --git a/src/jit.h b/src/jit.h index a085fe508..fc02e4aaa 100644 --- a/src/jit.h +++ b/src/jit.h @@ -48,7 +48,6 @@ typedef enum { PUSH, POP, ALLOC_STACK, - FREE_STACK, NATIVE_REG, PREFETCH, DEBUG_BREAK, @@ -95,10 +94,13 @@ typedef struct { #define VAL_NULL 0x80000000 #define FL_NATREG 0x40000000 +#define FL_NATMASK 0x70000000 #define FL_STACKREG (FL_NATREG | 0x20000000) +#define FL_STACKOFFS (FL_NATREG | 0x10000000) #define IS_NULL(e) ((e) == VAL_NULL) #define IS_NATREG(e) ((e) & FL_NATREG) #define MK_STACK_REG(v) (((v)&0xFFFFFFF) | FL_STACKREG) +#define MK_STACK_OFFS(v)(((v)&0xFFFFFFF) | FL_STACKOFFS) #define GET_STACK_OFFS(v) (((v) & 0x8000000) ? ((v) | 0xF0000000) : ((v)&0xFFFFFFF)); #define IS_CALL(op) ((op) == CALL_PTR || (op) == CALL_REG || (op) == CALL_FUN) diff --git a/src/jit_dump.c b/src/jit_dump.c index 081c86540..1f45427f6 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -45,7 +45,6 @@ static const char *op_names[] = { "push", "pop", "alloc-stack", - "free-stack", "native-reg", "prefetch", "debug-break", @@ -68,6 +67,12 @@ const char *hl_emit_regstr( ereg v ) { sprintf(fmt,"[ST%d]", index); else sprintf(fmt,"[ST+%d]", index); + } else if( (v&FL_STACKOFFS) == FL_STACKOFFS ) { + int index = GET_STACK_OFFS(v); + if( index < 0 ) + sprintf(fmt,"ST%d", index); + else + sprintf(fmt,"ST+%d]", index); } else if( IS_NATREG(v) ) sprintf(fmt,"%s", hl_natreg_str(v&(FL_NATREG-1))); else @@ -280,6 +285,7 @@ static void hl_dump_ptr_name( jit_ctx *ctx, void *ptr ) { void hl_emit_flush( jit_ctx *ctx ); void hl_regs_flush( jit_ctx *ctx ); +#define reg_str(r) val_str((((r) & FL_NATMASK) == FL_NATREG ? (((r)&~0xFF)|e->mode) : (r))) static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { printf("%s", op_names[e->op]); @@ -345,15 +351,15 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { printf(" [%s]", val_str(e->a)); else printf(" %s[%d]", val_str(e->a), offs); - printf(" = %s", val_str(e->b)); + printf(" = %s", reg_str(e->b)); //if( e->mode == 0 || e->mode != ctx->instrs[ctx->values_writes[e->b.index]].mode ) // printf(" ???"); } break; default: if( !IS_NULL(e->a) ) { - printf(" %s", val_str(e->a)); - if( !IS_NULL(e->b) ) printf(", %s", val_str(e->b)); + printf(" %s", reg_str(e->a)); + if( !IS_NULL(e->b) ) printf(", %s", reg_str(e->b)); } if( show_size && e->size_offs != 0 ) printf(" %d", e->size_offs); @@ -425,10 +431,10 @@ void hl_emit_dump( jit_ctx *ctx ) { dump_instr(ctx, e, i); while( rpos < ctx->reg_instr_count && rpos < ctx->reg_pos_map[i+1] ) { ereg out = ctx->reg_writes[rpos]; - einstr *er = ctx->reg_instrs + rpos; + e = ctx->reg_instrs + rpos; printf("\n\t\t\t\t@%X ",rpos); - if( !IS_NULL(out) ) printf("%s = ",val_str(out)); - dump_instr(ctx,er,rpos); + if( !IS_NULL(out) ) printf("%s = ",reg_str(out)); + dump_instr(ctx,e,rpos); rpos++; } printf("\n"); diff --git a/src/jit_emit.c b/src/jit_emit.c index 2f384ab05..7c7f579f6 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -23,17 +23,25 @@ #include #include "data_struct.h" +//#define EMIT_DEBUG + +#ifdef EMIT_DEBUG +# define emit_debug jit_debug +#else +# define emit_debug(...) +#endif + static void __ignore( void *value ) {} -int hl_emit_mode_sizes[] = {0,1,2,4,8,4,8,sizeof(void*),0,0}; +int hl_emit_mode_sizes[] = {0,1,2,4,8,4,8,HL_WSIZE,0,0}; typedef struct { hl_type *t; int id; + ereg ref; } vreg; #define MAX_TMP_ARGS 32 #define MAX_TRAPS 32 -#define MAX_REFS 512 // TODO : different impl typedef struct _linked_inf linked_inf; typedef struct _emit_block emit_block; @@ -123,14 +131,9 @@ struct _emit_ctx { ereg tmp_args[MAX_TMP_ARGS]; ereg traps[MAX_TRAPS]; - struct { - ereg r; - int reg; - } refs[MAX_REFS]; int *pos_map; int pos_map_size; int trap_count; - int ref_count; int_alloc args_data; int_alloc jump_regs; @@ -272,14 +275,6 @@ static ereg *get_tmp_args( emit_ctx *ctx, int count ) { return ctx->tmp_args; } -static ereg resolve_ref( emit_ctx *ctx, int reg ) { - for(int i=0;iref_count;i++) { - if( ctx->refs[i].reg == reg ) - return ctx->refs[i].r; - } - return VAL_NULL; -} - static unsigned char emit_get_mode( emit_ctx *ctx, ereg v ) { if( IS_NULL(v) ) jit_assert(); if( v < 0 ) @@ -403,7 +398,7 @@ static emit_block *alloc_block( emit_ctx *ctx ) { static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { blocks_add(b->preds,p); blocks_add(p->nexts,b); - jit_debug(" PRED #%d\n",p->id); + emit_debug(" PRED #%d\n",p->id); } static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { @@ -420,7 +415,7 @@ static void split_block( emit_ctx *ctx ) { b->sealed = true; b->id = ctx->current_block->id + 1; b->start_pos = ctx->emit_pos; - jit_debug("BLOCK #%d@%X[%X]\n",b->id,b->start_pos,ctx->op_pos); + emit_debug("BLOCK #%d@%X[%X]\n",b->id,b->start_pos,ctx->op_pos); while( ctx->arrival_points && ctx->arrival_points->id == ctx->op_pos ) { block_add_pred(ctx, b, (emit_block*)ctx->arrival_points->ptr); ctx->arrival_points = ctx->arrival_points->next; @@ -489,7 +484,7 @@ static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { static void phi_remove_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { ereg_remove(&p->vals,v); - jit_debug("%sPHI-REM-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); + emit_debug("%sPHI-REM-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); } static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { @@ -499,7 +494,7 @@ static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { return; if( !ereg_add(p->vals,v) ) return; - jit_debug("%sPHI-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); + emit_debug("%sPHI-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); if( v < 0 ) { tmp_phi *p2 = GET_PHI(v); phi_add(p2->ref_phis,p); @@ -525,7 +520,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { if( p->locked || p->opt ) jit_assert(); - jit_debug("%sPHI-OPT %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); + emit_debug("%sPHI-OPT %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); p->opt = true; ctx->phi_depth++; linked_inf *l = p->ref_blocks; @@ -545,7 +540,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { for(int i=0;iphi_depth--; - jit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); + emit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); return optimize_phi_rec(ctx,p); } @@ -567,7 +562,7 @@ static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { return v; if( !b->sealed ) { tmp_phi *p = alloc_phi(ctx,b,r); - jit_debug("%sPHI-SEALED %s = R%d\n",phi_prefix(ctx),val_str(p->value),r->id); + emit_debug("%sPHI-SEALED %s = R%d\n",phi_prefix(ctx),val_str(p->value),r->id); v = p->value; } else if( blocks_count(b->preds) == 1 ) v = emit_load_reg_block(ctx, blocks_get(b->preds,0), r); @@ -596,14 +591,13 @@ static void emit_walk_blocks( emit_ctx *ctx, void (*fun)(emit_ctx*,emit_block*) } static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { - ereg ref = resolve_ref(ctx, r->id); - if( ref >= 0 ) - return LOAD_MEM(ref,0,r->t); + if( !IS_NULL(r->ref) ) + return LOAD_MEM(r->ref,0,r->t); return emit_load_reg_block(ctx, ctx->current_block, r); } static void seal_block( emit_ctx *ctx, emit_block *b ) { - jit_debug(" SEAL #%d\n",b->id); + emit_debug(" SEAL #%d\n",b->id); for_iter(phi,p,b->phis) gather_phis(ctx, p); b->sealed = true; @@ -743,14 +737,13 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { return emit_phi(ctx, v1, v2); } bool need_dyn = dyn_need_type(dt); - ereg st = emit_gen_size(ctx, ALLOC_STACK, 1); + ereg st = emit_gen_size(ctx, ALLOC_STACK, HL_WSIZE); STORE_MEM(st, 0, v); ereg args[3]; args[0] = st; args[1] = LOAD_CONST_PTR(t); if( need_dyn ) args[2] = LOAD_CONST_PTR(dt); ereg r = emit_native_call(ctx, get_dyncast(dt), args, need_dyn ? 3 : 2, dt); - emit_gen_size(ctx, FREE_STACK, 1); return r; } @@ -925,7 +918,6 @@ void hl_emit_function( jit_ctx *jit ) { ctx->fun = f; ctx->emit_pos = 0; ctx->trap_count = 0; - ctx->ref_count = 0; ctx->phi_count = 0; ctx->flushed = false; int_alloc_reset(&ctx->args_data); @@ -934,7 +926,7 @@ void hl_emit_function( jit_ctx *jit ) { ctx->root_block = ctx->current_block = alloc_block(ctx); ctx->current_block->sealed = true; ctx->arrival_points = NULL; - jit_debug("---- begin [%X] ----\n",f->findex); + emit_debug("---- begin [%X] ----\n",f->findex); if( f->nregs > ctx->max_regs ) { free(ctx->vregs); ctx->vregs = (vreg*)malloc(sizeof(vreg) * (f->nregs + 1)); @@ -954,6 +946,7 @@ void hl_emit_function( jit_ctx *jit ) { for(i=0;inregs;i++) { vreg *r = R(i); r->t = f->regs[i]; + r->ref = VAL_NULL; } for(i=0;itype->fun->nargs;i++) { @@ -962,18 +955,6 @@ void hl_emit_function( jit_ctx *jit ) { STORE(R(i), emit_gen(ctx, LOAD_ARG, VAL_NULL, VAL_NULL, hl_type_mode(t))); } - for(i=f->nops-1;i>=0;i--) { - hl_opcode *o = f->ops + i; - if( o->op == ORef ) { - ereg ref = resolve_ref(ctx, o->p2); - if( ref >= 0 ) continue; - if( ctx->ref_count == MAX_REFS ) jit_error("Too many refs"); - ctx->refs[ctx->ref_count].r = emit_gen_size(ctx, ALLOC_STACK, hl_type_size(R(o->p2)->t)); - ctx->refs[ctx->ref_count].reg = o->p2; - ctx->ref_count++; - } - } - for(int op_pos=0;op_posnops;op_pos++) { ctx->op_pos = op_pos; ctx->pos_map[op_pos] = ctx->emit_pos; @@ -1069,7 +1050,7 @@ static void prepare_loop_block( emit_ctx *ctx ) { break; } if( offs < 0 && i + 1 + offs == ctx->op_pos ) { - jit_debug(" WAIT @%X\n",i); + emit_debug(" WAIT @%X\n",i); b->wait_nexts++; if( b->sealed ) { b->sealed = false; @@ -1307,7 +1288,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OCallClosure: if( ra->t->kind == HDYN ) { int i; - ereg st = emit_gen_size(ctx, ALLOC_STACK, o->p3); + ereg st = emit_gen_size(ctx, ALLOC_STACK, o->p3 * HL_WSIZE); for(i=0;ip3;i++) { vreg *r = R(o->extra[i]); if( !hl_is_dynamic(r->t) ) jit_assert(); @@ -1318,7 +1299,6 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[1] = st; args[2] = LOAD_CONST(o->p3,&hlt_i32); STORE(dst, emit_dyn_cast(ctx,emit_native_call(ctx,hl_dyn_call,args,3,dst->t),ra->t,dst->t)); - emit_gen_size(ctx, FREE_STACK, o->p3); } else { ereg r = LOAD(ra); ereg *args = get_tmp_args(ctx,o->p3+1); @@ -1524,11 +1504,11 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { patch_jump(ctx, jidx); nargs = o->p3 - 1; - ereg eargs = emit_gen_size(ctx, ALLOC_STACK, nargs); + ereg eargs = emit_gen_size(ctx, ALLOC_STACK, nargs * HL_WSIZE); for(i=0;iextra[i+1]))); bool need_dyn = !hl_is_ptr(dst->t) && dst->t->kind != HVOID; - int dyn_size = sizeof(vdynamic)/HL_WSIZE; + int dyn_size = sizeof(vdynamic); ereg edyn = need_dyn ? emit_gen_size(ctx, ALLOC_STACK, dyn_size) : LOAD_CONST_PTR(NULL); args = get_tmp_args(ctx, 4); @@ -1539,7 +1519,6 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 4, dst->t); - emit_gen_size(ctx, FREE_STACK, o->p3 + (need_dyn ? dyn_size : 0)); patch_jump(ctx, jend); if( dst->t->kind != HVOID ) STORE(dst, emit_phi(ctx, v1, v2)); @@ -1643,21 +1622,21 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case ORef: { - ereg ref = resolve_ref(ctx, ra->id); - if( IS_NULL(ref) ) jit_assert(); + if( IS_NULL(ra->ref) ) + ra->ref = emit_gen_size(ctx, ALLOC_STACK, hl_type_size(ra->t)); ereg r = vreg_find(ctx->current_block->written_vars, ra->id); if( !IS_NULL(r) ) { - STORE_MEM(ref, 0, LOAD(ra)); + STORE_MEM(ra->ref, 0, r); vreg_remove(&ctx->current_block->written_vars, ra->id); } - STORE(dst, ref); + STORE(dst,ra->ref); } break; case OUnref: - STORE(dst, LOAD_MEM(LOAD(ra),0,dst->t)); + STORE(dst, LOAD_MEM(ra->ref,0,dst->t)); break; case OSetref: - STORE_MEM(LOAD(dst),0,LOAD(ra)); + STORE_MEM(dst->ref,0,LOAD(ra)); break; case ORefData: switch( ra->t->kind ) { @@ -1846,7 +1825,6 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg ret = emit_native_call(ctx, fun, args, nargs, &hlt_i32); emit_test(ctx, ret, OJNull); int jskip = emit_jump(ctx, true); - emit_gen_size(ctx, FREE_STACK, sizeof(hl_trap_ctx)); STORE(dst, tinf ? LOAD_CONST_PTR(&tinf->exc_value) : LOAD_MEM_PTR(thread,(int)(int_val)&tinf->exc_value)); int jtrap = emit_jump(ctx, false); @@ -1887,8 +1865,6 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE_MEM(st, offset, LOAD_CONST_PTR(NULL)); } # endif - - emit_gen_size(ctx, FREE_STACK, sizeof(hl_trap_ctx)); } break; case OSwitch: diff --git a/src/jit_regs.c b/src/jit_regs.c index 0949e2ec5..9a4f8b1fc 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -36,7 +36,9 @@ typedef struct { int stack_pos; + int last_read; emit_mode mode; + ereg arg_reg; ereg current; bool saved; } value_info; @@ -55,6 +57,7 @@ struct _regs_ctx { int max_instrs; int cur_op; int emit_pos; + int stack_size; int persists_uses[2]; einstr *instrs; ereg *out_write; @@ -143,6 +146,20 @@ static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { // lookup available reg int mode = REG_MODE(v->mode); reg_config *cfg = &ctx->cfg[mode]; + if( !IS_NULL(v->arg_reg) ) { + bool free = true; + for_iter(values,v2,ctx->scratch) { + if( v2->current == v->arg_reg ) { + free = false; + break; + } + } + if( free ) { + v->current = v->arg_reg; + values_add(ctx->scratch,v); + return; + } + } for(int i=0;inscratchs;i++) { ereg r = cfg->scratch[i]; for_iter(values,v2,ctx->scratch) { @@ -173,9 +190,14 @@ static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { } } +static int regs_alloc_stack( regs_ctx *ctx, int size ) { + ctx->stack_size += size; + ctx->stack_size += jit_pad_size(ctx->stack_size,size); + return -ctx->stack_size; +} + static void regs_assign_stack_regs( regs_ctx *ctx ) { // dummy regs assign for testing : set all values on stack - int stack_pos = 0; int nargs = ctx->jit->fun->type->fun->nargs; call_regs regs = {0}; int args_size = 0; @@ -197,9 +219,7 @@ static void regs_assign_stack_regs( regs_ctx *ctx ) { continue; } } - stack_pos += size; - stack_pos += jit_pad_size(stack_pos,size); - v->stack_pos = MK_STACK_REG(-stack_pos); + v->stack_pos = MK_STACK_REG(regs_alloc_stack(ctx,size)); } for(int i=0;ijit->block_count;i++) { eblock *b = ctx->jit->blocks + i; @@ -207,9 +227,30 @@ static void regs_assign_stack_regs( regs_ctx *ctx ) { ephi *p = b->phis + k; int size = hl_emit_mode_sizes[p->mode]; value_info *v = VAL(VIDX(p->value)); - stack_pos += size; - stack_pos += jit_pad_size(stack_pos,size); - v->stack_pos = MK_STACK_REG(-stack_pos); + v->stack_pos = MK_STACK_REG(regs_alloc_stack(ctx,size)); + } + } +} + +static void regs_write_live( regs_ctx *ctx, ereg *r ) { + value_info *v = VAL(VIDX(*r)); + v->last_read = ctx->cur_op; +} + +static void regs_compute_liveness( regs_ctx *ctx ) { + for(int i=0;ijit->instr_count;i++) { + einstr *e = ctx->jit->instrs + i; + ctx->cur_op = i; + hl_emit_reg_iter(ctx->jit,e,ctx,regs_write_live); + if( IS_CALL(e->op) ) { + // anticipate register usage in call so we can previlege this assign + ereg *r = hl_emit_get_args(ctx->jit->emit, e); + call_regs regs = {0}; + for(int k=0;knargs;k++) { + value_info *v = VAL(VIDX(r[k])); + if( !IS_NULL(v->arg_reg) ) continue; + v->arg_reg = get_call_reg(ctx,regs,v->mode); + } } } } @@ -228,7 +269,7 @@ void hl_regs_flush( jit_ctx *jit ) { jit->reg_instrs = ctx->instrs; jit->reg_writes = ctx->out_write; jit->reg_pos_map = ctx->pos_map; - ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; + if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; } void hl_regs_function( jit_ctx *jit ) { @@ -239,6 +280,7 @@ void hl_regs_function( jit_ctx *jit ) { ctx->pos_map = (int*)malloc((jit->instr_count + 1) * sizeof(int)); ctx->emit_pos = 0; ctx->cur_op = 0; + ctx->stack_size = 0; ctx->pos_map[0] = 0; values_reset(&ctx->scratch); ctx->values = malloc(sizeof(value_info) * nvalues); @@ -246,20 +288,27 @@ void hl_regs_function( jit_ctx *jit ) { for(int i=0;icurrent = VAL_NULL; + v->arg_reg = VAL_NULL; + v->last_read = -1; } + regs_compute_liveness(ctx); regs_assign_stack_regs(ctx); int write_index = 0; - for(int i=0;iinstr_count;i++) { - einstr e = jit->instrs[i]; + for(int cur_op=0;cur_opinstr_count;cur_op++) { + einstr e = jit->instrs[cur_op]; ereg *ret_val = NULL; int nread; - ctx->cur_op = i; - if( i > 0 ) - ctx->pos_map[i] = ctx->emit_pos; + ctx->cur_op = cur_op; + if( cur_op > 0 ) + ctx->pos_map[cur_op] = ctx->emit_pos; + for_iter_back(values,v,ctx->scratch) { + if( v->last_read < cur_op ) + values_remove(&ctx->scratch,v); + } if( IS_CALL(e.op) ) { ereg *args = hl_emit_get_args(ctx->jit->emit,&e); for_iter(values,v,ctx->scratch) { - if( !v->saved ) { + if( !v->saved && v->last_read > cur_op ) { v->saved = true; regs_emit_mov(ctx, MK_STACK_REG(v->stack_pos), v->current, v->mode); } @@ -293,9 +342,12 @@ void hl_regs_function( jit_ctx *jit ) { } } ereg out; - if( write_index < jit->value_count && jit->values_writes[write_index] == i ) { + if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) { value_info *v = VAL(write_index++); - regs_assign(ctx,v,ret_val); + if( e.op == ALLOC_STACK ) + v->current = MK_STACK_OFFS(regs_alloc_stack(ctx, e.size_offs)); + else + regs_assign(ctx,v,ret_val); out = v->current; } else out = VAL_NULL; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 9a2e5b1bd..486e91590 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -45,50 +45,47 @@ typedef enum { _UNUSED = 0xFF } Reg; -typedef enum { - K64 = 0, - K32 = 1, - K16 = 2, - K8 = 3, - KF32 = 4, - KF64 = 5 -} RegKind; - -#define R(id,k) ((id) | ((k) << 8) | FL_NATREG) -#define X(id) R(id,K64) +#define R(id,mode) ((mode) | ((id)<<8) | FL_NATREG) +#define X(id) R(id,M_NONE) +#define MMX(id) R(id+64,M_F64) const char *hl_natreg_str( int reg ) { - static char out[8]; + static char out[16]; static const char *regs_str[] = { "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI" }; - RegKind k = (reg >> 8) & 0xFF; - Reg r = (reg & 0xFF); + emit_mode k = (reg & 0xFF); + Reg r = ((reg >> 8) & 0xFF); switch( k ) { - case K64: - if( r < 8 ) - sprintf(out,"R%s",regs_str[r]); - else - sprintf(out,"R%d",r); - break; - case K32: + case M_I32: if( r < 8 ) sprintf(out,"E%s",regs_str[r]); else - sprintf(out,"R%dD",r); + sprintf(out,"R%dD%s",r,r<16?"":"???"); break; - case K16: + case M_UI16: if( r < 8 ) sprintf(out,"%s",regs_str[r]); else - sprintf(out,"R%dW",r); + sprintf(out,"R%dW%s",r,r<16?"":"???"); break; - case K8: + case M_UI8: if( r < 8 ) sprintf(out,"%s",regs_str[r]); else - sprintf(out,"R%dB",r); + sprintf(out,"R%dB%s",r,r<16?"":"???"); + break; + case M_F32: + r -= 64; + sprintf(out,"XMM%df%s",r,r >= 0 && r < 16 ? "" : "???"); + break; + case M_F64: + r -= 64; + sprintf(out,"XMM%d%s",r,r >= 0 && r < 16 ? "" : "???"); break; default: - sprintf(out,"XMM%d",r); + if( r < 8 ) + sprintf(out,"R%s",regs_str[r]); + else + sprintf(out,"R%d%s",r,r<16?"":"???"); break; } return out; @@ -109,10 +106,10 @@ void hl_jit_init_regs( regs_config cfg ) { cfg[1].nscratchs = 16; # endif static int floats[] = { - R(0,KF64), R(1,KF64), R(2,KF64), R(3,KF64), - R(4,KF64), R(5,KF64), R(6,KF64), R(7,KF64), - R(8,KF64), R(9,KF64), R(10,KF64), R(11,KF64), - R(12,KF64), R(13,KF64), R(14,KF64), R(15,KF64) + MMX(0), MMX(1), MMX(2), MMX(3), + MMX(4), MMX(5), MMX(6), MMX(7), + MMX(8), MMX(9), MMX(10), MMX(11), + MMX(12), MMX(13), MMX(14), MMX(15) }; cfg[0].ret = scratch_regs[0]; cfg[0].nscratchs = sizeof(scratch_regs) / sizeof(int); diff --git a/src/module.c b/src/module.c index e46f73f13..5861d1dcf 100644 --- a/src/module.c +++ b/src/module.c @@ -711,8 +711,32 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { if( ctx == NULL ) return 0; hl_jit_init(ctx, m); +# ifdef HL_DEBUG + bool dump = false; + int filter = -1; + for(i=0;ijit_code = hl_jit_code(ctx, m, &m->codesize, &m->jit_debug, NULL); From 0c69fb24e6592dbf16b91149ca2b1906a30b2f6a Mon Sep 17 00:00:00 2001 From: ncannasse Date: Mon, 13 Apr 2026 22:56:20 +0200 Subject: [PATCH 24/83] continue work on reg alloc and emit fixes --- src/data_struct.c | 13 ++++++++ src/data_struct.h | 1 + src/jit.c | 30 ------------------- src/jit.h | 10 ++----- src/jit_dump.c | 12 +++++--- src/jit_emit.c | 75 +++++++++++++++++++++++------------------------ src/jit_regs.c | 59 +++++++++++++++++++++++++++++-------- src/module.c | 4 +-- 8 files changed, 108 insertions(+), 96 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index a3410a3ed..bc8840d5f 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -104,6 +104,19 @@ static S_VALUE S_NAME(find)( S_TYPE st, S_KEY k ) { return st.values[i]; return (S_VALUE)0; } +#else +static S_VALUE *S_NAME(reserve_impl)( hl_alloc *alloc, S_TYPE *st, int count ) { + if( st->cur + count > st->max ) { + int n = st->max ? (st->max << 1) : STRUCT_DEF_SIZE; + while( n < st->cur + count ) n <<= 1; + S_KEY *keys = (S_KEY*)hl_malloc(alloc,sizeof(S_KEY) * n); + memcpy(keys,st->keys,sizeof(S_KEY) * st->cur); + st->keys = keys; + } + S_VALUE *ptr = st->keys + st->cur; + st->cur += count; + return ptr; +} #endif diff --git a/src/data_struct.h b/src/data_struct.h index 0dfe08a6d..43494f18e 100644 --- a/src/data_struct.h +++ b/src/data_struct.h @@ -47,6 +47,7 @@ #define S_VALUE int #include "data_struct.c" #define int_arr_add(set,v) int_arr_add_impl(DEF_ALLOC,&(set),v) +#define int_arr_reserve(set,v) int_arr_reserve_impl(DEF_ALLOC,&(set),v) #define S_SORTED diff --git a/src/jit.c b/src/jit.c index eff202199..a1f9838f3 100644 --- a/src/jit.c +++ b/src/jit.c @@ -37,36 +37,6 @@ void hl_jit_null_field_access() { jit_assert(); } void hl_jit_null_access() { jit_assert(); } void hl_jit_assert() { jit_assert(); } -void int_alloc_reset( int_alloc *a ) { - a->cur = 0; -} - -void int_alloc_free( int_alloc *a ) { - free(a->data); - a->cur = 0; - a->max = 0; - a->data = NULL; -} - -int *int_alloc_get( int_alloc *a, int count ) { - while( a->cur + count > a->max ) { - int next_size = a->max ? a->max << 1 : 128; - int *new_data = (int*)malloc(sizeof(int) * next_size); - if( new_data == NULL ) jit_error("Out of memory"); - memcpy(new_data, a->data, sizeof(int) * a->cur); - free(a->data); - a->data = new_data; - a->max = next_size; - } - int *ptr = a->data + a->cur; - a->cur += count; - return ptr; -} - -void int_alloc_store( int_alloc *a, int v ) { - *int_alloc_get(a,1) = v; -} - void hl_emit_alloc( jit_ctx *jit ); void hl_emit_free( jit_ctx *jit ); void hl_emit_function( jit_ctx *jit ); diff --git a/src/jit.h b/src/jit.h index fc02e4aaa..a467238ce 100644 --- a/src/jit.h +++ b/src/jit.h @@ -48,7 +48,6 @@ typedef enum { PUSH, POP, ALLOC_STACK, - NATIVE_REG, PREFETCH, DEBUG_BREAK, } emit_op; @@ -63,9 +62,9 @@ typedef enum { M_UI16, M_I32, M_I64, - M_F32, - M_F64, M_PTR, + M_F64, + M_F32, M_VOID, M_NORET, } emit_mode; @@ -179,11 +178,6 @@ void hl_jit_null_field_access(); void hl_jit_null_access(); void hl_jit_assert(); -void int_alloc_reset( int_alloc *a ); -void int_alloc_free( int_alloc *a ); -int *int_alloc_get( int_alloc *a, int count ); -void int_alloc_store( int_alloc *a, int v ); - // emit & dump void hl_emit_dump( jit_ctx *ctx ); const char *hl_emit_regstr( ereg v ); diff --git a/src/jit_dump.c b/src/jit_dump.c index 1f45427f6..ab2d27d04 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -45,7 +45,6 @@ static const char *op_names[] = { "push", "pop", "alloc-stack", - "native-reg", "prefetch", "debug-break", }; @@ -72,7 +71,7 @@ const char *hl_emit_regstr( ereg v ) { if( index < 0 ) sprintf(fmt,"ST%d", index); else - sprintf(fmt,"ST+%d]", index); + sprintf(fmt,"ST+%d", index); } else if( IS_NATREG(v) ) sprintf(fmt,"%s", hl_natreg_str(v&(FL_NATREG-1))); else @@ -130,7 +129,7 @@ static void hl_dump_op( hl_function *fun, hl_opcode *op ) { } printf(",def=@%X", op->p3 + pos + 1); } else { - if( count == 0xFF ) + if( count == 0xFF ) count = op->p3; else { printf("%d,%d,",op->p2,op->p3); @@ -285,7 +284,8 @@ static void hl_dump_ptr_name( jit_ctx *ctx, void *ptr ) { void hl_emit_flush( jit_ctx *ctx ); void hl_regs_flush( jit_ctx *ctx ); -#define reg_str(r) val_str((((r) & FL_NATMASK) == FL_NATREG ? (((r)&~0xFF)|e->mode) : (r))) +#define reg_mode(r,mode) (((r) & FL_NATMASK) == FL_NATREG ? (((r)&~0xFF)|(mode)) : (r)) +#define reg_str(r) val_str(reg_mode(r,e->mode)) static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { printf("%s", op_names[e->op]); @@ -356,6 +356,10 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { // printf(" ???"); } break; + case CONV: + case CONV_UNSIGNED: + printf("%s %s", emit_mode_str(e->size_offs), val_str(reg_mode(e->a,e->size_offs))); + break; default: if( !IS_NULL(e->a) ) { printf(" %s", reg_str(e->a)); diff --git a/src/jit_emit.c b/src/jit_emit.c index 7c7f579f6..a49f3bd10 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -32,7 +32,7 @@ #endif static void __ignore( void *value ) {} -int hl_emit_mode_sizes[] = {0,1,2,4,8,4,8,HL_WSIZE,0,0}; +int hl_emit_mode_sizes[] = {0,1,2,4,8,HL_WSIZE,8,4,0,0}; typedef struct { hl_type *t; @@ -135,10 +135,10 @@ struct _emit_ctx { int pos_map_size; int trap_count; - int_alloc args_data; - int_alloc jump_regs; - int_alloc values; - + int_arr args_data; + int_arr jump_regs; + int_arr values; + emit_block *root_block; emit_block *current_block; emit_block *wait_seal; @@ -265,8 +265,8 @@ static emit_mode hl_type_mode( hl_type *t ) { } static ereg new_value( emit_ctx *ctx ) { - ereg r = {ctx->values.cur}; - int_alloc_store(&ctx->values, ctx->emit_pos-1); + ereg r = int_arr_count(ctx->values); + int_arr_add(ctx->values, ctx->emit_pos-1); return r; } @@ -279,7 +279,7 @@ static unsigned char emit_get_mode( emit_ctx *ctx, ereg v ) { if( IS_NULL(v) ) jit_assert(); if( v < 0 ) return GET_PHI(v)->mode; - return ctx->instrs[ctx->values.data[v]].mode; + return ctx->instrs[int_arr_get(ctx->values,v)].mode; } static const char *phi_prefix( emit_ctx *ctx ) { @@ -326,8 +326,8 @@ static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { e->size_offs = args[0]; return; } - int *args_data = int_alloc_get(&ctx->args_data, count); - e->size_offs = (int)(args_data - ctx->args_data.data); + int *args_data = int_arr_reserve(ctx->args_data, count); + e->size_offs = (int)(args_data - ctx->args_data.values); memcpy(args_data, args, sizeof(int) * count); } @@ -336,7 +336,7 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ) { return NULL; if( e->nargs == 1 ) return (ereg*)&e->size_offs; - return (ereg*)(ctx->args_data.data + e->size_offs); + return (ereg*)(ctx->args_data.values + e->size_offs); } static ereg emit_gen_ext( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode, int size_offs ) { @@ -388,7 +388,7 @@ static int emit_jump( emit_ctx *ctx, bool cond ) { } static void patch_jump( emit_ctx *ctx, int jpos ) { - ctx->instrs[jpos].size_offs = ctx->emit_pos - (jpos + 1); + ctx->instrs[jpos].size_offs = ctx->emit_pos - (jpos + 1); } static emit_block *alloc_block( emit_ctx *ctx ) { @@ -403,7 +403,7 @@ static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { if( IS_NULL(v) ) jit_assert(); - vreg_replace(b->written_vars,r->id,v); + vreg_replace(b->written_vars,r->id,v); if( v < 0 ) { tmp_phi *p = GET_PHI(v); p->ref_blocks = link_add_sort_unique(ctx,b->id,b,p->ref_blocks); @@ -430,8 +430,8 @@ static void split_block( emit_ctx *ctx ) { static void register_jump( emit_ctx *ctx, int jpos, int offs ) { int target = offs + ctx->op_pos + 1; - int_alloc_store(&ctx->jump_regs, jpos); - int_alloc_store(&ctx->jump_regs, target); + int_arr_add(ctx->jump_regs, jpos); + int_arr_add(ctx->jump_regs, target); if( offs > 0 ) { ctx->arrival_points = link_add_sort_unique(ctx, target, ctx->current_block, ctx->arrival_points); if( ctx->arrival_points->id != ctx->op_pos + 1 && ctx->fun->ops[ctx->op_pos].op != OSwitch ) @@ -502,7 +502,7 @@ static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { } static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { - + if( p->locked ) jit_assert(); ereg same = VAL_NULL; for_iter(ereg,v,p->vals) { @@ -717,8 +717,8 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, } } -static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode mode, bool _unsigned ) { - return emit_gen(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, VAL_NULL, mode); +static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode from, emit_mode to, bool _unsigned ) { + return emit_gen_ext(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, VAL_NULL, to, from); } static bool dyn_need_type( hl_type *t ) { @@ -812,10 +812,10 @@ void hl_emit_flush( jit_ctx *jit ) { int i = 0; if( ctx->flushed ) return; ctx->flushed = true; - while( i < ctx->jump_regs.cur ) { - int pos = ctx->jump_regs.data[i++]; + while( i < int_arr_count(ctx->jump_regs) ) { + int pos = int_arr_get(ctx->jump_regs,i++); einstr *e = ctx->instrs + pos; - int target = ctx->jump_regs.data[i++]; + int target = int_arr_get(ctx->jump_regs,i++); e->size_offs = ctx->pos_map[target] - (pos + 1); } ctx->pos_map[ctx->fun->nops] = -1; @@ -828,8 +828,8 @@ void hl_emit_flush( jit_ctx *jit ) { jit->blocks = hl_zalloc(&jit->falloc,sizeof(eblock) * jit->block_count); for(i=0;iblock_count;i++) jit->blocks[i].id = -1; - jit->value_count = ctx->values.cur; - jit->values_writes = ctx->values.data; + jit->value_count = int_arr_count(ctx->values); + jit->values_writes = ctx->values.values; emit_walk_blocks(ctx,emit_write_block); } @@ -920,9 +920,9 @@ void hl_emit_function( jit_ctx *jit ) { ctx->trap_count = 0; ctx->phi_count = 0; ctx->flushed = false; - int_alloc_reset(&ctx->args_data); - int_alloc_reset(&ctx->jump_regs); - int_alloc_reset(&ctx->values); + int_arr_free(&ctx->args_data); + int_arr_free(&ctx->jump_regs); + int_arr_free(&ctx->values); ctx->root_block = ctx->current_block = alloc_block(ctx); ctx->current_block->sealed = true; ctx->arrival_points = NULL; @@ -981,9 +981,6 @@ void hl_emit_free( jit_ctx *jit ) { free(ctx->vregs); free(ctx->instrs); free(ctx->pos_map); - int_alloc_free(&ctx->jump_regs); - int_alloc_free(&ctx->args_data); - int_alloc_free(&ctx->values); free(ctx); jit->emit = NULL; } @@ -1212,7 +1209,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OToSFloat: case OToInt: case OToUFloat: - STORE(dst, emit_conv(ctx,LOAD(ra),hl_type_mode(dst->t), o->op == OToUFloat)); + STORE(dst, emit_conv(ctx,LOAD(ra),hl_type_mode(ra->t),hl_type_mode(dst->t), o->op == OToUFloat)); break; case ORet: emit_gen(ctx, RET, dst->t->kind == HVOID ? VAL_NULL : LOAD(dst), VAL_NULL, M_NORET); @@ -1311,7 +1308,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { for(i=0;ip3;i++) args[i+1] = LOAD(R(o->extra[i])); ereg v1 = emit_dyn_call(ctx,LOAD_MEM_PTR(r,HL_WSIZE),args,o->p3 + 1,dst->t); - int jend = emit_jump(ctx, false); + int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); for(i=0;ip3;i++) args[i] = LOAD(R(o->extra[i])); @@ -1548,7 +1545,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg offs = OFFSET(LOAD(ra),LOAD(rb),1,0); ereg val = LOAD_MEM(offs, 0, dst->t); - if( o->op != OGetMem ) val = emit_conv(ctx, val, M_I32, false); + if( o->op != OGetMem ) val = emit_conv(ctx, val, M_I32, hl_type_mode(dst->t), false); STORE(dst, val); } break; @@ -1558,7 +1555,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg offs = OFFSET(LOAD(dst), LOAD(ra),1,0); ereg val = LOAD(rb); - if( o->op != OSetMem ) val = emit_conv(ctx, val, M_I32, false); + if( o->op != OSetMem ) val = emit_conv(ctx, val, M_I32, hl_type_mode(dst->t), false); STORE_MEM(offs, 0, val); } break; @@ -1633,7 +1630,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case OUnref: - STORE(dst, LOAD_MEM(ra->ref,0,dst->t)); + STORE(dst, LOAD_MEM(LOAD(ra),0,dst->t)); break; case OSetref: STORE_MEM(dst->ref,0,LOAD(ra)); @@ -1667,7 +1664,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, en); hl_enum_construct *c = &dst->t->tenum->constructs[o->p2]; for(int i=0;inparams;i++) - STORE_MEM(en, c->offsets[i], LOAD(R(o->extra[i]))); + STORE_MEM(en, c->offsets[i], LOAD(R(o->extra[i]))); } break; case OEnumAlloc: @@ -1761,7 +1758,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OTrap: { ereg st = emit_gen_size(ctx, ALLOC_STACK, sizeof(hl_trap_ctx)); - + ereg thread, current_addr; static hl_thread_info *tinf = NULL; static hl_trap_ctx *trap = NULL; @@ -1812,12 +1809,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { void *fun = setjmp; ereg args[2]; int nargs = 1; - args[0] = OFFSET(st, VAL_NULL, 0, (int)(int_val)&trap->buf); + args[0] = st; #if defined(HL_WIN) && defined(HL_64) // On Win64 setjmp actually takes two arguments // the jump buffer and the frame pointer (or the stack pointer if there is no FP) nargs = 2; - args[1] = emit_gen(ctx, NATIVE_REG, VAL_NULL, VAL_NULL, REG_RBP); + args[1] = emit_gen(ctx,MOV,MK_STACK_OFFS(0),VAL_NULL,M_PTR); #endif #ifdef HL_MINGW fun = _setjmp; @@ -1850,7 +1847,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { thread = emit_native_call(ctx, hl_get_thread, NULL, 0, &hlt_bytes); current_addr = OFFSET(thread, VAL_NULL, 0, (int)(int_val)&tinf->trap_current); # endif - + STORE_MEM(current_addr, 0, LOAD_MEM_PTR(st,(int)(int_val)&trap->prev)); # ifdef HL_WIN diff --git a/src/jit_regs.c b/src/jit_regs.c index 9a4f8b1fc..43a49001a 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -31,7 +31,7 @@ # define IS_WINCALL64 0 #endif -#define VIDX(e) (((e) < 0) ? ctx->jit->value_count + (-(e)-1) : (e)) +#define VIDX(e) (((e) < 0) ? ctx->jit->value_count + (-(e)-1) : (e)) #define REG_MODE(m) (((m) == M_F32 || (m) == M_F64) ? 1 :0) typedef struct { @@ -54,6 +54,7 @@ struct _regs_ctx { regs_config cfg; value_info *values; values scratch; + int_arr jump_regs; int max_instrs; int cur_op; int emit_pos; @@ -62,6 +63,7 @@ struct _regs_ctx { einstr *instrs; ereg *out_write; int *pos_map; + bool flushed; }; typedef int call_regs[2]; @@ -128,7 +130,7 @@ static void regs_emit_todo_impl( regs_ctx *ctx, int line ) { static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { - if( forced ) { + if( forced ) { if( v->current == *forced ) return; // erase previous value @@ -149,7 +151,7 @@ static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { if( !IS_NULL(v->arg_reg) ) { bool free = true; for_iter(values,v2,ctx->scratch) { - if( v2->current == v->arg_reg ) { + if( v2->current == v->arg_reg ) { free = false; break; } @@ -182,7 +184,7 @@ static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { value_info *v2 = values_first(ctx->scratch); if( !v2->saved ) { regs_emit_mov(ctx, MK_STACK_REG(v2->stack_pos), v2->current, v2->mode); - v2->saved = true; + v2->saved = true; } v->current = v2->current; values_remove(&ctx->scratch, v2); @@ -233,6 +235,8 @@ static void regs_assign_stack_regs( regs_ctx *ctx ) { } static void regs_write_live( regs_ctx *ctx, ereg *r ) { + if( IS_NULL(*r) ) jit_assert(); + if( IS_NATREG(*r) ) return; value_info *v = VAL(VIDX(*r)); v->last_read = ctx->cur_op; } @@ -249,7 +253,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { for(int k=0;knargs;k++) { value_info *v = VAL(VIDX(r[k])); if( !IS_NULL(v->arg_reg) ) continue; - v->arg_reg = get_call_reg(ctx,regs,v->mode); + v->arg_reg = get_call_reg(ctx,regs,v->mode); } } } @@ -265,11 +269,20 @@ void hl_regs_alloc( jit_ctx *jit ) { void hl_regs_flush( jit_ctx *jit ) { regs_ctx *ctx = jit->regs; + if( ctx->flushed ) return; + ctx->flushed = true; jit->reg_instr_count = ctx->emit_pos; jit->reg_instrs = ctx->instrs; jit->reg_writes = ctx->out_write; jit->reg_pos_map = ctx->pos_map; if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; + int i = 0; + while( i < int_arr_count(ctx->jump_regs) ) { + int pos = int_arr_get(ctx->jump_regs,i++); + einstr *e = ctx->instrs + pos; + int target = int_arr_get(ctx->jump_regs,i++); + e->size_offs = ctx->pos_map[target] - (pos + 1); + } } void hl_regs_function( jit_ctx *jit ) { @@ -277,12 +290,15 @@ void hl_regs_function( jit_ctx *jit ) { int nvalues = jit->value_count + jit->phi_count; memset(ctx->persists_uses,0,sizeof(ctx->persists_uses)); free(ctx->pos_map); + ctx->flushed = false; ctx->pos_map = (int*)malloc((jit->instr_count + 1) * sizeof(int)); ctx->emit_pos = 0; ctx->cur_op = 0; ctx->stack_size = 0; ctx->pos_map[0] = 0; - values_reset(&ctx->scratch); + jit->reg_instrs = NULL; + values_free(&ctx->scratch); + int_arr_free(&ctx->jump_regs); ctx->values = malloc(sizeof(value_info) * nvalues); memset(ctx->values, 0, sizeof(value_info) * nvalues); for(int i=0;icurrent = VAL_NULL; v->arg_reg = VAL_NULL; v->last_read = -1; + if( i < jit->value_count ) + v->mode = jit->instrs[jit->values_writes[i]].mode; + else + v->mode = M_NONE; // TODO : phi mode } regs_compute_liveness(ctx); regs_assign_stack_regs(ctx); @@ -307,10 +327,13 @@ void hl_regs_function( jit_ctx *jit ) { } if( IS_CALL(e.op) ) { ereg *args = hl_emit_get_args(ctx->jit->emit,&e); - for_iter(values,v,ctx->scratch) { - if( !v->saved && v->last_read > cur_op ) { - v->saved = true; - regs_emit_mov(ctx, MK_STACK_REG(v->stack_pos), v->current, v->mode); + bool will_scratch = e.mode != M_NORET; + if( will_scratch ) { + for_iter(values,v,ctx->scratch) { + if( !v->saved && v->last_read > cur_op ) { + v->saved = true; + regs_emit_mov(ctx, MK_STACK_REG(v->stack_pos), v->current, v->mode); + } } } call_regs regs = {0}; @@ -324,15 +347,18 @@ void hl_regs_function( jit_ctx *jit ) { else regs_emit_mov(ctx, r, v->current, v->mode); } - for_iter(values,v2,ctx->scratch) - v2->current = VAL_NULL; - values_reset(&ctx->scratch); + if( will_scratch ) { + for_iter(values,v2,ctx->scratch) + v2->current = VAL_NULL; + values_reset(&ctx->scratch); + } e.nargs = 0xFF; ret_val = &ctx->cfg[REG_MODE(e.mode)].ret; } else { ereg **regs = hl_emit_get_regs(&e,&nread); for(int k=0;kcurrent) ) { regs_assign(ctx,v,NULL); @@ -352,6 +378,13 @@ void hl_regs_function( jit_ctx *jit ) { } else out = VAL_NULL; regs_write_instr(ctx, &e, out); + switch( e.op ) { + case JUMP: + case JCOND: + int_arr_add(ctx->jump_regs, ctx->emit_pos - 1); + int_arr_add(ctx->jump_regs, cur_op + 1 + e.size_offs); + break; + } } free(ctx->values); hl_regs_flush(ctx->jit); diff --git a/src/module.c b/src/module.c index 5861d1dcf..3002011d4 100644 --- a/src/module.c +++ b/src/module.c @@ -717,7 +717,7 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { for(i=0;i= '0' && arg[pos] <= '9' ) filter |= arg[pos] - '0'; else - filter |= arg[pos] - 'A'; + filter |= arg[pos] - 'A' + 10; pos++; } } From 0945ddf243204d021a0afcf0bcc0e784a64c7587 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 18 Apr 2026 00:52:18 +0200 Subject: [PATCH 25/83] start codegen x86_64 --- src/data_struct.c | 1 + src/jit.c | 8 + src/jit.h | 15 +- src/jit_dump.c | 42 ++- src/jit_emit.c | 240 ++++++++++++++- src/jit_x86_64.c | 761 +++++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 1032 insertions(+), 35 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index bc8840d5f..aa203a286 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -112,6 +112,7 @@ static S_VALUE *S_NAME(reserve_impl)( hl_alloc *alloc, S_TYPE *st, int count ) { S_KEY *keys = (S_KEY*)hl_malloc(alloc,sizeof(S_KEY) * n); memcpy(keys,st->keys,sizeof(S_KEY) * st->cur); st->keys = keys; + st->max = n; } S_VALUE *ptr = st->keys + st->cur; st->cur += count; diff --git a/src/jit.c b/src/jit.c index a1f9838f3..88aaa0c3e 100644 --- a/src/jit.c +++ b/src/jit.c @@ -46,6 +46,9 @@ void hl_regs_alloc( jit_ctx *jit ); void hl_regs_free( jit_ctx *jit ); void hl_regs_function( jit_ctx *jit ); +void hl_codegen_alloc( jit_ctx *jit ); +void hl_codegen_free( jit_ctx *jit ); +void hl_codegen_function( jit_ctx *jit ); jit_ctx *hl_jit_alloc() { jit_ctx *ctx = (jit_ctx*)malloc(sizeof(jit_ctx)); @@ -53,6 +56,7 @@ jit_ctx *hl_jit_alloc() { hl_alloc_init(&ctx->falloc); hl_emit_alloc(ctx); hl_regs_alloc(ctx); + hl_codegen_alloc(ctx); return ctx; } @@ -60,6 +64,7 @@ void hl_jit_init( jit_ctx *ctx, hl_module *m ) { } void hl_jit_free( jit_ctx *ctx, h_bool can_reset ) { + hl_codegen_free(ctx); hl_regs_free(ctx); hl_emit_free(ctx); hl_free(&ctx->falloc); @@ -73,9 +78,12 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { hl_free(&ctx->falloc); ctx->mod = m; ctx->fun = f; + ctx->reg_instr_count = 0; + ctx->code_size = 0; current_ctx = ctx; hl_emit_function(ctx); hl_regs_function(ctx); + hl_codegen_function(ctx); current_ctx = NULL; return 0; } diff --git a/src/jit.h b/src/jit.h index a467238ce..0b14050aa 100644 --- a/src/jit.h +++ b/src/jit.h @@ -100,7 +100,7 @@ typedef struct { #define IS_NATREG(e) ((e) & FL_NATREG) #define MK_STACK_REG(v) (((v)&0xFFFFFFF) | FL_STACKREG) #define MK_STACK_OFFS(v)(((v)&0xFFFFFFF) | FL_STACKOFFS) -#define GET_STACK_OFFS(v) (((v) & 0x8000000) ? ((v) | 0xF0000000) : ((v)&0xFFFFFFF)); +#define GET_STACK_OFFS(v) ((int)(((v) & 0x8000000) ? ((v) | 0xF0000000) : ((v)&0xFFFFFFF))) #define IS_CALL(op) ((op) == CALL_PTR || (op) == CALL_REG || (op) == CALL_FUN) typedef struct { @@ -132,7 +132,8 @@ typedef struct _eblock { typedef struct _emit_ctx emit_ctx; typedef struct _regs_ctx regs_ctx; -typedef struct _jit_ctx ji_ctx; +typedef struct _code_ctx code_ctx; +typedef struct _jit_ctx jit_ctx; typedef struct { int nscratchs; @@ -150,8 +151,10 @@ struct _jit_ctx { hl_module *mod; hl_function *fun; hl_alloc falloc; + hl_alloc galloc; emit_ctx *emit; regs_ctx *regs; + code_ctx *code; // emit output int instr_count; int block_count; @@ -166,6 +169,10 @@ struct _jit_ctx { einstr *reg_instrs; ereg *reg_writes; int *reg_pos_map; + // gen output + int code_size; + unsigned char *code_instrs; + int *code_pos_map; }; jit_ctx *hl_jit_alloc(); @@ -180,12 +187,12 @@ void hl_jit_assert(); // emit & dump void hl_emit_dump( jit_ctx *ctx ); -const char *hl_emit_regstr( ereg v ); +const char *hl_emit_regstr( ereg v, emit_mode m ); ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); ereg **hl_emit_get_regs( einstr *e, int *count ); void hl_emit_reg_iter( jit_ctx *jit, einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ); extern int hl_emit_mode_sizes[]; -#define val_str(v) hl_emit_regstr(v) +#define val_str(v,m) hl_emit_regstr(v,m) #ifdef HL_DEBUG # define JIT_DEBUG diff --git a/src/jit_dump.c b/src/jit_dump.c index ab2d27d04..c6c5b736c 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -49,9 +49,9 @@ static const char *op_names[] = { "debug-break", }; -const char *hl_natreg_str( int reg ); +const char *hl_natreg_str( int reg, emit_mode m ); -const char *hl_emit_regstr( ereg v ) { +const char *hl_emit_regstr( ereg v, emit_mode m ) { static char fmts[4][10]; static int flip = 0; // allow up to four concurrent val_str @@ -73,7 +73,7 @@ const char *hl_emit_regstr( ereg v ) { else sprintf(fmt,"ST+%d", index); } else if( IS_NATREG(v) ) - sprintf(fmt,"%s", hl_natreg_str(v&(FL_NATREG-1))); + sprintf(fmt,"%s", hl_natreg_str(v,m)); else sprintf(fmt,"V%d",v); return fmt; @@ -223,7 +223,7 @@ static void hl_dump_args( jit_ctx *ctx, einstr *e ) { printf("("); for(int i=0;inargs;i++) { if( i != 0 ) printf(","); - printf("%s", val_str(v[i])); + printf("%s", val_str(v[i],M_NONE)); } printf(")"); } @@ -283,9 +283,9 @@ static void hl_dump_ptr_name( jit_ctx *ctx, void *ptr ) { void hl_emit_flush( jit_ctx *ctx ); void hl_regs_flush( jit_ctx *ctx ); +void hl_codegen_flush( jit_ctx *ctx ); -#define reg_mode(r,mode) (((r) & FL_NATMASK) == FL_NATREG ? (((r)&~0xFF)|(mode)) : (r)) -#define reg_str(r) val_str(reg_mode(r,e->mode)) +#define reg_str(r) val_str(r,e->mode) static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { printf("%s", op_names[e->op]); @@ -321,7 +321,7 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { hl_dump_args(ctx,e); break; case CALL_REG: - printf(" %s", val_str(e->a)); + printf(" %s", val_str(e->a,M_PTR)); hl_dump_args(ctx,e); break; case CALL_PTR: @@ -340,17 +340,17 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { break; case LOAD_ADDR: if( (e->size_offs>>8) ) - printf(" %s[%Xh]", val_str(e->a), e->size_offs); + printf(" %s[%Xh]", val_str(e->a,M_PTR), e->size_offs); else - printf(" %s[%d]", val_str(e->a), e->size_offs); + printf(" %s[%d]", val_str(e->a,M_PTR), e->size_offs); break; case STORE: { int offs = e->size_offs; if( offs == 0 ) - printf(" [%s]", val_str(e->a)); + printf(" [%s]", val_str(e->a,M_PTR)); else - printf(" %s[%d]", val_str(e->a), offs); + printf(" %s[%d]", val_str(e->a,M_PTR), offs); printf(" = %s", reg_str(e->b)); //if( e->mode == 0 || e->mode != ctx->instrs[ctx->values_writes[e->b.index]].mode ) // printf(" ???"); @@ -358,7 +358,7 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { break; case CONV: case CONV_UNSIGNED: - printf("%s %s", emit_mode_str(e->size_offs), val_str(reg_mode(e->a,e->size_offs))); + printf("%s %s", emit_mode_str(e->size_offs), val_str(e->a,(emit_mode)e->size_offs)); break; default: if( !IS_NULL(e->a) ) { @@ -379,6 +379,7 @@ void hl_emit_dump( jit_ctx *ctx ) { // if it not was not before (in case of dump during process) hl_emit_flush(ctx); hl_regs_flush(ctx); + hl_codegen_flush(ctx); printf("function "); hl_dump_fun_name(f); printf("("); @@ -403,6 +404,7 @@ void hl_emit_dump( jit_ctx *ctx ) { // print instrs int vpos = 0; int rpos = 0; + int cpos = 0; cur = 0; for(i=0;iinstr_count;i++) { while( cur < ctx->block_count && ctx->blocks[cur].start_pos == i ) { @@ -410,10 +412,10 @@ void hl_emit_dump( jit_ctx *ctx ) { printf("--- BLOCK #%d ---\n", cur); for(int k=0;kphi_count;k++) { ephi *p = b->phis + k; - printf("\t\t@%X %s = phi%s(",i,val_str(p->value),emit_mode_str(p->mode)); + printf("\t\t@%X %s = phi%s(",i,val_str(p->value,p->mode),emit_mode_str(p->mode)); for(int n=0;nnvalues;n++) { if( n > 0 ) printf(","); - printf("%s",val_str(p->values[n])); + printf("%s",val_str(p->values[n],p->mode)); } printf(")"); if( p->nvalues <= 1 ) @@ -436,7 +438,17 @@ void hl_emit_dump( jit_ctx *ctx ) { while( rpos < ctx->reg_instr_count && rpos < ctx->reg_pos_map[i+1] ) { ereg out = ctx->reg_writes[rpos]; e = ctx->reg_instrs + rpos; - printf("\n\t\t\t\t@%X ",rpos); + printf("\n\t\t\t"); + int count = 0; + while( cpos < ctx->code_size && cpos < ctx->code_pos_map[rpos+1] ) { + printf("%.2X",ctx->code_instrs[cpos++]); + count++; + } + while( count++ < 6 ) + printf(" "); + + printf(" @%X ",rpos); + if( !IS_NULL(out) ) printf("%s = ",reg_str(out)); dump_instr(ctx,e,rpos); rpos++; diff --git a/src/jit_emit.c b/src/jit_emit.c index a49f3bd10..273bf9bcc 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -482,6 +482,11 @@ static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { patch_instr_mode(ctx, GET_MODE(v)); } +static void emit_cmp( emit_ctx *ctx, ereg a, ereg b, hl_op o ) { + emit_gen_ext(ctx, CMP, a, b, 0, o); + patch_instr_mode(ctx, GET_MODE(a)); +} + static void phi_remove_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { ereg_remove(&p->vals,v); emit_debug("%sPHI-REM-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); @@ -1059,6 +1064,230 @@ static void prepare_loop_block( emit_ctx *ctx ) { } } +#define HL_DYN_VALUE 8 + +static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type *bt, ereg b, int offset ) { + if( at->kind == HDYN || bt->kind == HDYN || at->kind == HFUN || bt->kind == HFUN ) { + ereg args[2] = { a, b }; + ereg ret = emit_native_call(ctx,hl_dyn_compare,args,2,&hlt_i32); + if( op == OJSGt || op == OJSGte ) { + emit_cmp(ctx, ret, LOAD_CONST(hl_invalid_comparison,&hlt_i32), OJEq); + int jinvalid = emit_jump(ctx, true); + emit_test(ctx, ret, op); + register_block_jump(ctx, offset, true); + patch_jump(ctx, jinvalid); + return; + } + emit_test(ctx, ret, op); + // continue + } else switch( at->kind ) { + case HTYPE: + { + ereg args[2] = { a, b }; + ereg ret = emit_native_call(ctx,hl_same_type,args,2,&hlt_bool); + emit_test(ctx, ret, op); + } + break; + case HNULL: + { + if( op == OJEq ) { + // if( a == b || (a && b && a->v == b->v) ) goto + emit_cmp(ctx,a,b,OJEq); + register_block_jump(ctx,offset,true); + emit_test(ctx,a,OJNull); + int ja = emit_jump(ctx,true); + emit_test(ctx,b,OJNull); + int jb = emit_jump(ctx,true); + hl_type *vt = at->tparam; + emit_cmp(ctx, LOAD_MEM(a,HL_DYN_VALUE,vt), LOAD_MEM(b,HL_DYN_VALUE,vt), OJEq); + register_block_jump(ctx,offset,true); + patch_jump(ctx,ja); + patch_jump(ctx,jb); + } else if( op == OJNotEq ) { + // if( a != b && (!a || !b || a->v != b->v) ) goto + emit_cmp(ctx,a,b,OJEq); + int jeq = emit_jump(ctx,true); + emit_test(ctx,a,OJEq); + register_block_jump(ctx,offset,true); + emit_test(ctx,b,OJEq); + register_block_jump(ctx,offset,true); + hl_type *vt = at->tparam; + emit_cmp(ctx, LOAD_MEM(a,HL_DYN_VALUE,vt), LOAD_MEM(b,HL_DYN_VALUE,vt), OJNull); + int jcmp = emit_jump(ctx,true); + register_block_jump(ctx,offset,true); + patch_jump(ctx,jcmp); + patch_jump(ctx,jeq); + } else + jit_assert(); + return; + } + case HVIRTUAL: + { + BREAK(); + /* + preg p; + preg *pa = alloc_cpu(ctx,a,true); + preg *pb = alloc_cpu(ctx,b,true); + int ja,jb,jav,jbv,jvalue; + if( b->t->kind == HOBJ ) { + if( op->op == OJEq ) { + // if( a ? (b && a->value == b) : (b == NULL) ) goto + op64(ctx,TEST,pa,pa); + XJump_small(JZero,ja); + op64(ctx,TEST,pb,pb); + XJump_small(JZero,jb); + op64(ctx,MOV,pa,pmem(&p,pa->id,HL_WSIZE)); + op64(ctx,CMP,pa,pb); + XJump_small(JAlways,jvalue); + patch_jump(ctx,ja); + op64(ctx,TEST,pb,pb); + patch_jump(ctx,jvalue); + register_jump(ctx,do_jump(ctx,OJEq,false),targetPos); + patch_jump(ctx,jb); + } else if( op->op == OJNotEq ) { + // if( a ? (b == NULL || a->value != b) : (b != NULL) ) goto + op64(ctx,TEST,pa,pa); + XJump_small(JZero,ja); + op64(ctx,TEST,pb,pb); + register_jump(ctx,do_jump(ctx,OJEq,false),targetPos); + op64(ctx,MOV,pa,pmem(&p,pa->id,HL_WSIZE)); + op64(ctx,CMP,pa,pb); + XJump_small(JAlways,jvalue); + patch_jump(ctx,ja); + op64(ctx,TEST,pb,pb); + patch_jump(ctx,jvalue); + register_jump(ctx,do_jump(ctx,OJNotEq,false),targetPos); + } else + ASSERT(op->op); + scratch(pa); + return; + } + op64(ctx,CMP,pa,pb); + if( op->op == OJEq ) { + // if( a == b || (a && b && a->value && b->value && a->value == b->value) ) goto + register_jump(ctx,do_jump(ctx,OJEq, false),targetPos); + op64(ctx,TEST,pa,pa); + XJump_small(JZero,ja); + op64(ctx,TEST,pb,pb); + XJump_small(JZero,jb); + op64(ctx,MOV,pa,pmem(&p,pa->id,HL_WSIZE)); + op64(ctx,TEST,pa,pa); + XJump_small(JZero,jav); + op64(ctx,MOV,pb,pmem(&p,pb->id,HL_WSIZE)); + op64(ctx,TEST,pb,pb); + XJump_small(JZero,jbv); + op64(ctx,CMP,pa,pb); + XJump_small(JNeq,jvalue); + register_jump(ctx,do_jump(ctx,OJEq, false),targetPos); + patch_jump(ctx,ja); + patch_jump(ctx,jb); + patch_jump(ctx,jav); + patch_jump(ctx,jbv); + patch_jump(ctx,jvalue); + } else if( op->op == OJNotEq ) { + int jnext; + // if( a != b && (!a || !b || !a->value || !b->value || a->value != b->value) ) goto + XJump_small(JEq,jnext); + op64(ctx,TEST,pa,pa); + XJump_small(JZero,ja); + op64(ctx,TEST,pb,pb); + XJump_small(JZero,jb); + op64(ctx,MOV,pa,pmem(&p,pa->id,HL_WSIZE)); + op64(ctx,TEST,pa,pa); + XJump_small(JZero,jav); + op64(ctx,MOV,pb,pmem(&p,pb->id,HL_WSIZE)); + op64(ctx,TEST,pb,pb); + XJump_small(JZero,jbv); + op64(ctx,CMP,pa,pb); + XJump_small(JEq,jvalue); + patch_jump(ctx,ja); + patch_jump(ctx,jb); + patch_jump(ctx,jav); + patch_jump(ctx,jbv); + register_jump(ctx,do_jump(ctx,OJAlways, false),targetPos); + patch_jump(ctx,jnext); + patch_jump(ctx,jvalue); + } else + ASSERT(op->op); + scratch(pa); + scratch(pb); + return; + */ + } + break; + case HOBJ: + case HSTRUCT: + if( bt->kind == HVIRTUAL ) { + emit_jump_dyn(ctx,op,bt,b,at,a,offset); // inverse + return; + } + BREAK(); + /* + if( hl_get_obj_rt(a->t)->compareFun ) { + preg *pa = alloc_cpu(ctx,a,true); + preg *pb = alloc_cpu(ctx,b,true); + preg p; + int jeq, ja, jb, jcmp; + int args[] = { a->stack.id, b->stack.id }; + switch( op->op ) { + case OJEq: + // if( a == b || (a && b && cmp(a,b) == 0) ) goto + op64(ctx,CMP,pa,pb); + XJump_small(JEq,jeq); + op64(ctx,TEST,pa,pa); + XJump_small(JZero,ja); + op64(ctx,TEST,pb,pb); + XJump_small(JZero,jb); + op_call_fun(ctx,NULL,(int)(int_val)a->t->obj->rt->compareFun,2,args); + op32(ctx,TEST,PEAX,PEAX); + XJump_small(JNotZero,jcmp); + patch_jump(ctx,jeq); + register_jump(ctx,do_jump(ctx,OJAlways,false),targetPos); + patch_jump(ctx,ja); + patch_jump(ctx,jb); + patch_jump(ctx,jcmp); + break; + case OJNotEq: + // if( a != b && (!a || !b || cmp(a,b) != 0) ) goto + op64(ctx,CMP,pa,pb); + XJump_small(JEq,jeq); + op64(ctx,TEST,pa,pa); + register_jump(ctx,do_jump(ctx,OJEq,false),targetPos); + op64(ctx,TEST,pb,pb); + register_jump(ctx,do_jump(ctx,OJEq,false),targetPos); + + op_call_fun(ctx,NULL,(int)(int_val)a->t->obj->rt->compareFun,2,args); + op32(ctx,TEST,PEAX,PEAX); + XJump_small(JZero,jcmp); + + register_jump(ctx,do_jump(ctx,OJNotEq,false),targetPos); + patch_jump(ctx,jcmp); + patch_jump(ctx,jeq); + break; + default: + // if( a && b && cmp(a,b) ?? 0 ) goto + op64(ctx,TEST,pa,pa); + XJump_small(JZero,ja); + op64(ctx,TEST,pb,pb); + XJump_small(JZero,jb); + op_call_fun(ctx,NULL,(int)(int_val)a->t->obj->rt->compareFun,2,args); + op32(ctx,CMP,PEAX,pconst(&p,0)); + register_jump(ctx,do_jump(ctx,op->op,false),targetPos); + patch_jump(ctx,ja); + patch_jump(ctx,jb); + break; + } + return; + } + */ + // fallthrough + default: + emit_cmp(ctx, a, b, op); + break; + } + register_block_jump(ctx, offset, true); +} + static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { vreg *dst = R(o->p1); vreg *ra = R(o->p2); @@ -1186,11 +1415,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OJUGte: case OJNotLt: case OJNotGte: - { - emit_gen_ext(ctx, CMP, LOAD(dst), LOAD(ra), 0, o->op); - patch_instr_mode(ctx, hl_type_mode(dst->t)); - register_block_jump(ctx, o->p3, true); - } + emit_jump_dyn(ctx,o->op,dst->t,LOAD(dst),ra->t,LOAD(ra),o->p3); break; case OJAlways: register_block_jump(ctx, o->p1, false); @@ -1814,7 +2039,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { // On Win64 setjmp actually takes two arguments // the jump buffer and the frame pointer (or the stack pointer if there is no FP) nargs = 2; - args[1] = emit_gen(ctx,MOV,MK_STACK_OFFS(0),VAL_NULL,M_PTR); + args[1] = emit_gen(ctx,LEA,MK_STACK_REG(0),VAL_NULL,M_PTR); #endif #ifdef HL_MINGW fun = _setjmp; @@ -1868,8 +2093,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg v = LOAD(dst); int count = o->p2; - emit_gen_ext(ctx, CMP, v, LOAD_CONST(count,&hlt_i32), 0, OJUGte); - patch_instr_mode(ctx, M_I32); + emit_cmp(ctx,v,LOAD_CONST(count,&hlt_i32),OJUGte); int jdefault = emit_jump(ctx, true); emit_gen_ext(ctx, JUMP_TABLE, v, VAL_NULL, 0, count); for(int i=0; i #include "data_struct.h" +#define S_TYPE byte_arr +#define S_NAME(name) byte_##name +#define S_VALUE unsigned char +#include "data_struct.c" +#define byte_reserve(set,count) byte_reserve_impl(DEF_ALLOC,&set,count) +#define VAL_CONST (VAL_NULL-1) +#define FL_MEM (FL_NATREG | 0x8000000) +#define VAL_MEM(reg) (FL_MEM | (reg)) + +#define IS_FLOAT(mode) ((mode) == M_F64 || (mode) == M_F32) + +#define UNUSED VAL_NULL + typedef enum { RAX = 0, RCX = 1, @@ -43,18 +56,254 @@ typedef enum { R15 = 15, #endif _UNUSED = 0xFF -} Reg; +} CpuReg; + +#define R(id) ((id) | FL_NATREG) +#define X(id) R(id) +#define MMX(id) R(id+64) + +typedef enum { + _MOV, + _LEA, + _PUSH, + ADD, + SUB, + IMUL, // only overflow flag changes compared to MUL + DIV, + IDIV, + CDQ, + CDQE, + _POP, + _RET, + _CALL, + AND, + OR, + XOR, + _CMP, + _TEST, + NOP, + SHL, + SHR, + SAR, + INC, + DEC, + JMP, + // FPU + FSTP, + FSTP32, + FLD, + FLD32, + FLDCW, + // SSE + MOVSD, + MOVSS, + COMISD, + COMISS, + ADDSD, + SUBSD, + MULSD, + DIVSD, + ADDSS, + SUBSS, + MULSS, + DIVSS, + XORPD, + CVTSI2SD, + CVTSI2SS, + CVTSD2SI, + CVTSD2SS, + CVTSS2SD, + CVTSS2SI, + STMXCSR, + LDMXCSR, + // 8-16 bits + MOV8, + CMP8, + TEST8, + PUSH8, + MOV16, + CMP16, + TEST16, + // prefetchs + PREFETCHT0, + PREFETCHT1, + PREFETCHT2, + PREFETCHNTA, + PREFETCHW, + // -- + _CPU_LAST +} CpuOp; + +#define JAlways 0xE9 +#define JAlways_short 0xEB +#define JOverflow 0x80 +#define JULt 0x82 +#define JUGte 0x83 +#define JEq 0x84 +#define JNeq 0x85 +#define JULte 0x86 +#define JUGt 0x87 +#define JParity 0x8A +#define JNParity 0x8B +#define JSLt 0x8C +#define JSGte 0x8D +#define JSLte 0x8E +#define JSGt 0x8F + +#define JCarry JLt +#define JZero JEq +#define JNotZero JNeq + +#define FLAG_LONGOP 0x80000000 +#define FLAG_16B 0x40000000 +#define FLAG_8B 0x20000000 +#define FLAG_DUAL 0x10000000 +#define FLAG_DEF64 0x08000000 + +#define RM(op,id) ((op) | (((id)+1)<<8)) +#define GET_RM(op) (((op) >> ((op) < 0 ? 24 : 8)) & 15) +#define SBYTE(op) ((op) << 16) +#define LONG_OP(op) ((op) | FLAG_LONGOP) +#define OP16(op) LONG_OP((op) | FLAG_16B) +#define LONG_RM(op,id) LONG_OP(op | (((id) + 1) << 24)) + +typedef struct { + const char *name; // single operand + int r_mem; // r32 / r/m32 r32 + int mem_r; // r/m32 / r32 r/m32 + int r_const; // r32 / imm32 imm32 + int r_i8; // r32 / imm8 imm8 + int mem_const; // r/m32 / imm32 N/A +} opform; + +static opform OP_FORMS[] = { + { "MOV", 0x8B, 0x89, 0xB8, 0, RM(0xC7,0) }, + { "LEA", 0x8D }, + { "PUSH", 0x50 | FLAG_DEF64, RM(0xFF,6), 0x68, 0x6A }, + { "ADD", 0x03, 0x01, RM(0x81,0), RM(0x83,0) }, + { "SUB", 0x2B, 0x29, RM(0x81,5), RM(0x83,5) }, + { "IMUL", LONG_OP(0x0FAF), 0, 0x69 | FLAG_DUAL, 0x6B | FLAG_DUAL }, + { "DIV", RM(0xF7,6), RM(0xF7,6) }, + { "IDIV", RM(0xF7,7), RM(0xF7,7) }, + { "CDQ", 0x99 }, + { "CDQE", 0x98 }, + { "POP", 0x58 | FLAG_DEF64, RM(0x8F,0) }, + { "RET", 0xC3 }, + { "CALL", RM(0xFF,2) | FLAG_DEF64, RM(0xFF,2), 0xE8 }, + { "AND", 0x23, 0x21, RM(0x81,4), RM(0x83,4) }, + { "OR", 0x0B, 0x09, RM(0x81,1), RM(0x83,1) }, + { "XOR", 0x33, 0x31, RM(0x81,6), RM(0x83,6) }, + { "CMP", 0x3B, 0x39, RM(0x81,7), RM(0x83,7) }, + { "TEST", 0x85, 0x85/*SWP?*/, RM(0xF7,0) }, + { "NOP", 0x90 }, + { "SHL", RM(0xD3,4), 0, 0, RM(0xC1,4) }, + { "SHR", RM(0xD3,5), 0, 0, RM(0xC1,5) }, + { "SAR", RM(0xD3,7), 0, 0, RM(0xC1,7) }, + { "INC", IS_64 ? RM(0xFF,0) : 0x40, RM(0xFF,0) }, + { "DEC", IS_64 ? RM(0xFF,1) : 0x48, RM(0xFF,1) }, + { "JMP", RM(0xFF,4) }, + // FPU + { "FSTP", 0, RM(0xDD,3) }, + { "FSTP32", 0, RM(0xD9,3) }, + { "FLD", 0, RM(0xDD,0) }, + { "FLD32", 0, RM(0xD9,0) }, + { "FLDCW", 0, RM(0xD9, 5) }, + // SSE + { "MOVSD", 0xF20F10, 0xF20F11 }, + { "MOVSS", 0xF30F10, 0xF30F11 }, + { "COMISD", 0x660F2F }, + { "COMISS", LONG_OP(0x0F2F) }, + { "ADDSD", 0xF20F58 }, + { "SUBSD", 0xF20F5C }, + { "MULSD", 0xF20F59 }, + { "DIVSD", 0xF20F5E }, + { "ADDSS", 0xF30F58 }, + { "SUBSS", 0xF30F5C }, + { "MULSS", 0xF30F59 }, + { "DIVSS", 0xF30F5E }, + { "XORPD", 0x660F57 }, + { "CVTSI2SD", 0xF20F2A }, + { "CVTSI2SS", 0xF30F2A }, + { "CVTSD2SI", 0xF20F2D }, + { "CVTSD2SS", 0xF20F5A }, + { "CVTSS2SD", 0xF30F5A }, + { "CVTSS2SI", 0xF30F2D }, + { "STMXCSR", 0, LONG_RM(0x0FAE,3) }, + { "LDMXCSR", 0, LONG_RM(0x0FAE,2) }, + // 8 bits, + { "MOV8", 0x8A, 0x88, 0, 0xB0, RM(0xC6,0) }, + { "CMP8", 0x3A, 0x38, 0, RM(0x80,7) }, + { "TEST8", 0x84, 0x84, RM(0xF6,0) }, + { "PUSH8", FLAG_DEF64, 0, 0x6A | FLAG_8B }, + { "MOV16", OP16(0x8B), OP16(0x89), OP16(0xB8) }, + { "CMP16", OP16(0x3B), OP16(0x39) }, + { "TEST16", OP16(0x85) }, + // prefetchs + { "PREFETCHT0", FLAG_DEF64, LONG_RM(0x0F18,1) }, + { "PREFETCHT1", FLAG_DEF64, LONG_RM(0x0F18,2) }, + { "PREFETCHT2", FLAG_DEF64, LONG_RM(0x0F18,3) }, + { "PREFETCHNTA", FLAG_DEF64, LONG_RM(0x0F18,0) }, + { "PREFETCHW", FLAG_DEF64, LONG_RM(0x0F0D,1) }, +}; + +#ifdef HL_64 +# define REX() if( r64 ) B(r64 | 0x40) +#else +# define REX() +#endif + +static const int SIB_MULT[] = {-1, 0, 1, -1, 2, -1, -1, -1, 3}; + +#define B(v) ctx->code.values[ctx->code.cur++] = (unsigned char)(v) +#define W(wv) *(int*)&ctx->code.values[_incr(&ctx->code.cur,4)] = wv +#define W64(v64) *(int_val*)&ctx->code.values[_incr(&ctx->code.cur,8)] = v64 + +#define MOD_RM(mod,reg,rm) B(((mod) << 6) | (((reg)&7) << 3) | ((rm)&7)) +#define SIB(mult,rmult,rbase) B((SIB_MULT[mult]<<6) | (((rmult)&7)<<3) | ((rbase)&7)) +#define IS_SBYTE(c) ( (c) >= -128 && (c) < 128 ) -#define R(id,mode) ((mode) | ((id)<<8) | FL_NATREG) -#define X(id) R(id,M_NONE) -#define MMX(id) R(id+64,M_F64) +#define BREAK() B(0xCC) -const char *hl_natreg_str( int reg ) { +#define OP(b) \ + if( (b) & 0xFF0000 ) { \ + B((b)>>16); \ + if( r64 ) B(r64 | 0x40); /* also in 32 bits mode */ \ + B((b)>>8); \ + B(b); \ + } else { \ + if( (b) & FLAG_16B ) { \ + B(0x66); \ + REX(); \ + } else {\ + REX(); \ + if( (b) & FLAG_LONGOP ) B((b)>>8); \ + }\ + B(b); \ + } + +struct _code_ctx { + jit_ctx *jit; + byte_arr code; + int_arr funs; + int_arr short_jumps; + int_arr near_jumps; + int *pos_map; + int cur_op; + int fun_start_pos; + bool flushed; +}; + +static int _incr( int*v, int n ) { + int k = *v; + *v += n; + return k; +} + +const char *hl_natreg_str( int reg, emit_mode m ) { static char out[16]; static const char *regs_str[] = { "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI" }; - emit_mode k = (reg & 0xFF); - Reg r = ((reg >> 8) & 0xFF); - switch( k ) { + CpuReg r = (reg & 0xFF); + switch( m ) { case M_I32: if( r < 8 ) sprintf(out,"E%s",regs_str[r]); @@ -124,3 +373,499 @@ void hl_jit_init_regs( regs_config cfg ) { cfg[1].persist = (ereg*)floats+cfg[1].nscratchs; cfg[1].npersists = 16 - cfg[1].nscratchs; } + +#define EMIT(op,a,b,mode) emit_ext(ctx,op,a,b,mode,0) +#define ID2(a,b) ((a) | ((b)<<8)) + +typedef enum { + RCPU = 0, + RFPU = 1, + RSTACK = 2, + RCONST = 3, + RADDR = 4, + RMEM = 5, + RUNUSED = 6, +} preg_kind; + +typedef struct { + preg_kind kind; + CpuReg reg; + int value; +} preg; + +#define ERRIF(v) if( v ) jit_assert() + +static preg make_reg( ereg r, emit_mode m ) { + preg p; + if( IS_NULL(r) ) { + p.kind = RUNUSED; + return p; + } + if( r == VAL_CONST ) { + p.kind = RCONST; + return p; + } + if( (r & (FL_MEM | FL_NATMASK)) == FL_MEM ) { + p.kind = RMEM; + p.reg = (r&0xFFFF); + p.value = 0; + return p; + } + ERRIF(!IS_NATREG(r)); + ERRIF((r&FL_NATMASK) == FL_STACKOFFS); + if( (r & FL_NATMASK) == FL_STACKREG ) { + p.kind = RSTACK; + p.value = GET_STACK_OFFS(r); + } else if( m == M_F32 || m == M_F64 ) { + p.kind = RFPU; + p.reg = (r&0xFF) - 64; + } else { + p.kind = RCPU; + p.reg = (r&0xFF); + } + return p; +} + +static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, int_val value ) { + opform *f = &OP_FORMS[op]; + int r64 = (mode == M_PTR || mode == M_I64) && (f->r_mem&FLAG_DEF64) == 0 ? 8 : 0; + preg a = make_reg(_a,mode), b = make_reg(_b,mode); + switch( ID2(a.kind,b.kind) ) { + case ID2(RUNUSED,RUNUSED): + ERRIF(f->r_mem == 0); + OP(f->r_mem); + break; + case ID2(RCPU,RCPU): + case ID2(RFPU,RFPU): + ERRIF( f->r_mem == 0 && f->mem_r == 0 ); + if( a.reg > 7 ) r64 |= 4; + if( b.reg > 7 ) r64 |= 1; + OP(f->mem_r ? f->mem_r : f->r_mem); + MOD_RM(3,a.reg,b.reg); + break; + case ID2(RCPU,RFPU): + case ID2(RFPU,RCPU): + ERRIF( (f->r_mem>>16) == 0 ); + if( a.reg > 7 ) r64 |= 4; + if( b.reg > 7 ) r64 |= 1; + OP(f->r_mem); + MOD_RM(3,a.reg,b.reg); + break; + case ID2(RCPU,RUNUSED): + ERRIF( f->r_mem == 0 ); + if( a.reg > 7 ) r64 |= 1; + if( GET_RM(f->r_mem) > 0 ) { + OP(f->r_mem); + MOD_RM(3, GET_RM(f->r_mem)-1, a.reg); + } else + OP(f->r_mem + (a.reg&7)); + break; + case ID2(RSTACK,RUNUSED): + ERRIF( f->mem_r == 0 || GET_RM(f->mem_r) == 0 ); + { + OP(f->mem_r); + if( IS_SBYTE(a.value) ) { + MOD_RM(1,GET_RM(f->mem_r)-1,RBP); + B(a.value); + } else { + MOD_RM(2,GET_RM(f->mem_r)-1,RBP); + W(a.value); + } + } + break; + case ID2(RCPU,RCONST): + ERRIF( f->r_const == 0 && f->r_i8 == 0 ); + if( a.reg > 7 ) r64 |= 1; + { + // short byte form + if( f->r_i8 && IS_SBYTE(value) ) { + if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; + OP(f->r_i8); + if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_i8)-1,a.reg); + B(value); + } else if( GET_RM(f->r_const) > 0 || (f->r_const&FLAG_DUAL) ) { + if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; + OP(f->r_const&0xFF); + if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_const)-1,a.reg); + if( r64 && IS_64 && op == _MOV ) W64(value); else W((int)value); + } else { + ERRIF( f->r_const == 0); + OP((f->r_const&0xFF) + (a.reg&7)); + if( r64 && IS_64 && op == _MOV ) W64(value); else W((int)value); + } + } + break; + case ID2(RSTACK,RCPU): + case ID2(RSTACK,RFPU): + ERRIF( f->mem_r == 0 ); + if( b.reg > 7 ) r64 |= 4; + { + OP(f->mem_r); + if( IS_SBYTE(a.value) ) { + MOD_RM(1,b.reg,RBP); + B(a.value); + } else { + MOD_RM(2,b.reg,RBP); + W(a.value); + } + } + break; + case ID2(RCPU,RSTACK): + case ID2(RFPU,RSTACK): + ERRIF( f->r_mem == 0 ); + if( a.reg > 7 ) r64 |= 4; + { + OP(f->r_mem); + if( IS_SBYTE(b.value) ) { + MOD_RM(1,a.reg,RBP); + B(b.value); + } else { + MOD_RM(2,a.reg,RBP); + W(b.value); + } + } + break; + case ID2(RCONST,RUNUSED): + ERRIF( f->r_const == 0 ); + { + OP(f->r_const); + if( f->r_const & FLAG_8B ) B(value); else W((int)value); + } + break; + case ID2(RMEM,RUNUSED): + ERRIF( f->mem_r == 0 ); + { + int mult = a.value; + CpuReg reg = (a.reg & 0xFF); + if( mult == 0 ) { + if( reg > 7 ) r64 |= 1; + OP(f->mem_r); + if( value == 0 && (reg&7) != RBP ) { + MOD_RM(0,GET_RM(f->mem_r)-1,reg); + if( (reg&7) == RSP ) B(0x24); + } else if( IS_SBYTE(value) ) { + MOD_RM(1,GET_RM(f->mem_r)-1,reg); + if( (reg&7) == RSP ) B(0x24); + B(value); + } else { + MOD_RM(2,GET_RM(f->mem_r)-1,reg); + if( (reg&7) == RSP ) B(0x24); + W((int)value); + } + } else { + // [eax + ebx * M] + ERRIF(1); + } + } + break; + case ID2(RCPU, RMEM): + case ID2(RFPU, RMEM): + ERRIF( f->r_mem == 0 ); + { + int mult = b.value; + CpuReg reg = b.reg & 0xFF; + if( mult == 15 ) { + int pos; + if( a.reg > 7 ) r64 |= 4; + OP(f->r_mem); + MOD_RM(0,a.reg,5); + if( IS_64 ) { + // offset wrt current code + pos = ctx->code.cur + 4; + W((int)(value - pos)); + } else { + ERRIF(1); + } + } else if( mult == 0 ) { + if( a.reg > 7 ) r64 |= 4; + if( reg > 7 ) r64 |= 1; + OP(f->r_mem); + if( value == 0 && (reg&7) != RBP ) { + MOD_RM(0,a.reg,reg); + if( (reg&7) == RSP ) B(0x24); + } else if( IS_SBYTE(value) ) { + MOD_RM(1,a.reg,reg); + if( (reg&7) == RSP ) B(0x24); + B(value); + } else { + MOD_RM(2,a.reg,reg); + if( (reg&7) == RSP ) B(0x24); + W((int)value); + } + } else { + jit_assert(); + /* + if( a.reg > 7 ) r64 |= 4; + if( reg > 7 ) r64 |= 1; + if( regOrOffs > 7 ) r64 |= 2; + OP(f->r_mem); + MOD_RM(offset == 0 ? 0 : IS_SBYTE(offset) ? 1 : 2,a.reg,4); + SIB(mult,regOrOffs,reg); + if( offset ) { + if( IS_SBYTE(offset) ) B(offset); else W(offset); + } + */ + } + } + break; +# ifndef HL_64 + case ID2(RFPU,RADDR): +# endif + case ID2(RCPU,RADDR): + ERRIF( f->r_mem == 0 ); + if( a.reg > 7 ) r64 |= 4; + OP(f->r_mem); + MOD_RM(0,a.reg,5); + if( IS_64 ) + W64((int_val)value); + else + W((int)(int_val)value); + break; +# ifndef HL_64 + case ID2(RADDR,RFPU): +# endif + case ID2(RADDR,RCPU): + ERRIF( f->mem_r == 0 ); + if( b.reg > 7 ) r64 |= 4; + OP(f->mem_r); + MOD_RM(0,b.reg,5); + if( IS_64 ) + W64((int_val)value); + else + W((int)(int_val)value); + break; + case ID2(RMEM, RCPU): + case ID2(RMEM, RFPU): + ERRIF( f->mem_r == 0 ); + { + int mult = a.value; + CpuReg reg = a.reg & 0xFF; + if( mult == 15 ) { + int pos; + if( b.reg > 7 ) r64 |= 4; + OP(f->mem_r); + MOD_RM(0,b.reg,5); + if( IS_64 ) { + // offset wrt current code + pos = ctx->code.cur + 4; + W((int)(value - pos)); + } else { + ERRIF(1); + } + } else if( mult == 0 ) { + if( b.reg > 7 ) r64 |= 4; + if( reg > 7 ) r64 |= 1; + OP(f->mem_r); + if( value == 0 && (reg&7) != RBP ) { + MOD_RM(0,b.reg,reg); + if( (reg&7) == RSP ) B(0x24); + } else if( IS_SBYTE(value) ) { + MOD_RM(1,b.reg,reg); + if( (reg&7) == RSP ) B(0x24); + B(value); + } else { + MOD_RM(2,b.reg,reg); + if( (reg&7) == RSP ) B(0x24); + W((int)value); + } + } else { + jit_assert(); + /* + if( b.reg > 7 ) r64 |= 4; + if( reg > 7 ) r64 |= 1; + if( value > 7 ) r64 |= 2; + OP(f->mem_r); + MOD_RM(offset == 0 ? 0 : IS_SBYTE(offset) ? 1 : 2,b.reg,4); + SIB(mult,value,reg); + if( offset ) { + if( IS_SBYTE(offset) ) B(offset); else W(offset); + } + */ + } + } + break; + default: + ERRIF(1); + } +} + +static void emit_jump( code_ctx *ctx, int mode, int offset ) { + if( IS_SBYTE(offset*8) ) { + // assume it's ok to use short jump + B(mode == JAlways ? JAlways_short : mode - 0x10); + int_arr_add(ctx->short_jumps, byte_count(ctx->code)); + int_arr_add(ctx->short_jumps, ctx->cur_op + offset + 1); + B(-2); + } else { + B(mode); + int_arr_add(ctx->near_jumps, byte_count(ctx->code)); + int_arr_add(ctx->near_jumps, ctx->cur_op + offset + 1); + W(-5); + } +} + +void hl_codegen_flush( jit_ctx *jit ) { + code_ctx *ctx = jit->code; + if( ctx->flushed ) return; + ctx->flushed = true; + jit->code_size = ctx->code.cur; + jit->code_instrs = ctx->code.values; + jit->code_pos_map = ctx->pos_map; + if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->code.cur; +} + +void hl_codegen_function( jit_ctx *jit ) { + code_ctx *ctx = jit->code; + ctx->flushed = false; + byte_free(&ctx->code); + free(ctx->pos_map); + ctx->pos_map = (int*)malloc((jit->reg_instr_count + 1) * sizeof(int)); + for(int cur_pos=0;cur_posreg_instr_count;cur_pos++) { + einstr *e = jit->reg_instrs + cur_pos; + ereg out = jit->reg_writes[cur_pos]; + byte_reserve(ctx->code,64); + ctx->code.cur -= 64; + ctx->cur_op = cur_pos; + ctx->pos_map[cur_pos] = ctx->code.cur; + switch( e->op ) { + case LOAD_ARG: + case ALLOC_STACK: + continue; // nop + case MOV: + if( (e->a & FL_NATMASK) == FL_STACKOFFS ) + emit_ext(ctx,_LEA,out,VAL_MEM(R(RBP)),M_PTR,GET_STACK_OFFS(e->a)); + else + EMIT(_MOV, out, e->a, e->mode); + break; + case STORE: + if( (e->a & FL_NATMASK) == FL_STACKOFFS ) + emit_ext(ctx,_MOV, VAL_MEM(R(RBP)), e->b, e->mode, e->size_offs + GET_STACK_OFFS(e->a)); + else + emit_ext(ctx,_MOV, VAL_MEM(e->a), e->b, e->mode, e->size_offs); + break; + case PUSH_CONST: + emit_ext(ctx, _PUSH, VAL_CONST, UNUSED, e->mode, e->value); + break; + case DEBUG_BREAK: + BREAK(); + break; + case CALL_PTR: + emit_ext(ctx, _MOV, R(RAX), VAL_CONST, M_PTR, e->value); + EMIT(_CALL, R(RAX), UNUSED, M_NONE); + break; + case RET: + if( !IS_NULL(e->a) && e->a != R(RAX) ) + EMIT(_MOV, R(RAX), e->a, e->mode); + EMIT(_RET, UNUSED, UNUSED, M_NONE); + break; + case LOAD_CONST: + if( e->value == 0 ) + EMIT(XOR, out, out, e->mode); + else + emit_ext(ctx, _MOV, out, VAL_CONST, e->mode, e->value); + break; + case LOAD_ADDR: + if( (e->a & FL_NATMASK) == FL_STACKOFFS ) + emit_ext(ctx,_MOV,out,VAL_MEM(R(RBP)),e->mode, e->size_offs + GET_STACK_OFFS(e->a)); + else + emit_ext(ctx,_MOV,out,VAL_MEM(e->a),e->mode, e->size_offs); + break; + case CALL_FUN: + B(0xE8); + { + int pos = ctx->fun_start_pos + byte_count(ctx->code); + int fid = e->a; + int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,pos); + int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,fid); + } + W(0x80000000); + break; + case TEST: + EMIT(_TEST,e->a,e->a,e->mode); + break; + case CMP: + EMIT(_CMP,e->a,e->b,e->mode); + break; + case JCOND: + { + einstr *p = &e[-1]; + int op; + switch( p->size_offs ) { + case OJFalse: + case OJNull: + op = JZero; + break; + case OJTrue: + case OJNotNull: + op = JNotZero; + break; + case OJSGte: + op = IS_FLOAT(p->mode) ? JUGte : JSGte; + break; + case OJSGt: + op = IS_FLOAT(p->mode) ? JUGt : JSGt; + break; + case OJUGte: + op = JUGte; + break; + case OJSLt: + op = IS_FLOAT(p->mode) ? JULt : JSLt; + break; + case OJSLte: + op = IS_FLOAT(p->mode) ? JULte : JSLte; + break; + case OJULt: + op = JULt; + break; + case OJEq: + op = JEq; + break; + case OJNotEq: + op = JNeq; + break; + case OJNotLt: + op = JUGte; + break; + case OJNotGte: + op = JULt; + break; + default: + jit_assert(); + break; + } + emit_jump(ctx, op, e->size_offs); + } + break; + case JUMP: + emit_jump(ctx, JAlways, e->size_offs); + break; + case JUMP_TABLE: + BREAK(); + break; + case CONV: + BREAK(); + break; + default: + jit_assert(); + break; + } + if( ctx->code.cur > ctx->code.max ) jit_assert(); + } + hl_codegen_flush(jit); + ctx->fun_start_pos += byte_count(ctx->code); +} + + +void hl_codegen_alloc( jit_ctx *jit ) { + code_ctx *ctx = (code_ctx*)malloc(sizeof(code_ctx)); + memset(ctx,0,sizeof(code_ctx)); + jit->code = ctx; + ctx->jit = jit; +} + +void hl_codegen_free( jit_ctx *jit ) { + code_ctx *ctx = jit->code; + free(ctx->pos_map); + free(ctx); +} + From 15c1c08a32a83ce5addd886aa771b027b0794c53 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 18 Apr 2026 09:54:32 +0200 Subject: [PATCH 26/83] cleanup flags, changed VAL_NULL to UNUSED --- src/jit.h | 12 +++--- src/jit_dump.c | 19 +++------ src/jit_emit.c | 59 +++++++++++++------------ src/jit_regs.c | 34 +++++++-------- src/jit_x86_64.c | 109 ++++++++++++++++++++++++++++++++++++----------- 5 files changed, 145 insertions(+), 88 deletions(-) diff --git a/src/jit.h b/src/jit.h index 0b14050aa..37e4d12c5 100644 --- a/src/jit.h +++ b/src/jit.h @@ -91,17 +91,19 @@ typedef struct { }; } einstr; -#define VAL_NULL 0x80000000 -#define FL_NATREG 0x40000000 #define FL_NATMASK 0x70000000 -#define FL_STACKREG (FL_NATREG | 0x20000000) -#define FL_STACKOFFS (FL_NATREG | 0x10000000) -#define IS_NULL(e) ((e) == VAL_NULL) +#define FL_NATREG 0x40000000 +#define FL_MEMPTR 0x20000000 +#define FL_STACK 0x10000000 +#define FL_STACKREG (FL_NATREG | FL_MEMPTR | FL_STACK) +#define FL_STACKOFFS (FL_NATREG | FL_STACK) +#define IS_NULL(e) ((e) == 0) #define IS_NATREG(e) ((e) & FL_NATREG) #define MK_STACK_REG(v) (((v)&0xFFFFFFF) | FL_STACKREG) #define MK_STACK_OFFS(v)(((v)&0xFFFFFFF) | FL_STACKOFFS) #define GET_STACK_OFFS(v) ((int)(((v) & 0x8000000) ? ((v) | 0xF0000000) : ((v)&0xFFFFFFF))) #define IS_CALL(op) ((op) == CALL_PTR || (op) == CALL_REG || (op) == CALL_FUN) +#define UNUSED ((ereg)0) typedef struct { int *data; diff --git a/src/jit_dump.c b/src/jit_dump.c index c6c5b736c..ed55308e0 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -248,6 +248,8 @@ static void hl_dump_ptr_name( jit_ctx *ctx, void *ptr ) { N(hl_rethrow); N(hl_to_virtual); N(hl_alloc_enum); + N(hl_dyn_compare); + N(hl_same_type); DYN(f); DYN(d); DYN(i64); @@ -402,7 +404,7 @@ void hl_emit_dump( jit_ctx *ctx ) { if( cur != ctx->instr_count ) printf(" ??? MISSING BLOCK FOR RANGE %X-%X\n", cur, ctx->instr_count); // print instrs - int vpos = 0; + int vpos = 1; int rpos = 0; int cpos = 0; cur = 0; @@ -438,19 +440,12 @@ void hl_emit_dump( jit_ctx *ctx ) { while( rpos < ctx->reg_instr_count && rpos < ctx->reg_pos_map[i+1] ) { ereg out = ctx->reg_writes[rpos]; e = ctx->reg_instrs + rpos; - printf("\n\t\t\t"); - int count = 0; - while( cpos < ctx->code_size && cpos < ctx->code_pos_map[rpos+1] ) { - printf("%.2X",ctx->code_instrs[cpos++]); - count++; - } - while( count++ < 6 ) - printf(" "); - - printf(" @%X ",rpos); - + printf("\n\t\t\t\t@%X ",rpos); if( !IS_NULL(out) ) printf("%s = ",reg_str(out)); dump_instr(ctx,e,rpos); + printf("\033[80G"); + while( cpos < ctx->code_size && cpos < ctx->code_pos_map[rpos+1] ) + printf("%.2X",ctx->code_instrs[cpos++]); rpos++; } printf("\n"); diff --git a/src/jit_emit.c b/src/jit_emit.c index 273bf9bcc..a08aefa57 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -61,7 +61,6 @@ typedef struct _tmp_phi tmp_phi; #define S_SORTED -#define S_DEFVAL VAL_NULL #define S_TYPE ereg_map #define S_NAME(name) ereg_##name #define S_VALUE ereg @@ -70,7 +69,6 @@ typedef struct _tmp_phi tmp_phi; #define S_MAP -#define S_DEFVAL VAL_NULL #define S_TYPE vreg_map #define S_NAME(name) vreg_##name #define S_KEY int @@ -157,7 +155,7 @@ struct _emit_ctx { #define STORE_MEM(to, offs, v) emit_store_mem(ctx, to, offs, v) #define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR(obj,0),HL_WSIZE*2),HL_WSIZE*(id)) #define OFFSET(base,index,mult,offset) emit_gen_ext(ctx, LEA, base, index, M_PTR, (mult) | ((offset) << 8)) -#define BREAK() emit_gen(ctx, DEBUG_BREAK, VAL_NULL, VAL_NULL, 0) +#define BREAK() emit_gen(ctx, DEBUG_BREAK, UNUSED, UNUSED, 0) #define GET_MODE(r) emit_get_mode(ctx,r) #define GET_PHI(r) ctx->phis[-(r)-1] #define HDYN_VALUE 8 @@ -346,7 +344,7 @@ static ereg emit_gen_ext( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode, i e->size_offs = size_offs; e->a = a; e->b = b; - return mode == 0 || mode == M_NORET ? VAL_NULL : new_value(ctx); + return mode == 0 || mode == M_NORET ? UNUSED : new_value(ctx); } static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { @@ -354,7 +352,7 @@ static ereg emit_gen( emit_ctx *ctx, emit_op op, ereg a, ereg b, int mode ) { } static ereg emit_gen_size( emit_ctx *ctx, emit_op op, int size_offs ) { - return emit_gen_ext(ctx,op,VAL_NULL,VAL_NULL,op==ALLOC_STACK ? M_PTR : 0,size_offs); + return emit_gen_ext(ctx,op,UNUSED,UNUSED,op==ALLOC_STACK ? M_PTR : 0,size_offs); } static void patch_instr_mode( emit_ctx *ctx, int mode ) { @@ -383,7 +381,7 @@ static tmp_phi *alloc_phi( emit_ctx *ctx, emit_block *b, vreg *r ) { static int emit_jump( emit_ctx *ctx, bool cond ) { int p = ctx->emit_pos; - emit_gen(ctx, cond ? JCOND : JUMP, VAL_NULL, VAL_NULL, 0); + emit_gen(ctx, cond ? JCOND : JUMP, UNUSED, UNUSED, 0); return p; } @@ -450,7 +448,7 @@ static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) einstr *e = emit_instr(ctx, LOAD_ADDR); e->mode = hl_type_mode(size_t); e->a = v; - e->b = VAL_NULL; + e->b = UNUSED; e->size_offs = offset; return new_value(ctx); } @@ -466,7 +464,7 @@ static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int e->mode = (unsigned char)(ret ? hl_type_mode(ret) : M_NORET); e->value = (int_val)native_ptr; store_args(ctx, e, args, nargs); - return ret == NULL || e->mode == M_VOID ? VAL_NULL : new_value(ctx); + return ret == NULL || e->mode == M_VOID ? UNUSED : new_value(ctx); } static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_type *ret ) { @@ -474,11 +472,11 @@ static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_typ e->mode = hl_type_mode(ret); e->a = f; store_args(ctx, e, args, nargs); - return e->mode == M_VOID ? VAL_NULL : new_value(ctx); + return e->mode == M_VOID ? UNUSED : new_value(ctx); } static void emit_test( emit_ctx *ctx, ereg v, hl_op o ) { - emit_gen_ext(ctx, TEST, v, VAL_NULL, 0, o); + emit_gen_ext(ctx, TEST, v, UNUSED, 0, o); patch_instr_mode(ctx, GET_MODE(v)); } @@ -509,7 +507,7 @@ static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { if( p->locked ) jit_assert(); - ereg same = VAL_NULL; + ereg same = UNUSED; for_iter(ereg,v,p->vals) { if( v == same || v == p->value ) continue; @@ -632,7 +630,7 @@ static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int e->mode = hl_type_mode(dst->t); e->a = findex; store_args(ctx, e, args, count); - STORE(dst, e->mode == M_VOID ? VAL_NULL : new_value(ctx)); + STORE(dst, e->mode == M_VOID ? UNUSED : new_value(ctx)); } } @@ -723,7 +721,7 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, } static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode from, emit_mode to, bool _unsigned ) { - return emit_gen_ext(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, VAL_NULL, to, from); + return emit_gen_ext(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, UNUSED, to, from); } static bool dyn_need_type( hl_type *t ) { @@ -928,6 +926,7 @@ void hl_emit_function( jit_ctx *jit ) { int_arr_free(&ctx->args_data); int_arr_free(&ctx->jump_regs); int_arr_free(&ctx->values); + int_arr_add(ctx->values,-1); ctx->root_block = ctx->current_block = alloc_block(ctx); ctx->current_block->sealed = true; ctx->arrival_points = NULL; @@ -951,13 +950,13 @@ void hl_emit_function( jit_ctx *jit ) { for(i=0;inregs;i++) { vreg *r = R(i); r->t = f->regs[i]; - r->ref = VAL_NULL; + r->ref = 0; } for(i=0;itype->fun->nargs;i++) { hl_type *t = f->type->fun->args[i]; if( t->kind == HVOID ) continue; - STORE(R(i), emit_gen(ctx, LOAD_ARG, VAL_NULL, VAL_NULL, hl_type_mode(t))); + STORE(R(i), emit_gen(ctx, LOAD_ARG, UNUSED, UNUSED, hl_type_mode(t))); } for(int op_pos=0;op_posnops;op_pos++) { @@ -1300,7 +1299,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { switch( o->op ) { case OMov: case OUnsafeCast: - STORE(dst, emit_gen(ctx,MOV,LOAD(ra),VAL_NULL,hl_type_mode(ra->t))); + STORE(dst, emit_gen(ctx,MOV,LOAD(ra),UNUSED,hl_type_mode(ra->t))); break; case OInt: STORE(dst, LOAD_CONST(m->code->ints[o->p2], dst->t)); @@ -1394,7 +1393,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case ONeg: case ONot: - STORE(dst, emit_gen_ext(ctx, UNOP, LOAD(ra), VAL_NULL, hl_type_mode(dst->t), o->op)); + STORE(dst, emit_gen_ext(ctx, UNOP, LOAD(ra), UNUSED, hl_type_mode(dst->t), o->op)); break; case OJFalse: case OJTrue: @@ -1437,7 +1436,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, emit_conv(ctx,LOAD(ra),hl_type_mode(ra->t),hl_type_mode(dst->t), o->op == OToUFloat)); break; case ORet: - emit_gen(ctx, RET, dst->t->kind == HVOID ? VAL_NULL : LOAD(dst), VAL_NULL, M_NORET); + emit_gen(ctx, RET, dst->t->kind == HVOID ? UNUSED : LOAD(dst), 0, M_NORET); patch_instr_mode(ctx, hl_type_mode(dst->t)); break; case OIncr: @@ -1446,13 +1445,13 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( IS_FLOAT(dst->t) ) { jit_assert(); } else { - STORE(dst, emit_gen_ext(ctx,UNOP,LOAD(dst),VAL_NULL,hl_type_mode(dst->t),o->op)); + STORE(dst, emit_gen_ext(ctx,UNOP,LOAD(dst),UNUSED,hl_type_mode(dst->t),o->op)); } } break; case ONew: { - ereg arg = VAL_NULL; + ereg arg = UNUSED; void *allocFun = NULL; int nargs = 1; switch( dst->t->kind ) { @@ -1559,7 +1558,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( dst->t->kind == HSTRUCT ) { hl_type *ft = hl_obj_field_fetch(ra->t,o->p3)->t; if( ft->kind == HPACKED ) { - STORE(dst,OFFSET(r, VAL_NULL, 0, rt->fields_indexes[o->p3])); + STORE(dst,OFFSET(r, UNUSED, 0, rt->fields_indexes[o->p3])); break; } } @@ -1652,7 +1651,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { if( dst->t->kind == HSTRUCT ) { hl_type *ft = hl_obj_field_fetch(r->t,o->p2)->t; if( ft->kind == HPACKED ) { - STORE(dst, OFFSET(obj, VAL_NULL, 0, field_pos)); + STORE(dst, OFFSET(obj, UNUSED, 0, field_pos)); break; } } @@ -1863,7 +1862,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case ORefData: switch( ra->t->kind ) { case HARRAY: - STORE(dst, OFFSET(LOAD(ra),VAL_NULL,0,sizeof(varray))); + STORE(dst, OFFSET(LOAD(ra),UNUSED,0,sizeof(varray))); break; default: jit_assert(); @@ -1948,7 +1947,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hashed_name = hl_hash_gen(name, true); } // ----------------------------------------- - ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : VAL_NULL; + ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : UNUSED; emit_native_call(ctx, null_field_access ? hl_jit_null_field_access : hl_jit_null_access, &arg, null_field_access ? 1 : 0, NULL); patch_jump(ctx, jok); } @@ -1992,7 +1991,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { current_addr = LOAD_CONST_PTR(&tinf->trap_current); # else thread = emit_native_call(ctx, hl_get_thread, NULL, 0, &hlt_bytes); - current_addr = OFFSET(thread, VAL_NULL, 0, (int)(int_val)&tinf->trap_current); + current_addr = OFFSET(thread, UNUSED, 0, (int)(int_val)&tinf->trap_current); # endif STORE_MEM(st, (int)(int_val)&trap->prev, LOAD_MEM_PTR(current_addr,0)); STORE_MEM(current_addr, 0, st); @@ -2039,7 +2038,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { // On Win64 setjmp actually takes two arguments // the jump buffer and the frame pointer (or the stack pointer if there is no FP) nargs = 2; - args[1] = emit_gen(ctx,LEA,MK_STACK_REG(0),VAL_NULL,M_PTR); + args[1] = emit_gen(ctx,LEA,MK_STACK_REG(0),UNUSED,M_PTR); #endif #ifdef HL_MINGW fun = _setjmp; @@ -2070,7 +2069,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { current_addr = LOAD_CONST_PTR(&tinf->trap_current); # else thread = emit_native_call(ctx, hl_get_thread, NULL, 0, &hlt_bytes); - current_addr = OFFSET(thread, VAL_NULL, 0, (int)(int_val)&tinf->trap_current); + current_addr = OFFSET(thread, UNUSED, 0, (int)(int_val)&tinf->trap_current); # endif STORE_MEM(current_addr, 0, LOAD_MEM_PTR(st,(int)(int_val)&trap->prev)); @@ -2095,7 +2094,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { int count = o->p2; emit_cmp(ctx,v,LOAD_CONST(count,&hlt_i32),OJUGte); int jdefault = emit_jump(ctx, true); - emit_gen_ext(ctx, JUMP_TABLE, v, VAL_NULL, 0, count); + emit_gen_ext(ctx, JUMP_TABLE, v, UNUSED, 0, count); for(int i=0; iextra[i]); @@ -2120,7 +2119,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case HSTRUCT: { hl_runtime_obj *rt = hl_get_obj_rt(dst->t); - r = OFFSET(r, VAL_NULL, 0, rt->fields_indexes[o->p2-1]); + r = OFFSET(r, UNUSED, 0, rt->fields_indexes[o->p2-1]); } break; default: @@ -2128,7 +2127,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; } } - emit_gen(ctx, PREFETCH, r, VAL_NULL, o->p3); + emit_gen(ctx, PREFETCH, r, UNUSED, o->p3); } break; case OAsm: diff --git a/src/jit_regs.c b/src/jit_regs.c index 43a49001a..460f5acf5 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -77,7 +77,7 @@ static ereg get_call_reg( regs_ctx *ctx, call_regs regs, emit_mode m ) { if( regs[mode] < cfg->nargs ) r = cfg->arg[regs[mode]++]; else - r = VAL_NULL; + r = UNUSED; return r; } @@ -109,7 +109,7 @@ static void regs_emit_mov( regs_ctx *ctx, ereg to, ereg from, emit_mode m ) { e.mode = m; e.size_offs = 0; e.a = from; - e.b = VAL_NULL; + e.b = 0; regs_write_instr(ctx, &e, to); } @@ -119,13 +119,13 @@ static void regs_emit_todo_impl( regs_ctx *ctx, int line ) { e.header = PUSH_CONST; e.mode = M_I32; e.value = line; - regs_write_instr(ctx, &e, VAL_NULL); + regs_write_instr(ctx, &e, UNUSED); einstr e2; e2.header = DEBUG_BREAK; e2.size_offs = 0; - e2.a = VAL_NULL; - e2.b = VAL_NULL; - regs_write_instr(ctx, &e2, VAL_NULL); + e2.a = UNUSED; + e2.b = UNUSED; + regs_write_instr(ctx, &e2, UNUSED); } @@ -137,7 +137,7 @@ static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { for_iter(values,v2,ctx->scratch) { if( v2->current == *forced ) { if( !v2->saved ) jit_assert(); - v2->current = VAL_NULL; + v2->current = UNUSED; values_remove(&ctx->scratch,v2); break; } @@ -166,7 +166,7 @@ static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { ereg r = cfg->scratch[i]; for_iter(values,v2,ctx->scratch) { if( v2->current == r ) { - r = VAL_NULL; + r = UNUSED; break; } } @@ -203,12 +203,12 @@ static void regs_assign_stack_regs( regs_ctx *ctx ) { int nargs = ctx->jit->fun->type->fun->nargs; call_regs regs = {0}; int args_size = 0; - for(int i=0;ijit->value_count;i++) { + for(int i=1;ijit->value_count;i++) { value_info *v = VAL(i); einstr *e = ctx->jit->instrs + ctx->jit->values_writes[i]; int size = hl_emit_mode_sizes[e->mode]; if( size <= 0 ) hl_jit_assert(); - if( i < nargs ) { + if( i <= nargs ) { ereg r = get_call_reg(ctx,regs,e->mode); if( !IS_NULL(r) ) { v->current = r; @@ -301,19 +301,19 @@ void hl_regs_function( jit_ctx *jit ) { int_arr_free(&ctx->jump_regs); ctx->values = malloc(sizeof(value_info) * nvalues); memset(ctx->values, 0, sizeof(value_info) * nvalues); - for(int i=0;icurrent = VAL_NULL; - v->arg_reg = VAL_NULL; + v->current = UNUSED; + v->arg_reg = UNUSED; v->last_read = -1; - if( i < jit->value_count ) + if( i > 0 && i < jit->value_count ) v->mode = jit->instrs[jit->values_writes[i]].mode; else v->mode = M_NONE; // TODO : phi mode } regs_compute_liveness(ctx); regs_assign_stack_regs(ctx); - int write_index = 0; + int write_index = 1; for(int cur_op=0;cur_opinstr_count;cur_op++) { einstr e = jit->instrs[cur_op]; ereg *ret_val = NULL; @@ -349,7 +349,7 @@ void hl_regs_function( jit_ctx *jit ) { } if( will_scratch ) { for_iter(values,v2,ctx->scratch) - v2->current = VAL_NULL; + v2->current = UNUSED; values_reset(&ctx->scratch); } e.nargs = 0xFF; @@ -376,7 +376,7 @@ void hl_regs_function( jit_ctx *jit ) { regs_assign(ctx,v,ret_val); out = v->current; } else - out = VAL_NULL; + out = UNUSED; regs_write_instr(ctx, &e, out); switch( e.op ) { case JUMP: diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 5618a04e2..92fc9d2f3 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -28,14 +28,11 @@ #define S_VALUE unsigned char #include "data_struct.c" #define byte_reserve(set,count) byte_reserve_impl(DEF_ALLOC,&set,count) -#define VAL_CONST (VAL_NULL-1) -#define FL_MEM (FL_NATREG | 0x8000000) -#define VAL_MEM(reg) (FL_MEM | (reg)) +#define VAL_CONST 0x80000000 +#define VAL_MEM(reg) (FL_MEMPTR | (reg)) #define IS_FLOAT(mode) ((mode) == M_F64 || (mode) == M_F32) -#define UNUSED VAL_NULL - typedef enum { RAX = 0, RCX = 1, @@ -405,7 +402,7 @@ static preg make_reg( ereg r, emit_mode m ) { p.kind = RCONST; return p; } - if( (r & (FL_MEM | FL_NATMASK)) == FL_MEM ) { + if( (r & FL_NATMASK) == FL_MEMPTR ) { p.kind = RMEM; p.reg = (r&0xFFFF); p.value = 0; @@ -437,11 +434,18 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, break; case ID2(RCPU,RCPU): case ID2(RFPU,RFPU): - ERRIF( f->r_mem == 0 && f->mem_r == 0 ); - if( a.reg > 7 ) r64 |= 4; - if( b.reg > 7 ) r64 |= 1; - OP(f->mem_r ? f->mem_r : f->r_mem); - MOD_RM(3,a.reg,b.reg); + if( f->mem_r ) { + if( a.reg > 7 ) r64 |= 1; + if( b.reg > 7 ) r64 |= 4; + OP(f->mem_r); + MOD_RM(3,b.reg,a.reg); + } else { + ERRIF( f->r_mem == 0 ); + if( a.reg > 7 ) r64 |= 4; + if( b.reg > 7 ) r64 |= 1; + OP(f->r_mem); + MOD_RM(3,a.reg,b.reg); + } break; case ID2(RCPU,RFPU): case ID2(RFPU,RCPU): @@ -704,6 +708,39 @@ static void emit_jump( code_ctx *ctx, int mode, int offset ) { } } +static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode, int_val value ) { + if( (val & FL_NATMASK) == FL_STACKOFFS ) { + emit_mov(ctx,out,VAL_MEM(R(RBP)),mode, value + GET_STACK_OFFS(val)); + return; + } + if( (out & FL_NATMASK) == FL_STACKOFFS ) { + emit_mov(ctx,VAL_MEM(R(RBP)),val,mode, value + GET_STACK_OFFS(val)); + return; + } + emit_ext(ctx,mode == M_F32 ? MOVSS : mode == M_F64 ? MOVSD : _MOV,out,val,mode,value); +} + +static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_mode mode ) { + CpuOp cop; +# define F_OP(iop,f32,f64) cop = mode == M_F32 ? f32 : (mode == M_F64 ? f64 : iop); + switch( op ) { + case OAdd: + F_OP(ADD,ADDSS,ADDSD); + break; + case OIncr: + cop = INC; + break; + case ODecr: + cop = DEC; + break; + default: + jit_assert(); + break; + } + emit_mov(ctx, out, a, mode, 0); + EMIT(cop,out,b,mode); +} + void hl_codegen_flush( jit_ctx *jit ) { code_ctx *ctx = jit->code; if( ctx->flushed ) return; @@ -718,15 +755,22 @@ void hl_codegen_function( jit_ctx *jit ) { code_ctx *ctx = jit->code; ctx->flushed = false; byte_free(&ctx->code); + int_arr_free(&ctx->near_jumps); + int_arr_free(&ctx->short_jumps); free(ctx->pos_map); ctx->pos_map = (int*)malloc((jit->reg_instr_count + 1) * sizeof(int)); + ctx->pos_map[0] = 0; + byte_reserve(ctx->code,64); + ctx->code.cur -= 64; + EMIT(_PUSH,R(RBP),UNUSED,M_PTR); + EMIT(_MOV,R(RBP),R(RSP),M_PTR); for(int cur_pos=0;cur_posreg_instr_count;cur_pos++) { einstr *e = jit->reg_instrs + cur_pos; ereg out = jit->reg_writes[cur_pos]; byte_reserve(ctx->code,64); ctx->code.cur -= 64; ctx->cur_op = cur_pos; - ctx->pos_map[cur_pos] = ctx->code.cur; + if( cur_pos > 0 ) ctx->pos_map[cur_pos] = ctx->code.cur; switch( e->op ) { case LOAD_ARG: case ALLOC_STACK: @@ -735,13 +779,10 @@ void hl_codegen_function( jit_ctx *jit ) { if( (e->a & FL_NATMASK) == FL_STACKOFFS ) emit_ext(ctx,_LEA,out,VAL_MEM(R(RBP)),M_PTR,GET_STACK_OFFS(e->a)); else - EMIT(_MOV, out, e->a, e->mode); + emit_mov(ctx, out, e->a, e->mode, 0); break; case STORE: - if( (e->a & FL_NATMASK) == FL_STACKOFFS ) - emit_ext(ctx,_MOV, VAL_MEM(R(RBP)), e->b, e->mode, e->size_offs + GET_STACK_OFFS(e->a)); - else - emit_ext(ctx,_MOV, VAL_MEM(e->a), e->b, e->mode, e->size_offs); + emit_mov(ctx, VAL_MEM(e->a), e->b, e->mode, e->size_offs); break; case PUSH_CONST: emit_ext(ctx, _PUSH, VAL_CONST, UNUSED, e->mode, e->value); @@ -754,8 +795,11 @@ void hl_codegen_function( jit_ctx *jit ) { EMIT(_CALL, R(RAX), UNUSED, M_NONE); break; case RET: - if( !IS_NULL(e->a) && e->a != R(RAX) ) - EMIT(_MOV, R(RAX), e->a, e->mode); + if( !IS_NULL(e->a) ) { + ereg ret = IS_FLOAT(e->mode) ? MMX(0) : R(RAX); + if( e->a != ret ) emit_mov(ctx, ret, e->a, e->mode, 0); + } + EMIT(_POP,R(RBP),UNUSED,M_PTR); EMIT(_RET, UNUSED, UNUSED, M_NONE); break; case LOAD_CONST: @@ -765,10 +809,7 @@ void hl_codegen_function( jit_ctx *jit ) { emit_ext(ctx, _MOV, out, VAL_CONST, e->mode, e->value); break; case LOAD_ADDR: - if( (e->a & FL_NATMASK) == FL_STACKOFFS ) - emit_ext(ctx,_MOV,out,VAL_MEM(R(RBP)),e->mode, e->size_offs + GET_STACK_OFFS(e->a)); - else - emit_ext(ctx,_MOV,out,VAL_MEM(e->a),e->mode, e->size_offs); + emit_mov(ctx,out,VAL_MEM(e->a),e->mode, e->size_offs); break; case CALL_FUN: B(0xE8); @@ -784,7 +825,23 @@ void hl_codegen_function( jit_ctx *jit ) { EMIT(_TEST,e->a,e->a,e->mode); break; case CMP: - EMIT(_CMP,e->a,e->b,e->mode); + switch( e->mode ) { + case M_UI8: + EMIT(CMP8,e->a,e->b,e->mode); + break; + case M_UI16: + EMIT(CMP16,e->a,e->b,e->mode); + break; + case M_F32: + EMIT(COMISS,e->a,e->b,e->mode); + break; + case M_F64: + EMIT(COMISD,e->a,e->b,e->mode); + break; + default: + EMIT(_CMP,e->a,e->b,e->mode); + break; + } break; case JCOND: { @@ -845,6 +902,10 @@ void hl_codegen_function( jit_ctx *jit ) { case CONV: BREAK(); break; + case BINOP: + case UNOP: + emit_anyop(ctx, e->size_offs, out, e->a, e->b, e->mode); + break; default: jit_assert(); break; From 2af1e4253d0f2cf3ed4a73674d72a5d0b835f979 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 18 Apr 2026 22:28:18 +0200 Subject: [PATCH 27/83] insert BLOCK op, completed regs assign and phi mov inserts --- src/data_struct.c | 8 + src/jit.h | 5 +- src/jit_dump.c | 47 ++-- src/jit_emit.c | 67 +++--- src/jit_regs.c | 561 ++++++++++++++++++++++++++++++---------------- 5 files changed, 441 insertions(+), 247 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index aa203a286..9436ad571 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -97,6 +97,14 @@ INLINE static bool S_NAME(remove)( S_TYPE *st, S_KEY k ) { return false; } +INLINE static void S_NAME(remove_range)( S_TYPE *st, int pos, int count ) { + memmove(st->keys + pos, st->keys + pos + count, (st->cur - pos - count) * sizeof(S_KEY)); +# ifdef S_MAP + memmove(st->values + pos, st->values + pos + count, (st->cur - pos - count) * sizeof(S_VALUE)); +# endif + st->cur -= count; +} + #ifdef S_MAP static S_VALUE S_NAME(find)( S_TYPE st, S_KEY k ) { for(int i=0;isize_offs); break; + case BLOCK: + printf(" #%d", e->size_offs); + break; case LOAD_CONST: case PUSH_CONST: printf(" "); @@ -396,10 +400,9 @@ void hl_emit_dump( jit_ctx *ctx ) { int cur = 0; for(i=0;iblock_count;i++) { eblock *b = ctx->blocks + i; - if( b->id != i ) printf(" ??? BLOCK @%d ID is %d\n",i,b->id); if( b->start_pos != cur ) printf(" ??? BLOCK %d START AT %X != %X\n", i, b->start_pos, cur); if( b->end_pos < b->start_pos ) printf(" ??? BLOCK %d RANGE [%X,%X]\n", i, b->start_pos, b->end_pos); - cur = b->end_pos + 1; + cur = b->end_pos; } if( cur != ctx->instr_count ) printf(" ??? MISSING BLOCK FOR RANGE %X-%X\n", cur, ctx->instr_count); @@ -409,23 +412,6 @@ void hl_emit_dump( jit_ctx *ctx ) { int cpos = 0; cur = 0; for(i=0;iinstr_count;i++) { - while( cur < ctx->block_count && ctx->blocks[cur].start_pos == i ) { - eblock *b = &ctx->blocks[cur]; - printf("--- BLOCK #%d ---\n", cur); - for(int k=0;kphi_count;k++) { - ephi *p = b->phis + k; - printf("\t\t@%X %s = phi%s(",i,val_str(p->value,p->mode),emit_mode_str(p->mode)); - for(int n=0;nnvalues;n++) { - if( n > 0 ) printf(","); - printf("%s",val_str(p->values[n],p->mode)); - } - printf(")"); - if( p->nvalues <= 1 ) - printf(" ???"); - printf("\n"); - } - cur++; - } while( ctx->emit_pos_map[cur_op] == i ) { printf("@%X ", cur_op); hl_dump_op(ctx->fun, f->ops + cur_op); @@ -437,15 +423,34 @@ void hl_emit_dump( jit_ctx *ctx ) { if( vpos < ctx->value_count && ctx->values_writes[vpos] == i ) printf("V%d = ", vpos++); dump_instr(ctx, e, i); + if( e->op == BLOCK ) { + eblock *b = &ctx->blocks[e->size_offs]; + for(int k=0;kphi_count;k++) { + ephi *p = b->phis + k; + printf("\n\t\t@%X %s = phi%s(",i,val_str(p->value,p->mode),emit_mode_str(p->mode)); + for(int n=0;nnvalues;n++) { + if( n > 0 ) printf(","); + printf("%s",val_str(p->values[n],p->mode)); + } + printf(")"); + if( p->nvalues <= 1 ) + printf(" ???"); + } + } while( rpos < ctx->reg_instr_count && rpos < ctx->reg_pos_map[i+1] ) { ereg out = ctx->reg_writes[rpos]; e = ctx->reg_instrs + rpos; printf("\n\t\t\t\t@%X ",rpos); if( !IS_NULL(out) ) printf("%s = ",reg_str(out)); dump_instr(ctx,e,rpos); - printf("\033[80G"); - while( cpos < ctx->code_size && cpos < ctx->code_pos_map[rpos+1] ) + bool first = true; + while( cpos < ctx->code_size && cpos < ctx->code_pos_map[rpos+1] ) { + if( first ) { + printf("\033[80G"); + first = false; + } printf("%.2X",ctx->code_instrs[cpos++]); + } rpos++; } printf("\n"); diff --git a/src/jit_emit.c b/src/jit_emit.c index a08aefa57..1e0e01b44 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -87,7 +87,6 @@ struct _emit_block { int start_pos; int end_pos; int wait_nexts; - int mark; bool sealed; blocks nexts; blocks preds; @@ -137,7 +136,7 @@ struct _emit_ctx { int_arr jump_regs; int_arr values; - emit_block *root_block; + blocks blocks; emit_block *current_block; emit_block *wait_seal; linked_inf *arrival_points; @@ -259,6 +258,10 @@ static emit_mode hl_type_mode( hl_type *t ) { return sizeof(bool) == 1 ? M_UI8 : M_I32; if( t->kind == HGUID ) return M_I64; + if( t->kind == HF32 ) + return M_F32; + if( t->kind == HF64 ) + return M_F64; return M_PTR; } @@ -390,7 +393,12 @@ static void patch_jump( emit_ctx *ctx, int jpos ) { } static emit_block *alloc_block( emit_ctx *ctx ) { - return hl_zalloc(&ctx->jit->falloc, sizeof(emit_block)); + emit_block *b = hl_zalloc(&ctx->jit->falloc, sizeof(emit_block)); + b->id = blocks_count(ctx->blocks); + b->start_pos = ctx->emit_pos; + blocks_add(ctx->blocks, b); + if( b->id > 0 ) emit_gen_size(ctx, BLOCK, b->id); + return b; } static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { @@ -411,18 +419,16 @@ static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { static void split_block( emit_ctx *ctx ) { emit_block *b = alloc_block(ctx); b->sealed = true; - b->id = ctx->current_block->id + 1; - b->start_pos = ctx->emit_pos; emit_debug("BLOCK #%d@%X[%X]\n",b->id,b->start_pos,ctx->op_pos); while( ctx->arrival_points && ctx->arrival_points->id == ctx->op_pos ) { block_add_pred(ctx, b, (emit_block*)ctx->arrival_points->ptr); ctx->arrival_points = ctx->arrival_points->next; } bool dead_code = blocks_count(b->preds) == 0; // if we have no reach, force previous block dependency, this is rare dead code emit by compiler - einstr *eprev = &ctx->instrs[ctx->emit_pos-1]; + einstr *eprev = &ctx->instrs[b->start_pos-1]; if( (eprev->op != JUMP && eprev->op != RET && eprev->mode != M_NORET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) block_add_pred(ctx, b, ctx->current_block); - ctx->current_block->end_pos = ctx->emit_pos - 1; + ctx->current_block->end_pos = b->start_pos; ctx->current_block = b; } @@ -432,7 +438,7 @@ static void register_jump( emit_ctx *ctx, int jpos, int offs ) { int_arr_add(ctx->jump_regs, target); if( offs > 0 ) { ctx->arrival_points = link_add_sort_unique(ctx, target, ctx->current_block, ctx->arrival_points); - if( ctx->arrival_points->id != ctx->op_pos + 1 && ctx->fun->ops[ctx->op_pos].op != OSwitch ) + if( ctx->arrival_points->id != ctx->op_pos + 1 && ctx->fun->ops[ctx->op_pos].op != OSwitch && ctx->fun->ops[ctx->op_pos+1].op != OLabel ) split_block(ctx); } } @@ -487,7 +493,7 @@ static void emit_cmp( emit_ctx *ctx, ereg a, ereg b, hl_op o ) { static void phi_remove_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { ereg_remove(&p->vals,v); - emit_debug("%sPHI-REM-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); + emit_debug("%sPHI-REM-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(v,p->mode)); } static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { @@ -497,7 +503,7 @@ static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { return; if( !ereg_add(p->vals,v) ) return; - emit_debug("%sPHI-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(v)); + emit_debug("%sPHI-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(v,p->mode)); if( v < 0 ) { tmp_phi *p2 = GET_PHI(v); phi_add(p2->ref_phis,p); @@ -523,7 +529,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { if( p->locked || p->opt ) jit_assert(); - emit_debug("%sPHI-OPT %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); + emit_debug("%sPHI-OPT %s = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(same,p->mode)); p->opt = true; ctx->phi_depth++; linked_inf *l = p->ref_blocks; @@ -543,7 +549,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { for(int i=0;iphi_depth--; - emit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value), val_str(same)); + emit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(same,p->mode)); return optimize_phi_rec(ctx,p); } @@ -565,7 +571,7 @@ static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { return v; if( !b->sealed ) { tmp_phi *p = alloc_phi(ctx,b,r); - emit_debug("%sPHI-SEALED %s = R%d\n",phi_prefix(ctx),val_str(p->value),r->id); + emit_debug("%sPHI-SEALED %s = R%d\n",phi_prefix(ctx),val_str(p->value,p->mode),r->id); v = p->value; } else if( blocks_count(b->preds) == 1 ) v = emit_load_reg_block(ctx, blocks_get(b->preds,0), r); @@ -578,21 +584,6 @@ static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { return v; } -static void emit_walk_blocks_rec( emit_ctx *ctx, emit_block *b, int mark, void (*fun)(emit_ctx*,emit_block*) ) { - if( b->mark == mark ) return; - b->mark = mark; - fun(ctx, b); - for_iter(blocks,n,b->nexts) - emit_walk_blocks_rec(ctx,n,mark,fun); -} - -static void emit_walk_blocks( emit_ctx *ctx, void (*fun)(emit_ctx*,emit_block*) ) { - static int MARK_UID = 0; - int mark = ++MARK_UID; - if( mark == 0 ) mark = ++MARK_UID; - emit_walk_blocks_rec(ctx, ctx->root_block, mark, fun); -} - static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { if( !IS_NULL(r->ref) ) return LOAD_MEM(r->ref,0,r->t); @@ -771,8 +762,7 @@ static void remap_phi_reg( emit_ctx *ctx, ereg *r ) { static void emit_write_block( emit_ctx *ctx, emit_block *b ) { jit_ctx *jit = ctx->jit; eblock *bl = jit->blocks + b->id; - bl->id = b->id; - bl->start_pos = b->start_pos; + bl->start_pos = b->id == 0 ? 0 : b->start_pos; bl->end_pos = b->end_pos; bl->pred_count = blocks_count(b->preds); bl->next_count = blocks_count(b->nexts); @@ -822,18 +812,17 @@ void hl_emit_flush( jit_ctx *jit ) { e->size_offs = ctx->pos_map[target] - (pos + 1); } ctx->pos_map[ctx->fun->nops] = -1; - ctx->current_block->end_pos = ctx->emit_pos - 1; + ctx->current_block->end_pos = ctx->emit_pos; jit->instrs = ctx->instrs; jit->instr_count = ctx->emit_pos; jit->emit_pos_map = ctx->pos_map; jit->phi_count = 0; jit->block_count = ctx->current_block->id + 1; jit->blocks = hl_zalloc(&jit->falloc,sizeof(eblock) * jit->block_count); - for(i=0;iblock_count;i++) - jit->blocks[i].id = -1; jit->value_count = int_arr_count(ctx->values); jit->values_writes = ctx->values.values; - emit_walk_blocks(ctx,emit_write_block); + for_iter(blocks,b,ctx->blocks) + emit_write_block(ctx,b); } void hl_emit_reg_iter( jit_ctx *jit, einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ) { @@ -926,8 +915,9 @@ void hl_emit_function( jit_ctx *jit ) { int_arr_free(&ctx->args_data); int_arr_free(&ctx->jump_regs); int_arr_free(&ctx->values); + blocks_free(&ctx->blocks); int_arr_add(ctx->values,-1); - ctx->root_block = ctx->current_block = alloc_block(ctx); + ctx->current_block = alloc_block(ctx); ctx->current_block->sealed = true; ctx->arrival_points = NULL; emit_debug("---- begin [%X] ----\n",f->findex); @@ -962,6 +952,10 @@ void hl_emit_function( jit_ctx *jit ) { for(int op_pos=0;op_posnops;op_pos++) { ctx->op_pos = op_pos; ctx->pos_map[op_pos] = ctx->emit_pos; + if( op_pos == 0 ) { + ctx->current_block->start_pos = ctx->emit_pos; + emit_gen_size(ctx, BLOCK, 0); + } if( ctx->arrival_points && ctx->arrival_points->id == op_pos ) split_block(ctx); emit_opcode(ctx,f->ops + op_pos); @@ -969,6 +963,7 @@ void hl_emit_function( jit_ctx *jit ) { hl_emit_clean_phis(ctx); hl_emit_flush(ctx->jit); + if( ctx->wait_seal ) jit_assert(); } void hl_emit_alloc( jit_ctx *jit ) { @@ -1759,7 +1754,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case OLabel: - if( ctx->current_block->start_pos != ctx->emit_pos ) + if( ctx->current_block->start_pos != ctx->emit_pos-1 ) split_block(ctx); prepare_loop_block(ctx); break; diff --git a/src/jit_regs.c b/src/jit_regs.c index 460f5acf5..c2a51afbd 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -31,16 +31,28 @@ # define IS_WINCALL64 0 #endif + +//#define REGS_DEBUG + +#ifdef REGS_DEBUG +# define regs_debug jit_debug +#else +# define regs_debug(...) +#endif + +#define INVALID 0x80000000 + #define VIDX(e) (((e) < 0) ? ctx->jit->value_count + (-(e)-1) : (e)) +#define VAL_REG(e) VAL(VIDX(e)) #define REG_MODE(m) (((m) == M_F32 || (m) == M_F64) ? 1 :0) +#define IS_PURE(e) ((e) != UNUSED && ((e)&(FL_MEMPTR | FL_STACK)) == 0) typedef struct { int stack_pos; int last_read; emit_mode mode; - ereg arg_reg; - ereg current; - bool saved; + ereg pref_reg; + ereg reg; } value_info; #define S_TYPE values @@ -55,11 +67,12 @@ struct _regs_ctx { value_info *values; values scratch; int_arr jump_regs; + int_arr phi_movs; + int_arr *blocks_phis; int max_instrs; int cur_op; int emit_pos; int stack_size; - int persists_uses[2]; einstr *instrs; ereg *out_write; int *pos_map; @@ -128,229 +141,345 @@ static void regs_emit_todo_impl( regs_ctx *ctx, int line ) { regs_write_instr(ctx, &e2, UNUSED); } +static int regs_alloc_stack( regs_ctx *ctx, int size ) { + ctx->stack_size += size; + ctx->stack_size += jit_pad_size(ctx->stack_size,size); + return -ctx->stack_size; +} + +#define value_str(v) value_to_str(ctx,v) + +static const char *value_to_str( regs_ctx *ctx, value_info *v ) { + static char out[20]; + int idx = (int)(v - ctx->values); + ereg e = (ereg)(idx >= ctx->jit->value_count ? -(idx-ctx->jit->value_count) - 1 : idx); + sprintf(out,"%s:%s", val_str(e,v->mode), val_str(v->reg,v->mode)); + return out; +} + +static void spill( regs_ctx *ctx, value_info *v ) { + if( v->reg == UNUSED ) jit_assert(); + if( v->stack_pos == INVALID ) v->stack_pos = regs_alloc_stack(ctx, hl_emit_mode_sizes[v->mode]); + v->reg = MK_STACK_REG(v->stack_pos); + values_remove(&ctx->scratch,v); + regs_debug("REG SPILL %s @%X\n",value_str(v),ctx->cur_op); +} -static void regs_assign( regs_ctx *ctx, value_info *v, ereg *forced ) { - if( forced ) { - if( v->current == *forced ) - return; - // erase previous value +static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { + // lookup available reg + int mode = REG_MODE(v->mode); + reg_config *cfg = &ctx->cfg[mode]; + if( !IS_NULL(v->pref_reg) ) { + bool free = true; for_iter(values,v2,ctx->scratch) { - if( v2->current == *forced ) { - if( !v2->saved ) jit_assert(); - v2->current = UNUSED; - values_remove(&ctx->scratch,v2); + if( v2->reg == v->pref_reg ) { + free = false; break; } } - v->current = *forced; - values_add(ctx->scratch,v); - } else { - // lookup available reg - int mode = REG_MODE(v->mode); - reg_config *cfg = &ctx->cfg[mode]; - if( !IS_NULL(v->arg_reg) ) { - bool free = true; - for_iter(values,v2,ctx->scratch) { - if( v2->current == v->arg_reg ) { - free = false; - break; - } - } - if( free ) { - v->current = v->arg_reg; - values_add(ctx->scratch,v); - return; - } + if( free ) { + v->reg = v->pref_reg; + return true; } - for(int i=0;inscratchs;i++) { - ereg r = cfg->scratch[i]; - for_iter(values,v2,ctx->scratch) { - if( v2->current == r ) { - r = UNUSED; - break; - } - } - if( !IS_NULL(r) ) { - v->current = r; - values_add(ctx->scratch,v); - return; + } + for(int i=0;inscratchs;i++) { + ereg r = cfg->scratch[i]; + for_iter(values,v2,ctx->scratch) { + if( v2->reg == r ) { + r = UNUSED; + break; } } - if( ctx->persists_uses[mode] < cfg->npersists ) { - v->current = cfg->persist[ctx->persists_uses[mode]++]; - return; + if( !IS_NULL(r) ) { + v->reg = r; + return true; } - // free the oldest scratch reg - value_info *v2 = values_first(ctx->scratch); - if( !v2->saved ) { - regs_emit_mov(ctx, MK_STACK_REG(v2->stack_pos), v2->current, v2->mode); - v2->saved = true; - } - v->current = v2->current; - values_remove(&ctx->scratch, v2); - values_add(ctx->scratch, v); } + if( ctx->jit->persists_uses[mode] < cfg->npersists ) { + v->reg = cfg->persist[ctx->jit->persists_uses[mode]++]; + return false; + } + // free the oldest scratch reg + value_info *v2 = values_first(ctx->scratch); + v->reg = v2->reg; + spill(ctx, v2); + return true; } -static int regs_alloc_stack( regs_ctx *ctx, int size ) { - ctx->stack_size += size; - ctx->stack_size += jit_pad_size(ctx->stack_size,size); - return -ctx->stack_size; -} - -static void regs_assign_stack_regs( regs_ctx *ctx ) { - // dummy regs assign for testing : set all values on stack - int nargs = ctx->jit->fun->type->fun->nargs; - call_regs regs = {0}; - int args_size = 0; - for(int i=1;ijit->value_count;i++) { - value_info *v = VAL(i); - einstr *e = ctx->jit->instrs + ctx->jit->values_writes[i]; - int size = hl_emit_mode_sizes[e->mode]; - if( size <= 0 ) hl_jit_assert(); - if( i <= nargs ) { - ereg r = get_call_reg(ctx,regs,e->mode); - if( !IS_NULL(r) ) { - v->current = r; - values_add(ctx->scratch,v); - } - if( IS_NULL(r) || IS_WINCALL64 ) { - // use existing stack storage - v->stack_pos = args_size + HL_WSIZE*2; - args_size += size < 4 ? 4 : size; - continue; - } - } - v->stack_pos = MK_STACK_REG(regs_alloc_stack(ctx,size)); - } - for(int i=0;ijit->block_count;i++) { - eblock *b = ctx->jit->blocks + i; - for(int k=0;kphi_count;k++) { - ephi *p = b->phis + k; - int size = hl_emit_mode_sizes[p->mode]; - value_info *v = VAL(VIDX(p->value)); - v->stack_pos = MK_STACK_REG(regs_alloc_stack(ctx,size)); - } - } +static void regs_assign( regs_ctx *ctx, value_info *v ) { + if( v->reg != UNUSED ) jit_assert(); + if( regs_alloc_reg(ctx, v) ) + values_add(ctx->scratch, v); + regs_debug("REG ASSIGN %s @%X\n",value_str(v),ctx->cur_op); } static void regs_write_live( regs_ctx *ctx, ereg *r ) { if( IS_NULL(*r) ) jit_assert(); if( IS_NATREG(*r) ) return; - value_info *v = VAL(VIDX(*r)); + value_info *v = VAL_REG(*r); v->last_read = ctx->cur_op; } +static value_info *regs_current( regs_ctx *ctx, ereg r ) { + for_iter(values,v,ctx->scratch) { + if( v->reg == r ) + return v; + } + return NULL; +} + +static eblock *resolve_block_value( regs_ctx *ctx, ereg v ) { + if( v < 0 ) { + for(int i=0;ijit->block_count;i++) { + eblock *b = ctx->jit->blocks + i; + for(int p=0;pphi_count;p++) { + if( b->phis[p].value == v ) + return b; + } + } + } else { + if( IS_NATREG(v) ) jit_assert(); + int idx = ctx->jit->values_writes[v]; + int min = 0; + int max = ctx->jit->block_count; + while( min < max ) { + int med = (min + max) >> 1; + eblock *b = ctx->jit->blocks + med; + if( idx < b->start_pos ) { + max = med; + } else if( idx >= b->end_pos ) { + min = med; + } else + return b; + } + } + jit_assert(); +} + static void regs_compute_liveness( regs_ctx *ctx ) { - for(int i=0;ijit->instr_count;i++) { - einstr *e = ctx->jit->instrs + i; - ctx->cur_op = i; - hl_emit_reg_iter(ctx->jit,e,ctx,regs_write_live); + int write_index = 1; + jit_ctx *jit = ctx->jit; + hl_type *tret = ctx->jit->fun->type->fun->ret; + emit_mode mret = tret->kind == HF32 || tret->kind == HF64 ? M_F64 : M_PTR; + ereg ret = ctx->cfg[REG_MODE(mret)].ret; + for(int cur_op=0;cur_opinstr_count;cur_op++) { + einstr *e = jit->instrs + cur_op; + value_info *write = NULL; + + if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) + write = VAL(write_index++); + + ctx->cur_op = cur_op; + hl_emit_reg_iter(jit,e,ctx,regs_write_live); if( IS_CALL(e->op) ) { // anticipate register usage in call so we can previlege this assign - ereg *r = hl_emit_get_args(ctx->jit->emit, e); + ereg *r = hl_emit_get_args(jit->emit, e); call_regs regs = {0}; for(int k=0;knargs;k++) { - value_info *v = VAL(VIDX(r[k])); - if( !IS_NULL(v->arg_reg) ) continue; - v->arg_reg = get_call_reg(ctx,regs,v->mode); + value_info *v = VAL_REG(r[k]); + if( !IS_NULL(v->pref_reg) ) continue; + v->pref_reg = get_call_reg(ctx,regs,v->mode); } + if( write && IS_NULL(write->pref_reg) ) + write->pref_reg = ctx->cfg[REG_MODE(e->mode)].ret; + } else switch( e->op ) { + case RET: + if( e->a ) { + value_info *v = VAL_REG(e->a); + if( v->pref_reg == UNUSED ) v->pref_reg = ret; + } + break; + default: + break; } } -} - -void hl_regs_alloc( jit_ctx *jit ) { - regs_ctx *ctx = malloc(sizeof(regs_ctx)); - memset(ctx,0,sizeof(regs_ctx)); - ctx->jit = jit; - jit->regs = ctx; - hl_jit_init_regs(ctx->cfg); -} - -void hl_regs_flush( jit_ctx *jit ) { - regs_ctx *ctx = jit->regs; - if( ctx->flushed ) return; - ctx->flushed = true; - jit->reg_instr_count = ctx->emit_pos; - jit->reg_instrs = ctx->instrs; - jit->reg_writes = ctx->out_write; - jit->reg_pos_map = ctx->pos_map; - if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; - int i = 0; - while( i < int_arr_count(ctx->jump_regs) ) { - int pos = int_arr_get(ctx->jump_regs,i++); - einstr *e = ctx->instrs + pos; - int target = int_arr_get(ctx->jump_regs,i++); - e->size_offs = ctx->pos_map[target] - (pos + 1); + // compute reverse phis + for(int b=0;bblock_count;b++) { + eblock *bl = jit->blocks + b; + for(int p=0;pphi_count;p++) { + ephi *ph = bl->phis + p; + for(int k=0;knvalues;k++) { + ereg v = ph->values[k]; + eblock *b2 = resolve_block_value(ctx, v); + int_arr *arr = &ctx->blocks_phis[b2 - jit->blocks]; + regs_debug("ADD PHI %s:=%s to #%d\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks)); + int_arr_add(*arr,v); + int_arr_add(*arr,ph->value); + } + } } } -void hl_regs_function( jit_ctx *jit ) { - regs_ctx *ctx = jit->regs; - int nvalues = jit->value_count + jit->phi_count; - memset(ctx->persists_uses,0,sizeof(ctx->persists_uses)); - free(ctx->pos_map); - ctx->flushed = false; - ctx->pos_map = (int*)malloc((jit->instr_count + 1) * sizeof(int)); - ctx->emit_pos = 0; - ctx->cur_op = 0; - ctx->stack_size = 0; - ctx->pos_map[0] = 0; - jit->reg_instrs = NULL; - values_free(&ctx->scratch); - int_arr_free(&ctx->jump_regs); - ctx->values = malloc(sizeof(value_info) * nvalues); - memset(ctx->values, 0, sizeof(value_info) * nvalues); - for(int i=1;ijit; + // assign args + call_regs regs = {0}; + int args_size = 0; + for(int i=1;i<=ctx->jit->fun->type->fun->nargs;i++) { value_info *v = VAL(i); - v->current = UNUSED; - v->arg_reg = UNUSED; - v->last_read = -1; - if( i > 0 && i < jit->value_count ) - v->mode = jit->instrs[jit->values_writes[i]].mode; - else - v->mode = M_NONE; // TODO : phi mode + einstr *e = ctx->jit->instrs + ctx->jit->values_writes[i]; + int size = hl_emit_mode_sizes[e->mode]; + if( size <= 0 ) hl_jit_assert(); + ereg r = get_call_reg(ctx,regs,e->mode); + if( !IS_NULL(r) ) { + v->reg = r; + values_add(ctx->scratch,v); + } + if( IS_NULL(r) || IS_WINCALL64 ) { + // use existing stack storage + v->stack_pos = args_size + HL_WSIZE*2; + args_size += size < 4 ? 4 : size; + } } - regs_compute_liveness(ctx); - regs_assign_stack_regs(ctx); + // assign registers int write_index = 1; for(int cur_op=0;cur_opinstr_count;cur_op++) { einstr e = jit->instrs[cur_op]; - ereg *ret_val = NULL; - int nread; ctx->cur_op = cur_op; - if( cur_op > 0 ) - ctx->pos_map[cur_op] = ctx->emit_pos; for_iter_back(values,v,ctx->scratch) { if( v->last_read < cur_op ) values_remove(&ctx->scratch,v); } if( IS_CALL(e.op) ) { ereg *args = hl_emit_get_args(ctx->jit->emit,&e); + call_regs regs = {0}; bool will_scratch = e.mode != M_NORET; if( will_scratch ) { - for_iter(values,v,ctx->scratch) { - if( !v->saved && v->last_read > cur_op ) { - v->saved = true; - regs_emit_mov(ctx, MK_STACK_REG(v->stack_pos), v->current, v->mode); + for_iter_back(values,v2,ctx->scratch) { + if( v2->last_read > cur_op ) + spill(ctx,v2); + } + } + for(int k=0;kmode); + if( !IS_NULL(r) ) { + value_info *cur = regs_current(ctx,r); + if( cur && cur != v ) + spill(ctx,cur); + } + } + if( will_scratch ) values_reset(&ctx->scratch); + } + if( e.op == BLOCK ) { + eblock *bl = jit->blocks + e.size_offs; + for(int k=0;kphi_count;k++) { + ephi *p = bl->phis + k; + value_info *v = VAL_REG(p->value); + for(int n=0;nnvalues;n++) { + value_info *vn = VAL_REG(p->values[n]); + // ignore previously set pref_reg (minimize moves) + if( IS_PURE(vn->reg) && !regs_current(ctx,vn->reg) ) { + v->pref_reg = vn->reg; + break; + } + } + regs_assign(ctx, v); + } + } + if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) { + value_info *w = VAL(write_index++); + switch( e.op ) { + case ALLOC_STACK: + w->reg = MK_STACK_OFFS(regs_alloc_stack(ctx, e.size_offs)); + break; + case LOAD_ARG: + if( w->reg == UNUSED ) + regs_assign(ctx, w); // assign for stack reg + break; + case UNOP: + case BINOP: + if( w->pref_reg == UNUSED ) { + value_info *v = VAL_REG(e.a); + if( IS_PURE(v->reg) ) { + values_remove(&ctx->scratch,v); + w->pref_reg = v->reg; } } + // fallback + default: + regs_assign(ctx, w); + break; + } + } + } +} + +static void flush_phis( regs_ctx *ctx, eblock *b ) { + if( !b ) return; + jit_ctx *jit = ctx->jit; + int bid = (int)(b - jit->blocks); + int_arr arr = ctx->blocks_phis[bid]; + int idx = 0; + int_arr movs = ctx->phi_movs; + while( idx < int_arr_count(arr) ) { + ereg a = int_arr_get(arr,idx++); + ereg b = int_arr_get(arr,idx++); + value_info *from = VAL_REG(a); + value_info *to = VAL_REG(b); + if( from->reg == to->reg ) continue; + int size = int_arr_count(movs); + bool dup = false; + for(int k=0;kreg == to->reg && int_arr_get(movs,k+1) == from->reg ) { + dup = true; + break; + } + } + if( !dup ) { + int_arr_add(movs, b); + int_arr_add(movs, from->reg); + } + } + int_arr_free(&ctx->blocks_phis[bid]); + while( true ) { + int size = int_arr_count(movs); + if( !size ) break; + bool cycle = true; + for(int k=0;kreg ) { + read = true; + break; + } + } + if( !read ) { + regs_emit_mov(ctx,to->reg,int_arr_get(movs,k+1),to->mode); + int_arr_remove_range(&movs,k,2); + cycle = false; + break; } + } + if( cycle ) + jit_assert(); + } + ctx->phi_movs = movs; + int_arr_reset(&ctx->phi_movs); +} + +static void regs_emit_instrs( regs_ctx *ctx ) { + jit_ctx *jit = ctx->jit; + eblock *cur_block = NULL; + int write_index = 1; + ctx->pos_map[0] = 0; + for(int cur_op=0;cur_opinstr_count;cur_op++) { + einstr e = jit->instrs[cur_op]; + ereg *ret_val = NULL; + int nread; + ctx->cur_op = cur_op; + if( IS_CALL(e.op) ) { + ereg *args = hl_emit_get_args(ctx->jit->emit,&e); call_regs regs = {0}; for(int k=0;kmode); if( IS_NULL(r) ) { regs_todo(); - } else if( IS_NULL(v->current) ) - regs_emit_mov(ctx, r, MK_STACK_REG(v->stack_pos), v->mode); - else - regs_emit_mov(ctx, r, v->current, v->mode); - } - if( will_scratch ) { - for_iter(values,v2,ctx->scratch) - v2->current = UNUSED; - values_reset(&ctx->scratch); + } else + regs_emit_mov(ctx, r, v->reg, v->mode); } e.nargs = 0xFF; ret_val = &ctx->cfg[REG_MODE(e.mode)].ret; @@ -360,24 +489,21 @@ void hl_regs_function( jit_ctx *jit ) { ereg *r = regs[k]; if( IS_NATREG(*r) ) continue; value_info *v = VAL(VIDX(*r)); - if( IS_NULL(v->current) ) { - regs_assign(ctx,v,NULL); - regs_emit_mov(ctx, v->current, MK_STACK_REG(v->stack_pos), v->mode); - } - *r = v->current; + *r = v->reg; } } ereg out; - if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) { - value_info *v = VAL(write_index++); - if( e.op == ALLOC_STACK ) - v->current = MK_STACK_OFFS(regs_alloc_stack(ctx, e.size_offs)); - else - regs_assign(ctx,v,ret_val); - out = v->current; - } else + if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) + out = VAL(write_index++)->reg; + else out = UNUSED; - regs_write_instr(ctx, &e, out); + if( e.op == JUMP || e.op == JCOND ) + flush_phis(ctx,cur_block); + if( e.op == BLOCK ) { + flush_phis(ctx,cur_block); + cur_block = jit->blocks + e.size_offs; + } else + regs_write_instr(ctx, &e, out); switch( e.op ) { case JUMP: case JCOND: @@ -385,11 +511,70 @@ void hl_regs_function( jit_ctx *jit ) { int_arr_add(ctx->jump_regs, cur_op + 1 + e.size_offs); break; } + ctx->pos_map[cur_op+1] = ctx->emit_pos; + } +} + +void hl_regs_flush( jit_ctx *jit ) { + regs_ctx *ctx = jit->regs; + if( ctx->flushed ) return; + ctx->flushed = true; + jit->reg_instr_count = ctx->emit_pos; + jit->reg_instrs = ctx->instrs; + jit->reg_writes = ctx->out_write; + jit->reg_pos_map = ctx->pos_map; + if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; + int i = 0; + while( i < int_arr_count(ctx->jump_regs) ) { + int pos = int_arr_get(ctx->jump_regs,i++); + einstr *e = ctx->instrs + pos; + int target = int_arr_get(ctx->jump_regs,i++); + e->size_offs = ctx->pos_map[target] - (pos + 1); } - free(ctx->values); +} + +void hl_regs_function( jit_ctx *jit ) { + regs_ctx *ctx = jit->regs; + int nvalues = jit->value_count + jit->phi_count; + memset(jit->persists_uses,0,sizeof(jit->persists_uses)); + free(ctx->pos_map); + ctx->flushed = false; + ctx->pos_map = (int*)malloc((jit->instr_count + 1) * sizeof(int)); + ctx->emit_pos = 0; + ctx->cur_op = 0; + ctx->stack_size = 0; + jit->reg_instrs = NULL; + values_free(&ctx->scratch); + int_arr_free(&ctx->jump_regs); + int_arr_free(&ctx->phi_movs); + ctx->blocks_phis = (int_arr*)hl_zalloc(&jit->falloc,sizeof(int_arr) * jit->block_count); + ctx->values = (value_info*)hl_zalloc(&jit->falloc,sizeof(value_info) * nvalues); + for(int i=1;ireg = UNUSED; + v->pref_reg = UNUSED; + v->stack_pos = INVALID; + v->last_read = -1; + if( i < jit->value_count ) + v->mode = jit->instrs[jit->values_writes[i]].mode; + else + v->mode = M_NONE; // TODO : phi mode + } + regs_compute_liveness(ctx); + regs_assign_regs(ctx); + regs_emit_instrs(ctx); hl_regs_flush(ctx->jit); } + +void hl_regs_alloc( jit_ctx *jit ) { + regs_ctx *ctx = malloc(sizeof(regs_ctx)); + memset(ctx,0,sizeof(regs_ctx)); + ctx->jit = jit; + jit->regs = ctx; + hl_jit_init_regs(ctx->cfg); +} + void hl_regs_free( regs_ctx *ctx ) { free(ctx->pos_map); free(ctx->instrs); From 0720e2aae132973e35b17a86278a182e393d904f Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 19 Apr 2026 10:42:52 +0200 Subject: [PATCH 28/83] more work on code emit and reg assign --- src/jit.h | 19 +++- src/jit_dump.c | 30 +++++- src/jit_emit.c | 8 +- src/jit_regs.c | 75 ++++++++++----- src/jit_x86_64.c | 239 +++++++++++++++++++++++++++++++++++++---------- src/module.c | 1 + 6 files changed, 289 insertions(+), 83 deletions(-) diff --git a/src/jit.h b/src/jit.h index 737e4694e..a384f433f 100644 --- a/src/jit.h +++ b/src/jit.h @@ -100,6 +100,7 @@ typedef struct { #define FL_STACKOFFS (FL_NATREG | FL_STACK) #define IS_NULL(e) ((e) == 0) #define IS_NATREG(e) (((e) & (0x80000000 | FL_NATREG)) == FL_NATREG) +#define IS_PURE(e) ((e) != UNUSED && ((e)&(FL_MEMPTR | FL_STACK)) == 0) #define MK_STACK_REG(v) (((v)&0xFFFFFFF) | FL_STACKREG) #define MK_STACK_OFFS(v)(((v)&0xFFFFFFF) | FL_STACKOFFS) #define GET_STACK_OFFS(v) ((int)(((v) & 0x8000000) ? ((v) | 0xF0000000) : ((v)&0xFFFFFFF))) @@ -147,7 +148,13 @@ typedef struct { ereg *arg; } reg_config; -typedef reg_config regs_config[2]; +typedef struct { + reg_config regs; + reg_config floats; + ereg req_bit_shifts; + ereg req_div_a; + ereg req_div_b; +} regs_config; struct _jit_ctx { hl_module *mod; @@ -168,6 +175,7 @@ struct _jit_ctx { int *emit_pos_map; // regs output int reg_instr_count; + int reg_stack_usage; einstr *reg_instrs; ereg *reg_writes; int *reg_pos_map; @@ -195,6 +203,7 @@ ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); ereg **hl_emit_get_regs( einstr *e, int *count ); void hl_emit_reg_iter( jit_ctx *jit, einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ); extern int hl_emit_mode_sizes[]; +extern bool hl_jit_dump_bin; #define val_str(v,m) hl_emit_regstr(v,m) #ifdef HL_DEBUG @@ -210,10 +219,18 @@ extern int hl_emit_mode_sizes[]; # define jit_debug(...) #endif +#if defined(HL_WIN_CALL) && defined(HL_64) +# define IS_WINCALL64 1 +#else +# define IS_WINCALL64 0 +#endif + #define DEF_ALLOC &ctx->jit->falloc #define jit_pad_size(size,k) ((k == 0) ? 0 : ((-(size)) & (k - 1))) +static void __ignore( void *value ) {} + void hl_jit_error( const char *msg, const char *func, int line ); void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); diff --git a/src/jit_dump.c b/src/jit_dump.c index 2b4a9a0ac..eda6e396c 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -50,6 +50,8 @@ static const char *op_names[] = { "block", }; +bool hl_jit_dump_bin = false; + const char *hl_natreg_str( int reg, emit_mode m ); const char *hl_emit_regstr( ereg v, emit_mode m ) { @@ -64,15 +66,15 @@ const char *hl_emit_regstr( ereg v, emit_mode m ) { else if( (v&FL_STACKREG) == FL_STACKREG ) { int index = GET_STACK_OFFS(v); if( index < 0 ) - sprintf(fmt,"[ST%d]", index); + sprintf(fmt,"[ST-%Xh]", -index); else - sprintf(fmt,"[ST+%d]", index); + sprintf(fmt,"[ST+%Xh]", index); } else if( (v&FL_STACKOFFS) == FL_STACKOFFS ) { int index = GET_STACK_OFFS(v); if( index < 0 ) - sprintf(fmt,"ST%d", index); + sprintf(fmt,"ST-%Xh", -index); else - sprintf(fmt,"ST+%d", index); + sprintf(fmt,"ST+%Xh", index); } else if( IS_NATREG(v) ) sprintf(fmt,"%s", hl_natreg_str(v,m)); else @@ -366,6 +368,11 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { case CONV_UNSIGNED: printf("%s %s", emit_mode_str(e->size_offs), val_str(e->a,(emit_mode)e->size_offs)); break; + case LEA: + printf(" %s", reg_str(e->a)); + if( !IS_NULL(e->b) ) printf("+%s", reg_str(e->b)); + if( e->size_offs > 1 ) printf("*%d",e->size_offs); + break; default: if( !IS_NULL(e->a) ) { printf(" %s", reg_str(e->a)); @@ -446,7 +453,10 @@ void hl_emit_dump( jit_ctx *ctx ) { bool first = true; while( cpos < ctx->code_size && cpos < ctx->code_pos_map[rpos+1] ) { if( first ) { - printf("\033[80G"); + if( hl_jit_dump_bin ) + printf("\t\t\t"); + else + printf("\033[80G"); first = false; } printf("%.2X",ctx->code_instrs[cpos++]); @@ -464,6 +474,16 @@ void hl_emit_dump( jit_ctx *ctx ) { hl_dump_op(ctx->fun, f->ops + cur_op); printf("\n\t\t...\n"); } + if( cpos == ctx->code_size && cpos > 0 ) { + int n = 1; + for(int i=0;icode_pos_map[n] == i ) { + if( (n & 15) == 0 ) printf("\n"); else printf(" "); + n++; + } + printf("%.2X", ctx->code_instrs[i]); + } + } printf("\n\n"); fflush(stdout); } diff --git a/src/jit_emit.c b/src/jit_emit.c index 1e0e01b44..4054fa421 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -31,7 +31,6 @@ # define emit_debug(...) #endif -static void __ignore( void *value ) {} int hl_emit_mode_sizes[] = {0,1,2,4,8,HL_WSIZE,8,4,0,0}; typedef struct { @@ -722,12 +721,15 @@ static bool dyn_need_type( hl_type *t ) { static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { if( t->kind == HNULL && t->tparam->kind == dt->kind ) { emit_test(ctx, v, OJNotNull); - int jnot = emit_jump(ctx, false); + int jnot = emit_jump(ctx, true); + split_block(ctx); ereg v1 = LOAD_CONST(0,dt); - int jend = emit_jump(ctx, true); + int jend = emit_jump(ctx, false); patch_jump(ctx, jnot); + split_block(ctx); ereg v2 = LOAD_MEM(v,0,dt); patch_jump(ctx, jend); + split_block(ctx); return emit_phi(ctx, v1, v2); } bool need_dyn = dyn_need_type(dt); diff --git a/src/jit_regs.c b/src/jit_regs.c index c2a51afbd..f94f66945 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -25,13 +25,6 @@ #define VAL(k) (ctx->values + (k)) -#if defined(HL_WIN_CALL) && defined(HL_64) -# define IS_WINCALL64 1 -#else -# define IS_WINCALL64 0 -#endif - - //#define REGS_DEBUG #ifdef REGS_DEBUG @@ -45,7 +38,7 @@ #define VIDX(e) (((e) < 0) ? ctx->jit->value_count + (-(e)-1) : (e)) #define VAL_REG(e) VAL(VIDX(e)) #define REG_MODE(m) (((m) == M_F32 || (m) == M_F64) ? 1 :0) -#define IS_PURE(e) ((e) != UNUSED && ((e)&(FL_MEMPTR | FL_STACK)) == 0) +#define REG_CFG(m) (m ? &ctx->cfg.floats : &ctx->cfg.regs) typedef struct { int stack_pos; @@ -81,12 +74,12 @@ struct _regs_ctx { typedef int call_regs[2]; -void hl_jit_init_regs( regs_config cfg ); +void hl_jit_init_regs( regs_config *cfg ); static ereg get_call_reg( regs_ctx *ctx, call_regs regs, emit_mode m ) { ereg r; int mode = REG_MODE(m); - reg_config *cfg = &ctx->cfg[mode]; + reg_config *cfg = REG_CFG(mode); if( regs[mode] < cfg->nargs ) r = cfg->arg[regs[mode]++]; else @@ -168,7 +161,7 @@ static void spill( regs_ctx *ctx, value_info *v ) { static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { // lookup available reg int mode = REG_MODE(v->mode); - reg_config *cfg = &ctx->cfg[mode]; + reg_config *cfg = REG_CFG(mode); if( !IS_NULL(v->pref_reg) ) { bool free = true; for_iter(values,v2,ctx->scratch) { @@ -201,6 +194,7 @@ static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { } // free the oldest scratch reg value_info *v2 = values_first(ctx->scratch); + if( !v2 ) jit_assert(); v->reg = v2->reg; spill(ctx, v2); return true; @@ -261,7 +255,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { jit_ctx *jit = ctx->jit; hl_type *tret = ctx->jit->fun->type->fun->ret; emit_mode mret = tret->kind == HF32 || tret->kind == HF64 ? M_F64 : M_PTR; - ereg ret = ctx->cfg[REG_MODE(mret)].ret; + ereg ret = REG_CFG(REG_MODE(mret))->ret; for(int cur_op=0;cur_opinstr_count;cur_op++) { einstr *e = jit->instrs + cur_op; value_info *write = NULL; @@ -281,7 +275,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { v->pref_reg = get_call_reg(ctx,regs,v->mode); } if( write && IS_NULL(write->pref_reg) ) - write->pref_reg = ctx->cfg[REG_MODE(e->mode)].ret; + write->pref_reg = REG_CFG(REG_MODE(e->mode))->ret; } else switch( e->op ) { case RET: if( e->a ) { @@ -289,6 +283,22 @@ static void regs_compute_liveness( regs_ctx *ctx ) { if( v->pref_reg == UNUSED ) v->pref_reg = ret; } break; + case BINOP: + switch( e->size_offs ) { + case OSShr: + case OUShr: + case OShl: + if( ctx->cfg.req_bit_shifts ) VAL_REG(e->b)->pref_reg = ctx->cfg.req_bit_shifts; + break; + case OSDiv: + case OUDiv: + case OSMod: + case OUMod: + if( ctx->cfg.req_div_a ) VAL_REG(e->a)->pref_reg = ctx->cfg.req_div_a; + if( ctx->cfg.req_div_b ) VAL_REG(e->b)->pref_reg = ctx->cfg.req_div_b; + break; + } + break; default: break; } @@ -463,6 +473,7 @@ static void flush_phis( regs_ctx *ctx, eblock *b ) { static void regs_emit_instrs( regs_ctx *ctx ) { jit_ctx *jit = ctx->jit; eblock *cur_block = NULL; + call_regs regs = {0}; int write_index = 1; ctx->pos_map[0] = 0; for(int cur_op=0;cur_opinstr_count;cur_op++) { @@ -482,7 +493,9 @@ static void regs_emit_instrs( regs_ctx *ctx ) { regs_emit_mov(ctx, r, v->reg, v->mode); } e.nargs = 0xFF; - ret_val = &ctx->cfg[REG_MODE(e.mode)].ret; + ret_val = ®_CFG(REG_MODE(e.mode))->ret; + if( e.op == CALL_REG ) + e.a = VAL(VIDX(e.a))->reg; } else { ereg **regs = hl_emit_get_regs(&e,&nread); for(int k=0;kreg; } } - ereg out; + ereg out = UNUSED; if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) out = VAL(write_index++)->reg; - else - out = UNUSED; - if( e.op == JUMP || e.op == JCOND ) - flush_phis(ctx,cur_block); - if( e.op == BLOCK ) { + switch( e.op ) { + case BLOCK: flush_phis(ctx,cur_block); cur_block = jit->blocks + e.size_offs; - } else - regs_write_instr(ctx, &e, out); - switch( e.op ) { + break; + case LOAD_ARG: + { + ereg def = get_call_reg(ctx,regs,e.mode); + if( def && out != def ) + regs_emit_mov(ctx,out,def,e.mode); + else + regs_write_instr(ctx, &e, out); + } + break; case JUMP: case JCOND: + flush_phis(ctx,cur_block); + regs_write_instr(ctx, &e, out); int_arr_add(ctx->jump_regs, ctx->emit_pos - 1); int_arr_add(ctx->jump_regs, cur_op + 1 + e.size_offs); break; + default: + if( ret_val && out ) { + regs_write_instr(ctx, &e, *ret_val); + regs_emit_mov(ctx, out, *ret_val, e.mode); + } else + regs_write_instr(ctx, &e, out); + break; } ctx->pos_map[cur_op+1] = ctx->emit_pos; } @@ -523,6 +549,7 @@ void hl_regs_flush( jit_ctx *jit ) { jit->reg_instrs = ctx->instrs; jit->reg_writes = ctx->out_write; jit->reg_pos_map = ctx->pos_map; + jit->reg_stack_usage = ctx->stack_size; if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; int i = 0; while( i < int_arr_count(ctx->jump_regs) ) { @@ -572,7 +599,7 @@ void hl_regs_alloc( jit_ctx *jit ) { memset(ctx,0,sizeof(regs_ctx)); ctx->jit = jit; jit->regs = ctx; - hl_jit_init_regs(ctx->cfg); + hl_jit_init_regs(&ctx->cfg); } void hl_regs_free( regs_ctx *ctx ) { diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 92fc9d2f3..1ee3c47cb 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -337,38 +337,51 @@ const char *hl_natreg_str( int reg, emit_mode m ) { return out; } -void hl_jit_init_regs( regs_config cfg ) { +static int scratch_float_reg = -1; + +void hl_jit_init_regs( regs_config *cfg ) { + // exclude R11 at it's use as temporary for various ops # ifdef HL_WIN_CALL - static int scratch_regs[] = { X(RAX), X(RCX), X(RDX), X(R8), X(R9), X(R10), X(R11) }; + static int scratch_regs[] = { X(RAX), X(RCX), X(RDX), X(R8), X(R9), X(R10), /*X(R11)*/ }; static int free_regs[] = { X(RSI), X(RDI), X(RBX), X(R12), X(R13), X(R14), X(R15) }; static int call_regs[] = { X(RCX), X(RDX), X(R8), X(R9) }; - cfg[1].nargs = 4; - cfg[1].nscratchs = 6; # else - static int scratch_regs[] = { X(RAX), X(RCX), X(RDX), X(RSI), X(RDI), X(R8), X(R9), X(R10), X(R11) }; + static int scratch_regs[] = { X(RAX), X(RCX), X(RDX), X(RSI), X(RDI), X(R8), X(R9), X(R10), /*X(R11)*/ }; static int free_regs[] = { X(RBX), X(R12), X(R13), X(R14), X(R15) }; static int call_regs[] = { X(RDI), X(RSI), X(RDX), X(RCX), X(R8), X(R9) }; - cfg[1].nargs = 8; - cfg[1].nscratchs = 16; # endif + cfg->regs.ret = scratch_regs[0]; + cfg->regs.nscratchs = sizeof(scratch_regs) / sizeof(int); + cfg->regs.npersists = sizeof(free_regs) / sizeof(int); + cfg->regs.nargs = sizeof(call_regs) / sizeof(int); + cfg->regs.scratch = (ereg*)scratch_regs; + cfg->regs.persist = (ereg*)free_regs; + cfg->regs.arg = (ereg*)call_regs; + // floats static int floats[] = { MMX(0), MMX(1), MMX(2), MMX(3), MMX(4), MMX(5), MMX(6), MMX(7), MMX(8), MMX(9), MMX(10), MMX(11), MMX(12), MMX(13), MMX(14), MMX(15) }; - cfg[0].ret = scratch_regs[0]; - cfg[0].nscratchs = sizeof(scratch_regs) / sizeof(int); - cfg[0].npersists = sizeof(free_regs) / sizeof(int); - cfg[0].nargs = sizeof(call_regs) / sizeof(int); - cfg[0].scratch = (ereg*)scratch_regs; - cfg[0].persist = (ereg*)free_regs; - cfg[0].arg = (ereg*)call_regs; - cfg[1].ret = floats[0]; - cfg[1].scratch = (ereg*)floats; - cfg[1].arg = (ereg*)floats; - cfg[1].persist = (ereg*)floats+cfg[1].nscratchs; - cfg[1].npersists = 16 - cfg[1].nscratchs; +# ifdef HL_WIN_CALL + cfg->floats.nargs = 4; + cfg->floats.nscratchs = 6; +# else + cfg.floats.nargs = 8; + cfg.floats.nscratchs = 16; +# endif + scratch_float_reg = cfg->floats.nscratchs - 1; + cfg->floats.nscratchs--; + cfg->floats.ret = floats[0]; + cfg->floats.scratch = (ereg*)floats; + cfg->floats.arg = (ereg*)floats; + cfg->floats.persist = (ereg*)floats + cfg->floats.nscratchs + 1; + cfg->floats.npersists = 15 - cfg->floats.nscratchs; + // extra + cfg->req_bit_shifts = R(RCX); + cfg->req_div_a = R(RAX); + cfg->req_div_b = R(RDX); } #define EMIT(op,a,b,mode) emit_ext(ctx,op,a,b,mode,0) @@ -402,7 +415,7 @@ static preg make_reg( ereg r, emit_mode m ) { p.kind = RCONST; return p; } - if( (r & FL_NATMASK) == FL_MEMPTR ) { + if( (r & FL_NATMASK) == (FL_NATREG | FL_MEMPTR) ) { p.kind = RMEM; p.reg = (r&0xFFFF); p.value = 0; @@ -708,7 +721,15 @@ static void emit_jump( code_ctx *ctx, int mode, int offset ) { } } +static ereg get_tmp( emit_mode mode ) { + if( IS_FLOAT(mode) ) + return MMX(scratch_float_reg); + return R(R11); +} + static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode, int_val value ) { + if( out == val ) + return; if( (val & FL_NATMASK) == FL_STACKOFFS ) { emit_mov(ctx,out,VAL_MEM(R(RBP)),mode, value + GET_STACK_OFFS(val)); return; @@ -717,7 +738,12 @@ static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode, int_val emit_mov(ctx,VAL_MEM(R(RBP)),val,mode, value + GET_STACK_OFFS(val)); return; } - emit_ext(ctx,mode == M_F32 ? MOVSS : mode == M_F64 ? MOVSD : _MOV,out,val,mode,value); + if( !IS_PURE(out) && !IS_PURE(val) ) { + ereg tmp = get_tmp(mode); + emit_mov(ctx, tmp, val, mode, value); + emit_mov(ctx, out, tmp, mode, value); + } else + emit_ext(ctx,mode == M_F32 ? MOVSS : mode == M_F64 ? MOVSD : _MOV,out,val,mode,value); } static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_mode mode ) { @@ -727,18 +753,75 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ case OAdd: F_OP(ADD,ADDSS,ADDSD); break; + case OSub: + F_OP(SUB,SUBSS,SUBSD); + break; + case OMul: + F_OP(IMUL,MULSS,MULSD); + break; case OIncr: cop = INC; break; case ODecr: cop = DEC; break; + case OAnd: + cop = AND; + break; + case OOr: + cop = OR; + break; + case OXor: + cop = XOR; + break; + case OShl: + case OSShr: + case OUShr: + { + ereg f = R(RCX); + if( b != f ) { + ereg tmp = get_tmp(M_I32); + if( a == f ) { + EMIT(_MOV,tmp,a,M_I32); + a = tmp; + } + EMIT(_PUSH,f,UNUSED,M_I32); + EMIT(_MOV,f,b,M_I32); + emit_anyop(ctx, op, out, a, f, mode); + EMIT(_POP,out == f ? tmp : f,UNUSED,M_I32); + return; + } + } + b = UNUSED; + cop = (op == OShl ? SHL : (op == OSShr ? SAR : SHR)); + break; + case OSMod: + case OSDiv: + case OUMod: + case OUDiv: + BREAK(); + return; default: jit_assert(); break; } - emit_mov(ctx, out, a, mode, 0); - EMIT(cop,out,b,mode); + if( !IS_PURE(out) ) { + ereg tmp = get_tmp(mode); + emit_mov(ctx, tmp, a, mode, 0); + EMIT(cop,tmp,b,mode); + emit_mov(ctx, out, tmp, mode, 0); + } else { + emit_mov(ctx, out, a, mode, 0); + EMIT(cop,out,b,mode); + } +} + +static void before_call( code_ctx *ctx, einstr *e ) { + if( IS_WINCALL64 && e->mode != M_NORET ) emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,32); +} + +static void after_call( code_ctx *ctx, einstr *e ) { + if( IS_WINCALL64 && e->mode != M_NORET ) emit_ext(ctx,ADD,R(RSP),VAL_CONST,M_PTR,32); } void hl_codegen_flush( jit_ctx *jit ) { @@ -753,6 +836,10 @@ void hl_codegen_flush( jit_ctx *jit ) { void hl_codegen_function( jit_ctx *jit ) { code_ctx *ctx = jit->code; + int stack_offset = jit->reg_stack_usage; + int push_size = HL_WSIZE * 2; // RIP + RBP save + int align = (stack_offset + push_size) & 15; + if( align ) stack_offset += 16 - align; ctx->flushed = false; byte_free(&ctx->code); int_arr_free(&ctx->near_jumps); @@ -762,8 +849,8 @@ void hl_codegen_function( jit_ctx *jit ) { ctx->pos_map[0] = 0; byte_reserve(ctx->code,64); ctx->code.cur -= 64; - EMIT(_PUSH,R(RBP),UNUSED,M_PTR); EMIT(_MOV,R(RBP),R(RSP),M_PTR); + if( stack_offset ) emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,stack_offset); for(int cur_pos=0;cur_posreg_instr_count;cur_pos++) { einstr *e = jit->reg_instrs + cur_pos; ereg out = jit->reg_writes[cur_pos]; @@ -771,6 +858,10 @@ void hl_codegen_function( jit_ctx *jit ) { ctx->code.cur -= 64; ctx->cur_op = cur_pos; if( cur_pos > 0 ) ctx->pos_map[cur_pos] = ctx->code.cur; +# ifdef HL_DEBUG + int uid = cur_pos | (jit->fun->findex << 16); + __ignore(&uid); +# endif switch( e->op ) { case LOAD_ARG: case ALLOC_STACK: @@ -790,28 +881,31 @@ void hl_codegen_function( jit_ctx *jit ) { case DEBUG_BREAK: BREAK(); break; - case CALL_PTR: - emit_ext(ctx, _MOV, R(RAX), VAL_CONST, M_PTR, e->value); - EMIT(_CALL, R(RAX), UNUSED, M_NONE); - break; case RET: if( !IS_NULL(e->a) ) { ereg ret = IS_FLOAT(e->mode) ? MMX(0) : R(RAX); if( e->a != ret ) emit_mov(ctx, ret, e->a, e->mode, 0); } + if( stack_offset ) emit_ext(ctx,ADD,R(RSP),VAL_CONST,M_PTR,stack_offset); EMIT(_POP,R(RBP),UNUSED,M_PTR); EMIT(_RET, UNUSED, UNUSED, M_NONE); break; case LOAD_CONST: - if( e->value == 0 ) - EMIT(XOR, out, out, e->mode); - else - emit_ext(ctx, _MOV, out, VAL_CONST, e->mode, e->value); + { + ereg w = IS_PURE(out) ? out : get_tmp(e->mode); + if( e->value == 0 ) + EMIT(XOR, w, w, e->mode); + else + emit_mov(ctx, w, VAL_CONST, e->mode, e->value); + if( w != out ) + emit_mov(ctx, out, w, e->mode, 0); + } break; case LOAD_ADDR: emit_mov(ctx,out,VAL_MEM(e->a),e->mode, e->size_offs); break; case CALL_FUN: + before_call(ctx,e); B(0xE8); { int pos = ctx->fun_start_pos + byte_count(ctx->code); @@ -820,32 +914,54 @@ void hl_codegen_function( jit_ctx *jit ) { int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,fid); } W(0x80000000); + after_call(ctx,e); + break; + case CALL_PTR: + before_call(ctx,e); + emit_ext(ctx, _MOV, R(RAX), VAL_CONST, M_PTR, e->value); + EMIT(_CALL, R(RAX), UNUSED, M_NONE); + after_call(ctx,e); + break; + case CALL_REG: + before_call(ctx,e); + EMIT(_CALL, e->a, UNUSED, M_NONE); + after_call(ctx,e); break; case TEST: - EMIT(_TEST,e->a,e->a,e->mode); + if( IS_FLOAT(e->mode) ) + jit_assert(); + if( !IS_PURE(e->a) ) { + ereg tmp = get_tmp(e->mode); + emit_mov(ctx, tmp, e->a, e->mode, 0); + EMIT(_TEST,tmp,tmp,e->mode); + } else + EMIT(_TEST,e->a,e->a,e->mode); break; case CMP: - switch( e->mode ) { - case M_UI8: - EMIT(CMP8,e->a,e->b,e->mode); - break; - case M_UI16: - EMIT(CMP16,e->a,e->b,e->mode); - break; - case M_F32: - EMIT(COMISS,e->a,e->b,e->mode); - break; - case M_F64: - EMIT(COMISD,e->a,e->b,e->mode); - break; - default: - EMIT(_CMP,e->a,e->b,e->mode); - break; + { + CpuOp op; + switch( e->mode ) { + case M_UI8: op = CMP8; break; + case M_UI16: op = CMP16; break; + case M_F32: op = COMISS; break; + case M_F64: op = COMISD; break; + default: op = _CMP; break; + } + if( !IS_PURE(e->a) && !IS_PURE(e->b) ) { + ereg tmp = get_tmp(e->mode); + emit_mov(ctx, tmp, e->b, e->mode, 0); + EMIT(op,e->a,tmp,e->mode); + } else + EMIT(op,e->a,e->b,e->mode); } break; case JCOND: { - einstr *p = &e[-1]; + int prev = 0; + einstr *p; + do { + p = &e[--prev]; + } while( p->op == MOV ); int op; switch( p->size_offs ) { case OJFalse: @@ -906,14 +1022,37 @@ void hl_codegen_function( jit_ctx *jit ) { case UNOP: emit_anyop(ctx, e->size_offs, out, e->a, e->b, e->mode); break; + case LEA: + if( !IS_PURE(out) ) { + ereg tmp = get_tmp(e->mode); + EMIT(_LEA,tmp,e->a,e->mode); + emit_mov(ctx,out,tmp,e->mode,0); + } else + EMIT(_LEA,out,e->a,e->mode); + break; default: jit_assert(); break; } if( ctx->code.cur > ctx->code.max ) jit_assert(); } + while( byte_count(ctx->code) & 7 ) + byte_add_impl(&jit->falloc,&ctx->code,0xCC); // breaks hl_codegen_flush(jit); ctx->fun_start_pos += byte_count(ctx->code); + for(int i=0;ishort_jumps);i+=2) { + int pos = int_arr_get(ctx->short_jumps,i); + int target = int_arr_get(ctx->short_jumps,i+1); + int offset = ctx->pos_map[target] - (pos + 1); + if( !IS_SBYTE(offset) ) jit_assert(); + *(char*)&ctx->code.values[pos] = (char)offset; + } + for(int i=0;inear_jumps);i+=2) { + int pos = int_arr_get(ctx->near_jumps,i); + int target = int_arr_get(ctx->near_jumps,i+1); + int offset = ctx->pos_map[target] - (pos + 4); + *(int*)&ctx->code.values[pos] = offset; + } } diff --git a/src/module.c b/src/module.c index 3002011d4..e31db1411 100644 --- a/src/module.c +++ b/src/module.c @@ -717,6 +717,7 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { for(i=0;i Date: Sun, 19 Apr 2026 13:40:31 +0200 Subject: [PATCH 29/83] move stack handling from codegen to regs, boot ok --- src/jit.c | 32 +++++++++++++++--- src/jit.h | 21 +++++++----- src/jit_dump.c | 10 +++++- src/jit_emit.c | 15 ++++++++- src/jit_regs.c | 58 +++++++++++++++++++++++++++---- src/jit_x86_64.c | 88 ++++++++++++++++++++++++++++++------------------ src/std/fun.c | 4 +++ 7 files changed, 173 insertions(+), 55 deletions(-) diff --git a/src/jit.c b/src/jit.c index 88aaa0c3e..45920d3dd 100644 --- a/src/jit.c +++ b/src/jit.c @@ -40,7 +40,7 @@ void hl_jit_assert() { jit_assert(); } void hl_emit_alloc( jit_ctx *jit ); void hl_emit_free( jit_ctx *jit ); void hl_emit_function( jit_ctx *jit ); - +void hl_emit_final( jit_ctx *jit ); void hl_regs_alloc( jit_ctx *jit ); void hl_regs_free( jit_ctx *jit ); @@ -49,6 +49,7 @@ void hl_regs_function( jit_ctx *jit ); void hl_codegen_alloc( jit_ctx *jit ); void hl_codegen_free( jit_ctx *jit ); void hl_codegen_function( jit_ctx *jit ); +void hl_codegen_final( jit_ctx *jit ); jit_ctx *hl_jit_alloc() { jit_ctx *ctx = (jit_ctx*)malloc(sizeof(jit_ctx)); @@ -84,14 +85,35 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { hl_emit_function(ctx); hl_regs_function(ctx); hl_codegen_function(ctx); + int pos = ctx->out_pos; + if( pos + ctx->code_size > ctx->out_max ) { + int nsize = ctx->out_max ? ctx->out_max * 3 : 4096; + while( pos + ctx->code_size > nsize ) nsize *= 3; + unsigned char *nout = malloc(nsize); + if( !nout ) return -1; + memcpy(nout,ctx->output,pos); + free(ctx->output); + ctx->output = nout; + ctx->out_max = nsize; + } + memcpy(ctx->output + pos, ctx->code_instrs, ctx->code_size); + ctx->out_pos += ctx->code_size; current_ctx = NULL; - return 0; + return pos; } void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { - printf("TODO:emit_code\n"); - exit(0); - return NULL; + int size = ctx->out_pos; + if( size & 4095 ) size += 4096 - (size&4095); + unsigned char *code = (unsigned char*)hl_alloc_executable_memory(size); + if( code == NULL ) return NULL; + memcpy(code,ctx->output,size); + *codesize = size; + *debug = NULL; + ctx->final_code = code; + hl_emit_final(ctx); + hl_codegen_final(ctx); + return code; } void hl_jit_patch_method( void*fun, void**newt ) { diff --git a/src/jit.h b/src/jit.h index a384f433f..ef475e78e 100644 --- a/src/jit.h +++ b/src/jit.h @@ -51,6 +51,8 @@ typedef enum { PREFETCH, DEBUG_BREAK, BLOCK, + ENTER, + STACK_OFFS, } emit_op; typedef enum { @@ -151,6 +153,10 @@ typedef struct { typedef struct { reg_config regs; reg_config floats; + ereg stack_reg; + ereg stack_pos; + int stack_align; + int stack_align_offset; ereg req_bit_shifts; ereg req_div_a; ereg req_div_b; @@ -175,15 +181,18 @@ struct _jit_ctx { int *emit_pos_map; // regs output int reg_instr_count; - int reg_stack_usage; einstr *reg_instrs; ereg *reg_writes; int *reg_pos_map; - int persists_uses[2]; - // gen output + // codegen output int code_size; unsigned char *code_instrs; int *code_pos_map; + // accum output + int out_pos; + int out_max; + unsigned char *output; + unsigned char *final_code; }; jit_ctx *hl_jit_alloc(); @@ -219,12 +228,6 @@ extern bool hl_jit_dump_bin; # define jit_debug(...) #endif -#if defined(HL_WIN_CALL) && defined(HL_64) -# define IS_WINCALL64 1 -#else -# define IS_WINCALL64 0 -#endif - #define DEF_ALLOC &ctx->jit->falloc #define jit_pad_size(size,k) ((k == 0) ? 0 : ((-(size)) & (k - 1))) diff --git a/src/jit_dump.c b/src/jit_dump.c index eda6e396c..e83b506b6 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -48,6 +48,8 @@ static const char *op_names[] = { "prefetch", "debug-break", "block", + "enter", + "stack", }; bool hl_jit_dump_bin = false; @@ -341,6 +343,12 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { case BLOCK: printf(" #%d", e->size_offs); break; + case STACK_OFFS: + if( e->size_offs >= 0 ) + printf(" +%Xh", e->size_offs); + else + printf(" -%Xh", -e->size_offs); + break; case LOAD_CONST: case PUSH_CONST: printf(" "); @@ -477,7 +485,7 @@ void hl_emit_dump( jit_ctx *ctx ) { if( cpos == ctx->code_size && cpos > 0 ) { int n = 1; for(int i=0;icode_pos_map[n] == i ) { + while( ctx->code_pos_map[n] == i ) { if( (n & 15) == 0 ) printf("\n"); else printf(" "); n++; } diff --git a/src/jit_emit.c b/src/jit_emit.c index 4054fa421..f8f5e6108 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -139,7 +139,7 @@ struct _emit_ctx { emit_block *current_block; emit_block *wait_seal; linked_inf *arrival_points; - void *closure_list; // TODO : patch with good addresses + vclosure *closure_list; }; #define R(i) (ctx->vregs + (i)) @@ -945,6 +945,7 @@ void hl_emit_function( jit_ctx *jit ) { r->ref = 0; } + emit_gen(ctx,ENTER,UNUSED,UNUSED,M_NONE); for(i=0;itype->fun->nargs;i++) { hl_type *t = f->type->fun->args[i]; if( t->kind == HVOID ) continue; @@ -986,6 +987,18 @@ void hl_emit_free( jit_ctx *jit ) { jit->emit = NULL; } +void hl_emit_final( jit_ctx *jit ) { + emit_ctx *ctx = jit->emit; + vclosure *l = ctx->closure_list; + while( l ) { + vclosure *n = (vclosure*)l->value; + l->value = NULL; + l->fun = jit->final_code + (int_val)jit->mod->functions_ptrs[(int_val)l->fun]; + l = n; + } + ctx->closure_list = NULL; +} + static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { if( b->start_pos < target ) return false; diff --git a/src/jit_regs.c b/src/jit_regs.c index f94f66945..b891a544e 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -40,6 +40,12 @@ #define REG_MODE(m) (((m) == M_F32 || (m) == M_F64) ? 1 :0) #define REG_CFG(m) (m ? &ctx->cfg.floats : &ctx->cfg.regs) +#if defined(HL_WIN_CALL) && defined(HL_64) +# define IS_WINCALL64 1 +#else +# define IS_WINCALL64 0 +#endif + typedef struct { int stack_pos; int last_read; @@ -70,6 +76,8 @@ struct _regs_ctx { ereg *out_write; int *pos_map; bool flushed; + bool has_direct_call; + int persists_uses[2]; }; typedef int call_regs[2]; @@ -188,8 +196,8 @@ static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { return true; } } - if( ctx->jit->persists_uses[mode] < cfg->npersists ) { - v->reg = cfg->persist[ctx->jit->persists_uses[mode]++]; + if( ctx->persists_uses[mode] < cfg->npersists ) { + v->reg = cfg->persist[ctx->persists_uses[mode]++]; return false; } // free the oldest scratch reg @@ -269,11 +277,16 @@ static void regs_compute_liveness( regs_ctx *ctx ) { // anticipate register usage in call so we can previlege this assign ereg *r = hl_emit_get_args(jit->emit, e); call_regs regs = {0}; + bool needs_push = false; for(int k=0;knargs;k++) { value_info *v = VAL_REG(r[k]); - if( !IS_NULL(v->pref_reg) ) continue; + if( !IS_NULL(v->pref_reg) ) { + needs_push = true; + continue; + } v->pref_reg = get_call_reg(ctx,regs,v->mode); } + if( !needs_push && e->mode != M_NORET ) ctx->has_direct_call = true; if( write && IS_NULL(write->pref_reg) ) write->pref_reg = REG_CFG(REG_MODE(e->mode))->ret; } else switch( e->op ) { @@ -470,12 +483,31 @@ static void flush_phis( regs_ctx *ctx, eblock *b ) { int_arr_reset(&ctx->phi_movs); } +static void regs_emit( regs_ctx *ctx, ereg out, emit_op op, ereg a, ereg b, emit_mode m, int size_offs ) { + einstr e; + e.header = op; + e.mode = m; + e.a = a; + e.b = b; + e.size_offs = size_offs; + regs_write_instr(ctx, &e, out); +} + static void regs_emit_instrs( regs_ctx *ctx ) { jit_ctx *jit = ctx->jit; eblock *cur_block = NULL; call_regs regs = {0}; int write_index = 1; ctx->pos_map[0] = 0; + + int stack_offset = ctx->stack_size; + int push_size = HL_WSIZE * 2; // RIP + RBP save + if( IS_WINCALL64 && ctx->has_direct_call ) stack_offset += 0x20; // reserve + if( ctx->cfg.stack_align ) { + int align = (stack_offset + push_size + ctx->cfg.stack_align_offset) % ctx->cfg.stack_align; + if( align ) stack_offset += ctx->cfg.stack_align - align; + } + for(int cur_op=0;cur_opinstr_count;cur_op++) { einstr e = jit->instrs[cur_op]; ereg *ret_val = NULL; @@ -509,6 +541,8 @@ static void regs_emit_instrs( regs_ctx *ctx ) { if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) out = VAL(write_index++)->reg; switch( e.op ) { + case ALLOC_STACK: + break; case BLOCK: flush_phis(ctx,cur_block); cur_block = jit->blocks + e.size_offs; @@ -522,6 +556,13 @@ static void regs_emit_instrs( regs_ctx *ctx ) { regs_write_instr(ctx, &e, out); } break; + case ENTER: + { + regs_emit(ctx,UNUSED,PUSH,ctx->cfg.stack_pos,UNUSED,M_PTR,0); + regs_emit_mov(ctx,ctx->cfg.stack_pos,ctx->cfg.stack_reg,M_PTR); + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-stack_offset); + } + break; case JUMP: case JCOND: flush_phis(ctx,cur_block); @@ -529,6 +570,10 @@ static void regs_emit_instrs( regs_ctx *ctx ) { int_arr_add(ctx->jump_regs, ctx->emit_pos - 1); int_arr_add(ctx->jump_regs, cur_op + 1 + e.size_offs); break; + case RET: + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); + regs_emit(ctx,UNUSED,POP,ctx->cfg.stack_pos,UNUSED,M_PTR,0); + // fallback default: if( ret_val && out ) { regs_write_instr(ctx, &e, *ret_val); @@ -549,7 +594,6 @@ void hl_regs_flush( jit_ctx *jit ) { jit->reg_instrs = ctx->instrs; jit->reg_writes = ctx->out_write; jit->reg_pos_map = ctx->pos_map; - jit->reg_stack_usage = ctx->stack_size; if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; int i = 0; while( i < int_arr_count(ctx->jump_regs) ) { @@ -563,9 +607,10 @@ void hl_regs_flush( jit_ctx *jit ) { void hl_regs_function( jit_ctx *jit ) { regs_ctx *ctx = jit->regs; int nvalues = jit->value_count + jit->phi_count; - memset(jit->persists_uses,0,sizeof(jit->persists_uses)); + memset(ctx->persists_uses,0,sizeof(ctx->persists_uses)); free(ctx->pos_map); ctx->flushed = false; + ctx->has_direct_call = false; ctx->pos_map = (int*)malloc((jit->instr_count + 1) * sizeof(int)); ctx->emit_pos = 0; ctx->cur_op = 0; @@ -602,7 +647,8 @@ void hl_regs_alloc( jit_ctx *jit ) { hl_jit_init_regs(&ctx->cfg); } -void hl_regs_free( regs_ctx *ctx ) { +void hl_regs_free( jit_ctx *jit ) { + regs_ctx *ctx = jit->regs; free(ctx->pos_map); free(ctx->instrs); free(ctx->out_write); diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 1ee3c47cb..324324603 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -286,7 +286,6 @@ struct _code_ctx { int_arr near_jumps; int *pos_map; int cur_op; - int fun_start_pos; bool flushed; }; @@ -382,6 +381,12 @@ void hl_jit_init_regs( regs_config *cfg ) { cfg->req_bit_shifts = R(RCX); cfg->req_div_a = R(RAX); cfg->req_div_b = R(RDX); + cfg->stack_reg = R(RSP); + cfg->stack_pos = R(RBP); + cfg->stack_align = 16; +# ifdef HL_WIN_CALL + cfg->stack_align_offset = 8; +# endif } #define EMIT(op,a,b,mode) emit_ext(ctx,op,a,b,mode,0) @@ -438,7 +443,8 @@ static preg make_reg( ereg r, emit_mode m ) { static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, int_val value ) { opform *f = &OP_FORMS[op]; - int r64 = (mode == M_PTR || mode == M_I64) && (f->r_mem&FLAG_DEF64) == 0 ? 8 : 0; + int mode64 = (mode == M_PTR || mode == M_I64) && (f->r_mem&FLAG_DEF64) == 0 ? 8 : 0; + int r64 = mode64; preg a = make_reg(_a,mode), b = make_reg(_b,mode); switch( ID2(a.kind,b.kind) ) { case ID2(RUNUSED,RUNUSED): @@ -504,11 +510,11 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; OP(f->r_const&0xFF); if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_const)-1,a.reg); - if( r64 && IS_64 && op == _MOV ) W64(value); else W((int)value); + if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); } else { ERRIF( f->r_const == 0); OP((f->r_const&0xFF) + (a.reg&7)); - if( r64 && IS_64 && op == _MOV ) W64(value); else W((int)value); + if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); } } break; @@ -707,13 +713,20 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, } static void emit_jump( code_ctx *ctx, int mode, int offset ) { - if( IS_SBYTE(offset*8) ) { + int op_mult; +# ifdef HL_DEBUG + op_mult = 16; // additional debug info per op +# else + op_mult = 8; +# endif + if( IS_SBYTE(offset*op_mult) ) { // assume it's ok to use short jump B(mode == JAlways ? JAlways_short : mode - 0x10); int_arr_add(ctx->short_jumps, byte_count(ctx->code)); int_arr_add(ctx->short_jumps, ctx->cur_op + offset + 1); B(-2); } else { + B(0x0F); B(mode); int_arr_add(ctx->near_jumps, byte_count(ctx->code)); int_arr_add(ctx->near_jumps, ctx->cur_op + offset + 1); @@ -816,14 +829,6 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ } } -static void before_call( code_ctx *ctx, einstr *e ) { - if( IS_WINCALL64 && e->mode != M_NORET ) emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,32); -} - -static void after_call( code_ctx *ctx, einstr *e ) { - if( IS_WINCALL64 && e->mode != M_NORET ) emit_ext(ctx,ADD,R(RSP),VAL_CONST,M_PTR,32); -} - void hl_codegen_flush( jit_ctx *jit ) { code_ctx *ctx = jit->code; if( ctx->flushed ) return; @@ -836,10 +841,6 @@ void hl_codegen_flush( jit_ctx *jit ) { void hl_codegen_function( jit_ctx *jit ) { code_ctx *ctx = jit->code; - int stack_offset = jit->reg_stack_usage; - int push_size = HL_WSIZE * 2; // RIP + RBP save - int align = (stack_offset + push_size) & 15; - if( align ) stack_offset += 16 - align; ctx->flushed = false; byte_free(&ctx->code); int_arr_free(&ctx->near_jumps); @@ -849,8 +850,10 @@ void hl_codegen_function( jit_ctx *jit ) { ctx->pos_map[0] = 0; byte_reserve(ctx->code,64); ctx->code.cur -= 64; - EMIT(_MOV,R(RBP),R(RSP),M_PTR); - if( stack_offset ) emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,stack_offset); +# ifdef HL_DEBUG + int reg_index = 0; + int emit_index = 0; +# endif for(int cur_pos=0;cur_posreg_instr_count;cur_pos++) { einstr *e = jit->reg_instrs + cur_pos; ereg out = jit->reg_writes[cur_pos]; @@ -859,12 +862,18 @@ void hl_codegen_function( jit_ctx *jit ) { ctx->cur_op = cur_pos; if( cur_pos > 0 ) ctx->pos_map[cur_pos] = ctx->code.cur; # ifdef HL_DEBUG - int uid = cur_pos | (jit->fun->findex << 16); - __ignore(&uid); + int rid = cur_pos | (jit->fun->findex << 16); + while( reg_index < jit->instr_count && jit->reg_pos_map[reg_index] <= cur_pos ) reg_index++; + if( emit_index < jit->fun->nops && jit->emit_pos_map[emit_index] < reg_index ) { + int uid = emit_index | (jit->fun->findex << 16); + emit_ext(ctx,_MOV,get_tmp(M_I32),VAL_CONST,M_I32,uid); + __ignore(&uid); + __ignore(&rid); + emit_index++; + } # endif switch( e->op ) { case LOAD_ARG: - case ALLOC_STACK: continue; // nop case MOV: if( (e->a & FL_NATMASK) == FL_STACKOFFS ) @@ -875,6 +884,12 @@ void hl_codegen_function( jit_ctx *jit ) { case STORE: emit_mov(ctx, VAL_MEM(e->a), e->b, e->mode, e->size_offs); break; + case PUSH: + emit_ext(ctx, _PUSH, e->a, UNUSED, e->mode, 0); + break; + case POP: + emit_ext(ctx, _POP, e->a, UNUSED, e->mode, 0); + break; case PUSH_CONST: emit_ext(ctx, _PUSH, VAL_CONST, UNUSED, e->mode, e->value); break; @@ -886,8 +901,6 @@ void hl_codegen_function( jit_ctx *jit ) { ereg ret = IS_FLOAT(e->mode) ? MMX(0) : R(RAX); if( e->a != ret ) emit_mov(ctx, ret, e->a, e->mode, 0); } - if( stack_offset ) emit_ext(ctx,ADD,R(RSP),VAL_CONST,M_PTR,stack_offset); - EMIT(_POP,R(RBP),UNUSED,M_PTR); EMIT(_RET, UNUSED, UNUSED, M_NONE); break; case LOAD_CONST: @@ -905,27 +918,21 @@ void hl_codegen_function( jit_ctx *jit ) { emit_mov(ctx,out,VAL_MEM(e->a),e->mode, e->size_offs); break; case CALL_FUN: - before_call(ctx,e); B(0xE8); { - int pos = ctx->fun_start_pos + byte_count(ctx->code); + int pos = jit->out_pos + byte_count(ctx->code); int fid = e->a; int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,pos); int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,fid); + W(0xBADBAD00 - (pos + 4)); } - W(0x80000000); - after_call(ctx,e); break; case CALL_PTR: - before_call(ctx,e); emit_ext(ctx, _MOV, R(RAX), VAL_CONST, M_PTR, e->value); EMIT(_CALL, R(RAX), UNUSED, M_NONE); - after_call(ctx,e); break; case CALL_REG: - before_call(ctx,e); EMIT(_CALL, e->a, UNUSED, M_NONE); - after_call(ctx,e); break; case TEST: if( IS_FLOAT(e->mode) ) @@ -1030,6 +1037,12 @@ void hl_codegen_function( jit_ctx *jit ) { } else EMIT(_LEA,out,e->a,e->mode); break; + case STACK_OFFS: + if( e->size_offs >= 0 ) + emit_ext(ctx,ADD,R(RSP),VAL_CONST,M_PTR,e->size_offs); + else + emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,-e->size_offs); + break; default: jit_assert(); break; @@ -1039,7 +1052,6 @@ void hl_codegen_function( jit_ctx *jit ) { while( byte_count(ctx->code) & 7 ) byte_add_impl(&jit->falloc,&ctx->code,0xCC); // breaks hl_codegen_flush(jit); - ctx->fun_start_pos += byte_count(ctx->code); for(int i=0;ishort_jumps);i+=2) { int pos = int_arr_get(ctx->short_jumps,i); int target = int_arr_get(ctx->short_jumps,i+1); @@ -1069,3 +1081,13 @@ void hl_codegen_free( jit_ctx *jit ) { free(ctx); } +void hl_codegen_final( jit_ctx *jit ) { + code_ctx *ctx = jit->code; + for(int i=0;ifuns);i+=2) { + int pos = int_arr_get(ctx->funs,i); + int fid = int_arr_get(ctx->funs,i+1); + int offset = (int)(int_val)jit->mod->functions_ptrs[fid] - (pos + 4); + *(int*)(jit->final_code + pos) = offset; + } + int_arr_reset(&ctx->funs); +} diff --git a/src/std/fun.c b/src/std/fun.c index fafa8177e..ee611e8a8 100644 --- a/src/std/fun.c +++ b/src/std/fun.c @@ -221,6 +221,10 @@ HL_PRIM vdynamic *hl_dyn_call( vclosure *c, vdynamic **args, int nargs ) { tmp.args[i+1] = args[i]; c = &ctmp; } else { + if( nargs == 0 && c->t->fun->ret->kind == HVOID ) { + ((void (*)(void))c->fun)(); + return NULL; + } for(i=0;i Date: Sun, 19 Apr 2026 18:43:01 +0200 Subject: [PATCH 30/83] added unwind tables and a few fixes --- src/jit.c | 50 +++++++++++++++++++++++++++++++++++--------- src/jit.h | 1 - src/jit_regs.c | 10 +++++++-- src/jit_x86_64.c | 54 ++++++++++++++++++++++++++++++++++++++++-------- src/module.c | 7 +++++++ 5 files changed, 100 insertions(+), 22 deletions(-) diff --git a/src/jit.c b/src/jit.c index 45920d3dd..e3e6fdad0 100644 --- a/src/jit.c +++ b/src/jit.c @@ -61,7 +61,40 @@ jit_ctx *hl_jit_alloc() { return ctx; } +static bool jit_code_reserve( jit_ctx *ctx, int size ) { + int pos = ctx->out_pos; + if( pos + size > ctx->out_max ) { + int nsize = ctx->out_max ? ctx->out_max * 3 : 4096; + while( pos + ctx->code_size > nsize ) nsize *= 3; + unsigned char *nout = malloc(nsize); + if( !nout ) return false; + memcpy(nout,ctx->output,pos); + free(ctx->output); + ctx->output = nout; + ctx->out_max = nsize; + } + return true; +} + void hl_jit_init( jit_ctx *ctx, hl_module *m ) { +#ifdef WIN64_UNWIND_TABLES + unsigned char version = 1; + unsigned char flags = 0; + unsigned char CountOfCodes = 2; + unsigned char SizeOfProlog = 4; + unsigned char FrameRegister = 5; // RBP + unsigned char FrameOffset = 0; + jit_code_reserve(ctx,64); +# define B(v) ctx->output[ctx->out_pos++] = v +# define UW(offs,code,inf) B(offs); B((code) | (inf) << 4) + B((version) | (flags) << 3); + B(SizeOfProlog); + B(CountOfCodes); + B((FrameRegister) | (FrameOffset) << 4); + UW(4, 3 /*UWOP_SET_FPREG*/, 0); + UW(1, 0 /*UWOP_PUSH_NONVOL*/, 5); + while( ctx->out_pos & 15 ) B(0); +#endif } void hl_jit_free( jit_ctx *ctx, h_bool can_reset ) { @@ -85,18 +118,15 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { hl_emit_function(ctx); hl_regs_function(ctx); hl_codegen_function(ctx); + if( !jit_code_reserve(ctx, ctx->code_size) ) + return -1; int pos = ctx->out_pos; - if( pos + ctx->code_size > ctx->out_max ) { - int nsize = ctx->out_max ? ctx->out_max * 3 : 4096; - while( pos + ctx->code_size > nsize ) nsize *= 3; - unsigned char *nout = malloc(nsize); - if( !nout ) return -1; - memcpy(nout,ctx->output,pos); - free(ctx->output); - ctx->output = nout; - ctx->out_max = nsize; - } memcpy(ctx->output + pos, ctx->code_instrs, ctx->code_size); +#ifdef WIN64_UNWIND_TABLES + int fid = (int)(f - ctx->mod->code->functions); + ctx->mod->unwind_table[fid].BeginAddress = ctx->out_pos; + ctx->mod->unwind_table[fid].EndAddress = ctx->out_pos + ctx->code_size; +#endif ctx->out_pos += ctx->code_size; current_ctx = NULL; return pos; diff --git a/src/jit.h b/src/jit.h index ef475e78e..5ab9d5289 100644 --- a/src/jit.h +++ b/src/jit.h @@ -156,7 +156,6 @@ typedef struct { ereg stack_reg; ereg stack_pos; int stack_align; - int stack_align_offset; ereg req_bit_shifts; ereg req_div_a; ereg req_div_b; diff --git a/src/jit_regs.c b/src/jit_regs.c index b891a544e..dc369514e 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -504,7 +504,7 @@ static void regs_emit_instrs( regs_ctx *ctx ) { int push_size = HL_WSIZE * 2; // RIP + RBP save if( IS_WINCALL64 && ctx->has_direct_call ) stack_offset += 0x20; // reserve if( ctx->cfg.stack_align ) { - int align = (stack_offset + push_size + ctx->cfg.stack_align_offset) % ctx->cfg.stack_align; + int align = (stack_offset + push_size) % ctx->cfg.stack_align; if( align ) stack_offset += ctx->cfg.stack_align - align; } @@ -571,9 +571,15 @@ static void regs_emit_instrs( regs_ctx *ctx ) { int_arr_add(ctx->jump_regs, cur_op + 1 + e.size_offs); break; case RET: + if( e.a ) { + ereg ret = REG_CFG(REG_MODE(e.mode))->ret; + if( e.a != ret ) + regs_emit_mov(ctx, ret, e.a, e.mode); + } regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); regs_emit(ctx,UNUSED,POP,ctx->cfg.stack_pos,UNUSED,M_PTR,0); - // fallback + regs_emit(ctx,UNUSED,RET,UNUSED,UNUSED,M_NONE,0); + break; default: if( ret_val && out ) { regs_write_instr(ctx, &e, *ret_val); diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 324324603..6350469f2 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -228,7 +228,7 @@ static opform OP_FORMS[] = { { "STMXCSR", 0, LONG_RM(0x0FAE,3) }, { "LDMXCSR", 0, LONG_RM(0x0FAE,2) }, // 8 bits, - { "MOV8", 0x8A, 0x88, 0, 0xB0, RM(0xC6,0) }, + { "MOV8", 0x8A, 0x88, 0, RM(0xC6,0) }, { "CMP8", 0x3A, 0x38, 0, RM(0x80,7) }, { "TEST8", 0x84, 0x84, RM(0xF6,0) }, { "PUSH8", FLAG_DEF64, 0, 0x6A | FLAG_8B }, @@ -384,9 +384,6 @@ void hl_jit_init_regs( regs_config *cfg ) { cfg->stack_reg = R(RSP); cfg->stack_pos = R(RBP); cfg->stack_align = 16; -# ifdef HL_WIN_CALL - cfg->stack_align_offset = 8; -# endif } #define EMIT(op,a,b,mode) emit_ext(ctx,op,a,b,mode,0) @@ -755,8 +752,10 @@ static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode, int_val ereg tmp = get_tmp(mode); emit_mov(ctx, tmp, val, mode, value); emit_mov(ctx, out, tmp, mode, value); - } else - emit_ext(ctx,mode == M_F32 ? MOVSS : mode == M_F64 ? MOVSD : _MOV,out,val,mode,value); + } else { + static CpuOp MOV_OP[] = {_MOV,MOV8,MOV16,_MOV,_MOV,_MOV,MOVSD,MOVSS,_MOV,_MOV}; + emit_ext(ctx, MOV_OP[mode],out,val,mode,value); + } } static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_mode mode ) { @@ -839,6 +838,37 @@ void hl_codegen_flush( jit_ctx *jit ) { if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->code.cur; } +static void emit_nop( code_ctx *ctx, int size ) { + byte_reserve(ctx->code,size); + ctx->code.cur -= size; + if( size >= 8 ) { + W(0x841F0F); + W(0); + return; + } + if( size >= 5 ) { + W(0x441F0F); + B(0); + return; + } + if( size >= 4 ) { + W(0x401F0F); + return; + } + if( size >= 3 ) { + B(0x0F); + B(0x1F); + B(0x00); + return; + } + if( size >= 2 ) { + B(0x66); + B(0x90); + return; + } + B(0x90); +} + void hl_codegen_function( jit_ctx *jit ) { code_ctx *ctx = jit->code; ctx->flushed = false; @@ -882,7 +912,13 @@ void hl_codegen_function( jit_ctx *jit ) { emit_mov(ctx, out, e->a, e->mode, 0); break; case STORE: - emit_mov(ctx, VAL_MEM(e->a), e->b, e->mode, e->size_offs); + if( !IS_PURE(e->a) && !IS_PURE(e->b) && (e->a & FL_NATMASK) != FL_STACKOFFS ) { + EMIT(_PUSH,e->b,UNUSED,e->mode); + ereg tmp = get_tmp(M_PTR); + emit_mov(ctx, tmp, e->a, M_PTR, 0); + emit_ext(ctx, _POP,VAL_MEM(tmp), UNUSED, e->mode, e->size_offs); + } else + emit_mov(ctx, VAL_MEM(e->a), e->b, e->mode, e->size_offs); break; case PUSH: emit_ext(ctx, _PUSH, e->a, UNUSED, e->mode, 0); @@ -1049,8 +1085,8 @@ void hl_codegen_function( jit_ctx *jit ) { } if( ctx->code.cur > ctx->code.max ) jit_assert(); } - while( byte_count(ctx->code) & 7 ) - byte_add_impl(&jit->falloc,&ctx->code,0xCC); // breaks + while( byte_count(ctx->code) & 15 ) + emit_nop(ctx,16 - (byte_count(ctx->code) & 15)); hl_codegen_flush(jit); for(int i=0;ishort_jumps);i+=2) { int pos = int_arr_get(ctx->short_jumps,i); diff --git a/src/module.c b/src/module.c index e31db1411..dca9c76be 100644 --- a/src/module.c +++ b/src/module.c @@ -706,6 +706,10 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { if( hot_reload ) m->hash = hl_code_hash_alloc(m->code); hl_module_init_natives(m); hl_module_init_indexes(m); +# ifdef WIN64_UNWIND_TABLES + m->unwind_table = malloc(sizeof(RUNTIME_FUNCTION) * m->code->nfunctions); + memset(m->unwind_table, 0, sizeof(RUNTIME_FUNCTION) * m->code->nfunctions); +# endif // JIT ctx = hl_jit_alloc(); if( ctx == NULL ) @@ -764,6 +768,9 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { hl_gc_set_dump_types(hl_module_types_dump); # ifdef HL_VTUNE hl_setup.vtune_init = modules_init_vtune; +# endif +# ifdef WIN64_UNWIND_TABLES + RtlAddFunctionTable(m->unwind_table, m->code->nfunctions, (DWORD64)m->jit_code); # endif hl_jit_free(ctx, hot_reload); if( hot_reload ) { From 248ef74922d17121e69f6751f0dd459fa69a392e Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 19 Apr 2026 23:43:01 +0200 Subject: [PATCH 31/83] added null access trampolines (shorter code) and codegen fixes/cleanup/wip --- src/hlmodule.h | 1 + src/jit.c | 52 +++++-- src/jit.h | 3 +- src/jit_dump.c | 14 +- src/jit_emit.c | 9 +- src/jit_regs.c | 4 +- src/jit_x86_64.c | 376 +++++++++++++++++++++-------------------------- src/module.c | 7 +- 8 files changed, 228 insertions(+), 238 deletions(-) diff --git a/src/hlmodule.h b/src/hlmodule.h index 01ab8be2e..59e8106a9 100644 --- a/src/hlmodule.h +++ b/src/hlmodule.h @@ -136,6 +136,7 @@ typedef struct { jit_ctx *jit_ctx; hl_module_context ctx; #ifdef WIN64_UNWIND_TABLES + int unwind_table_size; PRUNTIME_FUNCTION unwind_table; #endif } hl_module; diff --git a/src/jit.c b/src/jit.c index e3e6fdad0..91483e1dc 100644 --- a/src/jit.c +++ b/src/jit.c @@ -33,9 +33,21 @@ void hl_jit_error( const char *msg, const char *func, int line ) { fflush(stdout); } -void hl_jit_null_field_access() { jit_assert(); } -void hl_jit_null_access() { jit_assert(); } -void hl_jit_assert() { jit_assert(); } +void hl_jit_null_field_access( int fhash ) { + vbyte *field = hl_field_name(fhash); + hl_buffer *b = hl_alloc_buffer(); + hl_buffer_str(b, USTR("Null access .")); + hl_buffer_str(b, (uchar*)field); + vdynamic *d = hl_alloc_dynamic(&hlt_bytes); + d->v.ptr = hl_buffer_content(b,NULL); + hl_throw(d); +} + +void hl_jit_assert() { + vdynamic *d = hl_alloc_dynamic(&hlt_bytes); + d->v.ptr = USTR("Assert"); + hl_throw(d); +} void hl_emit_alloc( jit_ctx *jit ); void hl_emit_free( jit_ctx *jit ); @@ -47,6 +59,7 @@ void hl_regs_free( jit_ctx *jit ); void hl_regs_function( jit_ctx *jit ); void hl_codegen_alloc( jit_ctx *jit ); +void hl_codegen_init( jit_ctx *jit ); void hl_codegen_free( jit_ctx *jit ); void hl_codegen_function( jit_ctx *jit ); void hl_codegen_final( jit_ctx *jit ); @@ -61,6 +74,15 @@ jit_ctx *hl_jit_alloc() { return ctx; } +void hl_jit_define_function( jit_ctx *ctx, int start, int size ) { +#ifdef WIN64_UNWIND_TABLES + int fid = ctx->fdef_index++; + if( fid >= ctx->mod->unwind_table_size ) jit_assert(); + ctx->mod->unwind_table[fid].BeginAddress = start; + ctx->mod->unwind_table[fid].EndAddress = start + size; +#endif +} + static bool jit_code_reserve( jit_ctx *ctx, int size ) { int pos = ctx->out_pos; if( pos + size > ctx->out_max ) { @@ -76,7 +98,17 @@ static bool jit_code_reserve( jit_ctx *ctx, int size ) { return true; } +static bool jit_code_append( jit_ctx *ctx ) { + if( !jit_code_reserve(ctx,ctx->code_size) ) + return false; + int pos = ctx->out_pos; + memcpy(ctx->output + pos, ctx->code_instrs, ctx->code_size); + ctx->out_pos += ctx->code_size; + return true; +} + void hl_jit_init( jit_ctx *ctx, hl_module *m ) { + ctx->mod = m; #ifdef WIN64_UNWIND_TABLES unsigned char version = 1; unsigned char flags = 0; @@ -95,6 +127,8 @@ void hl_jit_init( jit_ctx *ctx, hl_module *m ) { UW(1, 0 /*UWOP_PUSH_NONVOL*/, 5); while( ctx->out_pos & 15 ) B(0); #endif + hl_codegen_init(ctx); + jit_code_append(ctx); } void hl_jit_free( jit_ctx *ctx, h_bool can_reset ) { @@ -118,16 +152,10 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { hl_emit_function(ctx); hl_regs_function(ctx); hl_codegen_function(ctx); - if( !jit_code_reserve(ctx, ctx->code_size) ) - return -1; int pos = ctx->out_pos; - memcpy(ctx->output + pos, ctx->code_instrs, ctx->code_size); -#ifdef WIN64_UNWIND_TABLES - int fid = (int)(f - ctx->mod->code->functions); - ctx->mod->unwind_table[fid].BeginAddress = ctx->out_pos; - ctx->mod->unwind_table[fid].EndAddress = ctx->out_pos + ctx->code_size; -#endif - ctx->out_pos += ctx->code_size; + hl_jit_define_function(ctx, pos, ctx->code_size); + if( !jit_code_append(ctx) ) + return -1; current_ctx = NULL; return pos; } diff --git a/src/jit.h b/src/jit.h index 5ab9d5289..9e4adf43f 100644 --- a/src/jit.h +++ b/src/jit.h @@ -188,6 +188,7 @@ struct _jit_ctx { unsigned char *code_instrs; int *code_pos_map; // accum output + int fdef_index; int out_pos; int out_max; unsigned char *output; @@ -199,9 +200,9 @@ void hl_jit_free( jit_ctx *ctx, h_bool can_reset ); void hl_jit_reset( jit_ctx *ctx, hl_module *m ); void hl_jit_init( jit_ctx *ctx, hl_module *m ); int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ); +void hl_jit_define_function( jit_ctx *ctx, int start, int size ); void hl_jit_null_field_access(); -void hl_jit_null_access(); void hl_jit_assert(); // emit & dump diff --git a/src/jit_dump.c b/src/jit_dump.c index e83b506b6..83e6bfa3d 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -261,7 +261,7 @@ static void hl_dump_ptr_name( jit_ctx *ctx, void *ptr ) { DYN(i); DYN(p); N2("null_field",hl_jit_null_field_access); - N2("null_access",hl_jit_null_access); + N2("null_access",hl_null_access); N(hl_get_thread); N(setjmp); N(_setjmp); @@ -355,10 +355,7 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { dump_value(ctx, e->value, e->mode); break; case LOAD_ADDR: - if( (e->size_offs>>8) ) - printf(" %s[%Xh]", val_str(e->a,M_PTR), e->size_offs); - else - printf(" %s[%d]", val_str(e->a,M_PTR), e->size_offs); + printf(" %s[%Xh]", val_str(e->a,M_PTR), e->size_offs); break; case STORE: { @@ -366,10 +363,8 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { if( offs == 0 ) printf(" [%s]", val_str(e->a,M_PTR)); else - printf(" %s[%d]", val_str(e->a,M_PTR), offs); + printf(" %s[%Xh]", val_str(e->a,M_PTR), offs); printf(" = %s", reg_str(e->b)); - //if( e->mode == 0 || e->mode != ctx->instrs[ctx->values_writes[e->b.index]].mode ) - // printf(" ???"); } break; case CONV: @@ -379,7 +374,8 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { case LEA: printf(" %s", reg_str(e->a)); if( !IS_NULL(e->b) ) printf("+%s", reg_str(e->b)); - if( e->size_offs > 1 ) printf("*%d",e->size_offs); + if( (e->size_offs&0xFF) > 1 ) printf("*%d",e->size_offs&0xFF); + if( e->size_offs >> 8 ) printf("+%Xh", e->size_offs>>8); break; default: if( !IS_NULL(e->a) ) { diff --git a/src/jit_emit.c b/src/jit_emit.c index f8f5e6108..c6c6a0c25 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -1957,8 +1957,13 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hashed_name = hl_hash_gen(name, true); } // ----------------------------------------- - ereg arg = null_field_access ? LOAD_CONST(hashed_name,&hlt_i32) : UNUSED; - emit_native_call(ctx, null_field_access ? hl_jit_null_field_access : hl_jit_null_access, &arg, null_field_access ? 1 : 0, NULL); + null_field_access = false; + if( null_field_access ) { + einstr *e = emit_instr(ctx, PUSH_CONST); + e->mode = M_PTR; + e->value = hashed_name; + } + emit_native_call(ctx, null_field_access ? hl_jit_null_field_access : hl_null_access, NULL, 0, NULL); patch_jump(ctx, jok); } break; diff --git a/src/jit_regs.c b/src/jit_regs.c index dc369514e..be16faf97 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -131,8 +131,8 @@ static void regs_emit_mov( regs_ctx *ctx, ereg to, ereg from, emit_mode m ) { static void regs_emit_todo_impl( regs_ctx *ctx, int line ) { einstr e; e.header = PUSH_CONST; - e.mode = M_I32; - e.value = line; + e.mode = M_PTR; + e.size_offs = line; regs_write_instr(ctx, &e, UNUSED); einstr e2; e2.header = DEBUG_BREAK; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 6350469f2..e0f3d363b 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -287,6 +287,8 @@ struct _code_ctx { int *pos_map; int cur_op; bool flushed; + int null_access_pos; + int null_field_pos; }; static int _incr( int*v, int n ) { @@ -394,9 +396,8 @@ typedef enum { RFPU = 1, RSTACK = 2, RCONST = 3, - RADDR = 4, - RMEM = 5, - RUNUSED = 6, + RMEM = 4, + RUNUSED = 5, } preg_kind; typedef struct { @@ -451,6 +452,7 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, case ID2(RCPU,RCPU): case ID2(RFPU,RFPU): if( f->mem_r ) { + // canonical form if( a.reg > 7 ) r64 |= 1; if( b.reg > 7 ) r64 |= 4; OP(f->mem_r); @@ -482,226 +484,118 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, break; case ID2(RSTACK,RUNUSED): ERRIF( f->mem_r == 0 || GET_RM(f->mem_r) == 0 ); - { - OP(f->mem_r); - if( IS_SBYTE(a.value) ) { - MOD_RM(1,GET_RM(f->mem_r)-1,RBP); - B(a.value); - } else { - MOD_RM(2,GET_RM(f->mem_r)-1,RBP); - W(a.value); - } + OP(f->mem_r); + if( IS_SBYTE(a.value) ) { + MOD_RM(1,GET_RM(f->mem_r)-1,RBP); + B(a.value); + } else { + MOD_RM(2,GET_RM(f->mem_r)-1,RBP); + W(a.value); } break; case ID2(RCPU,RCONST): ERRIF( f->r_const == 0 && f->r_i8 == 0 ); if( a.reg > 7 ) r64 |= 1; - { - // short byte form - if( f->r_i8 && IS_SBYTE(value) ) { - if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; - OP(f->r_i8); - if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_i8)-1,a.reg); - B(value); - } else if( GET_RM(f->r_const) > 0 || (f->r_const&FLAG_DUAL) ) { - if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; - OP(f->r_const&0xFF); - if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_const)-1,a.reg); - if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); - } else { - ERRIF( f->r_const == 0); - OP((f->r_const&0xFF) + (a.reg&7)); - if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); - } + if( f->r_i8 && IS_SBYTE(value) ) { + if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; + OP(f->r_i8); + if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_i8)-1,a.reg); + B(value); + } else if( GET_RM(f->r_const) > 0 || (f->r_const&FLAG_DUAL) ) { + if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; + OP(f->r_const&0xFF); + if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_const)-1,a.reg); + if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); + } else { + ERRIF( f->r_const == 0); + OP((f->r_const&0xFF) + (a.reg&7)); + if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); } break; case ID2(RSTACK,RCPU): case ID2(RSTACK,RFPU): ERRIF( f->mem_r == 0 ); if( b.reg > 7 ) r64 |= 4; - { - OP(f->mem_r); - if( IS_SBYTE(a.value) ) { - MOD_RM(1,b.reg,RBP); - B(a.value); - } else { - MOD_RM(2,b.reg,RBP); - W(a.value); - } + OP(f->mem_r); + if( IS_SBYTE(a.value) ) { + MOD_RM(1,b.reg,RBP); + B(a.value); + } else { + MOD_RM(2,b.reg,RBP); + W(a.value); } break; case ID2(RCPU,RSTACK): case ID2(RFPU,RSTACK): ERRIF( f->r_mem == 0 ); if( a.reg > 7 ) r64 |= 4; - { - OP(f->r_mem); - if( IS_SBYTE(b.value) ) { - MOD_RM(1,a.reg,RBP); - B(b.value); - } else { - MOD_RM(2,a.reg,RBP); - W(b.value); - } + OP(f->r_mem); + if( IS_SBYTE(b.value) ) { + MOD_RM(1,a.reg,RBP); + B(b.value); + } else { + MOD_RM(2,a.reg,RBP); + W(b.value); } break; case ID2(RCONST,RUNUSED): ERRIF( f->r_const == 0 ); - { - OP(f->r_const); - if( f->r_const & FLAG_8B ) B(value); else W((int)value); - } + OP(f->r_const); + if( f->r_const & FLAG_8B ) B(value); else W((int)value); break; case ID2(RMEM,RUNUSED): ERRIF( f->mem_r == 0 ); - { - int mult = a.value; - CpuReg reg = (a.reg & 0xFF); - if( mult == 0 ) { - if( reg > 7 ) r64 |= 1; - OP(f->mem_r); - if( value == 0 && (reg&7) != RBP ) { - MOD_RM(0,GET_RM(f->mem_r)-1,reg); - if( (reg&7) == RSP ) B(0x24); - } else if( IS_SBYTE(value) ) { - MOD_RM(1,GET_RM(f->mem_r)-1,reg); - if( (reg&7) == RSP ) B(0x24); - B(value); - } else { - MOD_RM(2,GET_RM(f->mem_r)-1,reg); - if( (reg&7) == RSP ) B(0x24); - W((int)value); - } - } else { - // [eax + ebx * M] - ERRIF(1); - } + if( a.reg > 7 ) r64 |= 1; + OP(f->mem_r); + if( value == 0 && (a.reg&7) != RBP ) { + MOD_RM(0,GET_RM(f->mem_r)-1,a.reg); + if( (a.reg&7) == RSP ) B(0x24); + } else if( IS_SBYTE(value) ) { + MOD_RM(1,GET_RM(f->mem_r)-1,a.reg); + if( (a.reg&7) == RSP ) B(0x24); + B(value); + } else { + MOD_RM(2,GET_RM(f->mem_r)-1,a.reg); + if( (a.reg&7) == RSP ) B(0x24); + W((int)value); } break; case ID2(RCPU, RMEM): case ID2(RFPU, RMEM): - ERRIF( f->r_mem == 0 ); - { - int mult = b.value; - CpuReg reg = b.reg & 0xFF; - if( mult == 15 ) { - int pos; - if( a.reg > 7 ) r64 |= 4; - OP(f->r_mem); - MOD_RM(0,a.reg,5); - if( IS_64 ) { - // offset wrt current code - pos = ctx->code.cur + 4; - W((int)(value - pos)); - } else { - ERRIF(1); - } - } else if( mult == 0 ) { - if( a.reg > 7 ) r64 |= 4; - if( reg > 7 ) r64 |= 1; - OP(f->r_mem); - if( value == 0 && (reg&7) != RBP ) { - MOD_RM(0,a.reg,reg); - if( (reg&7) == RSP ) B(0x24); - } else if( IS_SBYTE(value) ) { - MOD_RM(1,a.reg,reg); - if( (reg&7) == RSP ) B(0x24); - B(value); - } else { - MOD_RM(2,a.reg,reg); - if( (reg&7) == RSP ) B(0x24); - W((int)value); - } - } else { - jit_assert(); - /* - if( a.reg > 7 ) r64 |= 4; - if( reg > 7 ) r64 |= 1; - if( regOrOffs > 7 ) r64 |= 2; - OP(f->r_mem); - MOD_RM(offset == 0 ? 0 : IS_SBYTE(offset) ? 1 : 2,a.reg,4); - SIB(mult,regOrOffs,reg); - if( offset ) { - if( IS_SBYTE(offset) ) B(offset); else W(offset); - } - */ - } - } - break; -# ifndef HL_64 - case ID2(RFPU,RADDR): -# endif - case ID2(RCPU,RADDR): ERRIF( f->r_mem == 0 ); if( a.reg > 7 ) r64 |= 4; + if( b.reg > 7 ) r64 |= 1; OP(f->r_mem); - MOD_RM(0,a.reg,5); - if( IS_64 ) - W64((int_val)value); - else - W((int)(int_val)value); - break; -# ifndef HL_64 - case ID2(RADDR,RFPU): -# endif - case ID2(RADDR,RCPU): - ERRIF( f->mem_r == 0 ); - if( b.reg > 7 ) r64 |= 4; - OP(f->mem_r); - MOD_RM(0,b.reg,5); - if( IS_64 ) - W64((int_val)value); - else - W((int)(int_val)value); + if( value == 0 && (b.reg&7) != RBP ) { + MOD_RM(0,a.reg,b.reg); + if( (b.reg&7) == RSP ) B(0x24); + } else if( IS_SBYTE(value) ) { + MOD_RM(1,a.reg,b.reg); + if( (b.reg&7) == RSP ) B(0x24); + B(value); + } else { + MOD_RM(2,a.reg,b.reg); + if( (b.reg&7) == RSP ) B(0x24); + W((int)value); + } break; case ID2(RMEM, RCPU): case ID2(RMEM, RFPU): ERRIF( f->mem_r == 0 ); - { - int mult = a.value; - CpuReg reg = a.reg & 0xFF; - if( mult == 15 ) { - int pos; - if( b.reg > 7 ) r64 |= 4; - OP(f->mem_r); - MOD_RM(0,b.reg,5); - if( IS_64 ) { - // offset wrt current code - pos = ctx->code.cur + 4; - W((int)(value - pos)); - } else { - ERRIF(1); - } - } else if( mult == 0 ) { - if( b.reg > 7 ) r64 |= 4; - if( reg > 7 ) r64 |= 1; - OP(f->mem_r); - if( value == 0 && (reg&7) != RBP ) { - MOD_RM(0,b.reg,reg); - if( (reg&7) == RSP ) B(0x24); - } else if( IS_SBYTE(value) ) { - MOD_RM(1,b.reg,reg); - if( (reg&7) == RSP ) B(0x24); - B(value); - } else { - MOD_RM(2,b.reg,reg); - if( (reg&7) == RSP ) B(0x24); - W((int)value); - } - } else { - jit_assert(); - /* - if( b.reg > 7 ) r64 |= 4; - if( reg > 7 ) r64 |= 1; - if( value > 7 ) r64 |= 2; - OP(f->mem_r); - MOD_RM(offset == 0 ? 0 : IS_SBYTE(offset) ? 1 : 2,b.reg,4); - SIB(mult,value,reg); - if( offset ) { - if( IS_SBYTE(offset) ) B(offset); else W(offset); - } - */ - } + if( a.reg > 7 ) r64 |= 1; + if( b.reg > 7 ) r64 |= 4; + OP(f->mem_r); + if( value == 0 && (a.reg&7) != RBP ) { + MOD_RM(0,b.reg,a.reg); + if( (a.reg&7) == RSP ) B(0x24); + } else if( IS_SBYTE(value) ) { + MOD_RM(1,b.reg,a.reg); + if( (a.reg&7) == RSP ) B(0x24); + B(value); + } else { + MOD_RM(2,b.reg,a.reg); + if( (a.reg&7) == RSP ) B(0x24); + W((int)value); } break; default: @@ -731,10 +625,11 @@ static void emit_jump( code_ctx *ctx, int mode, int offset ) { } } +#define RTMP R(R11) static ereg get_tmp( emit_mode mode ) { if( IS_FLOAT(mode) ) return MMX(scratch_float_reg); - return R(R11); + return RTMP; } static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode, int_val value ) { @@ -751,7 +646,7 @@ static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode, int_val if( !IS_PURE(out) && !IS_PURE(val) ) { ereg tmp = get_tmp(mode); emit_mov(ctx, tmp, val, mode, value); - emit_mov(ctx, out, tmp, mode, value); + emit_mov(ctx, out, tmp, mode, 0); } else { static CpuOp MOV_OP[] = {_MOV,MOV8,MOV16,_MOV,_MOV,_MOV,MOVSD,MOVSS,_MOV,_MOV}; emit_ext(ctx, MOV_OP[mode],out,val,mode,value); @@ -792,15 +687,14 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ { ereg f = R(RCX); if( b != f ) { - ereg tmp = get_tmp(M_I32); if( a == f ) { - EMIT(_MOV,tmp,a,M_I32); - a = tmp; + EMIT(_MOV,RTMP,a,M_I32); + a = RTMP; } EMIT(_PUSH,f,UNUSED,M_I32); EMIT(_MOV,f,b,M_I32); emit_anyop(ctx, op, out, a, f, mode); - EMIT(_POP,out == f ? tmp : f,UNUSED,M_I32); + EMIT(_POP,out == f ? RTMP : f,UNUSED,M_I32); return; } } @@ -869,6 +763,29 @@ static void emit_nop( code_ctx *ctx, int size ) { B(0x90); } +static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { + einstr e = *_e; + if( e.a && (e.a & FL_NATMASK) == FL_STACKOFFS ) { + e.value += GET_STACK_OFFS(e.a); + e.a = R(RBP); + } + if( !IS_PURE(e.a) ) { + emit_mov(ctx, RTMP, e.a, e.mode, 0); + e.a = RTMP; + //if( !IS_PURE(e.b) ) jit_assert(); + } else if( e.b && !IS_PURE(e.b) ) { + emit_mov(ctx, RTMP, e.b, e.mode, 0); + e.b = RTMP; + } + BREAK(); + //emit_ext(ctx,_LEA,out,e.a,e.mode,e.value | ((e.b & 0xFF) << 16)); +} + +static void align_function( code_ctx *ctx ) { + while( byte_count(ctx->code) & 15 ) + emit_nop(ctx,16 - (byte_count(ctx->code) & 15)); +} + void hl_codegen_function( jit_ctx *jit ) { code_ctx *ctx = jit->code; ctx->flushed = false; @@ -894,9 +811,9 @@ void hl_codegen_function( jit_ctx *jit ) { # ifdef HL_DEBUG int rid = cur_pos | (jit->fun->findex << 16); while( reg_index < jit->instr_count && jit->reg_pos_map[reg_index] <= cur_pos ) reg_index++; + int uid = emit_index | (jit->fun->findex << 16); if( emit_index < jit->fun->nops && jit->emit_pos_map[emit_index] < reg_index ) { - int uid = emit_index | (jit->fun->findex << 16); - emit_ext(ctx,_MOV,get_tmp(M_I32),VAL_CONST,M_I32,uid); + emit_ext(ctx,_MOV,RTMP,VAL_CONST,M_I32,uid); __ignore(&uid); __ignore(&rid); emit_index++; @@ -914,9 +831,8 @@ void hl_codegen_function( jit_ctx *jit ) { case STORE: if( !IS_PURE(e->a) && !IS_PURE(e->b) && (e->a & FL_NATMASK) != FL_STACKOFFS ) { EMIT(_PUSH,e->b,UNUSED,e->mode); - ereg tmp = get_tmp(M_PTR); - emit_mov(ctx, tmp, e->a, M_PTR, 0); - emit_ext(ctx, _POP,VAL_MEM(tmp), UNUSED, e->mode, e->size_offs); + emit_mov(ctx, RTMP, e->a, M_PTR, 0); + emit_ext(ctx, _POP,VAL_MEM(RTMP), UNUSED, e->mode, e->size_offs); } else emit_mov(ctx, VAL_MEM(e->a), e->b, e->mode, e->size_offs); break; @@ -927,7 +843,7 @@ void hl_codegen_function( jit_ctx *jit ) { emit_ext(ctx, _POP, e->a, UNUSED, e->mode, 0); break; case PUSH_CONST: - emit_ext(ctx, _PUSH, VAL_CONST, UNUSED, e->mode, e->value); + emit_ext(ctx, (e->value&0xFF) == e->value && e->mode == M_PTR ? PUSH8 : _PUSH, VAL_CONST, UNUSED, e->mode, e->value); break; case DEBUG_BREAK: BREAK(); @@ -943,7 +859,7 @@ void hl_codegen_function( jit_ctx *jit ) { { ereg w = IS_PURE(out) ? out : get_tmp(e->mode); if( e->value == 0 ) - EMIT(XOR, w, w, e->mode); + EMIT(IS_FLOAT(e->mode) ? XORPD : XOR, w, w, e->mode); else emit_mov(ctx, w, VAL_CONST, e->mode, e->value); if( w != out ) @@ -951,7 +867,16 @@ void hl_codegen_function( jit_ctx *jit ) { } break; case LOAD_ADDR: - emit_mov(ctx,out,VAL_MEM(e->a),e->mode, e->size_offs); + { + ereg addr; + if( (e->a & FL_STACKREG) == FL_STACKREG ) { + emit_mov(ctx,RTMP,e->a,M_PTR,0); + addr = VAL_MEM(RTMP); + } else { + addr = VAL_MEM(e->a); + } + emit_mov(ctx,out,addr,e->mode,e->size_offs); + } break; case CALL_FUN: B(0xE8); @@ -960,12 +885,19 @@ void hl_codegen_function( jit_ctx *jit ) { int fid = e->a; int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,pos); int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,fid); - W(0xBADBAD00 - (pos + 4)); + W(0); } break; case CALL_PTR: - emit_ext(ctx, _MOV, R(RAX), VAL_CONST, M_PTR, e->value); - EMIT(_CALL, R(RAX), UNUSED, M_NONE); + if( e->value == (uint64)hl_null_access || e->value == (uint64)hl_jit_null_field_access ) { + // call near + int target = e->value == (uint64)hl_null_access ? ctx->null_access_pos : ctx->null_field_pos; + B(0xE8); + W(target - (jit->out_pos + byte_count(ctx->code) + 4)); + } else { + emit_ext(ctx, _MOV, R(RAX), VAL_CONST, M_PTR, e->value); + EMIT(_CALL, R(RAX), UNUSED, M_NONE); + } break; case CALL_REG: EMIT(_CALL, e->a, UNUSED, M_NONE); @@ -1068,10 +1000,10 @@ void hl_codegen_function( jit_ctx *jit ) { case LEA: if( !IS_PURE(out) ) { ereg tmp = get_tmp(e->mode); - EMIT(_LEA,tmp,e->a,e->mode); + emit_lea(ctx,tmp,e); emit_mov(ctx,out,tmp,e->mode,0); } else - EMIT(_LEA,out,e->a,e->mode); + emit_lea(ctx,out,e); break; case STACK_OFFS: if( e->size_offs >= 0 ) @@ -1085,8 +1017,7 @@ void hl_codegen_function( jit_ctx *jit ) { } if( ctx->code.cur > ctx->code.max ) jit_assert(); } - while( byte_count(ctx->code) & 15 ) - emit_nop(ctx,16 - (byte_count(ctx->code) & 15)); + align_function(ctx); hl_codegen_flush(jit); for(int i=0;ishort_jumps);i+=2) { int pos = int_arr_get(ctx->short_jumps,i); @@ -1111,6 +1042,33 @@ void hl_codegen_alloc( jit_ctx *jit ) { ctx->jit = jit; } +void hl_codegen_init( jit_ctx *jit ) { + code_ctx *ctx = jit->code; + byte_reserve(ctx->code,256); + ctx->code.cur -= 256; + ctx->null_access_pos = jit->out_pos + byte_count(ctx->code); + EMIT(_PUSH,R(RBP),UNUSED,M_PTR); + EMIT(_MOV,R(RBP),R(RSP),M_PTR); + emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x20); + emit_mov(ctx,R(RAX),VAL_CONST,M_PTR,(int_val)hl_null_access); + EMIT(_CALL,R(RAX),UNUSED,M_PTR); + BREAK(); + hl_jit_define_function(jit, ctx->null_access_pos, jit->out_pos + byte_count(ctx->code) - ctx->null_access_pos); + align_function(ctx); + ctx->null_field_pos = jit->out_pos + byte_count(ctx->code); + EMIT(_PUSH,R(RBP),UNUSED,M_PTR); + EMIT(_MOV,R(RBP),R(RSP),M_PTR); + emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x28); + emit_mov(ctx,R(RCX),VAL_MEM(R(RBP)),M_I32,HL_WSIZE*2); + emit_mov(ctx,R(RAX),VAL_CONST,M_PTR,(int_val)hl_jit_null_field_access); + EMIT(_CALL,R(RAX),UNUSED,M_PTR); + BREAK(); + hl_jit_define_function(jit, ctx->null_field_pos, jit->out_pos + byte_count(ctx->code) - ctx->null_field_pos); + align_function(ctx); + if( byte_count(ctx->code) > ctx->code.max ) jit_assert(); + hl_codegen_flush(jit); +} + void hl_codegen_free( jit_ctx *jit ) { code_ctx *ctx = jit->code; free(ctx->pos_map); diff --git a/src/module.c b/src/module.c index dca9c76be..420875b48 100644 --- a/src/module.c +++ b/src/module.c @@ -707,8 +707,9 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { hl_module_init_natives(m); hl_module_init_indexes(m); # ifdef WIN64_UNWIND_TABLES - m->unwind_table = malloc(sizeof(RUNTIME_FUNCTION) * m->code->nfunctions); - memset(m->unwind_table, 0, sizeof(RUNTIME_FUNCTION) * m->code->nfunctions); + m->unwind_table_size = m->code->nfunctions + 10; // extra space for jit internals + m->unwind_table = malloc(sizeof(RUNTIME_FUNCTION) * m->unwind_table_size); + memset(m->unwind_table, 0, sizeof(RUNTIME_FUNCTION) * m->unwind_table_size); # endif // JIT ctx = hl_jit_alloc(); @@ -770,7 +771,7 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { hl_setup.vtune_init = modules_init_vtune; # endif # ifdef WIN64_UNWIND_TABLES - RtlAddFunctionTable(m->unwind_table, m->code->nfunctions, (DWORD64)m->jit_code); + RtlAddFunctionTable(m->unwind_table, m->unwind_table_size, (DWORD64)m->jit_code); # endif hl_jit_free(ctx, hot_reload); if( hot_reload ) { From afe37f426d9c06b0d7d20d6e2979bceca25bb109 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Tue, 21 Apr 2026 21:55:26 +0200 Subject: [PATCH 32/83] rework jump tables and emit local jumps, use parallel moves for call args --- src/jit.c | 3 + src/jit.h | 5 ++ src/jit_dump.c | 18 +++++ src/jit_emit.c | 112 ++++++++++++++++++--------- src/jit_regs.c | 193 ++++++++++++++++++++++++++++------------------- src/jit_x86_64.c | 22 +++++- src/module.c | 3 + 7 files changed, 242 insertions(+), 114 deletions(-) diff --git a/src/jit.c b/src/jit.c index 91483e1dc..fddfec0b1 100644 --- a/src/jit.c +++ b/src/jit.c @@ -64,9 +64,12 @@ void hl_codegen_free( jit_ctx *jit ); void hl_codegen_function( jit_ctx *jit ); void hl_codegen_final( jit_ctx *jit ); +void hl_jit_init_regs( regs_config *cfg ); + jit_ctx *hl_jit_alloc() { jit_ctx *ctx = (jit_ctx*)malloc(sizeof(jit_ctx)); memset(ctx,0,sizeof(jit_ctx)); + hl_jit_init_regs(&ctx->cfg); hl_alloc_init(&ctx->falloc); hl_emit_alloc(ctx); hl_regs_alloc(ctx); diff --git a/src/jit.h b/src/jit.h index 9e4adf43f..4aad5493c 100644 --- a/src/jit.h +++ b/src/jit.h @@ -53,6 +53,7 @@ typedef enum { BLOCK, ENTER, STACK_OFFS, + XCHG, } emit_op; typedef enum { @@ -156,6 +157,7 @@ typedef struct { ereg stack_reg; ereg stack_pos; int stack_align; + int debug_prefix_size; ereg req_bit_shifts; ereg req_div_a; ereg req_div_b; @@ -169,6 +171,7 @@ struct _jit_ctx { emit_ctx *emit; regs_ctx *regs; code_ctx *code; + regs_config cfg; // emit output int instr_count; int block_count; @@ -208,6 +211,8 @@ void hl_jit_assert(); // emit & dump void hl_emit_dump( jit_ctx *ctx ); const char *hl_emit_regstr( ereg v, emit_mode m ); +void hl_emit_store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ); +void hl_emit_remap_jumps( emit_ctx *ctx, void *jumps, einstr *instrs, int *pos_map ); ereg *hl_emit_get_args( emit_ctx *ctx, einstr *e ); ereg **hl_emit_get_regs( einstr *e, int *count ); void hl_emit_reg_iter( jit_ctx *jit, einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ); diff --git a/src/jit_dump.c b/src/jit_dump.c index 83e6bfa3d..5c1d72e34 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -50,6 +50,7 @@ static const char *op_names[] = { "block", "enter", "stack", + "xchg", }; bool hl_jit_dump_bin = false; @@ -340,6 +341,17 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { case JCOND: printf(" @%X", cur_pos + 1 + e->size_offs); break; + case JUMP_TABLE: + { + int *offsets = hl_emit_get_args(ctx->emit, e); + printf(" %s (", reg_str(e->a)); + for(int k=0;knargs;k++) { + if( k > 0 ) printf(","); + printf("@%X", cur_pos + 1 + offsets[k]); + } + printf(")"); + } + break; case BLOCK: printf(" #%d", e->size_offs); break; @@ -421,12 +433,14 @@ void hl_emit_dump( jit_ctx *ctx ) { int vpos = 1; int rpos = 0; int cpos = 0; + bool new_op = false; cur = 0; for(i=0;iinstr_count;i++) { while( ctx->emit_pos_map[cur_op] == i ) { printf("@%X ", cur_op); hl_dump_op(ctx->fun, f->ops + cur_op); printf("\n"); + new_op = true; cur_op++; } einstr *e = ctx->instrs + i; @@ -462,6 +476,10 @@ void hl_emit_dump( jit_ctx *ctx ) { else printf("\033[80G"); first = false; + if( new_op ) { + new_op = false; + cpos += ctx->cfg.debug_prefix_size; + } } printf("%.2X",ctx->code_instrs[cpos++]); } diff --git a/src/jit_emit.c b/src/jit_emit.c index c6c6a0c25..7f9781052 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -317,7 +317,8 @@ static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { e->b = from; } -static void store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { +#define store_args hl_emit_store_args +void hl_emit_store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { if( count < 0 ) jit_assert(); if( count > 64 ) jit_error("Too many arguments"); e->nargs = (unsigned char)count; @@ -381,16 +382,6 @@ static tmp_phi *alloc_phi( emit_ctx *ctx, emit_block *b, vreg *r ) { return p; } -static int emit_jump( emit_ctx *ctx, bool cond ) { - int p = ctx->emit_pos; - emit_gen(ctx, cond ? JCOND : JUMP, UNUSED, UNUSED, 0); - return p; -} - -static void patch_jump( emit_ctx *ctx, int jpos ) { - ctx->instrs[jpos].size_offs = ctx->emit_pos - (jpos + 1); -} - static emit_block *alloc_block( emit_ctx *ctx ) { emit_block *b = hl_zalloc(&ctx->jit->falloc, sizeof(emit_block)); b->id = blocks_count(ctx->blocks); @@ -415,7 +406,9 @@ static void store_block_var( emit_ctx *ctx, emit_block *b, vreg *r, ereg v ) { } } -static void split_block( emit_ctx *ctx ) { +static bool split_block( emit_ctx *ctx ) { + if( ctx->current_block->start_pos == ctx->emit_pos-1 ) + return false; emit_block *b = alloc_block(ctx); b->sealed = true; emit_debug("BLOCK #%d@%X[%X]\n",b->id,b->start_pos,ctx->op_pos); @@ -425,10 +418,43 @@ static void split_block( emit_ctx *ctx ) { } bool dead_code = blocks_count(b->preds) == 0; // if we have no reach, force previous block dependency, this is rare dead code emit by compiler einstr *eprev = &ctx->instrs[b->start_pos-1]; - if( (eprev->op != JUMP && eprev->op != RET && eprev->mode != M_NORET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) + if( (eprev->op != JUMP && eprev->op != JUMP_TABLE && eprev->op != RET && eprev->mode != M_NORET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) block_add_pred(ctx, b, ctx->current_block); ctx->current_block->end_pos = b->start_pos; ctx->current_block = b; + return true; +} + +static int emit_jump( emit_ctx *ctx, bool cond ) { + int p = ctx->emit_pos; + emit_gen(ctx, cond ? JCOND : JUMP, UNUSED, UNUSED, 0); + split_block(ctx); + return p; +} + +static void patch_jump( emit_ctx *ctx, int jpos ) { + emit_block *b = NULL; + // find the block or initial jump was + for_iter_back(blocks,b2,ctx->blocks) { + if( b2->start_pos <= jpos ) { + b = b2; + break; + } + } + if( !b ) jit_assert(); + // patch opcode + ctx->instrs[jpos].size_offs = ctx->emit_pos - (jpos + 1); + if( ctx->current_block->start_pos == ctx->emit_pos-1 ) { + block_add_pred(ctx, ctx->current_block, b); + } else { + ctx->arrival_points = link_add_sort_unique(ctx, ctx->op_pos, b, ctx->arrival_points); + if( !split_block(ctx) ) jit_assert(); + } +} + +static void add_jump_target( emit_ctx *ctx, int jpos, int offs ) { + int target = offs + ctx->op_pos + 1; + ctx->arrival_points = link_add_sort_unique(ctx, target, ctx->current_block, ctx->arrival_points); } static void register_jump( emit_ctx *ctx, int jpos, int offs ) { @@ -436,9 +462,9 @@ static void register_jump( emit_ctx *ctx, int jpos, int offs ) { int_arr_add(ctx->jump_regs, jpos); int_arr_add(ctx->jump_regs, target); if( offs > 0 ) { - ctx->arrival_points = link_add_sort_unique(ctx, target, ctx->current_block, ctx->arrival_points); + add_jump_target(ctx, jpos, offs); if( ctx->arrival_points->id != ctx->op_pos + 1 && ctx->fun->ops[ctx->op_pos].op != OSwitch && ctx->fun->ops[ctx->op_pos+1].op != OLabel ) - split_block(ctx); + ctx->arrival_points = link_add_sort_unique(ctx, ctx->op_pos + 1, ctx->current_block, ctx->arrival_points); } } @@ -722,14 +748,11 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { if( t->kind == HNULL && t->tparam->kind == dt->kind ) { emit_test(ctx, v, OJNotNull); int jnot = emit_jump(ctx, true); - split_block(ctx); ereg v1 = LOAD_CONST(0,dt); int jend = emit_jump(ctx, false); patch_jump(ctx, jnot); - split_block(ctx); ereg v2 = LOAD_MEM(v,0,dt); patch_jump(ctx, jend); - split_block(ctx); return emit_phi(ctx, v1, v2); } bool need_dyn = dyn_need_type(dt); @@ -802,19 +825,30 @@ static void emit_write_block( emit_ctx *ctx, emit_block *b ) { } } +void hl_emit_remap_jumps( emit_ctx *ctx, void *_jumps, einstr *instrs, int *pos_map ) { + int_arr jumps = *(int_arr*)_jumps; + int i = 0; + while( i < int_arr_count(jumps) ) { + int pos = int_arr_get(jumps,i++); + int target = int_arr_get(jumps,i++); + einstr *e = instrs + pos; + if( e->op == JUMP_TABLE ) { + int *args = (int*)hl_emit_get_args(ctx, e); + for(int k=0;knargs;k++) + args[k] = pos_map[target + args[k]] - (pos + 1); + } else + e->size_offs = pos_map[target] - (pos + 1); + } + int_arr_reset((int_arr*)_jumps); +} + void hl_emit_flush( jit_ctx *jit ) { emit_ctx *ctx = jit->emit; - int i = 0; if( ctx->flushed ) return; ctx->flushed = true; - while( i < int_arr_count(ctx->jump_regs) ) { - int pos = int_arr_get(ctx->jump_regs,i++); - einstr *e = ctx->instrs + pos; - int target = int_arr_get(ctx->jump_regs,i++); - e->size_offs = ctx->pos_map[target] - (pos + 1); - } ctx->pos_map[ctx->fun->nops] = -1; ctx->current_block->end_pos = ctx->emit_pos; + hl_emit_remap_jumps(ctx,&ctx->jump_regs, ctx->instrs, ctx->pos_map); jit->instrs = ctx->instrs; jit->instr_count = ctx->emit_pos; jit->emit_pos_map = ctx->pos_map; @@ -954,7 +988,10 @@ void hl_emit_function( jit_ctx *jit ) { for(int op_pos=0;op_posnops;op_pos++) { ctx->op_pos = op_pos; - ctx->pos_map[op_pos] = ctx->emit_pos; + if( ctx->emit_pos > 0 && ctx->instrs[ctx->emit_pos-1].op == BLOCK ) + ctx->pos_map[op_pos] = ctx->emit_pos-1; + else + ctx->pos_map[op_pos] = ctx->emit_pos; if( op_pos == 0 ) { ctx->current_block->start_pos = ctx->emit_pos; emit_gen_size(ctx, BLOCK, 0); @@ -1019,7 +1056,8 @@ static bool seal_block_rec( emit_ctx *ctx, emit_block *b, int target ) { } static void register_block_jump( emit_ctx *ctx, int offs, bool cond ) { - int jidx = emit_jump(ctx, cond); + int jidx = ctx->emit_pos; + emit_gen(ctx, cond ? JCOND : JUMP, UNUSED, UNUSED, 0); register_jump(ctx, jidx, offs); if( offs < 0 ) { int target = ctx->pos_map[ctx->op_pos + 1 + offs]; @@ -1769,8 +1807,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case OLabel: - if( ctx->current_block->start_pos != ctx->emit_pos-1 ) - split_block(ctx); + split_block(ctx); prepare_loop_block(ctx); break; case OGetI8: @@ -1957,7 +1994,6 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hashed_name = hl_hash_gen(name, true); } // ----------------------------------------- - null_field_access = false; if( null_field_access ) { einstr *e = emit_instr(ctx, PUSH_CONST); e->mode = M_PTR; @@ -2109,10 +2145,16 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { int count = o->p2; emit_cmp(ctx,v,LOAD_CONST(count,&hlt_i32),OJUGte); int jdefault = emit_jump(ctx, true); - emit_gen_ext(ctx, JUMP_TABLE, v, UNUSED, 0, count); - for(int i=0; iextra[i]); + int pos = ctx->emit_pos; + einstr *e = emit_instr(ctx, JUMP_TABLE); + e->a = v; + store_args(ctx,e,(ereg*)o->extra,count); + register_jump(ctx, pos, 0); + for(int k=0;kextra[k]; + if( offs < 0 ) jit_assert(); + if( offs == 0 ) continue; + add_jump_target(ctx, pos, offs); } patch_jump(ctx, jdefault); } @@ -2121,7 +2163,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, LOAD_MEM(LOAD(ra),0,&hlt_i32)); break; case OAssert: - emit_native_call(ctx, hl_jit_assert, NULL, 0, &hlt_void); + emit_native_call(ctx, hl_jit_assert, NULL, 0, NULL); break; case ONop: break; diff --git a/src/jit_regs.c b/src/jit_regs.c index be16faf97..37f057f0d 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -38,7 +38,7 @@ #define VIDX(e) (((e) < 0) ? ctx->jit->value_count + (-(e)-1) : (e)) #define VAL_REG(e) VAL(VIDX(e)) #define REG_MODE(m) (((m) == M_F32 || (m) == M_F64) ? 1 :0) -#define REG_CFG(m) (m ? &ctx->cfg.floats : &ctx->cfg.regs) +#define REG_CFG(m) (m ? &ctx->jit->cfg.floats : &ctx->jit->cfg.regs) #if defined(HL_WIN_CALL) && defined(HL_64) # define IS_WINCALL64 1 @@ -62,16 +62,16 @@ typedef struct { struct _regs_ctx { jit_ctx *jit; - regs_config cfg; value_info *values; values scratch; int_arr jump_regs; - int_arr phi_movs; + int_arr pack_movs; int_arr *blocks_phis; int max_instrs; int cur_op; int emit_pos; int stack_size; + int stack_offset; einstr *instrs; ereg *out_write; int *pos_map; @@ -82,8 +82,6 @@ struct _regs_ctx { typedef int call_regs[2]; -void hl_jit_init_regs( regs_config *cfg ); - static ereg get_call_reg( regs_ctx *ctx, call_regs regs, emit_mode m ) { ereg r; int mode = REG_MODE(m); @@ -116,15 +114,19 @@ static void regs_write_instr( regs_ctx *ctx, einstr *e, ereg out ) { ctx->instrs[ctx->emit_pos++] = *e; } -static void regs_emit_mov( regs_ctx *ctx, ereg to, ereg from, emit_mode m ) { - if( to == from ) return; +static void regs_emit( regs_ctx *ctx, ereg out, emit_op op, ereg a, ereg b, emit_mode m, int size_offs ) { einstr e; - e.header = MOV; + e.header = op; e.mode = m; - e.size_offs = 0; - e.a = from; - e.b = 0; - regs_write_instr(ctx, &e, to); + e.a = a; + e.b = b; + e.size_offs = size_offs; + regs_write_instr(ctx, &e, out); +} + +static void regs_emit_mov( regs_ctx *ctx, ereg to, ereg from, emit_mode m ) { + if( to == from ) return; + regs_emit(ctx,to,MOV,from,UNUSED,m,0); } #define regs_todo() regs_emit_todo_impl(ctx,__LINE__) @@ -164,6 +166,7 @@ static void spill( regs_ctx *ctx, value_info *v ) { v->reg = MK_STACK_REG(v->stack_pos); values_remove(&ctx->scratch,v); regs_debug("REG SPILL %s @%X\n",value_str(v),ctx->cur_op); + if( v->stack_pos < 0 ) v->reg = UNUSED; } static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { @@ -301,14 +304,14 @@ static void regs_compute_liveness( regs_ctx *ctx ) { case OSShr: case OUShr: case OShl: - if( ctx->cfg.req_bit_shifts ) VAL_REG(e->b)->pref_reg = ctx->cfg.req_bit_shifts; + if( jit->cfg.req_bit_shifts ) VAL_REG(e->b)->pref_reg = jit->cfg.req_bit_shifts; break; case OSDiv: case OUDiv: case OSMod: case OUMod: - if( ctx->cfg.req_div_a ) VAL_REG(e->a)->pref_reg = ctx->cfg.req_div_a; - if( ctx->cfg.req_div_b ) VAL_REG(e->b)->pref_reg = ctx->cfg.req_div_b; + if( jit->cfg.req_div_a ) VAL_REG(e->a)->pref_reg = jit->cfg.req_div_a; + if( jit->cfg.req_div_b ) VAL_REG(e->b)->pref_reg = jit->cfg.req_div_b; break; } break; @@ -321,6 +324,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { eblock *bl = jit->blocks + b; for(int p=0;pphi_count;p++) { ephi *ph = bl->phis + p; + VAL_REG(ph->value)->mode = ph->mode; for(int k=0;knvalues;k++) { ereg v = ph->values[k]; eblock *b2 = resolve_block_value(ctx, v); @@ -328,6 +332,8 @@ static void regs_compute_liveness( regs_ctx *ctx ) { regs_debug("ADD PHI %s:=%s to #%d\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks)); int_arr_add(*arr,v); int_arr_add(*arr,ph->value); + value_info *val = VAL_REG(v); + if( val->last_read < b2->end_pos ) val->last_read = b2->end_pos; } } } @@ -352,6 +358,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { // use existing stack storage v->stack_pos = args_size + HL_WSIZE*2; args_size += size < 4 ? 4 : size; + if( IS_NULL(r) ) v->reg = MK_STACK_REG(-v->stack_pos); } } // assign registers @@ -359,10 +366,12 @@ static void regs_assign_regs( regs_ctx *ctx ) { for(int cur_op=0;cur_opinstr_count;cur_op++) { einstr e = jit->instrs[cur_op]; ctx->cur_op = cur_op; + for_iter_back(values,v,ctx->scratch) { - if( v->last_read < cur_op ) + if( v->last_read <= cur_op ) values_remove(&ctx->scratch,v); } + if( IS_CALL(e.op) ) { ereg *args = hl_emit_get_args(ctx->jit->emit,&e); call_regs regs = {0}; @@ -385,6 +394,10 @@ static void regs_assign_regs( regs_ctx *ctx ) { if( will_scratch ) values_reset(&ctx->scratch); } if( e.op == BLOCK ) { + for_iter_back(values,v,ctx->scratch) { + if( v->last_read == cur_op ) + values_remove(&ctx->scratch,v); + } eblock *bl = jit->blocks + e.size_offs; for(int k=0;kphi_count;k++) { ephi *p = bl->phis + k; @@ -426,6 +439,57 @@ static void regs_assign_regs( regs_ctx *ctx ) { } } } + // assign stack regs + int nvalues = jit->value_count + jit->phi_count; + ctx->stack_offset = (ctx->persists_uses[0] + ctx->persists_uses[1]) * 8; + for(int i=0;ivalues + i; + if( v->reg == UNUSED ) v->reg = MK_STACK_REG(v->stack_pos);// + ctx->stack_offset); + } +} + +static void flush_movs( regs_ctx *ctx ) { + int_arr movs = ctx->pack_movs; + while( true ) { + int size = int_arr_count(movs); + if( !size ) break; + bool cycle = true; + for(int k=0;kpack_movs = movs; + int_arr_reset(&ctx->pack_movs); } static void flush_phis( regs_ctx *ctx, eblock *b ) { @@ -434,7 +498,7 @@ static void flush_phis( regs_ctx *ctx, eblock *b ) { int bid = (int)(b - jit->blocks); int_arr arr = ctx->blocks_phis[bid]; int idx = 0; - int_arr movs = ctx->phi_movs; + int_arr movs = ctx->pack_movs; while( idx < int_arr_count(arr) ) { ereg a = int_arr_get(arr,idx++); ereg b = int_arr_get(arr,idx++); @@ -443,54 +507,21 @@ static void flush_phis( regs_ctx *ctx, eblock *b ) { if( from->reg == to->reg ) continue; int size = int_arr_count(movs); bool dup = false; - for(int k=0;kreg == to->reg && int_arr_get(movs,k+1) == from->reg ) { + for(int k=0;kreg && int_arr_get(movs,k+1) == from->reg ) { dup = true; break; } } if( !dup ) { - int_arr_add(movs, b); + int_arr_add(movs, to->reg); int_arr_add(movs, from->reg); + int_arr_add(movs, from->mode); } } + ctx->pack_movs = movs; int_arr_free(&ctx->blocks_phis[bid]); - while( true ) { - int size = int_arr_count(movs); - if( !size ) break; - bool cycle = true; - for(int k=0;kreg ) { - read = true; - break; - } - } - if( !read ) { - regs_emit_mov(ctx,to->reg,int_arr_get(movs,k+1),to->mode); - int_arr_remove_range(&movs,k,2); - cycle = false; - break; - } - } - if( cycle ) - jit_assert(); - } - ctx->phi_movs = movs; - int_arr_reset(&ctx->phi_movs); -} - -static void regs_emit( regs_ctx *ctx, ereg out, emit_op op, ereg a, ereg b, emit_mode m, int size_offs ) { - einstr e; - e.header = op; - e.mode = m; - e.a = a; - e.b = b; - e.size_offs = size_offs; - regs_write_instr(ctx, &e, out); + flush_movs(ctx); } static void regs_emit_instrs( regs_ctx *ctx ) { @@ -501,11 +532,11 @@ static void regs_emit_instrs( regs_ctx *ctx ) { ctx->pos_map[0] = 0; int stack_offset = ctx->stack_size; - int push_size = HL_WSIZE * 2; // RIP + RBP save + int push_size = HL_WSIZE * 2 + ctx->stack_offset; // RIP + RBP save if( IS_WINCALL64 && ctx->has_direct_call ) stack_offset += 0x20; // reserve - if( ctx->cfg.stack_align ) { - int align = (stack_offset + push_size) % ctx->cfg.stack_align; - if( align ) stack_offset += ctx->cfg.stack_align - align; + if( jit->cfg.stack_align ) { + int align = (stack_offset + push_size) % jit->cfg.stack_align; + if( align ) stack_offset += jit->cfg.stack_align - align; } for(int cur_op=0;cur_opinstr_count;cur_op++) { @@ -517,13 +548,17 @@ static void regs_emit_instrs( regs_ctx *ctx ) { ereg *args = hl_emit_get_args(ctx->jit->emit,&e); call_regs regs = {0}; for(int k=0;kmode); if( IS_NULL(r) ) { regs_todo(); - } else - regs_emit_mov(ctx, r, v->reg, v->mode); + } else if( r != v->reg ) { + int_arr_add(ctx->pack_movs,r); + int_arr_add(ctx->pack_movs,v->reg); + int_arr_add(ctx->pack_movs,v->mode); + } } + flush_movs(ctx); e.nargs = 0xFF; ret_val = ®_CFG(REG_MODE(e.mode))->ret; if( e.op == CALL_REG ) @@ -558,17 +593,26 @@ static void regs_emit_instrs( regs_ctx *ctx ) { break; case ENTER: { - regs_emit(ctx,UNUSED,PUSH,ctx->cfg.stack_pos,UNUSED,M_PTR,0); - regs_emit_mov(ctx,ctx->cfg.stack_pos,ctx->cfg.stack_reg,M_PTR); + regs_emit(ctx,UNUSED,PUSH,jit->cfg.stack_pos,UNUSED,M_PTR,0); + regs_emit_mov(ctx,jit->cfg.stack_pos,jit->cfg.stack_reg,M_PTR); + for(int i=0;ipersists_uses[0];i++) + regs_emit(ctx,UNUSED,PUSH,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR,0); + for(int i=0;ipersists_uses[1];i++) + regs_emit(ctx,UNUSED,PUSH,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64,0); regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-stack_offset); } break; case JUMP: case JCOND: + case JUMP_TABLE: flush_phis(ctx,cur_block); + if( e.op == JUMP_TABLE ) { + // copy args (remap later) + hl_emit_store_args(jit->emit,&e,hl_emit_get_args(jit->emit,&e),e.nargs); + } regs_write_instr(ctx, &e, out); int_arr_add(ctx->jump_regs, ctx->emit_pos - 1); - int_arr_add(ctx->jump_regs, cur_op + 1 + e.size_offs); + int_arr_add(ctx->jump_regs, cur_op + 1 + (e.op == JUMP_TABLE ? 0 : e.size_offs)); break; case RET: if( e.a ) { @@ -577,7 +621,11 @@ static void regs_emit_instrs( regs_ctx *ctx ) { regs_emit_mov(ctx, ret, e.a, e.mode); } regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); - regs_emit(ctx,UNUSED,POP,ctx->cfg.stack_pos,UNUSED,M_PTR,0); + for(int i=ctx->persists_uses[1]-1;i>=0;i--) + regs_emit(ctx,UNUSED,POP,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64,0); + for(int i=ctx->persists_uses[0]-1;i>=0;i--) + regs_emit(ctx,UNUSED,POP,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR,0); + regs_emit(ctx,UNUSED,POP,jit->cfg.stack_pos,UNUSED,M_PTR,0); regs_emit(ctx,UNUSED,RET,UNUSED,UNUSED,M_NONE,0); break; default: @@ -601,13 +649,7 @@ void hl_regs_flush( jit_ctx *jit ) { jit->reg_writes = ctx->out_write; jit->reg_pos_map = ctx->pos_map; if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; - int i = 0; - while( i < int_arr_count(ctx->jump_regs) ) { - int pos = int_arr_get(ctx->jump_regs,i++); - einstr *e = ctx->instrs + pos; - int target = int_arr_get(ctx->jump_regs,i++); - e->size_offs = ctx->pos_map[target] - (pos + 1); - } + hl_emit_remap_jumps(jit->emit, &ctx->jump_regs, ctx->instrs, ctx->pos_map); } void hl_regs_function( jit_ctx *jit ) { @@ -624,7 +666,7 @@ void hl_regs_function( jit_ctx *jit ) { jit->reg_instrs = NULL; values_free(&ctx->scratch); int_arr_free(&ctx->jump_regs); - int_arr_free(&ctx->phi_movs); + int_arr_free(&ctx->pack_movs); ctx->blocks_phis = (int_arr*)hl_zalloc(&jit->falloc,sizeof(int_arr) * jit->block_count); ctx->values = (value_info*)hl_zalloc(&jit->falloc,sizeof(value_info) * nvalues); for(int i=1;ijit = jit; jit->regs = ctx; - hl_jit_init_regs(&ctx->cfg); } void hl_regs_free( jit_ctx *jit ) { diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index e0f3d363b..32a19d5d6 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -386,6 +386,9 @@ void hl_jit_init_regs( regs_config *cfg ) { cfg->stack_reg = R(RSP); cfg->stack_pos = R(RBP); cfg->stack_align = 16; +# ifdef HL_DEBUG + cfg->debug_prefix_size = 6; +# endif } #define EMIT(op,a,b,mode) emit_ext(ctx,op,a,b,mode,0) @@ -811,12 +814,15 @@ void hl_codegen_function( jit_ctx *jit ) { # ifdef HL_DEBUG int rid = cur_pos | (jit->fun->findex << 16); while( reg_index < jit->instr_count && jit->reg_pos_map[reg_index] <= cur_pos ) reg_index++; - int uid = emit_index | (jit->fun->findex << 16); - if( emit_index < jit->fun->nops && jit->emit_pos_map[emit_index] < reg_index ) { - emit_ext(ctx,_MOV,RTMP,VAL_CONST,M_I32,uid); + int uid; + while( emit_index < jit->fun->nops && jit->emit_pos_map[emit_index] < reg_index ) { + uid = emit_index | (jit->fun->findex << 16); __ignore(&uid); __ignore(&rid); emit_index++; + if( emit_index >= jit->fun->nops || jit->emit_pos_map[emit_index] >= reg_index ) + emit_ext(ctx,_MOV,RTMP,VAL_CONST,M_I32,uid); + if( uid == 0x19A0000 ) BREAK(); } # endif switch( e->op ) { @@ -828,6 +834,16 @@ void hl_codegen_function( jit_ctx *jit ) { else emit_mov(ctx, out, e->a, e->mode, 0); break; + case XCHG: + { + ereg tmp = get_tmp(e->mode); + if( !IS_PURE(e->a) && !IS_PURE(e->b) ) + jit_assert(); + emit_mov(ctx, tmp, e->a, e->mode, 0); + emit_mov(ctx, e->a, e->b, e->mode, 0); + emit_mov(ctx, e->b, tmp, e->mode, 0); + } + break; case STORE: if( !IS_PURE(e->a) && !IS_PURE(e->b) && (e->a & FL_NATMASK) != FL_STACKOFFS ) { EMIT(_PUSH,e->b,UNUSED,e->mode); diff --git a/src/module.c b/src/module.c index 420875b48..793fc1854 100644 --- a/src/module.c +++ b/src/module.c @@ -754,6 +754,9 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { # endif } m->jit_code = hl_jit_code(ctx, m, &m->codesize, &m->jit_debug, NULL); +# ifdef HL_DEBUG + if( filter >= 0 ) exit(0); +# endif for(i=0;icode->nfunctions;i++) { hl_function *f = m->code->functions + i; m->functions_ptrs[f->findex] = ((unsigned char*)m->jit_code) + ((int_val)m->functions_ptrs[f->findex]); From fda19b736859380b63040fb621c0bbc98cba7d8b Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 22 Apr 2026 00:57:59 +0200 Subject: [PATCH 33/83] fixes in reg alloc and codegen --- src/jit_emit.c | 4 +-- src/jit_regs.c | 61 ++++++++++++++++++++++++------------- src/jit_x86_64.c | 78 +++++++++++++++++++++++++++++++++++------------- 3 files changed, 100 insertions(+), 43 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 7f9781052..b892c1ec2 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -794,9 +794,9 @@ static void emit_write_block( emit_ctx *ctx, emit_block *b ) { bl->preds = (int*)hl_malloc(&jit->falloc,sizeof(int)*bl->pred_count); bl->nexts = (int*)hl_malloc(&jit->falloc,sizeof(int)*bl->next_count); for(int i=0;ipred_count;i++) - bl->preds[i++] = blocks_get(b->preds,i)->id; + bl->preds[i] = blocks_get(b->preds,i)->id; for(int i=0;inext_count;i++) - bl->nexts[i++] = blocks_get(b->nexts,i)->id; + bl->nexts[i] = blocks_get(b->nexts,i)->id; // write phis { for_iter(phi,p,b->phis) diff --git a/src/jit_regs.c b/src/jit_regs.c index 37f057f0d..26d889497 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -49,6 +49,7 @@ typedef struct { int stack_pos; int last_read; + int tot_reads; emit_mode mode; ereg pref_reg; ereg reg; @@ -223,6 +224,7 @@ static void regs_write_live( regs_ctx *ctx, ereg *r ) { if( IS_NATREG(*r) ) return; value_info *v = VAL_REG(*r); v->last_read = ctx->cur_op; + v->tot_reads++; } static value_info *regs_current( regs_ctx *ctx, ereg r ) { @@ -328,12 +330,30 @@ static void regs_compute_liveness( regs_ctx *ctx ) { for(int k=0;knvalues;k++) { ereg v = ph->values[k]; eblock *b2 = resolve_block_value(ctx, v); - int_arr *arr = &ctx->blocks_phis[b2 - jit->blocks]; - regs_debug("ADD PHI %s:=%s to #%d\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks)); - int_arr_add(*arr,v); - int_arr_add(*arr,ph->value); value_info *val = VAL_REG(v); - if( val->last_read < b2->end_pos ) val->last_read = b2->end_pos; + if( b2->start_pos >= bl->start_pos ) { + // loop : insert in all predecesors + for(int i=0;ipred_count;i++) { + b2 = &jit->blocks[bl->preds[i]]; + if( b2->start_pos >= bl->start_pos ) { + int_arr *arr = &ctx->blocks_phis[b2 - jit->blocks]; + regs_debug("ADD PHI %s:=%s to #%d@%X\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks),b2->end_pos-1); + int_arr_add(*arr,v); + int_arr_add(*arr,ph->value); + val->tot_reads++; + if( val->last_read < b2->end_pos ) + val->last_read = b2->end_pos; + } + } + } else { + int_arr *arr = &ctx->blocks_phis[b2 - jit->blocks]; + regs_debug("ADD PHI %s:=%s to #%d@%X\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks),b2->end_pos-1); + int_arr_add(*arr,v); + int_arr_add(*arr,ph->value); + val->tot_reads++; + if( val->last_read < b2->end_pos ) + val->last_read = b2->end_pos; + } } } } @@ -365,8 +385,18 @@ static void regs_assign_regs( regs_ctx *ctx ) { int write_index = 1; for(int cur_op=0;cur_opinstr_count;cur_op++) { einstr e = jit->instrs[cur_op]; + value_info *write = NULL; ctx->cur_op = cur_op; + if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) { + write = VAL(write_index++); + // try to preserve ops in the from A = A op B + if( (e.op == UNOP || e.op == BINOP) && write->pref_reg == UNUSED ) { + value_info *v = VAL_REG(e.a); + if( IS_PURE(v->reg) ) write->pref_reg = v->reg; + } + } + for_iter_back(values,v,ctx->scratch) { if( v->last_read <= cur_op ) values_remove(&ctx->scratch,v); @@ -413,28 +443,17 @@ static void regs_assign_regs( regs_ctx *ctx ) { regs_assign(ctx, v); } } - if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) { - value_info *w = VAL(write_index++); + if( write ) { switch( e.op ) { case ALLOC_STACK: - w->reg = MK_STACK_OFFS(regs_alloc_stack(ctx, e.size_offs)); + write->reg = MK_STACK_OFFS(regs_alloc_stack(ctx, e.size_offs)); break; case LOAD_ARG: - if( w->reg == UNUSED ) - regs_assign(ctx, w); // assign for stack reg + if( write->reg == UNUSED ) + regs_assign(ctx, write); // assign for stack reg break; - case UNOP: - case BINOP: - if( w->pref_reg == UNUSED ) { - value_info *v = VAL_REG(e.a); - if( IS_PURE(v->reg) ) { - values_remove(&ctx->scratch,v); - w->pref_reg = v->reg; - } - } - // fallback default: - regs_assign(ctx, w); + regs_assign(ctx, write); break; } } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 32a19d5d6..149404c12 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -23,6 +23,10 @@ #include #include "data_struct.h" +#ifdef HL_DEBUG +# define GEN_DEBUG +#endif + #define S_TYPE byte_arr #define S_NAME(name) byte_##name #define S_VALUE unsigned char @@ -170,11 +174,10 @@ typedef struct { int mem_r; // r/m32 / r32 r/m32 int r_const; // r32 / imm32 imm32 int r_i8; // r32 / imm8 imm8 - int mem_const; // r/m32 / imm32 N/A } opform; static opform OP_FORMS[] = { - { "MOV", 0x8B, 0x89, 0xB8, 0, RM(0xC7,0) }, + { "MOV", 0x8B, 0x89, 0xB8, 0 }, { "LEA", 0x8D }, { "PUSH", 0x50 | FLAG_DEF64, RM(0xFF,6), 0x68, 0x6A }, { "ADD", 0x03, 0x01, RM(0x81,0), RM(0x83,0) }, @@ -386,7 +389,7 @@ void hl_jit_init_regs( regs_config *cfg ) { cfg->stack_reg = R(RSP); cfg->stack_pos = R(RBP); cfg->stack_align = 16; -# ifdef HL_DEBUG +# ifdef GEN_DEBUG cfg->debug_prefix_size = 6; # endif } @@ -608,7 +611,7 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, static void emit_jump( code_ctx *ctx, int mode, int offset ) { int op_mult; -# ifdef HL_DEBUG +# ifdef GEN_DEBUG op_mult = 16; // additional debug info per op # else op_mult = 8; @@ -620,7 +623,7 @@ static void emit_jump( code_ctx *ctx, int mode, int offset ) { int_arr_add(ctx->short_jumps, ctx->cur_op + offset + 1); B(-2); } else { - B(0x0F); + if( mode != JAlways ) B(0x0F); B(mode); int_arr_add(ctx->near_jumps, byte_count(ctx->code)); int_arr_add(ctx->near_jumps, ctx->cur_op + offset + 1); @@ -690,14 +693,20 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ { ereg f = R(RCX); if( b != f ) { - if( a == f ) { + if( a == f || out == f ) { EMIT(_MOV,RTMP,a,M_I32); a = RTMP; } - EMIT(_PUSH,f,UNUSED,M_I32); - EMIT(_MOV,f,b,M_I32); - emit_anyop(ctx, op, out, a, f, mode); - EMIT(_POP,out == f ? RTMP : f,UNUSED,M_I32); + if( out == f ) { + EMIT(_MOV,f,b,M_I32); + emit_anyop(ctx, op, RTMP, RTMP, f, mode); + EMIT(_MOV,f,RTMP,M_I32); + } else { + EMIT(_PUSH,f,UNUSED,M_I32); + EMIT(_MOV,f,b,M_I32); + emit_anyop(ctx, op, out, a, f, mode); + EMIT(_POP,f,UNUSED,M_I32); + } return; } } @@ -773,15 +782,41 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { e.a = R(RBP); } if( !IS_PURE(e.a) ) { - emit_mov(ctx, RTMP, e.a, e.mode, 0); + // a is always a mem address ! + emit_mov(ctx, RTMP, e.a, M_PTR, 0); e.a = RTMP; - //if( !IS_PURE(e.b) ) jit_assert(); + if( !IS_PURE(e.b) ) { + BREAK(); // TODO ! + return; + } } else if( e.b && !IS_PURE(e.b) ) { - emit_mov(ctx, RTMP, e.b, e.mode, 0); + // b is always an int index ! + emit_mov(ctx, RTMP, e.b, M_I32, 0); e.b = RTMP; } - BREAK(); - //emit_ext(ctx,_LEA,out,e.a,e.mode,e.value | ((e.b & 0xFF) << 16)); + int mult = e.size_offs & 0xFF; + int offs = e.size_offs >> 8; + if( mult == 0 ) { + BREAK(); + emit_ext(ctx,_LEA,out,e.a,M_PTR,offs); + return; + } + int r64 = 8 | ((out&8) ? 4 : 0) | ((e.b&8) ? 2 : 0) | ((e.a & 8) ? 1 : 0); + REX(); + B(0x8D); + MOD_RM(1,out&7,4); + switch( mult ) { + case 1: mult = 0; break; + case 2: mult = 1; break; + case 4: mult = 2; break; + case 8: mult = 3; break; + default: jit_assert(); + } + SIB(mult,e.b&7,e.a&7); + if( IS_SBYTE(offs) ) + B(offs); + else + jit_assert(); } static void align_function( code_ctx *ctx ) { @@ -800,7 +835,7 @@ void hl_codegen_function( jit_ctx *jit ) { ctx->pos_map[0] = 0; byte_reserve(ctx->code,64); ctx->code.cur -= 64; -# ifdef HL_DEBUG +# ifdef GEN_DEBUG int reg_index = 0; int emit_index = 0; # endif @@ -811,7 +846,7 @@ void hl_codegen_function( jit_ctx *jit ) { ctx->code.cur -= 64; ctx->cur_op = cur_pos; if( cur_pos > 0 ) ctx->pos_map[cur_pos] = ctx->code.cur; -# ifdef HL_DEBUG +# ifdef GEN_DEBUG int rid = cur_pos | (jit->fun->findex << 16); while( reg_index < jit->instr_count && jit->reg_pos_map[reg_index] <= cur_pos ) reg_index++; int uid; @@ -822,16 +857,16 @@ void hl_codegen_function( jit_ctx *jit ) { emit_index++; if( emit_index >= jit->fun->nops || jit->emit_pos_map[emit_index] >= reg_index ) emit_ext(ctx,_MOV,RTMP,VAL_CONST,M_I32,uid); - if( uid == 0x19A0000 ) BREAK(); } # endif switch( e->op ) { case LOAD_ARG: continue; // nop case MOV: - if( (e->a & FL_NATMASK) == FL_STACKOFFS ) + if( (e->a & FL_NATMASK) == FL_STACKOFFS ) { + if( !IS_PURE(out) ) jit_assert(); emit_ext(ctx,_LEA,out,VAL_MEM(R(RBP)),M_PTR,GET_STACK_OFFS(e->a)); - else + } else emit_mov(ctx, out, e->a, e->mode, 0); break; case XCHG: @@ -849,6 +884,9 @@ void hl_codegen_function( jit_ctx *jit ) { EMIT(_PUSH,e->b,UNUSED,e->mode); emit_mov(ctx, RTMP, e->a, M_PTR, 0); emit_ext(ctx, _POP,VAL_MEM(RTMP), UNUSED, e->mode, e->size_offs); + } else if( (e->a & FL_NATMASK) == FL_STACKREG ) { + emit_mov(ctx, RTMP, e->a, M_PTR, 0); + emit_mov(ctx, VAL_MEM(RTMP), e->b, e->mode, e->size_offs); } else emit_mov(ctx, VAL_MEM(e->a), e->b, e->mode, e->size_offs); break; From 539d2679bc92646e4880f3cc391fa5dbec07423d Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 22 Apr 2026 07:45:15 +0200 Subject: [PATCH 34/83] fixed mov impl --- src/jit_dump.c | 4 +-- src/jit_emit.c | 17 +++++++------ src/jit_x86_64.c | 64 +++++++++++++++++++++++++----------------------- 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/jit_dump.c b/src/jit_dump.c index 5c1d72e34..6385796d0 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -204,8 +204,8 @@ static void dump_value( jit_ctx *ctx, uint64 value, emit_mode mode ) { printf("NULL"); else if( mode == M_PTR && value >= (uint64)code->types && value < (uint64)(code->types + code->ntypes) ) uprintf(USTR("<%s>"),hl_type_str((hl_type*)value)); - else if( mode == M_PTR && value >= (uint64)mod->globals_data && value < (uint64)(mod->globals_data + mod->globals_size) ) - printf("",(int)(value - (uint64)mod->globals_data)); + else if( mode == M_PTR && value == (uint64)mod->globals_data ) + printf("",(int)(value - (uint64)mod->globals_data)); else if( value == (uint64)&hlt_void ) printf(""); else diff --git a/src/jit_emit.c b/src/jit_emit.c index b892c1ec2..6afbf450c 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -1383,14 +1383,14 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case OGetGlobal: { - void *addr = m->globals_data + m->globals_indexes[o->p2]; - STORE(dst, LOAD_MEM_PTR(LOAD_CONST_PTR(addr),0)); + int offs = m->globals_indexes[o->p2]; + STORE(dst, LOAD_MEM_PTR(LOAD_CONST_PTR(m->globals_data),offs)); } break; case OSetGlobal: { - void *addr = m->globals_data + m->globals_indexes[o->p1]; - STORE_MEM(LOAD_CONST_PTR(addr),0,LOAD(ra)); + int offs = m->globals_indexes[o->p1]; + STORE_MEM(LOAD_CONST_PTR(m->globals_data),offs,LOAD(ra)); } break; case OCall0: @@ -2072,14 +2072,17 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hl_opcode *next = f->ops + ctx->op_pos + 1 + o->p2; hl_opcode *next2 = f->ops + ctx->op_pos + 2 + o->p2; void *addr = NULL; + int offs = 0; if( cat->op == OCatch || (next->op == OGetGlobal && next2->op == OCall2 && next2->p3 == next->p1 && dst->id == (int)(int_val)next2->extra) ) { int gindex = cat->op == OCatch ? cat->p1 : next->p2; hl_type *gt = m->code->globals[gindex]; while( gt->kind == HOBJ && gt->obj->super ) gt = gt->obj->super; - if( gt->kind == HOBJ && gt->obj->nfields && gt->obj->fields[0].t->kind == HTYPE ) - addr = m->globals_data + m->globals_indexes[gindex]; + if( gt->kind == HOBJ && gt->obj->nfields && gt->obj->fields[0].t->kind == HTYPE ) { + addr = m->globals_data; + offs = m->globals_indexes[gindex]; + } } - STORE_MEM(st, (int)(int_val)&trap->tcheck, addr ? LOAD_MEM_PTR(LOAD_CONST_PTR(addr),0) : LOAD_CONST_PTR(NULL)); + STORE_MEM(st, (int)(int_val)&trap->tcheck, addr ? LOAD_MEM_PTR(LOAD_CONST_PTR(addr),offs) : LOAD_CONST_PTR(NULL)); void *fun = setjmp; ereg args[2]; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 149404c12..68604d444 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -638,27 +638,31 @@ static ereg get_tmp( emit_mode mode ) { return RTMP; } -static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode, int_val value ) { +static void emit_mov_ext( code_ctx *ctx, ereg out, int out_val, ereg val, int in_val, emit_mode mode ) { if( out == val ) return; if( (val & FL_NATMASK) == FL_STACKOFFS ) { - emit_mov(ctx,out,VAL_MEM(R(RBP)),mode, value + GET_STACK_OFFS(val)); + emit_mov_ext(ctx,out,out_val,VAL_MEM(R(RBP)),in_val + GET_STACK_OFFS(val),mode); return; } if( (out & FL_NATMASK) == FL_STACKOFFS ) { - emit_mov(ctx,VAL_MEM(R(RBP)),val,mode, value + GET_STACK_OFFS(val)); + emit_mov_ext(ctx,VAL_MEM(R(RBP)),out_val + GET_STACK_OFFS(val),val,in_val,mode); return; } if( !IS_PURE(out) && !IS_PURE(val) ) { ereg tmp = get_tmp(mode); - emit_mov(ctx, tmp, val, mode, value); - emit_mov(ctx, out, tmp, mode, 0); + emit_mov_ext(ctx, tmp, 0, val, in_val, mode); + emit_mov_ext(ctx, out, out_val, tmp, 0, mode); } else { static CpuOp MOV_OP[] = {_MOV,MOV8,MOV16,_MOV,_MOV,_MOV,MOVSD,MOVSS,_MOV,_MOV}; - emit_ext(ctx, MOV_OP[mode],out,val,mode,value); + emit_ext(ctx, MOV_OP[mode],out,val,mode,IS_PURE(out) ? in_val : out_val); } } +static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode ) { + emit_mov_ext(ctx,out,0,val,0,mode); +} + static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_mode mode ) { CpuOp cop; # define F_OP(iop,f32,f64) cop = mode == M_F32 ? f32 : (mode == M_F64 ? f64 : iop); @@ -725,11 +729,11 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ } if( !IS_PURE(out) ) { ereg tmp = get_tmp(mode); - emit_mov(ctx, tmp, a, mode, 0); + emit_mov(ctx, tmp, a, mode); EMIT(cop,tmp,b,mode); - emit_mov(ctx, out, tmp, mode, 0); + emit_mov(ctx, out, tmp, mode); } else { - emit_mov(ctx, out, a, mode, 0); + emit_mov(ctx, out, a, mode); EMIT(cop,out,b,mode); } } @@ -783,7 +787,7 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { } if( !IS_PURE(e.a) ) { // a is always a mem address ! - emit_mov(ctx, RTMP, e.a, M_PTR, 0); + emit_mov(ctx, RTMP, e.a, M_PTR); e.a = RTMP; if( !IS_PURE(e.b) ) { BREAK(); // TODO ! @@ -791,7 +795,7 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { } } else if( e.b && !IS_PURE(e.b) ) { // b is always an int index ! - emit_mov(ctx, RTMP, e.b, M_I32, 0); + emit_mov(ctx, RTMP, e.b, M_I32); e.b = RTMP; } int mult = e.size_offs & 0xFF; @@ -867,28 +871,28 @@ void hl_codegen_function( jit_ctx *jit ) { if( !IS_PURE(out) ) jit_assert(); emit_ext(ctx,_LEA,out,VAL_MEM(R(RBP)),M_PTR,GET_STACK_OFFS(e->a)); } else - emit_mov(ctx, out, e->a, e->mode, 0); + emit_mov(ctx, out, e->a, e->mode); break; case XCHG: { ereg tmp = get_tmp(e->mode); if( !IS_PURE(e->a) && !IS_PURE(e->b) ) jit_assert(); - emit_mov(ctx, tmp, e->a, e->mode, 0); - emit_mov(ctx, e->a, e->b, e->mode, 0); - emit_mov(ctx, e->b, tmp, e->mode, 0); + emit_mov(ctx, tmp, e->a, e->mode); + emit_mov(ctx, e->a, e->b, e->mode); + emit_mov(ctx, e->b, tmp, e->mode); } break; case STORE: if( !IS_PURE(e->a) && !IS_PURE(e->b) && (e->a & FL_NATMASK) != FL_STACKOFFS ) { EMIT(_PUSH,e->b,UNUSED,e->mode); - emit_mov(ctx, RTMP, e->a, M_PTR, 0); + emit_mov(ctx, RTMP, e->a, M_PTR); emit_ext(ctx, _POP,VAL_MEM(RTMP), UNUSED, e->mode, e->size_offs); } else if( (e->a & FL_NATMASK) == FL_STACKREG ) { - emit_mov(ctx, RTMP, e->a, M_PTR, 0); - emit_mov(ctx, VAL_MEM(RTMP), e->b, e->mode, e->size_offs); + emit_mov(ctx, RTMP, e->a, M_PTR); + emit_mov_ext(ctx, VAL_MEM(RTMP), e->size_offs, e->b, 0, e->mode); } else - emit_mov(ctx, VAL_MEM(e->a), e->b, e->mode, e->size_offs); + emit_mov_ext(ctx, VAL_MEM(e->a), e->size_offs, e->b, 0, e->mode); break; case PUSH: emit_ext(ctx, _PUSH, e->a, UNUSED, e->mode, 0); @@ -905,7 +909,7 @@ void hl_codegen_function( jit_ctx *jit ) { case RET: if( !IS_NULL(e->a) ) { ereg ret = IS_FLOAT(e->mode) ? MMX(0) : R(RAX); - if( e->a != ret ) emit_mov(ctx, ret, e->a, e->mode, 0); + if( e->a != ret ) emit_mov(ctx, ret, e->a, e->mode); } EMIT(_RET, UNUSED, UNUSED, M_NONE); break; @@ -915,21 +919,21 @@ void hl_codegen_function( jit_ctx *jit ) { if( e->value == 0 ) EMIT(IS_FLOAT(e->mode) ? XORPD : XOR, w, w, e->mode); else - emit_mov(ctx, w, VAL_CONST, e->mode, e->value); + emit_ext(ctx, _MOV, w, VAL_CONST, e->mode, e->value); if( w != out ) - emit_mov(ctx, out, w, e->mode, 0); + emit_mov(ctx, out, w, e->mode); } break; case LOAD_ADDR: { ereg addr; if( (e->a & FL_STACKREG) == FL_STACKREG ) { - emit_mov(ctx,RTMP,e->a,M_PTR,0); + emit_mov(ctx,RTMP,e->a,M_PTR); addr = VAL_MEM(RTMP); } else { addr = VAL_MEM(e->a); } - emit_mov(ctx,out,addr,e->mode,e->size_offs); + emit_mov_ext(ctx,out,0,addr,e->size_offs,e->mode); } break; case CALL_FUN: @@ -961,7 +965,7 @@ void hl_codegen_function( jit_ctx *jit ) { jit_assert(); if( !IS_PURE(e->a) ) { ereg tmp = get_tmp(e->mode); - emit_mov(ctx, tmp, e->a, e->mode, 0); + emit_mov(ctx, tmp, e->a, e->mode); EMIT(_TEST,tmp,tmp,e->mode); } else EMIT(_TEST,e->a,e->a,e->mode); @@ -978,7 +982,7 @@ void hl_codegen_function( jit_ctx *jit ) { } if( !IS_PURE(e->a) && !IS_PURE(e->b) ) { ereg tmp = get_tmp(e->mode); - emit_mov(ctx, tmp, e->b, e->mode, 0); + emit_mov(ctx, tmp, e->b, e->mode); EMIT(op,e->a,tmp,e->mode); } else EMIT(op,e->a,e->b,e->mode); @@ -1055,7 +1059,7 @@ void hl_codegen_function( jit_ctx *jit ) { if( !IS_PURE(out) ) { ereg tmp = get_tmp(e->mode); emit_lea(ctx,tmp,e); - emit_mov(ctx,out,tmp,e->mode,0); + emit_mov(ctx,out,tmp,e->mode); } else emit_lea(ctx,out,e); break; @@ -1104,7 +1108,7 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_PUSH,R(RBP),UNUSED,M_PTR); EMIT(_MOV,R(RBP),R(RSP),M_PTR); emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x20); - emit_mov(ctx,R(RAX),VAL_CONST,M_PTR,(int_val)hl_null_access); + emit_ext(ctx,_MOV,R(RAX),VAL_CONST,M_PTR,(int_val)hl_null_access); EMIT(_CALL,R(RAX),UNUSED,M_PTR); BREAK(); hl_jit_define_function(jit, ctx->null_access_pos, jit->out_pos + byte_count(ctx->code) - ctx->null_access_pos); @@ -1113,8 +1117,8 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_PUSH,R(RBP),UNUSED,M_PTR); EMIT(_MOV,R(RBP),R(RSP),M_PTR); emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x28); - emit_mov(ctx,R(RCX),VAL_MEM(R(RBP)),M_I32,HL_WSIZE*2); - emit_mov(ctx,R(RAX),VAL_CONST,M_PTR,(int_val)hl_jit_null_field_access); + emit_ext(ctx,_MOV,R(RCX),VAL_MEM(R(RBP)),M_I32,HL_WSIZE*2); + emit_ext(ctx,_MOV,R(RAX),VAL_CONST,M_PTR,(int_val)hl_jit_null_field_access); EMIT(_CALL,R(RAX),UNUSED,M_PTR); BREAK(); hl_jit_define_function(jit, ctx->null_field_pos, jit->out_pos + byte_count(ctx->code) - ctx->null_field_pos); From 9ea4835456deeffc36756ce6ddd5c48383b11913 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 22 Apr 2026 22:51:57 +0200 Subject: [PATCH 35/83] clear gc allocated mem to 0xCD in GC_DEBUG --- src/allocator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allocator.c b/src/allocator.c index 47dfc8f41..f9bd63420 100644 --- a/src/allocator.c +++ b/src/allocator.c @@ -313,6 +313,8 @@ static void *gc_alloc_fixed( int part, int kind ) { for(i=0;iblock_size;i++) if( ptr[i] != 0xDD ) hl_fatal("assert"); + else + ptr[i] = 0xCD; } # endif gc_free_pages[pid] = ph; @@ -367,6 +369,8 @@ static void *gc_alloc_var( int part, int size, int kind ) { for(i=0;ibmp ) { From da98c13c33698ea3ad642963bf58fc47459598b2 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 22 Apr 2026 23:34:49 +0200 Subject: [PATCH 36/83] various fixes --- src/jit_emit.c | 25 +++++++++++------ src/jit_regs.c | 72 +++++++++++++++++++++++++++++++----------------- src/jit_x86_64.c | 47 +++++++++++++++++++------------ 3 files changed, 92 insertions(+), 52 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 6afbf450c..bce2a43a7 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -251,8 +251,6 @@ static linked_inf *link_sort_remove( linked_inf *head, int id ) { static emit_mode hl_type_mode( hl_type *t ) { if( t->kind == HVOID ) return M_VOID; - if( t->kind < HBOOL ) - return (emit_mode)t->kind; if( t->kind == HBOOL ) return sizeof(bool) == 1 ? M_UI8 : M_I32; if( t->kind == HGUID ) @@ -261,6 +259,8 @@ static emit_mode hl_type_mode( hl_type *t ) { return M_F32; if( t->kind == HF64 ) return M_F64; + if( t->kind < HBOOL ) + return (emit_mode)t->kind; return M_PTR; } @@ -736,8 +736,14 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, } } + static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode from, emit_mode to, bool _unsigned ) { - return emit_gen_ext(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, UNUSED, to, from); + if( (from == M_F32 || from == M_F64) != (to == M_F32 || to == M_F64) ) + return emit_gen_ext(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, UNUSED, to, from); + if( from != to && (from == M_F32 || from == M_F64) && (to == M_F32 || to == M_F64) ) + return emit_gen_ext(ctx, CONV, v, UNUSED, to, from); + // no-op + return emit_gen(ctx,MOV,v,UNUSED,to); } static bool dyn_need_type( hl_type *t ) { @@ -1365,9 +1371,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { double d; uint64 i; } v; - if( dst->t->kind == HF32 ) + if( dst->t->kind == HF32 ) { + v.i = 0; v.f = (float)m->code->floats[o->p2]; - else + } else v.d = m->code->floats[o->p2]; STORE(dst, LOAD_CONST(v.i, dst->t)); } @@ -1816,7 +1823,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg offs = OFFSET(LOAD(ra),LOAD(rb),1,0); ereg val = LOAD_MEM(offs, 0, dst->t); - if( o->op != OGetMem ) val = emit_conv(ctx, val, M_I32, hl_type_mode(dst->t), false); + if( o->op != OGetMem ) patch_instr_mode(ctx, o->op == OGetI8 ? M_UI8 : M_UI16); STORE(dst, val); } break; @@ -1826,8 +1833,8 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg offs = OFFSET(LOAD(dst), LOAD(ra),1,0); ereg val = LOAD(rb); - if( o->op != OSetMem ) val = emit_conv(ctx, val, M_I32, hl_type_mode(dst->t), false); STORE_MEM(offs, 0, val); + if( o->op != OSetMem ) patch_instr_mode(ctx, o->op == OSetI8 ? M_UI8 : M_UI16); } break; case OType: @@ -1921,8 +1928,8 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OToVirtual: { ereg args[2]; - args[0] = LOAD(ra); - args[1] = LOAD_CONST_PTR(dst->t); + args[0] = LOAD_CONST_PTR(dst->t); + args[1] = LOAD(ra); STORE(dst, emit_native_call(ctx,hl_to_virtual,args,2, dst->t)); } break; diff --git a/src/jit_regs.c b/src/jit_regs.c index 26d889497..adaf2794f 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -40,6 +40,9 @@ #define REG_MODE(m) (((m) == M_F32 || (m) == M_F64) ? 1 :0) #define REG_CFG(m) (m ? &ctx->jit->cfg.floats : &ctx->jit->cfg.regs) +#define EMIT(r,a,b,m) regs_emit(ctx,UNUSED,r,a,b,m,0) +#define BREAK() EMIT(DEBUG_BREAK,UNUSED,UNUSED,0) + #if defined(HL_WIN_CALL) && defined(HL_64) # define IS_WINCALL64 1 #else @@ -94,6 +97,12 @@ static ereg get_call_reg( regs_ctx *ctx, call_regs regs, emit_mode m ) { return r; } +static int get_stack_size( emit_mode m ) { + int size = hl_emit_mode_sizes[m]; + if( size < HL_WSIZE ) size = HL_WSIZE; + return size; +} + static void regs_write_instr( regs_ctx *ctx, einstr *e, ereg out ) { if( ctx->emit_pos == ctx->max_instrs ) { int pos = ctx->emit_pos; @@ -130,21 +139,6 @@ static void regs_emit_mov( regs_ctx *ctx, ereg to, ereg from, emit_mode m ) { regs_emit(ctx,to,MOV,from,UNUSED,m,0); } -#define regs_todo() regs_emit_todo_impl(ctx,__LINE__) -static void regs_emit_todo_impl( regs_ctx *ctx, int line ) { - einstr e; - e.header = PUSH_CONST; - e.mode = M_PTR; - e.size_offs = line; - regs_write_instr(ctx, &e, UNUSED); - einstr e2; - e2.header = DEBUG_BREAK; - e2.size_offs = 0; - e2.a = UNUSED; - e2.b = UNUSED; - regs_write_instr(ctx, &e2, UNUSED); -} - static int regs_alloc_stack( regs_ctx *ctx, int size ) { ctx->stack_size += size; ctx->stack_size += jit_pad_size(ctx->stack_size,size); @@ -496,7 +490,7 @@ static void flush_movs( regs_ctx *ctx ) { ereg to = int_arr_get(movs,0); ereg from = int_arr_get(movs,1); int mode = int_arr_get(movs,2); - regs_emit(ctx,UNUSED,XCHG,to,from,mode,0); + EMIT(XCHG,to,from,mode); int_arr_remove_range(&movs,0,3); size -= 3; for(int k=0;kinstrs[cur_op]; ereg *ret_val = NULL; int nread; + int instr_stack_offset = 0; ctx->cur_op = cur_op; if( IS_CALL(e.op) ) { ereg *args = hl_emit_get_args(ctx->jit->emit,&e); call_regs regs = {0}; + int stack_args = 0; + int stack_bits = 0; for(int k=0;kmode); if( IS_NULL(r) ) { - regs_todo(); + stack_args += get_stack_size(v->mode); + stack_bits |= 1 << k; } else if( r != v->reg ) { int_arr_add(ctx->pack_movs,r); int_arr_add(ctx->pack_movs,v->reg); int_arr_add(ctx->pack_movs,v->mode); } } + if( stack_args > 0 ) { + int offset = 0; + if( jit->cfg.stack_align ) { + int align = stack_args % jit->cfg.stack_align; + if( align ) offset = jit->cfg.stack_align - align; + } + if( offset ) + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,0,-offset); + for(int k=e.nargs-1;k>=0;k--) { + if( stack_bits & (1 << k) ) + EMIT(PUSH,VAL_REG(args[k])->reg,UNUSED,M_PTR); + } + if( IS_WINCALL64 ) { + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,0,-0x20); + offset += 0x20; + } + instr_stack_offset = stack_args+offset; + } flush_movs(ctx); e.nargs = 0xFF; ret_val = ®_CFG(REG_MODE(e.mode))->ret; @@ -598,7 +614,6 @@ static void regs_emit_instrs( regs_ctx *ctx ) { case ALLOC_STACK: break; case BLOCK: - flush_phis(ctx,cur_block); cur_block = jit->blocks + e.size_offs; break; case LOAD_ARG: @@ -612,12 +627,12 @@ static void regs_emit_instrs( regs_ctx *ctx ) { break; case ENTER: { - regs_emit(ctx,UNUSED,PUSH,jit->cfg.stack_pos,UNUSED,M_PTR,0); + EMIT(PUSH,jit->cfg.stack_pos,UNUSED,M_PTR); regs_emit_mov(ctx,jit->cfg.stack_pos,jit->cfg.stack_reg,M_PTR); for(int i=0;ipersists_uses[0];i++) - regs_emit(ctx,UNUSED,PUSH,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR,0); + EMIT(PUSH,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR); for(int i=0;ipersists_uses[1];i++) - regs_emit(ctx,UNUSED,PUSH,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64,0); + EMIT(PUSH,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-stack_offset); } break; @@ -641,12 +656,15 @@ static void regs_emit_instrs( regs_ctx *ctx ) { } regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); for(int i=ctx->persists_uses[1]-1;i>=0;i--) - regs_emit(ctx,UNUSED,POP,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64,0); + EMIT(POP,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); for(int i=ctx->persists_uses[0]-1;i>=0;i--) - regs_emit(ctx,UNUSED,POP,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR,0); - regs_emit(ctx,UNUSED,POP,jit->cfg.stack_pos,UNUSED,M_PTR,0); - regs_emit(ctx,UNUSED,RET,UNUSED,UNUSED,M_NONE,0); + EMIT(POP,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR); + EMIT(POP,jit->cfg.stack_pos,UNUSED,M_PTR); + EMIT(RET,UNUSED,UNUSED,M_NONE); break; + case MOV: + if( out == e.a ) break; + // fallthrough default: if( ret_val && out ) { regs_write_instr(ctx, &e, *ret_val); @@ -655,6 +673,10 @@ static void regs_emit_instrs( regs_ctx *ctx ) { regs_write_instr(ctx, &e, out); break; } + if( instr_stack_offset ) + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,instr_stack_offset); + if( cur_block && cur_block->end_pos == cur_op+1 ) + flush_phis(ctx,cur_block); ctx->pos_map[cur_op+1] = ctx->emit_pos; } } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 68604d444..3edf18fa1 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -108,6 +108,7 @@ typedef enum { SUBSS, MULSS, DIVSS, + XORPS, XORPD, CVTSI2SD, CVTSI2SS, @@ -119,10 +120,12 @@ typedef enum { LDMXCSR, // 8-16 bits MOV8, + MOVZX8, CMP8, TEST8, PUSH8, MOV16, + MOVZX16, CMP16, TEST16, // prefetchs @@ -221,6 +224,7 @@ static opform OP_FORMS[] = { { "SUBSS", 0xF30F5C }, { "MULSS", 0xF30F59 }, { "DIVSS", 0xF30F5E }, + { "XORPS", 0x0F57 }, { "XORPD", 0x660F57 }, { "CVTSI2SD", 0xF20F2A }, { "CVTSI2SS", 0xF30F2A }, @@ -232,10 +236,12 @@ static opform OP_FORMS[] = { { "LDMXCSR", 0, LONG_RM(0x0FAE,2) }, // 8 bits, { "MOV8", 0x8A, 0x88, 0, RM(0xC6,0) }, + { "MOVZX8", LONG_OP(0x0FB6) }, { "CMP8", 0x3A, 0x38, 0, RM(0x80,7) }, { "TEST8", 0x84, 0x84, RM(0xF6,0) }, { "PUSH8", FLAG_DEF64, 0, 0x6A | FLAG_8B }, { "MOV16", OP16(0x8B), OP16(0x89), OP16(0xB8) }, + { "MOVZX16", LONG_OP(0x0FB7) }, { "CMP16", OP16(0x3B), OP16(0x39) }, { "TEST16", OP16(0x85) }, // prefetchs @@ -303,6 +309,7 @@ static int _incr( int*v, int n ) { const char *hl_natreg_str( int reg, emit_mode m ) { static char out[16]; static const char *regs_str[] = { "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI" }; + static const char *regs_str8[] = { "AL", "CL", "DL", "BL", "SPL", "BPL", "SIL", "DIL" }; CpuReg r = (reg & 0xFF); switch( m ) { case M_I32: @@ -319,7 +326,7 @@ const char *hl_natreg_str( int reg, emit_mode m ) { break; case M_UI8: if( r < 8 ) - sprintf(out,"%s",regs_str[r]); + sprintf(out,"%s",regs_str8[r]); else sprintf(out,"R%dB%s",r,r<16?"":"???"); break; @@ -655,7 +662,12 @@ static void emit_mov_ext( code_ctx *ctx, ereg out, int out_val, ereg val, int in emit_mov_ext(ctx, out, out_val, tmp, 0, mode); } else { static CpuOp MOV_OP[] = {_MOV,MOV8,MOV16,_MOV,_MOV,_MOV,MOVSD,MOVSS,_MOV,_MOV}; - emit_ext(ctx, MOV_OP[mode],out,val,mode,IS_PURE(out) ? in_val : out_val); + CpuOp op = MOV_OP[mode]; + if( (mode == M_UI8 || mode == M_UI16) && IS_PURE(out) ) { + op++; // MOVZX + mode = M_I64; + } + emit_ext(ctx,op,out,val,mode,IS_PURE(out) ? in_val : out_val); } } @@ -789,9 +801,10 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { // a is always a mem address ! emit_mov(ctx, RTMP, e.a, M_PTR); e.a = RTMP; - if( !IS_PURE(e.b) ) { - BREAK(); // TODO ! - return; + if( e.b && !IS_PURE(e.b) ) { + if( !IS_PURE(out) ) jit_assert(); + emit_mov(ctx, out, e.b, M_I32); + e.b = out; } } else if( e.b && !IS_PURE(e.b) ) { // b is always an int index ! @@ -801,6 +814,7 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { int mult = e.size_offs & 0xFF; int offs = e.size_offs >> 8; if( mult == 0 ) { + // no index BREAK(); emit_ext(ctx,_LEA,out,e.a,M_PTR,offs); return; @@ -808,19 +822,12 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { int r64 = 8 | ((out&8) ? 4 : 0) | ((e.b&8) ? 2 : 0) | ((e.a & 8) ? 1 : 0); REX(); B(0x8D); - MOD_RM(1,out&7,4); - switch( mult ) { - case 1: mult = 0; break; - case 2: mult = 1; break; - case 4: mult = 2; break; - case 8: mult = 3; break; - default: jit_assert(); - } + MOD_RM(offs == 0 ? 0 : 1,out&7,4); SIB(mult,e.b&7,e.a&7); - if( IS_SBYTE(offs) ) + if( offs != 0 ) { + if( !IS_SBYTE(offs) ) jit_assert(); B(offs); - else - jit_assert(); + } } static void align_function( code_ctx *ctx ) { @@ -895,9 +902,11 @@ void hl_codegen_function( jit_ctx *jit ) { emit_mov_ext(ctx, VAL_MEM(e->a), e->size_offs, e->b, 0, e->mode); break; case PUSH: + if( IS_FLOAT(e->mode) ) BREAK(); emit_ext(ctx, _PUSH, e->a, UNUSED, e->mode, 0); break; case POP: + if( IS_FLOAT(e->mode) ) BREAK(); emit_ext(ctx, _POP, e->a, UNUSED, e->mode, 0); break; case PUSH_CONST: @@ -917,9 +926,11 @@ void hl_codegen_function( jit_ctx *jit ) { { ereg w = IS_PURE(out) ? out : get_tmp(e->mode); if( e->value == 0 ) - EMIT(IS_FLOAT(e->mode) ? XORPD : XOR, w, w, e->mode); - else + EMIT(e->mode == M_F32 ? XORPS : e->mode == M_F64 ? XORPD : XOR, w, w, e->mode); + else { + if( IS_FLOAT(e->mode) ) BREAK(); emit_ext(ctx, _MOV, w, VAL_CONST, e->mode, e->value); + } if( w != out ) emit_mov(ctx, out, w, e->mode); } From b8b51aca8f1da2a20e949cfa208d2a96fcd3ebcd Mon Sep 17 00:00:00 2001 From: Nicolas Cannasse Date: Thu, 23 Apr 2026 19:14:59 +0200 Subject: [PATCH 37/83] fixed generation for haxe unit tests, added floats constant pool --- src/jit.c | 3 +- src/jit.h | 1 + src/jit_dump.c | 2 +- src/jit_emit.c | 26 ++++------ src/jit_regs.c | 16 +++--- src/jit_x86_64.c | 127 ++++++++++++++++++++++++++++++++++++++++------- 6 files changed, 134 insertions(+), 41 deletions(-) diff --git a/src/jit.c b/src/jit.c index fddfec0b1..624007b95 100644 --- a/src/jit.c +++ b/src/jit.c @@ -164,6 +164,8 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { } void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { + hl_codegen_final(ctx); + jit_code_append(ctx); int size = ctx->out_pos; if( size & 4095 ) size += 4096 - (size&4095); unsigned char *code = (unsigned char*)hl_alloc_executable_memory(size); @@ -173,7 +175,6 @@ void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **d *debug = NULL; ctx->final_code = code; hl_emit_final(ctx); - hl_codegen_final(ctx); return code; } diff --git a/src/jit.h b/src/jit.h index 4aad5493c..e9267bfca 100644 --- a/src/jit.h +++ b/src/jit.h @@ -108,6 +108,7 @@ typedef struct { #define MK_STACK_OFFS(v)(((v)&0xFFFFFFF) | FL_STACKOFFS) #define GET_STACK_OFFS(v) ((int)(((v) & 0x8000000) ? ((v) | 0xF0000000) : ((v)&0xFFFFFFF))) #define IS_CALL(op) ((op) == CALL_PTR || (op) == CALL_REG || (op) == CALL_FUN) +#define IS_FLOAT(mode) ((mode) == M_F64 || (mode) == M_F32) #define UNUSED ((ereg)0) typedef struct { diff --git a/src/jit_dump.c b/src/jit_dump.c index 6385796d0..80d99d666 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -205,7 +205,7 @@ static void dump_value( jit_ctx *ctx, uint64 value, emit_mode mode ) { else if( mode == M_PTR && value >= (uint64)code->types && value < (uint64)(code->types + code->ntypes) ) uprintf(USTR("<%s>"),hl_type_str((hl_type*)value)); else if( mode == M_PTR && value == (uint64)mod->globals_data ) - printf("",(int)(value - (uint64)mod->globals_data)); + printf(""); else if( value == (uint64)&hlt_void ) printf(""); else diff --git a/src/jit_emit.c b/src/jit_emit.c index bce2a43a7..30eb67a5e 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -158,8 +158,6 @@ struct _emit_ctx { #define GET_PHI(r) ctx->phis[-(r)-1] #define HDYN_VALUE 8 -#define IS_FLOAT(t) ((t)->kind == HF64 || (t)->kind == HF32) - static hl_type hlt_ui8 = { HUI8, 0 }; static hl_type hlt_ui16 = { HUI16, 0 }; @@ -320,7 +318,7 @@ static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { #define store_args hl_emit_store_args void hl_emit_store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { if( count < 0 ) jit_assert(); - if( count > 64 ) jit_error("Too many arguments"); + if( count > 256 ) jit_error("Too many arguments"); e->nargs = (unsigned char)count; if( count == 0 ) return; if( count == 1 ) { @@ -738,16 +736,16 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode from, emit_mode to, bool _unsigned ) { - if( (from == M_F32 || from == M_F64) != (to == M_F32 || to == M_F64) ) + if( IS_FLOAT(from) != IS_FLOAT(to) ) return emit_gen_ext(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, UNUSED, to, from); - if( from != to && (from == M_F32 || from == M_F64) && (to == M_F32 || to == M_F64) ) + if( from != to && IS_FLOAT(from) && IS_FLOAT(to) ) return emit_gen_ext(ctx, CONV, v, UNUSED, to, from); // no-op return emit_gen(ctx,MOV,v,UNUSED,to); } static bool dyn_need_type( hl_type *t ) { - return !(IS_FLOAT(t) || t->kind == HI64 || t->kind == HGUID); + return !(t->kind == HF32 || t->kind == HF64 || t->kind == HI64 || t->kind == HGUID); } static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { @@ -982,13 +980,12 @@ void hl_emit_function( jit_ctx *jit ) { for(i=0;inregs;i++) { vreg *r = R(i); r->t = f->regs[i]; - r->ref = 0; + r->ref = UNUSED; } emit_gen(ctx,ENTER,UNUSED,UNUSED,M_NONE); for(i=0;itype->fun->nargs;i++) { hl_type *t = f->type->fun->args[i]; - if( t->kind == HVOID ) continue; STORE(R(i), emit_gen(ctx, LOAD_ARG, UNUSED, UNUSED, hl_type_mode(t))); } @@ -1496,13 +1493,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case OIncr: case ODecr: - { - if( IS_FLOAT(dst->t) ) { - jit_assert(); - } else { - STORE(dst, emit_gen_ext(ctx,UNOP,LOAD(dst),UNUSED,hl_type_mode(dst->t),o->op)); - } - } + STORE(dst, emit_gen_ext(ctx,UNOP,LOAD(dst),UNUSED,hl_type_mode(dst->t),o->op)); break; case ONew: { @@ -1911,7 +1902,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, LOAD_MEM(LOAD(ra),0,dst->t)); break; case OSetref: - STORE_MEM(dst->ref,0,LOAD(ra)); + if( dst->ref == UNUSED ) + STORE_MEM(LOAD(dst),0,LOAD(ra)); + else + STORE_MEM(dst->ref,0,LOAD(ra)); break; case ORefData: switch( ra->t->kind ) { diff --git a/src/jit_regs.c b/src/jit_regs.c index adaf2794f..23cc724d6 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -37,7 +37,7 @@ #define VIDX(e) (((e) < 0) ? ctx->jit->value_count + (-(e)-1) : (e)) #define VAL_REG(e) VAL(VIDX(e)) -#define REG_MODE(m) (((m) == M_F32 || (m) == M_F64) ? 1 :0) +#define REG_MODE(m) (IS_FLOAT(m) ? 1 :0) #define REG_CFG(m) (m ? &ctx->jit->cfg.floats : &ctx->jit->cfg.regs) #define EMIT(r,a,b,m) regs_emit(ctx,UNUSED,r,a,b,m,0) @@ -306,8 +306,10 @@ static void regs_compute_liveness( regs_ctx *ctx ) { case OUDiv: case OSMod: case OUMod: - if( jit->cfg.req_div_a ) VAL_REG(e->a)->pref_reg = jit->cfg.req_div_a; - if( jit->cfg.req_div_b ) VAL_REG(e->b)->pref_reg = jit->cfg.req_div_b; + if( !IS_FLOAT(e->mode) ) { + if( jit->cfg.req_div_a ) VAL_REG(e->a)->pref_reg = jit->cfg.req_div_a; + if( jit->cfg.req_div_b ) VAL_REG(e->b)->pref_reg = jit->cfg.req_div_b; + } break; } break; @@ -362,7 +364,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { value_info *v = VAL(i); einstr *e = ctx->jit->instrs + ctx->jit->values_writes[i]; int size = hl_emit_mode_sizes[e->mode]; - if( size <= 0 ) hl_jit_assert(); + if( size <= 0 && e->mode != M_VOID ) jit_assert(); ereg r = get_call_reg(ctx,regs,e->mode); if( !IS_NULL(r) ) { v->reg = r; @@ -633,7 +635,8 @@ static void regs_emit_instrs( regs_ctx *ctx ) { EMIT(PUSH,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR); for(int i=0;ipersists_uses[1];i++) EMIT(PUSH,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); - regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-stack_offset); + if( stack_offset ) + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-stack_offset); } break; case JUMP: @@ -654,7 +657,8 @@ static void regs_emit_instrs( regs_ctx *ctx ) { if( e.a != ret ) regs_emit_mov(ctx, ret, e.a, e.mode); } - regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); + if( stack_offset ) + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); for(int i=ctx->persists_uses[1]-1;i>=0;i--) EMIT(POP,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); for(int i=ctx->persists_uses[0]-1;i>=0;i--) diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 3edf18fa1..99a9c6590 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -35,7 +35,19 @@ #define VAL_CONST 0x80000000 #define VAL_MEM(reg) (FL_MEMPTR | (reg)) -#define IS_FLOAT(mode) ((mode) == M_F64 || (mode) == M_F32) +#define S_TYPE value_arr +#define S_NAME(name) value_arr_##name +#define S_VALUE uint64 +#include "data_struct.c" + +#define S_SORTED +#define S_MAP +#define S_TYPE value_map +#define S_NAME(name) value_map_##name +#define S_KEY uint64 +#define S_VALUE int +#define S_DEFVAL -1 +#include "data_struct.c" typedef enum { RAX = 0, @@ -72,6 +84,7 @@ typedef enum { IMUL, // only overflow flag changes compared to MUL DIV, IDIV, + NEG, CDQ, CDQE, _POP, @@ -188,6 +201,7 @@ static opform OP_FORMS[] = { { "IMUL", LONG_OP(0x0FAF), 0, 0x69 | FLAG_DUAL, 0x6B | FLAG_DUAL }, { "DIV", RM(0xF7,6), RM(0xF7,6) }, { "IDIV", RM(0xF7,7), RM(0xF7,7) }, + { "NEG", RM(0xF7,3) }, { "CDQ", 0x99 }, { "CDQE", 0x98 }, { "POP", 0x58 | FLAG_DEF64, RM(0x8F,0) }, @@ -214,8 +228,8 @@ static opform OP_FORMS[] = { // SSE { "MOVSD", 0xF20F10, 0xF20F11 }, { "MOVSS", 0xF30F10, 0xF30F11 }, - { "COMISD", 0x660F2F }, - { "COMISS", LONG_OP(0x0F2F) }, + { "COMISD", 0x660F2F, LONG_RM(0x660F2F,1) }, + { "COMISS", LONG_OP(0x0F2F), LONG_RM(0x0F2F,1) }, { "ADDSD", 0xF20F58 }, { "SUBSD", 0xF20F5C }, { "MULSD", 0xF20F59 }, @@ -293,6 +307,9 @@ struct _code_ctx { int_arr funs; int_arr short_jumps; int_arr near_jumps; + value_map const_table_lookup; + value_arr const_table; + int_arr const_refs; int *pos_map; int cur_op; bool flushed; @@ -442,7 +459,7 @@ static preg make_reg( ereg r, emit_mode m ) { if( (r & FL_NATMASK) == FL_STACKREG ) { p.kind = RSTACK; p.value = GET_STACK_OFFS(r); - } else if( m == M_F32 || m == M_F64 ) { + } else if( IS_FLOAT(m) ) { p.kind = RFPU; p.reg = (r&0xFF) - 64; } else { @@ -735,6 +752,28 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ case OUDiv: BREAK(); return; + case ONot: + BREAK(); + cop = XOR; + b = a; + break; + case ONeg: + if( IS_FLOAT(mode) ) { + BREAK(); + if( out != a && IS_PURE(out) ) { + EMIT(mode == M_F32 ? XORPS : XORPD, out, out, mode); + EMIT(mode == M_F32 ? SUBSS : SUBSD, out, a, mode); + } else { + ereg tmp = get_tmp(mode); + EMIT(mode == M_F32 ? XORPS : XORPD, tmp, tmp, mode); + EMIT(mode == M_F32 ? SUBSS : SUBSD, tmp, a, mode); + EMIT(mode == M_F32 ? MOVSS : MOVSD, out, tmp, mode); + } + return; + } + BREAK(); + cop = NEG; + break; default: jit_assert(); break; @@ -835,6 +874,17 @@ static void align_function( code_ctx *ctx ) { emit_nop(ctx,16 - (byte_count(ctx->code) & 15)); } +static void alloc_stored_value( code_ctx *ctx, uint64 value ) { + int idx = value_map_find(ctx->const_table_lookup, value); + if( idx < 0 ) { + idx = ctx->const_table.cur; + value_map_add_impl(&ctx->jit->galloc,&ctx->const_table_lookup,value,idx); + value_arr_add_impl(&ctx->jit->galloc,&ctx->const_table,value); + } + int_arr_add_impl(&ctx->jit->galloc,&ctx->const_refs,ctx->jit->out_pos + byte_count(ctx->code) - 4); + int_arr_add_impl(&ctx->jit->galloc,&ctx->const_refs,idx); +} + void hl_codegen_function( jit_ctx *jit ) { code_ctx *ctx = jit->code; ctx->flushed = false; @@ -875,8 +925,9 @@ void hl_codegen_function( jit_ctx *jit ) { continue; // nop case MOV: if( (e->a & FL_NATMASK) == FL_STACKOFFS ) { - if( !IS_PURE(out) ) jit_assert(); - emit_ext(ctx,_LEA,out,VAL_MEM(R(RBP)),M_PTR,GET_STACK_OFFS(e->a)); + ereg w = IS_PURE(out) ? out : RTMP; + emit_ext(ctx,_LEA,w,VAL_MEM(R(RBP)),M_PTR,GET_STACK_OFFS(e->a)); + if( w != out ) emit_mov(ctx, out, w, M_PTR); } else emit_mov(ctx, out, e->a, e->mode); break; @@ -903,7 +954,11 @@ void hl_codegen_function( jit_ctx *jit ) { break; case PUSH: if( IS_FLOAT(e->mode) ) BREAK(); - emit_ext(ctx, _PUSH, e->a, UNUSED, e->mode, 0); + if( (e->a&FL_NATMASK) == FL_STACKOFFS ) { + emit_mov(ctx, RTMP, e->a, e->mode); + emit_ext(ctx, _PUSH, RTMP, UNUSED, e->mode, 0); + } else + emit_ext(ctx, _PUSH, e->a, UNUSED, e->mode, 0); break; case POP: if( IS_FLOAT(e->mode) ) BREAK(); @@ -924,15 +979,25 @@ void hl_codegen_function( jit_ctx *jit ) { break; case LOAD_CONST: { - ereg w = IS_PURE(out) ? out : get_tmp(e->mode); + emit_mode mode = e->mode; + if( !IS_PURE(out) ) + mode = (mode == M_F32 ? M_I32 : mode == M_F64 ? M_I64 : mode); // don't use FP for stack ops + ereg w = IS_PURE(out) ? out : get_tmp(mode); if( e->value == 0 ) - EMIT(e->mode == M_F32 ? XORPS : e->mode == M_F64 ? XORPD : XOR, w, w, e->mode); - else { - if( IS_FLOAT(e->mode) ) BREAK(); - emit_ext(ctx, _MOV, w, VAL_CONST, e->mode, e->value); - } + EMIT(mode == M_F32 ? XORPS : mode == M_F64 ? XORPD : XOR, w, w, mode); + else if( IS_FLOAT(mode) ) { + // MOVSS / MOVSD with data relative + B(e->mode == M_F32 ? 0xF3 : 0xF2); + if( out&8 ) B(0x44); + B(0x0F); + B(0x10); + MOD_RM(0,out&7,5); + W(0); + alloc_stored_value(ctx, e->value); + } else + emit_ext(ctx, _MOV, w, VAL_CONST, mode, e->value); if( w != out ) - emit_mov(ctx, out, w, e->mode); + emit_mov(ctx, out, w, mode); } break; case LOAD_ADDR: @@ -994,6 +1059,7 @@ void hl_codegen_function( jit_ctx *jit ) { if( !IS_PURE(e->a) && !IS_PURE(e->b) ) { ereg tmp = get_tmp(e->mode); emit_mov(ctx, tmp, e->b, e->mode); + if( op == COMISS || op == COMISD ) BREAK(); EMIT(op,e->a,tmp,e->mode); } else EMIT(op,e->a,e->b,e->mode); @@ -1004,8 +1070,8 @@ void hl_codegen_function( jit_ctx *jit ) { int prev = 0; einstr *p; do { - p = &e[--prev]; - } while( p->op == MOV ); + p = jit->reg_instrs + ctx->cur_op - (++prev); + } while( p->op == MOV || p->op == JCOND ); int op; switch( p->size_offs ) { case OJFalse: @@ -1046,6 +1112,14 @@ void hl_codegen_function( jit_ctx *jit ) { case OJNotGte: op = JULt; break; + case 0: + if( p->op == DEBUG_BREAK ) { + // found a debug break ! + BREAK(); + op = JZero; + break; + } + // fallback default: jit_assert(); break; @@ -1060,6 +1134,7 @@ void hl_codegen_function( jit_ctx *jit ) { BREAK(); break; case CONV: + case CONV_UNSIGNED: BREAK(); break; case BINOP: @@ -1080,6 +1155,9 @@ void hl_codegen_function( jit_ctx *jit ) { else emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,-e->size_offs); break; + case PREFETCH: + BREAK(); + break; default: jit_assert(); break; @@ -1146,11 +1224,26 @@ void hl_codegen_free( jit_ctx *jit ) { void hl_codegen_final( jit_ctx *jit ) { code_ctx *ctx = jit->code; + // patch function offsets for(int i=0;ifuns);i+=2) { int pos = int_arr_get(ctx->funs,i); int fid = int_arr_get(ctx->funs,i+1); int offset = (int)(int_val)jit->mod->functions_ptrs[fid] - (pos + 4); - *(int*)(jit->final_code + pos) = offset; + *(int*)(jit->output + pos) = offset; } int_arr_reset(&ctx->funs); + // emit constant table + jit->code_size = value_arr_count(ctx->const_table) * sizeof(uint64); + jit->code_instrs = (unsigned char *)ctx->const_table.values; + // patch constant offsets + for(int i=0;iconst_refs);i+=2) { + int pos = int_arr_get(ctx->const_refs,i); + int coffs = int_arr_get(ctx->const_refs,i+1) * sizeof(uint64); + int offset = (jit->out_pos + coffs) - (pos + 4); + *(int*)(jit->output + pos) = offset; + } + int_arr_reset(&ctx->const_refs); + // cleanup + value_arr_free(&ctx->const_table); + value_map_free(&ctx->const_table_lookup); } From 9aa690dee87e5946a3f7651874598941c4d48aab Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 24 Apr 2026 08:04:18 +0200 Subject: [PATCH 38/83] more fixes and optims --- src/jit_regs.c | 19 +++++++++++++------ src/jit_x86_64.c | 32 ++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/jit_regs.c b/src/jit_regs.c index 23cc724d6..259c268a5 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -50,6 +50,7 @@ #endif typedef struct { + int id; int stack_pos; int last_read; int tot_reads; @@ -149,9 +150,7 @@ static int regs_alloc_stack( regs_ctx *ctx, int size ) { static const char *value_to_str( regs_ctx *ctx, value_info *v ) { static char out[20]; - int idx = (int)(v - ctx->values); - ereg e = (ereg)(idx >= ctx->jit->value_count ? -(idx-ctx->jit->value_count) - 1 : idx); - sprintf(out,"%s:%s", val_str(e,v->mode), val_str(v->reg,v->mode)); + sprintf(out,"%s:%s", val_str(v->id,v->mode), val_str(v->reg,v->mode)); return out; } @@ -382,8 +381,13 @@ static void regs_assign_regs( regs_ctx *ctx ) { for(int cur_op=0;cur_opinstr_count;cur_op++) { einstr e = jit->instrs[cur_op]; value_info *write = NULL; +# ifdef HL_DEBUG + int uid = (jit->fun->findex << 16) | cur_op; + __ignore(&uid); +# endif ctx->cur_op = cur_op; + if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) { write = VAL(write_index++); // try to preserve ops in the from A = A op B @@ -720,10 +724,13 @@ void hl_regs_function( jit_ctx *jit ) { v->pref_reg = UNUSED; v->stack_pos = INVALID; v->last_read = -1; - if( i < jit->value_count ) + if( i < jit->value_count ) { + v->id = i; v->mode = jit->instrs[jit->values_writes[i]].mode; - else - v->mode = M_NONE; // TODO : phi mode + } else { + v->id = -(i-jit->value_count) - 1; + v->mode = M_NONE; + } } regs_compute_liveness(ctx); regs_assign_regs(ctx); diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 99a9c6590..257623f58 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -743,11 +743,20 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ return; } } + if( out == b ) { + ereg r = get_tmp(mode); + emit_anyop(ctx,op,r,a,b,mode); + emit_mov(ctx,out,r,mode); + return; + } b = UNUSED; cop = (op == OShl ? SHL : (op == OSShr ? SAR : SHR)); break; - case OSMod: case OSDiv: + F_OP(IDIV,DIVSS,DIVSD); + if( !IS_FLOAT(mode) ) BREAK(); + break; + case OSMod: case OUMod: case OUDiv: BREAK(); @@ -778,7 +787,7 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ jit_assert(); break; } - if( !IS_PURE(out) ) { + if( !IS_PURE(out) || out == b ) { ereg tmp = get_tmp(mode); emit_mov(ctx, tmp, a, mode); EMIT(cop,tmp,b,mode); @@ -965,7 +974,13 @@ void hl_codegen_function( jit_ctx *jit ) { emit_ext(ctx, _POP, e->a, UNUSED, e->mode, 0); break; case PUSH_CONST: - emit_ext(ctx, (e->value&0xFF) == e->value && e->mode == M_PTR ? PUSH8 : _PUSH, VAL_CONST, UNUSED, e->mode, e->value); + if( e->mode != M_PTR ) jit_assert(); + if( (e->value&0xFF) == e->value ) + emit_ext(ctx,PUSH8, VAL_CONST, UNUSED, M_PTR, e->value); + else if( (e->value&0xFFFFFFFF) == e->value ) + emit_ext(ctx,_PUSH, VAL_CONST, UNUSED, M_I32, e->value); // will push 64bits + else + emit_ext(ctx,_PUSH, VAL_CONST, UNUSED, M_PTR, e->value); break; case DEBUG_BREAK: BREAK(); @@ -994,7 +1009,9 @@ void hl_codegen_function( jit_ctx *jit ) { MOD_RM(0,out&7,5); W(0); alloc_stored_value(ctx, e->value); - } else + } else if( (mode == M_PTR || mode == M_I64) && (e->value&0xFFFFFFFF) == e->value ) + emit_ext(ctx, _MOV, w, VAL_CONST, M_I32, e->value); + else emit_ext(ctx, _MOV, w, VAL_CONST, mode, e->value); if( w != out ) emit_mov(ctx, out, w, mode); @@ -1029,8 +1046,11 @@ void hl_codegen_function( jit_ctx *jit ) { B(0xE8); W(target - (jit->out_pos + byte_count(ctx->code) + 4)); } else { - emit_ext(ctx, _MOV, R(RAX), VAL_CONST, M_PTR, e->value); - EMIT(_CALL, R(RAX), UNUSED, M_NONE); + // call near indirect + B(0xFF); + B(0x15); + W(0); + alloc_stored_value(ctx, (uint64)e->value); } break; case CALL_REG: From 8656551906a7a6b2cfc08b4d3d6cfd8220f375bb Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 25 Apr 2026 07:34:17 +0200 Subject: [PATCH 39/83] added LOAD_FUN, CATCH and some other fixes --- src/jit.h | 2 ++ src/jit_dump.c | 2 ++ src/jit_emit.c | 22 +++++++++++++--------- src/jit_regs.c | 44 ++++++++++++++++++++++++++------------------ src/jit_x86_64.c | 16 ++++++++++++++++ 5 files changed, 59 insertions(+), 27 deletions(-) diff --git a/src/jit.h b/src/jit.h index e9267bfca..8231ad492 100644 --- a/src/jit.h +++ b/src/jit.h @@ -28,6 +28,7 @@ typedef enum { LOAD_ADDR, LOAD_CONST, LOAD_ARG, + LOAD_FUN, STORE, LEA, TEST, @@ -54,6 +55,7 @@ typedef enum { ENTER, STACK_OFFS, XCHG, + CATCH, } emit_op; typedef enum { diff --git a/src/jit_dump.c b/src/jit_dump.c index 80d99d666..72c63242b 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -25,6 +25,7 @@ static const char *op_names[] = { "load-addr", "load-const", "load-arg", + "load-fun", "store", "lea", "test", @@ -51,6 +52,7 @@ static const char *op_names[] = { "enter", "stack", "xchg", + "catch", }; bool hl_jit_dump_bin = false; diff --git a/src/jit_emit.c b/src/jit_emit.c index 30eb67a5e..a78f96507 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -1271,9 +1271,10 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type emit_jump_dyn(ctx,op,bt,b,at,a,offset); // inverse return; } - BREAK(); + if( hl_get_obj_rt(at)->compareFun ) { + BREAK(); /* - if( hl_get_obj_rt(a->t)->compareFun ) { + preg *pa = alloc_cpu(ctx,a,true); preg *pb = alloc_cpu(ctx,b,true); preg p; @@ -1327,9 +1328,9 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type patch_jump(ctx,jb); break; } + */ return; } - */ // fallthrough default: emit_cmp(ctx, a, b, op); @@ -1523,8 +1524,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg args[3]; args[0] = LOAD_CONST_PTR(m->code->functions[m->functions_indexes[o->p2]].type); - // TODO : WRITE (emit_pos + op_count) to process later and replace address ! - args[1] = LOAD_CONST_PTR(0); + einstr *e = emit_instr(ctx, LOAD_FUN); + e->mode = M_PTR; + e->size_offs = o->p2; + args[1] = new_value(ctx); args[2] = LOAD(rb); STORE(dst, emit_native_call(ctx,hl_alloc_closure_ptr,args,3,dst->t)); } @@ -1751,12 +1754,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case HVIRTUAL: - // code for : if( hl_vfields(o)[f] ) dst = *hl_vfields(o)[f](o->value,args...); else dst = hl_dyn_call_obj(o->value,field,args,&ret) + // code for : if( (fun=hl_vfields(o)[f]) ) dst = fun(o->value,args...); else dst = hl_dyn_call_obj(o->value,field,args,&ret) { vreg *_o = R(o->extra[0]); ereg obj = LOAD(_o); - ereg field = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p2); - emit_test(ctx, field, OJNull); + ereg fun = LOAD_MEM_PTR(obj,sizeof(vvirtual)+HL_WSIZE*o->p2); + emit_test(ctx, fun, OJNull); int jidx = emit_jump(ctx, true); int nargs = o->p3; @@ -1765,7 +1768,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[0] = LOAD_MEM_PTR(obj,HL_WSIZE); for(i=1;iextra[i])); - ereg v1 = emit_dyn_call(ctx,LOAD_MEM_PTR(field,0),args,nargs,dst->t); + ereg v1 = emit_dyn_call(ctx,fun,args,nargs,dst->t); int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); @@ -2141,6 +2144,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE_MEM(st, offset, LOAD_CONST_PTR(NULL)); } # endif + emit_instr(ctx, CATCH); } break; case OSwitch: diff --git a/src/jit_regs.c b/src/jit_regs.c index 259c268a5..232737f47 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -209,7 +209,7 @@ static void regs_assign( regs_ctx *ctx, value_info *v ) { if( v->reg != UNUSED ) jit_assert(); if( regs_alloc_reg(ctx, v) ) values_add(ctx->scratch, v); - regs_debug("REG ASSIGN %s @%X\n",value_str(v),ctx->cur_op); + regs_debug("REG ASSIGN %s @%X-@%X\n",value_str(v),ctx->cur_op,v->last_read); } static void regs_write_live( regs_ctx *ctx, ereg *r ) { @@ -406,6 +406,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { ereg *args = hl_emit_get_args(ctx->jit->emit,&e); call_regs regs = {0}; bool will_scratch = e.mode != M_NORET; + value_info *vcall = e.op == CALL_REG ? VAL_REG(e.a) : NULL; if( will_scratch ) { for_iter_back(values,v2,ctx->scratch) { if( v2->last_read > cur_op ) @@ -419,11 +420,14 @@ static void regs_assign_regs( regs_ctx *ctx ) { value_info *cur = regs_current(ctx,r); if( cur && cur != v ) spill(ctx,cur); + if( vcall && vcall->reg == r ) + spill(ctx,vcall); } } if( will_scratch ) values_reset(&ctx->scratch); } - if( e.op == BLOCK ) { + switch( e.op ) { + case BLOCK: for_iter_back(values,v,ctx->scratch) { if( v->last_read == cur_op ) values_remove(&ctx->scratch,v); @@ -442,20 +446,23 @@ static void regs_assign_regs( regs_ctx *ctx ) { } regs_assign(ctx, v); } - } - if( write ) { - switch( e.op ) { - case ALLOC_STACK: - write->reg = MK_STACK_OFFS(regs_alloc_stack(ctx, e.size_offs)); - break; - case LOAD_ARG: - if( write->reg == UNUSED ) - regs_assign(ctx, write); // assign for stack reg - break; - default: - regs_assign(ctx, write); - break; + break; + case CATCH: + { + for_iter_back(values,v2,ctx->scratch) + spill(ctx,v2); } + break; + case ALLOC_STACK: + write->reg = MK_STACK_OFFS(regs_alloc_stack(ctx, e.size_offs)); + break; + case LOAD_ARG: + if( write->reg == UNUSED ) + regs_assign(ctx, write); // assign for stack reg + break; + default: + if( write ) regs_assign(ctx, write); + break; } } // assign stack regs @@ -463,7 +470,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { ctx->stack_offset = (ctx->persists_uses[0] + ctx->persists_uses[1]) * 8; for(int i=0;ivalues + i; - if( v->reg == UNUSED ) v->reg = MK_STACK_REG(v->stack_pos);// + ctx->stack_offset); + if( v->reg == UNUSED ) v->reg = MK_STACK_REG(v->stack_pos); } } @@ -603,13 +610,13 @@ static void regs_emit_instrs( regs_ctx *ctx ) { e.nargs = 0xFF; ret_val = ®_CFG(REG_MODE(e.mode))->ret; if( e.op == CALL_REG ) - e.a = VAL(VIDX(e.a))->reg; + e.a = VAL_REG(e.a)->reg; } else { ereg **regs = hl_emit_get_regs(&e,&nread); for(int k=0;kreg; } } @@ -618,6 +625,7 @@ static void regs_emit_instrs( regs_ctx *ctx ) { out = VAL(write_index++)->reg; switch( e.op ) { case ALLOC_STACK: + case CATCH: break; case BLOCK: cur_block = jit->blocks + e.size_offs; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 257623f58..f609a4bfd 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1029,6 +1029,22 @@ void hl_codegen_function( jit_ctx *jit ) { emit_mov_ext(ctx,out,0,addr,e->size_offs,e->mode); } break; + case LOAD_FUN: + { + ereg w = IS_PURE(out) ? out : RTMP; + // LEA relative + B(0x48 + ((out & 8) ? 4 : 0)); + B(0x8D); + MOD_RM(0,out&7,5); + int pos = jit->out_pos + byte_count(ctx->code); + int fid = e->size_offs; + int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,pos); + int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,fid); + W(0); + if( w != out ) + emit_mov(ctx, out, w, M_PTR); + } + break; case CALL_FUN: B(0xE8); { From 7215193932beef286b49d94567fdbc90cad64df3 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 25 Apr 2026 12:37:09 +0200 Subject: [PATCH 40/83] fixed refs impl and loops liveness, added jump tables and fixes --- src/data_struct.c | 4 ++ src/jit.c | 4 +- src/jit.h | 1 + src/jit_dump.c | 4 +- src/jit_emit.c | 22 ++------ src/jit_regs.c | 70 +++++++++++++++++++++---- src/jit_x86_64.c | 130 +++++++++++++++++++++++++++++++++------------- 7 files changed, 168 insertions(+), 67 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index 9436ad571..367b70e59 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -245,6 +245,10 @@ INLINE static S_VALUE S_NAME(get)( S_TYPE st, int idx ) { return st.values[idx]; } +INLINE static S_VALUE *S_NAME(addr)( S_TYPE st, int idx ) { + return &st.values[idx]; +} + INLINE static S_VALUE S_NAME(first)( S_TYPE st ) { return st.cur == 0 ? S_DEFVAL : st.values[0]; } diff --git a/src/jit.c b/src/jit.c index 624007b95..3a3b80b68 100644 --- a/src/jit.c +++ b/src/jit.c @@ -61,6 +61,7 @@ void hl_regs_function( jit_ctx *jit ); void hl_codegen_alloc( jit_ctx *jit ); void hl_codegen_init( jit_ctx *jit ); void hl_codegen_free( jit_ctx *jit ); +void hl_codegen_flush_consts( jit_ctx *jit ); void hl_codegen_function( jit_ctx *jit ); void hl_codegen_final( jit_ctx *jit ); @@ -164,7 +165,7 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { } void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { - hl_codegen_final(ctx); + hl_codegen_flush_consts(ctx); jit_code_append(ctx); int size = ctx->out_pos; if( size & 4095 ) size += 4096 - (size&4095); @@ -175,6 +176,7 @@ void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **d *debug = NULL; ctx->final_code = code; hl_emit_final(ctx); + hl_codegen_final(ctx); return code; } diff --git a/src/jit.h b/src/jit.h index 8231ad492..eb0f03325 100644 --- a/src/jit.h +++ b/src/jit.h @@ -56,6 +56,7 @@ typedef enum { STACK_OFFS, XCHG, CATCH, + ADDRESS, } emit_op; typedef enum { diff --git a/src/jit_dump.c b/src/jit_dump.c index 72c63242b..0d60834c8 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -53,6 +53,7 @@ static const char *op_names[] = { "stack", "xchg", "catch", + "address", }; bool hl_jit_dump_bin = false; @@ -386,10 +387,11 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { printf("%s %s", emit_mode_str(e->size_offs), val_str(e->a,(emit_mode)e->size_offs)); break; case LEA: - printf(" %s", reg_str(e->a)); + printf(" [%s", reg_str(e->a)); if( !IS_NULL(e->b) ) printf("+%s", reg_str(e->b)); if( (e->size_offs&0xFF) > 1 ) printf("*%d",e->size_offs&0xFF); if( e->size_offs >> 8 ) printf("+%Xh", e->size_offs>>8); + printf("]"); break; default: if( !IS_NULL(e->a) ) { diff --git a/src/jit_emit.c b/src/jit_emit.c index a78f96507..626bc41c2 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -36,7 +36,6 @@ int hl_emit_mode_sizes[] = {0,1,2,4,8,HL_WSIZE,8,4,0,0}; typedef struct { hl_type *t; int id; - ereg ref; } vreg; #define MAX_TMP_ARGS 32 @@ -608,8 +607,6 @@ static ereg emit_load_reg_block( emit_ctx *ctx, emit_block *b, vreg *r ) { } static ereg emit_load_reg( emit_ctx *ctx, vreg *r ) { - if( !IS_NULL(r->ref) ) - return LOAD_MEM(r->ref,0,r->t); return emit_load_reg_block(ctx, ctx->current_block, r); } @@ -980,7 +977,6 @@ void hl_emit_function( jit_ctx *jit ) { for(i=0;inregs;i++) { vreg *r = R(i); r->t = f->regs[i]; - r->ref = UNUSED; } emit_gen(ctx,ENTER,UNUSED,UNUSED,M_NONE); @@ -1135,7 +1131,7 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type { ereg args[2] = { a, b }; ereg ret = emit_native_call(ctx,hl_same_type,args,2,&hlt_bool); - emit_test(ctx, ret, op); + emit_test(ctx, emit_gen_ext(ctx,UNOP,ret,UNUSED,M_I32,ONot), op); } break; case HNULL: @@ -1890,25 +1886,13 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE(dst, LOAD_MEM(LOAD(ra),HL_WSIZE*2,&hlt_i32)); break; case ORef: - { - if( IS_NULL(ra->ref) ) - ra->ref = emit_gen_size(ctx, ALLOC_STACK, hl_type_size(ra->t)); - ereg r = vreg_find(ctx->current_block->written_vars, ra->id); - if( !IS_NULL(r) ) { - STORE_MEM(ra->ref, 0, r); - vreg_remove(&ctx->current_block->written_vars, ra->id); - } - STORE(dst,ra->ref); - } + STORE(dst, emit_gen(ctx, ADDRESS, LOAD(ra), UNUSED, M_PTR)); break; case OUnref: STORE(dst, LOAD_MEM(LOAD(ra),0,dst->t)); break; case OSetref: - if( dst->ref == UNUSED ) - STORE_MEM(LOAD(dst),0,LOAD(ra)); - else - STORE_MEM(dst->ref,0,LOAD(ra)); + STORE_MEM(LOAD(dst),0,LOAD(ra)); break; case ORefData: switch( ra->t->kind ) { diff --git a/src/jit_regs.c b/src/jit_regs.c index 232737f47..bd9f0c2d3 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -77,6 +77,8 @@ struct _regs_ctx { int emit_pos; int stack_size; int stack_offset; + int loop_start; + int loop_end; einstr *instrs; ereg *out_write; int *pos_map; @@ -155,12 +157,10 @@ static const char *value_to_str( regs_ctx *ctx, value_info *v ) { } static void spill( regs_ctx *ctx, value_info *v ) { - if( v->reg == UNUSED ) jit_assert(); if( v->stack_pos == INVALID ) v->stack_pos = regs_alloc_stack(ctx, hl_emit_mode_sizes[v->mode]); v->reg = MK_STACK_REG(v->stack_pos); values_remove(&ctx->scratch,v); regs_debug("REG SPILL %s @%X\n",value_str(v),ctx->cur_op); - if( v->stack_pos < 0 ) v->reg = UNUSED; } static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { @@ -216,7 +216,7 @@ static void regs_write_live( regs_ctx *ctx, ereg *r ) { if( IS_NULL(*r) ) jit_assert(); if( IS_NATREG(*r) ) return; value_info *v = VAL_REG(*r); - v->last_read = ctx->cur_op; + v->last_read = ctx->loop_end && ctx->jit->values_writes[v->id] < ctx->loop_start ? ctx->loop_end : ctx->cur_op; v->tot_reads++; } @@ -257,6 +257,9 @@ static eblock *resolve_block_value( regs_ctx *ctx, ereg v ) { } static void regs_compute_liveness( regs_ctx *ctx ) { +# define MAX_LOOP_DEPTH 256 + int loop_saves[MAX_LOOP_DEPTH]; + int loop_count = 0; int write_index = 1; jit_ctx *jit = ctx->jit; hl_type *tret = ctx->jit->fun->type->fun->ret; @@ -266,6 +269,11 @@ static void regs_compute_liveness( regs_ctx *ctx ) { einstr *e = jit->instrs + cur_op; value_info *write = NULL; + if( ctx->loop_end == cur_op && cur_op ) { + ctx->loop_end = loop_saves[--loop_count]; + ctx->loop_start = loop_saves[--loop_count]; + } + if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) write = VAL(write_index++); @@ -312,10 +320,29 @@ static void regs_compute_liveness( regs_ctx *ctx ) { break; } break; + case BLOCK: + { + // are we in loop ? + eblock *bl = jit->blocks + e->size_offs; + int loop_end = -1; + for(int k=0;kpred_count;k++) { + eblock *b2 = jit->blocks + bl->preds[k]; + if( b2->start_pos > bl->start_pos && b2->end_pos >= loop_end ) + loop_end = b2->end_pos - 1; + } + if( loop_end > 0 ) { + loop_saves[loop_count++] = ctx->loop_start; + loop_saves[loop_count++] = ctx->loop_end; + ctx->loop_start = cur_op; + ctx->loop_end = loop_end; + } + } + break; default: break; } } + if( loop_count != 0 ) jit_assert(); // compute reverse phis for(int b=0;bblock_count;b++) { eblock *bl = jit->blocks + b; @@ -373,7 +400,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { // use existing stack storage v->stack_pos = args_size + HL_WSIZE*2; args_size += size < 4 ? 4 : size; - if( IS_NULL(r) ) v->reg = MK_STACK_REG(-v->stack_pos); + if( IS_NULL(r) ) v->reg = MK_STACK_REG(v->stack_pos); } } // assign registers @@ -455,15 +482,21 @@ static void regs_assign_regs( regs_ctx *ctx ) { break; case ALLOC_STACK: write->reg = MK_STACK_OFFS(regs_alloc_stack(ctx, e.size_offs)); - break; + continue; case LOAD_ARG: if( write->reg == UNUSED ) regs_assign(ctx, write); // assign for stack reg - break; + continue; + case ADDRESS: + { + value_info *v = VAL_REG(e.a); + spill(ctx, v); + break; + } default: - if( write ) regs_assign(ctx, write); break; } + if( write ) regs_assign(ctx, write); } // assign stack regs int nvalues = jit->value_count + jit->phi_count; @@ -571,6 +604,14 @@ static void regs_emit_instrs( regs_ctx *ctx ) { int nread; int instr_stack_offset = 0; ctx->cur_op = cur_op; + + value_info *vout = NULL; + ereg out = UNUSED; + if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) { + vout = VAL(write_index++); + out = vout->reg; + } + if( IS_CALL(e.op) ) { ereg *args = hl_emit_get_args(ctx->jit->emit,&e); call_regs regs = {0}; @@ -608,7 +649,12 @@ static void regs_emit_instrs( regs_ctx *ctx ) { } flush_movs(ctx); e.nargs = 0xFF; - ret_val = ®_CFG(REG_MODE(e.mode))->ret; + if( vout && vout->last_read > cur_op ) + ret_val = ®_CFG(REG_MODE(e.mode))->ret; + else if( e.mode != M_NORET ) { + e.mode = M_VOID; // ignore output + out = UNUSED; + } if( e.op == CALL_REG ) e.a = VAL_REG(e.a)->reg; } else { @@ -620,9 +666,6 @@ static void regs_emit_instrs( regs_ctx *ctx ) { *r = v->reg; } } - ereg out = UNUSED; - if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) - out = VAL(write_index++)->reg; switch( e.op ) { case ALLOC_STACK: case CATCH: @@ -682,6 +725,11 @@ static void regs_emit_instrs( regs_ctx *ctx ) { if( out == e.a ) break; // fallthrough default: + if( e.op == ADDRESS ) { + e.op = LEA; + if( !(e.a & FL_MEMPTR) ) jit_assert(); + e.a &= ~FL_MEMPTR; + } if( ret_val && out ) { regs_write_instr(ctx, &e, *ret_val); regs_emit_mov(ctx, out, *ret_val, e.mode); diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index f609a4bfd..25fe4ec01 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -308,11 +308,13 @@ struct _code_ctx { int_arr short_jumps; int_arr near_jumps; value_map const_table_lookup; - value_arr const_table; + byte_arr const_table; int_arr const_refs; + int_arr const_addr; int *pos_map; int cur_op; bool flushed; + int const_table_pos; int null_access_pos; int null_field_pos; }; @@ -762,10 +764,12 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ BREAK(); return; case ONot: - BREAK(); - cop = XOR; - b = a; - break; + if( IS_PURE(a) ) { + emit_ext(ctx,XOR,a,VAL_CONST,M_I32,1); + } else { + BREAK(); + } + return; case ONeg: if( IS_FLOAT(mode) ) { BREAK(); @@ -787,7 +791,9 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ jit_assert(); break; } - if( !IS_PURE(out) || out == b ) { + if( out == a && IS_PURE(a) ) { + EMIT(cop,out,b,mode); + } else if( !IS_PURE(out) || out == b ) { ereg tmp = get_tmp(mode); emit_mov(ctx, tmp, a, mode); EMIT(cop,tmp,b,mode); @@ -841,10 +847,15 @@ static void emit_nop( code_ctx *ctx, int size ) { static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { einstr e = *_e; - if( e.a && (e.a & FL_NATMASK) == FL_STACKOFFS ) { - e.value += GET_STACK_OFFS(e.a); + + int mult = e.size_offs & 0xFF; + int offs = e.size_offs >> 8; + + if( (e.a & FL_NATMASK) == FL_STACKOFFS ) { + offs += GET_STACK_OFFS(e.a); e.a = R(RBP); } + if( !IS_PURE(e.a) ) { // a is always a mem address ! emit_mov(ctx, RTMP, e.a, M_PTR); @@ -859,14 +870,14 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { emit_mov(ctx, RTMP, e.b, M_I32); e.b = RTMP; } - int mult = e.size_offs & 0xFF; - int offs = e.size_offs >> 8; + if( mult == 0 ) { + if( e.a & FL_MEMPTR ) jit_assert(); // no index - BREAK(); - emit_ext(ctx,_LEA,out,e.a,M_PTR,offs); + emit_ext(ctx,_LEA,out,e.a | FL_MEMPTR,M_PTR,offs); return; } + int r64 = 8 | ((out&8) ? 4 : 0) | ((e.b&8) ? 2 : 0) | ((e.a & 8) ? 1 : 0); REX(); B(0x8D); @@ -883,15 +894,37 @@ static void align_function( code_ctx *ctx ) { emit_nop(ctx,16 - (byte_count(ctx->code) & 15)); } -static void alloc_stored_value( code_ctx *ctx, uint64 value ) { - int idx = value_map_find(ctx->const_table_lookup, value); - if( idx < 0 ) { - idx = ctx->const_table.cur; - value_map_add_impl(&ctx->jit->galloc,&ctx->const_table_lookup,value,idx); - value_arr_add_impl(&ctx->jit->galloc,&ctx->const_table,value); +static int reserve_const_segment( code_ctx *ctx, int size, int align ) { + int pos = byte_count(ctx->const_table); + if( align ) { + int k = pos & (align-1); + if( k ) { + byte_reserve_impl(&ctx->jit->galloc,&ctx->const_table,align - k); + pos = byte_count(ctx->const_table); + } + } + byte_reserve_impl(&ctx->jit->galloc,&ctx->const_table,size); + return pos; +} + +static void alloc_const( code_ctx *ctx, uint64 value ) { + int pos = value_map_find(ctx->const_table_lookup, value); + if( pos < 0 ) { + pos = reserve_const_segment(ctx,8,8); + *(uint64*)byte_addr(ctx->const_table,pos) = value; + value_map_add_impl(&ctx->jit->galloc,&ctx->const_table_lookup,value,pos); } int_arr_add_impl(&ctx->jit->galloc,&ctx->const_refs,ctx->jit->out_pos + byte_count(ctx->code) - 4); - int_arr_add_impl(&ctx->jit->galloc,&ctx->const_refs,idx); + int_arr_add_impl(&ctx->jit->galloc,&ctx->const_refs,pos); +} + +static int emit_lea_rel( code_ctx *ctx, ereg out ) { + B(0x48 + ((out & 8) ? 4 : 0)); + B(0x8D); + MOD_RM(0,out&7,5); + int pos = ctx->jit->out_pos + byte_count(ctx->code); + W(0); + return pos; } void hl_codegen_function( jit_ctx *jit ) { @@ -951,7 +984,7 @@ void hl_codegen_function( jit_ctx *jit ) { } break; case STORE: - if( !IS_PURE(e->a) && !IS_PURE(e->b) && (e->a & FL_NATMASK) != FL_STACKOFFS ) { + if( !IS_PURE(e->a) && !IS_PURE(e->b) && (e->a & FL_NATMASK) != FL_STACKOFFS && (e->b & FL_NATMASK) != FL_STACKOFFS ) { EMIT(_PUSH,e->b,UNUSED,e->mode); emit_mov(ctx, RTMP, e->a, M_PTR); emit_ext(ctx, _POP,VAL_MEM(RTMP), UNUSED, e->mode, e->size_offs); @@ -1008,7 +1041,7 @@ void hl_codegen_function( jit_ctx *jit ) { B(0x10); MOD_RM(0,out&7,5); W(0); - alloc_stored_value(ctx, e->value); + alloc_const(ctx, e->value); } else if( (mode == M_PTR || mode == M_I64) && (e->value&0xFFFFFFFF) == e->value ) emit_ext(ctx, _MOV, w, VAL_CONST, M_I32, e->value); else @@ -1032,15 +1065,10 @@ void hl_codegen_function( jit_ctx *jit ) { case LOAD_FUN: { ereg w = IS_PURE(out) ? out : RTMP; - // LEA relative - B(0x48 + ((out & 8) ? 4 : 0)); - B(0x8D); - MOD_RM(0,out&7,5); - int pos = jit->out_pos + byte_count(ctx->code); + int pos = emit_lea_rel(ctx,w); int fid = e->size_offs; int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,pos); int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,fid); - W(0); if( w != out ) emit_mov(ctx, out, w, M_PTR); } @@ -1066,7 +1094,11 @@ void hl_codegen_function( jit_ctx *jit ) { B(0xFF); B(0x15); W(0); - alloc_stored_value(ctx, (uint64)e->value); + alloc_const(ctx, (uint64)e->value); + if( e->mode == M_UI8 || e->mode == M_UI16 ) { + // clear value upper bits + EMIT(e->mode == M_UI8 ? MOVZX8 : MOVZX16,R(RAX),R(RAX),M_PTR); + } } break; case CALL_REG: @@ -1167,7 +1199,23 @@ void hl_codegen_function( jit_ctx *jit ) { emit_jump(ctx, JAlways, e->size_offs); break; case JUMP_TABLE: - BREAK(); + { + int start = reserve_const_segment(ctx,HL_WSIZE * e->nargs,16); + int pos = emit_lea_rel(ctx, RTMP); + int_arr_add_impl(&ctx->jit->galloc,&ctx->const_refs,pos); + int_arr_add_impl(&ctx->jit->galloc,&ctx->const_refs,start); + ereg a = RTMP; + ereg b = e->a; + B(0x40 | ((a&8)?1:0) | ((b&8)?2:0)); + B(0xFF); + B(0x24); + SIB(3,(b&7),(a&7)); + int here = jit->out_pos + byte_count(ctx->code); + for(int k=0;knargs;k++) { + int_arr_add_impl(&jit->galloc,&ctx->const_addr,start + k * HL_WSIZE); + int_arr_add_impl(&jit->galloc,&ctx->const_addr,here); + } + } break; case CONV: case CONV_UNSIGNED: @@ -1258,7 +1306,7 @@ void hl_codegen_free( jit_ctx *jit ) { free(ctx); } -void hl_codegen_final( jit_ctx *jit ) { +void hl_codegen_flush_consts( jit_ctx *jit ) { code_ctx *ctx = jit->code; // patch function offsets for(int i=0;ifuns);i+=2) { @@ -1269,17 +1317,29 @@ void hl_codegen_final( jit_ctx *jit ) { } int_arr_reset(&ctx->funs); // emit constant table - jit->code_size = value_arr_count(ctx->const_table) * sizeof(uint64); - jit->code_instrs = (unsigned char *)ctx->const_table.values; + jit->code_size = byte_count(ctx->const_table); + jit->code_instrs = ctx->const_table.values; + ctx->const_table_pos = jit->out_pos; // patch constant offsets for(int i=0;iconst_refs);i+=2) { int pos = int_arr_get(ctx->const_refs,i); - int coffs = int_arr_get(ctx->const_refs,i+1) * sizeof(uint64); - int offset = (jit->out_pos + coffs) - (pos + 4); + int coffs = int_arr_get(ctx->const_refs,i+1); + int offset = (ctx->const_table_pos + coffs) - (pos + 4); *(int*)(jit->output + pos) = offset; } int_arr_reset(&ctx->const_refs); // cleanup - value_arr_free(&ctx->const_table); + byte_free(&ctx->const_table); value_map_free(&ctx->const_table_lookup); } + +void hl_codegen_final( jit_ctx *jit ) { + code_ctx *ctx = jit->code; + // patch absolute addresses + for(int i=0;iconst_addr);i+=2) { + int pos = int_arr_get(ctx->const_addr,i); + int offs = int_arr_get(ctx->const_addr,i+1); + *(void**)(jit->final_code + ctx->const_table_pos + pos) = jit->final_code + offs; + } + int_arr_free(&ctx->const_addr); +} \ No newline at end of file From 223d611db789fef09cad437e618b10230fdfe6f0 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 25 Apr 2026 15:24:37 +0200 Subject: [PATCH 41/83] complete emit for compareFun --- src/jit_emit.c | 117 ++++++++++++++++++++++++------------------------- src/jit_regs.c | 2 +- 2 files changed, 58 insertions(+), 61 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 626bc41c2..5d02af8a3 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -627,6 +627,14 @@ static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { return p->value; } +static ereg emit_call_fid( emit_ctx *ctx, int findex, ereg *args, int nargs, emit_mode mode ) { + einstr *e = emit_instr(ctx, CALL_FUN); + e->mode = mode; + e->a = findex; + store_args(ctx, e, args, nargs); + return mode == M_VOID ? UNUSED : new_value(ctx); +} + static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int *args_regs ) { hl_module *m = ctx->mod; int fid = m->functions_indexes[findex]; @@ -637,11 +645,8 @@ static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int if( isNative ) STORE(dst, emit_native_call(ctx, m->functions_ptrs[findex], args, count, dst->t)); else { - einstr *e = emit_instr(ctx, CALL_FUN); - e->mode = hl_type_mode(dst->t); - e->a = findex; - store_args(ctx, e, args, count); - STORE(dst, e->mode == M_VOID ? UNUSED : new_value(ctx)); + ereg out = emit_call_fid(ctx,findex,args,count,hl_type_mode(dst->t)); + if( out ) STORE(dst, out); } } @@ -1268,63 +1273,54 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type return; } if( hl_get_obj_rt(at)->compareFun ) { - BREAK(); - /* - - preg *pa = alloc_cpu(ctx,a,true); - preg *pb = alloc_cpu(ctx,b,true); - preg p; - int jeq, ja, jb, jcmp; - int args[] = { a->stack.id, b->stack.id }; - switch( op->op ) { + ereg args[] = {a,b}; + switch( op ) { case OJEq: - // if( a == b || (a && b && cmp(a,b) == 0) ) goto - op64(ctx,CMP,pa,pb); - XJump_small(JEq,jeq); - op64(ctx,TEST,pa,pa); - XJump_small(JZero,ja); - op64(ctx,TEST,pb,pb); - XJump_small(JZero,jb); - op_call_fun(ctx,NULL,(int)(int_val)a->t->obj->rt->compareFun,2,args); - op32(ctx,TEST,PEAX,PEAX); - XJump_small(JNotZero,jcmp); - patch_jump(ctx,jeq); - register_jump(ctx,do_jump(ctx,OJAlways,false),targetPos); - patch_jump(ctx,ja); - patch_jump(ctx,jb); - patch_jump(ctx,jcmp); + { + // if( a == b || (a && b && cmp(a,b) == 0) ) goto + emit_cmp(ctx,a,b,OJEq); + int jeq = emit_jump(ctx, true); + emit_test(ctx,a,OJNull); + int ja = emit_jump(ctx, true); + emit_test(ctx,b,OJNull); + int jb = emit_jump(ctx, true); + emit_test(ctx, emit_call_fid(ctx,(int)(int_val)at->obj->rt->compareFun,args,2,M_I32),OJNotNull); + int jcmp = emit_jump(ctx, true); + patch_jump(ctx, jeq); + register_block_jump(ctx, offset, false); + patch_jump(ctx, ja); + patch_jump(ctx, jb); + patch_jump(ctx, jcmp); + } break; case OJNotEq: - // if( a != b && (!a || !b || cmp(a,b) != 0) ) goto - op64(ctx,CMP,pa,pb); - XJump_small(JEq,jeq); - op64(ctx,TEST,pa,pa); - register_jump(ctx,do_jump(ctx,OJEq,false),targetPos); - op64(ctx,TEST,pb,pb); - register_jump(ctx,do_jump(ctx,OJEq,false),targetPos); - - op_call_fun(ctx,NULL,(int)(int_val)a->t->obj->rt->compareFun,2,args); - op32(ctx,TEST,PEAX,PEAX); - XJump_small(JZero,jcmp); - - register_jump(ctx,do_jump(ctx,OJNotEq,false),targetPos); - patch_jump(ctx,jcmp); - patch_jump(ctx,jeq); + { + // if( a != b && (!a || !b || cmp(a,b) != 0) ) goto + emit_cmp(ctx,a,b,OJEq); + int jeq = emit_jump(ctx, true); + emit_test(ctx,a,OJEq); + register_block_jump(ctx,offset,true); + emit_test(ctx,b,OJEq); + register_block_jump(ctx,offset,true); + emit_test(ctx, emit_call_fid(ctx,(int)(int_val)at->obj->rt->compareFun,args,2,M_I32),OJNotNull); + register_block_jump(ctx,offset,true); + patch_jump(ctx,jeq); + } break; default: - // if( a && b && cmp(a,b) ?? 0 ) goto - op64(ctx,TEST,pa,pa); - XJump_small(JZero,ja); - op64(ctx,TEST,pb,pb); - XJump_small(JZero,jb); - op_call_fun(ctx,NULL,(int)(int_val)a->t->obj->rt->compareFun,2,args); - op32(ctx,CMP,PEAX,pconst(&p,0)); - register_jump(ctx,do_jump(ctx,op->op,false),targetPos); - patch_jump(ctx,ja); - patch_jump(ctx,jb); + { + // if( a && b && cmp(a,b) ~op~ 0 ) goto + emit_test(ctx,a,OJNull); + int ja = emit_jump(ctx, true); + emit_test(ctx,b,OJNull); + int jb = emit_jump(ctx, true); + emit_cmp(ctx, emit_call_fid(ctx,(int)(int_val)at->obj->rt->compareFun,args,2,M_I32), LOAD_CONST(0,&hlt_i32),op); + register_block_jump(ctx,offset,true); + patch_jump(ctx,ja); + patch_jump(ctx,jb); + } break; } - */ return; } // fallthrough @@ -1750,7 +1746,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case HVIRTUAL: - // code for : if( (fun=hl_vfields(o)[f]) ) dst = fun(o->value,args...); else dst = hl_dyn_call_obj(o->value,field,args,&ret) + // code for : if( (fun=hl_vfields(o)[f]) ) dst = fun(o->value,args...); else dst = hl_dyn_call_obj(o->value,ft,field,args,&ret) { vreg *_o = R(o->extra[0]); ereg obj = LOAD(_o); @@ -1770,7 +1766,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { patch_jump(ctx, jidx); nargs = o->p3 - 1; - ereg eargs = emit_gen_size(ctx, ALLOC_STACK, nargs * HL_WSIZE); + ereg eargs = nargs == 0 ? LOAD_CONST_PTR(NULL) : emit_gen_size(ctx, ALLOC_STACK, nargs * HL_WSIZE); for(i=0;iextra[i+1]))); bool need_dyn = !hl_is_ptr(dst->t) && dst->t->kind != HVOID; @@ -1779,9 +1775,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args = get_tmp_args(ctx, 4); args[0] = LOAD_MEM_PTR(obj,HL_WSIZE); - args[1] = LOAD_CONST(_o->t->virt->fields[o->p2].hashed_name,&hlt_i32); - args[2] = eargs; - args[3] = edyn; + args[1] = LOAD_CONST_PTR(_o->t->virt->fields[o->p2].t); + args[2] = LOAD_CONST(_o->t->virt->fields[o->p2].hashed_name,&hlt_i32); + args[3] = eargs; + args[4] = edyn; ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 4, dst->t); diff --git a/src/jit_regs.c b/src/jit_regs.c index bd9f0c2d3..60279e17f 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -269,7 +269,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { einstr *e = jit->instrs + cur_op; value_info *write = NULL; - if( ctx->loop_end == cur_op && cur_op ) { + while( ctx->loop_end == cur_op && cur_op ) { ctx->loop_end = loop_saves[--loop_count]; ctx->loop_start = loop_saves[--loop_count]; } From 5e9ef7221bbb60e1cbaca20db6774a10bf8b53e8 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 25 Apr 2026 18:14:42 +0200 Subject: [PATCH 42/83] added call_c2hl trampoline --- src/jit.c | 57 ++++++++++++++++++++++++++++++++++ src/jit.h | 13 ++++++++ src/jit_regs.c | 6 ---- src/jit_x86_64.c | 80 +++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 142 insertions(+), 14 deletions(-) diff --git a/src/jit.c b/src/jit.c index 3a3b80b68..5dea0d44b 100644 --- a/src/jit.c +++ b/src/jit.c @@ -164,6 +164,58 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { return pos; } +static void *null_wrapper( hl_type *ft ) { + return hl_jit_assert; +} + +static void *call_jit_c2hl = hl_jit_assert; +static int arg_reg_count = 0; +static int arg_fp_count = 0; + +static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { + int nargs = t->fun->nargs; + if( nargs > MAX_ARGS ) + hl_error("Too many arguments for dynamic call"); + struct { + void *regs[MAX_ARGS]; + void *stack[MAX_ARGS]; + } vargs; + int rp = 0, fp = 0, sp = MAX_ARGS; + for(int i=0;ifun->nargs;i++) { + hl_type *at = t->fun->args[i]; + void *v = args[i]; + if( at->kind == HF32 || at->kind == HF64 ) { + if( fp < arg_fp_count ) + vargs.regs[arg_reg_count + fp++] = v; + else + vargs.stack[--sp] = v; + } else if( rp < arg_reg_count ) + vargs.regs[rp++] = v; + else + vargs.stack[--sp] = v; + } + switch( t->fun->ret->kind ) { + case HUI8: + case HUI16: + case HI32: + case HBOOL: + ret->v.i = ((int (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + return &ret->v.i; + case HI64: + case HGUID: + ret->v.i64 = ((int64 (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + return &ret->v.i64; + case HF32: + ret->v.f = ((float (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + return &ret->v.f; + case HF64: + ret->v.d = ((double (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + return &ret->v.d; + default: + return ((void *(*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + } +} + void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { hl_codegen_flush_consts(ctx); jit_code_append(ctx); @@ -177,6 +229,11 @@ void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **d ctx->final_code = code; hl_emit_final(ctx); hl_codegen_final(ctx); + arg_reg_count = ctx->cfg.regs.nargs; + arg_fp_count = ctx->cfg.floats.nargs; + call_jit_c2hl = ctx->final_code + ctx->code_funs.c2hl; + hl_setup.get_wrapper = null_wrapper; + hl_setup.static_call = callback_c2hl; return code; } diff --git a/src/jit.h b/src/jit.h index eb0f03325..5e056eee9 100644 --- a/src/jit.h +++ b/src/jit.h @@ -114,6 +114,14 @@ typedef struct { #define IS_FLOAT(mode) ((mode) == M_F64 || (mode) == M_F32) #define UNUSED ((ereg)0) +#define MAX_ARGS 16 + +#if defined(HL_WIN_CALL) && defined(HL_64) +# define IS_WINCALL64 1 +#else +# define IS_WINCALL64 0 +#endif + typedef struct { int *data; int max; @@ -167,6 +175,10 @@ typedef struct { ereg req_div_b; } regs_config; +typedef struct { + int c2hl; +} jit_special_funs; + struct _jit_ctx { hl_module *mod; hl_function *fun; @@ -194,6 +206,7 @@ struct _jit_ctx { int code_size; unsigned char *code_instrs; int *code_pos_map; + jit_special_funs code_funs; // accum output int fdef_index; int out_pos; diff --git a/src/jit_regs.c b/src/jit_regs.c index 60279e17f..3d713b9f2 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -43,12 +43,6 @@ #define EMIT(r,a,b,m) regs_emit(ctx,UNUSED,r,a,b,m,0) #define BREAK() EMIT(DEBUG_BREAK,UNUSED,UNUSED,0) -#if defined(HL_WIN_CALL) && defined(HL_64) -# define IS_WINCALL64 1 -#else -# define IS_WINCALL64 0 -#endif - typedef struct { int id; int stack_pos; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 25fe4ec01..1545fb8f4 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -369,6 +369,8 @@ const char *hl_natreg_str( int reg, emit_mode m ) { static int scratch_float_reg = -1; +static ereg scratch_not_param[] = { X(RAX), X(R10), X(R11) }; + void hl_jit_init_regs( regs_config *cfg ) { // exclude R11 at it's use as temporary for various ops # ifdef HL_WIN_CALL @@ -1273,10 +1275,36 @@ void hl_codegen_alloc( jit_ctx *jit ) { ctx->jit = jit; } +static void flush_function( code_ctx *ctx, int start ) { + hl_jit_define_function(ctx->jit, start, ctx->jit->out_pos + byte_count(ctx->code) - start); + align_function(ctx); + if( byte_count(ctx->code) > ctx->code.max ) jit_assert(); +} + +static int jump_near( code_ctx *ctx, int mode ) { + int pos = byte_count(ctx->code); + if( mode < 0 ) { + // backwards + int target = -mode; + B(JAlways_short); + B(target - (pos + 2)); + } else { + B(mode == JAlways ? JAlways_short : mode - 0x10); + B(0); + } + return pos; +} + +static void patch_jump_near( code_ctx *ctx, int jpos ) { + ctx->code.values[jpos + 1] = (unsigned char)(byte_count(ctx->code) - (jpos + 2)); +} + void hl_codegen_init( jit_ctx *jit ) { code_ctx *ctx = jit->code; - byte_reserve(ctx->code,256); - ctx->code.cur -= 256; + byte_reserve(ctx->code,1024); + ctx->code.cur -= 1024; + + // generate hl_null_access stub ctx->null_access_pos = jit->out_pos + byte_count(ctx->code); EMIT(_PUSH,R(RBP),UNUSED,M_PTR); EMIT(_MOV,R(RBP),R(RSP),M_PTR); @@ -1284,19 +1312,55 @@ void hl_codegen_init( jit_ctx *jit ) { emit_ext(ctx,_MOV,R(RAX),VAL_CONST,M_PTR,(int_val)hl_null_access); EMIT(_CALL,R(RAX),UNUSED,M_PTR); BREAK(); - hl_jit_define_function(jit, ctx->null_access_pos, jit->out_pos + byte_count(ctx->code) - ctx->null_access_pos); - align_function(ctx); + flush_function(ctx, ctx->null_access_pos); + + // generate hl_null_field access stub ctx->null_field_pos = jit->out_pos + byte_count(ctx->code); EMIT(_PUSH,R(RBP),UNUSED,M_PTR); EMIT(_MOV,R(RBP),R(RSP),M_PTR); emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x28); - emit_ext(ctx,_MOV,R(RCX),VAL_MEM(R(RBP)),M_I32,HL_WSIZE*2); + emit_ext(ctx,_MOV,jit->cfg.regs.arg[0],VAL_MEM(R(RBP)),M_I32,HL_WSIZE*2); emit_ext(ctx,_MOV,R(RAX),VAL_CONST,M_PTR,(int_val)hl_jit_null_field_access); EMIT(_CALL,R(RAX),UNUSED,M_PTR); BREAK(); - hl_jit_define_function(jit, ctx->null_field_pos, jit->out_pos + byte_count(ctx->code) - ctx->null_field_pos); - align_function(ctx); - if( byte_count(ctx->code) > ctx->code.max ) jit_assert(); + flush_function(ctx, ctx->null_field_pos); + + // generate c2hl stub + jit->code_funs.c2hl = jit->out_pos + byte_count(ctx->code); + regs_config *cfg = &jit->cfg; + EMIT(_PUSH,R(RBP),UNUSED,M_PTR); + EMIT(_MOV,R(RBP),R(RSP),M_PTR); + + ereg fptr = scratch_not_param[0]; + ereg vargs = scratch_not_param[1]; + ereg nargs = scratch_not_param[2]; + EMIT(_MOV,fptr,cfg->regs.arg[0],M_PTR); + EMIT(_MOV,vargs,cfg->regs.arg[1],M_PTR); + EMIT(_MOV,nargs,cfg->regs.arg[2],M_I32); + + for(int i=0;iregs.nargs;i++) + emit_ext(ctx, _MOV, cfg->regs.arg[i], vargs|FL_MEMPTR, M_PTR, i * 8); + for(int i=0;ifloats.nargs;i++) + emit_ext(ctx, MOVSD, cfg->floats.arg[i]-64, vargs|FL_MEMPTR, M_PTR, (i + cfg->regs.nargs) * 8); + + emit_ext(ctx,ADD,vargs,VAL_CONST,M_PTR,(MAX_ARGS - 1) * HL_WSIZE); + int begin = byte_count(ctx->code); + EMIT(_TEST,nargs,nargs,M_I32); + int pos = jump_near(ctx,JZero); + EMIT(_PUSH,vargs|FL_MEMPTR,UNUSED,M_PTR); + EMIT(DEC,nargs,UNUSED,M_I32); + jump_near(ctx,-begin); + patch_jump_near(ctx,pos); + + if( IS_WINCALL64 ) emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x20); + EMIT(_CALL, fptr, UNUSED, M_NONE); + + EMIT(_MOV,R(RSP),R(RBP),M_PTR); + EMIT(_POP,R(RBP),UNUSED,M_PTR); + EMIT(_RET,UNUSED,UNUSED,M_NONE); + + flush_function(ctx, ctx->null_field_pos); + hl_codegen_flush(jit); } From 76a66f225c44368bb442c5b2380fdf6e67819781 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 25 Apr 2026 21:46:59 +0200 Subject: [PATCH 43/83] added correct phi-from block tracking, emit cmov to prevent reg overwrite --- src/data_struct.c | 10 ++++++ src/data_struct.h | 1 + src/jit.h | 5 ++- src/jit_dump.c | 10 ++++-- src/jit_emit.c | 36 +++++++++++++-------- src/jit_regs.c | 81 ++++++++++++----------------------------------- src/jit_x86_64.c | 6 +++- 7 files changed, 71 insertions(+), 78 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index 367b70e59..313d708bc 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -49,6 +49,9 @@ typedef struct { } S_TYPE; typedef S_VALUE S_NAME(_value); +#ifdef S_MAP +typedef S_KEY S_NAME(_key); +#endif INLINE static void S_NAME(check_size)( hl_alloc *alloc, S_TYPE *st ) { if( st->cur == st->max ) { @@ -258,6 +261,13 @@ INLINE static bool S_NAME(iter_next)( S_TYPE st, S_VALUE *val, int idx ) { return idx < st.cur; } +#ifdef S_MAP +INLINE static bool S_NAME(iter_next_key)( S_TYPE st, S_KEY *key, int idx ) { + if( idx < st.cur ) *key = st.keys[idx]; + return idx < st.cur; +} +#endif + INLINE static bool S_NAME(iter_prev)( S_TYPE st, S_VALUE *val, int idx ) { if( idx >= 0 ) *val = st.values[idx]; return idx >= 0; diff --git a/src/data_struct.h b/src/data_struct.h index 43494f18e..5c5b9fe4e 100644 --- a/src/data_struct.h +++ b/src/data_struct.h @@ -34,6 +34,7 @@ #define STRUCT_DEF_SIZE 2 #define for_iter(name,var,set) name##__value var; for(int __idx=0;name##_iter_next(set,&var,__idx);__idx++) +#define for_iter_key(name,var,set) name##__key var; for(int __idx=0;name##_iter_next_key(set,&var,__idx);__idx++) #define for_iter_back(name,var,set) name##__value var; for(int __idx=(set).cur-1;name##_iter_prev(set,&var,__idx);__idx--) #define S_TYPE ptr_set diff --git a/src/jit.h b/src/jit.h index 5e056eee9..154eda309 100644 --- a/src/jit.h +++ b/src/jit.h @@ -45,6 +45,9 @@ typedef enum { CALL_REG, CALL_FUN, MOV, + CMOV, + XCHG, + CXCHG, PUSH_CONST, PUSH, POP, @@ -54,7 +57,6 @@ typedef enum { BLOCK, ENTER, STACK_OFFS, - XCHG, CATCH, ADDRESS, } emit_op; @@ -135,6 +137,7 @@ struct _ephi { int nvalues; emit_mode mode; ereg *values; + int *blocks; }; typedef struct _eblock { diff --git a/src/jit_dump.c b/src/jit_dump.c index 0d60834c8..cc02d9642 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -42,6 +42,9 @@ static const char *op_names[] = { "call", "call", "mov", + "cmov", + "xchg", + "cxhg", "push-const", "push", "pop", @@ -51,7 +54,6 @@ static const char *op_names[] = { "block", "enter", "stack", - "xchg", "catch", "address", }; @@ -452,6 +454,10 @@ void hl_emit_dump( jit_ctx *ctx ) { if( vpos < ctx->value_count && ctx->values_writes[vpos] == i ) printf("V%d = ", vpos++); dump_instr(ctx, e, i); + if( e->op == JCOND && (ctx->instrs[i+1].op != BLOCK || ctx->instrs[i+1+e->size_offs].op != BLOCK) ) + printf(" ???"); + if( e->op == JUMP && ctx->instrs[i+1+e->size_offs].op != BLOCK ) + printf(" ???"); if( e->op == BLOCK ) { eblock *b = &ctx->blocks[e->size_offs]; for(int k=0;kphi_count;k++) { @@ -459,7 +465,7 @@ void hl_emit_dump( jit_ctx *ctx ) { printf("\n\t\t@%X %s = phi%s(",i,val_str(p->value,p->mode),emit_mode_str(p->mode)); for(int n=0;nnvalues;n++) { if( n > 0 ) printf(","); - printf("%s",val_str(p->values[n],p->mode)); + printf("%s:%d",val_str(p->values[n],p->mode),p->blocks[n]); } printf(")"); if( p->nvalues <= 1 ) diff --git a/src/jit_emit.c b/src/jit_emit.c index 5d02af8a3..e36f6eda5 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -59,11 +59,13 @@ typedef struct _tmp_phi tmp_phi; #define S_SORTED +#define S_MAP #define S_TYPE ereg_map #define S_NAME(name) ereg_##name -#define S_VALUE ereg +#define S_KEY ereg +#define S_VALUE emit_block* #include "data_struct.c" -#define ereg_add(set,v) ereg_add_impl(DEF_ALLOC,&(set),v) +#define ereg_add(set,k,v) ereg_add_impl(DEF_ALLOC,&(set),k,v) #define S_MAP @@ -444,7 +446,6 @@ static void patch_jump( emit_ctx *ctx, int jpos ) { if( ctx->current_block->start_pos == ctx->emit_pos-1 ) { block_add_pred(ctx, ctx->current_block, b); } else { - ctx->arrival_points = link_add_sort_unique(ctx, ctx->op_pos, b, ctx->arrival_points); if( !split_block(ctx) ) jit_assert(); } } @@ -518,14 +519,14 @@ static void phi_remove_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { emit_debug("%sPHI-REM-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(v,p->mode)); } -static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v ) { +static void phi_add_val( emit_ctx *ctx, tmp_phi *p, ereg v, emit_block *from ) { if( !p->b ) jit_assert(); if( IS_NULL(v) ) jit_assert(); if( p->value == v ) return; - if( !ereg_add(p->vals,v) ) + if( !ereg_add(p->vals,v,from) ) return; - emit_debug("%sPHI-DEP %s = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(v,p->mode)); + emit_debug("%sPHI-DEP %s:#%d = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), from->id, val_str(v,p->mode)); if( v < 0 ) { tmp_phi *p2 = GET_PHI(v); phi_add(p2->ref_phis,p); @@ -536,7 +537,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { if( p->locked ) jit_assert(); ereg same = UNUSED; - for_iter(ereg,v,p->vals) { + for_iter_key(ereg,v,p->vals) { if( v == same || v == p->value ) continue; if( !IS_NULL(same) ) @@ -561,9 +562,10 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { store_block_var(ctx,b,p->r,same); l = l->next; } + emit_block *bsame = ereg_find(p->vals,same); for_iter(phi,p2,p->ref_phis) { phi_remove_val(ctx,p2,p->value); - phi_add_val(ctx,p2,same); + phi_add_val(ctx,p2,same,bsame); } p->ref_blocks = NULL; int count = phi_count(p->ref_phis); @@ -571,7 +573,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { for(int i=0;iphi_depth--; - emit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(same,p->mode)); + emit_debug("%sPHI-OPT-DONE %s = %s:#%d\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(same,p->mode), bsame->id); return optimize_phi_rec(ctx,p); } @@ -581,7 +583,7 @@ static ereg gather_phis( emit_ctx *ctx, tmp_phi *p ) { p->locked = true; for_iter(blocks,b,p->b->preds) { ereg r = p->r ? emit_load_reg_block(ctx, b, p->r) : p->value; - phi_add_val(ctx, p, r); + phi_add_val(ctx, p, r, b); } p->locked = false; return optimize_phi_rec(ctx, p); @@ -622,8 +624,8 @@ static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { if( mode != GET_MODE(v2) ) jit_assert(); tmp_phi *p = alloc_phi(ctx, ctx->current_block, NULL); p->mode = mode; - phi_add_val(ctx, p, v1); - phi_add_val(ctx, p, v2); + phi_add_val(ctx, p, v1, blocks_get(ctx->blocks,ctx->current_block->id - 2)); + phi_add_val(ctx, p, v2, blocks_get(ctx->blocks,ctx->current_block->id - 1)); return p->value; } @@ -823,11 +825,15 @@ static void emit_write_block( emit_ctx *ctx, emit_block *b ) { p2->mode = p->mode; p2->nvalues = ereg_count(p->vals); p2->values = (ereg*)hl_malloc(&jit->falloc,sizeof(ereg)*p2->nvalues); + p2->blocks = (ereg*)hl_malloc(&jit->falloc,sizeof(int)*p2->nvalues); int k = 0; - for_iter(ereg,v,p->vals) { + for_iter_key(ereg,v,p->vals) { remap_phi_reg(ctx, &v); p2->values[k++] = v; } + k = 0; + for_iter(ereg,bfrom,p->vals) + p2->blocks[k++] = bfrom->id; } } @@ -996,6 +1002,8 @@ void hl_emit_function( jit_ctx *jit ) { ctx->pos_map[op_pos] = ctx->emit_pos-1; else ctx->pos_map[op_pos] = ctx->emit_pos; + if( f->findex == 0x32B && op_pos == 0x48 ) + hl_debug_break(); if( op_pos == 0 ) { ctx->current_block->start_pos = ctx->emit_pos; emit_gen_size(ctx, BLOCK, 0); @@ -1004,6 +1012,8 @@ void hl_emit_function( jit_ctx *jit ) { split_block(ctx); emit_opcode(ctx,f->ops + op_pos); } + if( ctx->arrival_points ) + jit_assert(); hl_emit_clean_phis(ctx); hl_emit_flush(ctx->jit); diff --git a/src/jit_regs.c b/src/jit_regs.c index 3d713b9f2..c1ba483b1 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -222,34 +222,6 @@ static value_info *regs_current( regs_ctx *ctx, ereg r ) { return NULL; } -static eblock *resolve_block_value( regs_ctx *ctx, ereg v ) { - if( v < 0 ) { - for(int i=0;ijit->block_count;i++) { - eblock *b = ctx->jit->blocks + i; - for(int p=0;pphi_count;p++) { - if( b->phis[p].value == v ) - return b; - } - } - } else { - if( IS_NATREG(v) ) jit_assert(); - int idx = ctx->jit->values_writes[v]; - int min = 0; - int max = ctx->jit->block_count; - while( min < max ) { - int med = (min + max) >> 1; - eblock *b = ctx->jit->blocks + med; - if( idx < b->start_pos ) { - max = med; - } else if( idx >= b->end_pos ) { - min = med; - } else - return b; - } - } - jit_assert(); -} - static void regs_compute_liveness( regs_ctx *ctx ) { # define MAX_LOOP_DEPTH 256 int loop_saves[MAX_LOOP_DEPTH]; @@ -345,31 +317,15 @@ static void regs_compute_liveness( regs_ctx *ctx ) { VAL_REG(ph->value)->mode = ph->mode; for(int k=0;knvalues;k++) { ereg v = ph->values[k]; - eblock *b2 = resolve_block_value(ctx, v); + eblock *b2 = jit->blocks + ph->blocks[k]; value_info *val = VAL_REG(v); - if( b2->start_pos >= bl->start_pos ) { - // loop : insert in all predecesors - for(int i=0;ipred_count;i++) { - b2 = &jit->blocks[bl->preds[i]]; - if( b2->start_pos >= bl->start_pos ) { - int_arr *arr = &ctx->blocks_phis[b2 - jit->blocks]; - regs_debug("ADD PHI %s:=%s to #%d@%X\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks),b2->end_pos-1); - int_arr_add(*arr,v); - int_arr_add(*arr,ph->value); - val->tot_reads++; - if( val->last_read < b2->end_pos ) - val->last_read = b2->end_pos; - } - } - } else { - int_arr *arr = &ctx->blocks_phis[b2 - jit->blocks]; - regs_debug("ADD PHI %s:=%s to #%d@%X\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks),b2->end_pos-1); - int_arr_add(*arr,v); - int_arr_add(*arr,ph->value); - val->tot_reads++; - if( val->last_read < b2->end_pos ) - val->last_read = b2->end_pos; - } + int_arr *arr = &ctx->blocks_phis[b2 - jit->blocks]; + regs_debug("ADD PHI %s:=%s to #%d@%X\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks),b2->end_pos-1); + int_arr_add(*arr,v); + int_arr_add(*arr,ph->value); + val->tot_reads++; + if( val->last_read < b2->end_pos ) + val->last_read = b2->end_pos; } } } @@ -501,7 +457,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { } } -static void flush_movs( regs_ctx *ctx ) { +static void flush_movs( regs_ctx *ctx, bool cond ) { int_arr movs = ctx->pack_movs; while( true ) { int size = int_arr_count(movs); @@ -520,7 +476,8 @@ static void flush_movs( regs_ctx *ctx ) { if( !read ) { ereg from = int_arr_get(movs,k+1); int mode = int_arr_get(movs,k+2); - regs_emit_mov(ctx,to,from,mode); + bool cmov = cond && IS_PURE(to); + regs_emit(ctx,to,cmov?CMOV:MOV,from,UNUSED,mode,0); int_arr_remove_range(&movs,k,3); cycle = false; break; @@ -530,7 +487,8 @@ static void flush_movs( regs_ctx *ctx ) { ereg to = int_arr_get(movs,0); ereg from = int_arr_get(movs,1); int mode = int_arr_get(movs,2); - EMIT(XCHG,to,from,mode); + bool cmov = cond && (IS_PURE(to) || IS_PURE(from)); + regs_emit(ctx,UNUSED,cmov?CXCHG:XCHG,to,from,mode,0); int_arr_remove_range(&movs,0,3); size -= 3; for(int k=0;kpack_movs); } -static void flush_phis( regs_ctx *ctx, eblock *b ) { +static void flush_phis( regs_ctx *ctx, eblock *b, bool cond ) { if( !b ) return; jit_ctx *jit = ctx->jit; int bid = (int)(b - jit->blocks); int_arr arr = ctx->blocks_phis[bid]; int idx = 0; int_arr movs = ctx->pack_movs; + while( idx < int_arr_count(arr) ) { ereg a = int_arr_get(arr,idx++); ereg b = int_arr_get(arr,idx++); @@ -574,7 +533,7 @@ static void flush_phis( regs_ctx *ctx, eblock *b ) { } ctx->pack_movs = movs; int_arr_free(&ctx->blocks_phis[bid]); - flush_movs(ctx); + flush_movs(ctx, cond); } static void regs_emit_instrs( regs_ctx *ctx ) { @@ -641,7 +600,7 @@ static void regs_emit_instrs( regs_ctx *ctx ) { } instr_stack_offset = stack_args+offset; } - flush_movs(ctx); + flush_movs(ctx,0); e.nargs = 0xFF; if( vout && vout->last_read > cur_op ) ret_val = ®_CFG(REG_MODE(e.mode))->ret; @@ -688,10 +647,10 @@ static void regs_emit_instrs( regs_ctx *ctx ) { regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-stack_offset); } break; - case JUMP: case JCOND: + case JUMP: case JUMP_TABLE: - flush_phis(ctx,cur_block); + flush_phis(ctx,cur_block, e.op == JCOND); if( e.op == JUMP_TABLE ) { // copy args (remap later) hl_emit_store_args(jit->emit,&e,hl_emit_get_args(jit->emit,&e),e.nargs); @@ -734,7 +693,7 @@ static void regs_emit_instrs( regs_ctx *ctx ) { if( instr_stack_offset ) regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,instr_stack_offset); if( cur_block && cur_block->end_pos == cur_op+1 ) - flush_phis(ctx,cur_block); + flush_phis(ctx,cur_block,false); ctx->pos_map[cur_op+1] = ctx->emit_pos; } } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 1545fb8f4..c82c3f734 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1141,7 +1141,7 @@ void hl_codegen_function( jit_ctx *jit ) { einstr *p; do { p = jit->reg_instrs + ctx->cur_op - (++prev); - } while( p->op == MOV || p->op == JCOND ); + } while( p->op == MOV || p->op == JCOND || p->op == CMOV || p->op == XCHG || p->op == CXCHG ); int op; switch( p->size_offs ) { case OJFalse: @@ -1244,6 +1244,10 @@ void hl_codegen_function( jit_ctx *jit ) { case PREFETCH: BREAK(); break; + case CMOV: + case CXCHG: + BREAK(); + break; default: jit_assert(); break; From e4677bb05790c1d30684642a13ccd38955f99a92 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 26 Apr 2026 09:39:38 +0200 Subject: [PATCH 44/83] review and fixed block construction --- src/jit_dump.c | 67 ++++++++++++++++++++++++++++++++++++------------- src/jit_emit.c | 68 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 95 insertions(+), 40 deletions(-) diff --git a/src/jit_dump.c b/src/jit_dump.c index cc02d9642..c4e01fa93 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -359,6 +359,8 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { break; case BLOCK: printf(" #%d", e->size_offs); + if( e->size_offs && ctx->blocks[e->size_offs].pred_count == 0 ) + printf(" ???DEAD"); break; case STACK_OFFS: if( e->size_offs >= 0 ) @@ -407,8 +409,6 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { } void hl_emit_dump( jit_ctx *ctx ) { - int i; - int cur_op = 0; hl_function *f = ctx->fun; int nargs = f->type->fun->nargs; // if it not was not before (in case of dump during process) @@ -418,16 +418,16 @@ void hl_emit_dump( jit_ctx *ctx ) { printf("function "); hl_dump_fun_name(f); printf("("); - for(i=0;i 0 ) printf(","); uprintf(USTR("R%d"), i); } printf(")\n"); - for(i=0;inregs;i++) + for(int i=0;inregs;i++) uprintf(USTR("\tR%d : %s\n"),i, hl_type_str(f->regs[i])); // check blocks intervals int cur = 0; - for(i=0;iblock_count;i++) { + for(int i=0;iblock_count;i++) { eblock *b = ctx->blocks + i; if( b->start_pos != cur ) printf(" ??? BLOCK %d START AT %X != %X\n", i, b->start_pos, cur); if( b->end_pos < b->start_pos ) printf(" ??? BLOCK %d RANGE [%X,%X]\n", i, b->start_pos, b->end_pos); @@ -439,30 +439,60 @@ void hl_emit_dump( jit_ctx *ctx ) { int vpos = 1; int rpos = 0; int cpos = 0; + int cur_op = 0; bool new_op = false; - cur = 0; - for(i=0;iinstr_count;i++) { - while( ctx->emit_pos_map[cur_op] == i ) { + eblock *cur_block = NULL; + for(int icount=0;icountinstr_count;icount++) { + while( ctx->emit_pos_map[cur_op] == icount ) { printf("@%X ", cur_op); hl_dump_op(ctx->fun, f->ops + cur_op); printf("\n"); new_op = true; cur_op++; } - einstr *e = ctx->instrs + i; - printf("\t\t@%X ", i); - if( vpos < ctx->value_count && ctx->values_writes[vpos] == i ) + einstr *e = ctx->instrs + icount; + printf("\t\t@%X ", icount); + if( vpos < ctx->value_count && ctx->values_writes[vpos] == icount ) printf("V%d = ", vpos++); - dump_instr(ctx, e, i); - if( e->op == JCOND && (ctx->instrs[i+1].op != BLOCK || ctx->instrs[i+1+e->size_offs].op != BLOCK) ) - printf(" ???"); - if( e->op == JUMP && ctx->instrs[i+1+e->size_offs].op != BLOCK ) - printf(" ???"); + dump_instr(ctx, e, icount); + if( e->op == JCOND || e->op == JUMP ) { + int target = icount + 1 + e->size_offs; + bool bad = false; + if( icount + 1 >= ctx->instr_count || target < 0 || target >= ctx->instr_count ) + bad = true; + else if( ctx->instrs[target].op != BLOCK || (e->op == JCOND && ctx->instrs[icount+1].op != BLOCK) ) + bad = true; + else { + bool found = false; + for(int k=0;knext_count;k++) { + if( cur_block->nexts[k] == ctx->instrs[target].size_offs ) + found = true; + if( (e->op == JUMP || e->op == JUMP_TABLE) && ctx->instrs[icount+1].op == BLOCK && ctx->instrs[icount+1].size_offs == cur_block->nexts[k] ) + printf(" ???LEAK"); + } + if( !found ) printf(" ???NEXT"); + } + if( bad ) + printf(" ???"); + } if( e->op == BLOCK ) { eblock *b = &ctx->blocks[e->size_offs]; + for(int k=0;kpred_count;k++) { + eblock *p = &ctx->blocks[b->preds[k]]; + einstr *pe = &ctx->instrs[p->end_pos-1]; + if( p->end_pos == icount ) + continue; + bool bad = false; + if( (pe->op == JUMP || pe->op == JCOND) && pe->size_offs == icount - p->end_pos ) + bad = false; + else if( pe->op != JUMP_TABLE ) + bad = true; + if( bad ) + printf(" ???PREV#%d",b->preds[k]); + } for(int k=0;kphi_count;k++) { ephi *p = b->phis + k; - printf("\n\t\t@%X %s = phi%s(",i,val_str(p->value,p->mode),emit_mode_str(p->mode)); + printf("\n\t\t@%X %s = phi%s(",icount,val_str(p->value,p->mode),emit_mode_str(p->mode)); for(int n=0;nnvalues;n++) { if( n > 0 ) printf(","); printf("%s:%d",val_str(p->values[n],p->mode),p->blocks[n]); @@ -471,8 +501,9 @@ void hl_emit_dump( jit_ctx *ctx ) { if( p->nvalues <= 1 ) printf(" ???"); } + cur_block = b; } - while( rpos < ctx->reg_instr_count && rpos < ctx->reg_pos_map[i+1] ) { + while( rpos < ctx->reg_instr_count && rpos < ctx->reg_pos_map[icount+1] ) { ereg out = ctx->reg_writes[rpos]; e = ctx->reg_instrs + rpos; printf("\n\t\t\t\t@%X ",rpos); diff --git a/src/jit_emit.c b/src/jit_emit.c index e36f6eda5..f55dd2fb4 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -391,6 +391,9 @@ static emit_block *alloc_block( emit_ctx *ctx ) { } static void block_add_pred( emit_ctx *ctx, emit_block *b, emit_block *p ) { + for_iter(blocks,p2,b->preds) + if( p2 == p ) + return; blocks_add(b->preds,p); blocks_add(p->nexts,b); emit_debug(" PRED #%d\n",p->id); @@ -415,18 +418,25 @@ static bool split_block( emit_ctx *ctx ) { block_add_pred(ctx, b, (emit_block*)ctx->arrival_points->ptr); ctx->arrival_points = ctx->arrival_points->next; } - bool dead_code = blocks_count(b->preds) == 0; // if we have no reach, force previous block dependency, this is rare dead code emit by compiler einstr *eprev = &ctx->instrs[b->start_pos-1]; - if( (eprev->op != JUMP && eprev->op != JUMP_TABLE && eprev->op != RET && eprev->mode != M_NORET) || ctx->fun->ops[ctx->op_pos].op == OTrap || dead_code ) + if( eprev->op != JUMP && eprev->op != JUMP_TABLE && eprev->op != RET && eprev->mode != M_NORET ) block_add_pred(ctx, b, ctx->current_block); ctx->current_block->end_pos = b->start_pos; ctx->current_block = b; return true; } +static void add_jump_target( emit_ctx *ctx, int offs ) { + if( offs == 0 && ctx->current_block->start_pos == ctx->emit_pos-1 ) + return; + int target = offs + ctx->op_pos + 1; + ctx->arrival_points = link_add_sort_unique(ctx, target, ctx->current_block, ctx->arrival_points); +} + static int emit_jump( emit_ctx *ctx, bool cond ) { int p = ctx->emit_pos; emit_gen(ctx, cond ? JCOND : JUMP, UNUSED, UNUSED, 0); + if( !cond ) add_jump_target(ctx, 0); split_block(ctx); return p; } @@ -440,30 +450,22 @@ static void patch_jump( emit_ctx *ctx, int jpos ) { break; } } - if( !b ) jit_assert(); + if( !b || b == ctx->current_block ) jit_assert(); // patch opcode - ctx->instrs[jpos].size_offs = ctx->emit_pos - (jpos + 1); - if( ctx->current_block->start_pos == ctx->emit_pos-1 ) { + bool after_block = ctx->current_block->start_pos == ctx->emit_pos-1; + ctx->instrs[jpos].size_offs = ctx->emit_pos - (after_block?1:0) - (jpos + 1); + if( after_block ) { block_add_pred(ctx, ctx->current_block, b); } else { if( !split_block(ctx) ) jit_assert(); } } -static void add_jump_target( emit_ctx *ctx, int jpos, int offs ) { - int target = offs + ctx->op_pos + 1; - ctx->arrival_points = link_add_sort_unique(ctx, target, ctx->current_block, ctx->arrival_points); -} - static void register_jump( emit_ctx *ctx, int jpos, int offs ) { int target = offs + ctx->op_pos + 1; int_arr_add(ctx->jump_regs, jpos); int_arr_add(ctx->jump_regs, target); - if( offs > 0 ) { - add_jump_target(ctx, jpos, offs); - if( ctx->arrival_points->id != ctx->op_pos + 1 && ctx->fun->ops[ctx->op_pos].op != OSwitch && ctx->fun->ops[ctx->op_pos+1].op != OLabel ) - ctx->arrival_points = link_add_sort_unique(ctx, ctx->op_pos + 1, ctx->current_block, ctx->arrival_points); - } + if( offs > 0 ) add_jump_target(ctx, offs); } static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { @@ -1002,14 +1004,22 @@ void hl_emit_function( jit_ctx *jit ) { ctx->pos_map[op_pos] = ctx->emit_pos-1; else ctx->pos_map[op_pos] = ctx->emit_pos; - if( f->findex == 0x32B && op_pos == 0x48 ) - hl_debug_break(); if( op_pos == 0 ) { ctx->current_block->start_pos = ctx->emit_pos; emit_gen_size(ctx, BLOCK, 0); } - if( ctx->arrival_points && ctx->arrival_points->id == op_pos ) - split_block(ctx); + if( ctx->arrival_points ) { + if( ctx->arrival_points->id < op_pos ) + jit_assert(); + while( ctx->arrival_points && ctx->arrival_points->id == op_pos && !split_block(ctx) ) { + emit_block *b = ctx->arrival_points->ptr; + for_iter(blocks,bp,ctx->current_block->preds) { + if( b == bp ) { b = NULL; break; } + } + if( b ) block_add_pred(ctx, ctx->current_block, b); + ctx->arrival_points = ctx->arrival_points->next; + } + } emit_opcode(ctx,f->ops + op_pos); } if( ctx->arrival_points ) @@ -1170,10 +1180,13 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type int jeq = emit_jump(ctx,true); emit_test(ctx,a,OJEq); register_block_jump(ctx,offset,true); + split_block(ctx); emit_test(ctx,b,OJEq); register_block_jump(ctx,offset,true); + split_block(ctx); hl_type *vt = at->tparam; emit_cmp(ctx, LOAD_MEM(a,HL_DYN_VALUE,vt), LOAD_MEM(b,HL_DYN_VALUE,vt), OJNull); + add_jump_target(ctx, 0); int jcmp = emit_jump(ctx,true); register_block_jump(ctx,offset,true); patch_jump(ctx,jcmp); @@ -1291,13 +1304,14 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type emit_cmp(ctx,a,b,OJEq); int jeq = emit_jump(ctx, true); emit_test(ctx,a,OJNull); - int ja = emit_jump(ctx, true); + int ja = emit_jump(ctx, true); emit_test(ctx,b,OJNull); int jb = emit_jump(ctx, true); emit_test(ctx, emit_call_fid(ctx,(int)(int_val)at->obj->rt->compareFun,args,2,M_I32),OJNotNull); int jcmp = emit_jump(ctx, true); patch_jump(ctx, jeq); register_block_jump(ctx, offset, false); + split_block(ctx); patch_jump(ctx, ja); patch_jump(ctx, jb); patch_jump(ctx, jcmp); @@ -1307,11 +1321,14 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type { // if( a != b && (!a || !b || cmp(a,b) != 0) ) goto emit_cmp(ctx,a,b,OJEq); + add_jump_target(ctx, 0); int jeq = emit_jump(ctx, true); emit_test(ctx,a,OJEq); register_block_jump(ctx,offset,true); + split_block(ctx); emit_test(ctx,b,OJEq); register_block_jump(ctx,offset,true); + split_block(ctx); emit_test(ctx, emit_call_fid(ctx,(int)(int_val)at->obj->rt->compareFun,args,2,M_I32),OJNotNull); register_block_jump(ctx,offset,true); patch_jump(ctx,jeq); @@ -1457,6 +1474,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { emit_test(ctx, LOAD(dst), o->op); register_block_jump(ctx, o->p2, true); + add_jump_target(ctx, 0); } break; case OJEq: @@ -1470,6 +1488,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OJNotLt: case OJNotGte: emit_jump_dyn(ctx,o->op,dst->t,LOAD(dst),ra->t,LOAD(ra),o->p3); + add_jump_target(ctx, 0); break; case OJAlways: register_block_jump(ctx, o->p1, false); @@ -1960,6 +1979,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case ONullCheck: { emit_test(ctx, LOAD(dst), OJNotNull); + add_jump_target(ctx, 0); int jok = emit_jump(ctx, true); // ----- DETECT FIELD ACCESS ---------------- @@ -2097,8 +2117,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { int jskip = emit_jump(ctx, true); STORE(dst, tinf ? LOAD_CONST_PTR(&tinf->exc_value) : LOAD_MEM_PTR(thread,(int)(int_val)&tinf->exc_value)); - int jtrap = emit_jump(ctx, false); + int jtrap = ctx->emit_pos; + emit_gen(ctx, JUMP, UNUSED, UNUSED, 0); register_jump(ctx, jtrap, o->p2); + split_block(ctx); patch_jump(ctx, jskip); if( ctx->trap_count == MAX_TRAPS ) jit_error("Too many try/catch depth"); @@ -2143,17 +2165,19 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg v = LOAD(dst); int count = o->p2; emit_cmp(ctx,v,LOAD_CONST(count,&hlt_i32),OJUGte); + add_jump_target(ctx, 0); int jdefault = emit_jump(ctx, true); int pos = ctx->emit_pos; einstr *e = emit_instr(ctx, JUMP_TABLE); e->a = v; + patch_instr_mode(ctx, M_NORET); store_args(ctx,e,(ereg*)o->extra,count); register_jump(ctx, pos, 0); for(int k=0;kextra[k]; if( offs < 0 ) jit_assert(); if( offs == 0 ) continue; - add_jump_target(ctx, pos, offs); + add_jump_target(ctx, offs); } patch_jump(ctx, jdefault); } From 4fbb1f0c825d2d88d373f9ff3a3b54d9104e694f Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 26 Apr 2026 13:37:29 +0200 Subject: [PATCH 45/83] added div and conv, fixed phis (again) --- src/data_struct.c | 22 +++++ src/jit.h | 1 - src/jit_dump.c | 1 - src/jit_emit.c | 28 +++---- src/jit_x86_64.c | 208 +++++++++++++++++++++++++++++++++++----------- 5 files changed, 195 insertions(+), 65 deletions(-) diff --git a/src/data_struct.c b/src/data_struct.c index 313d708bc..ed417770e 100644 --- a/src/data_struct.c +++ b/src/data_struct.c @@ -178,6 +178,28 @@ INLINE static void S_NAME(replace_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { st->values[pos] = v; st->cur++; } + +INLINE static bool S_NAME(add_pair_impl)( hl_alloc *alloc, S_TYPE *st, S_ARGS ) { + int min = 0; + int max = st->cur; + int pos; + while( min < max ) { + int mid = (min + max) >> 1; + S_KEY k2 = st->keys[mid]; + if( k2 < k ) min = mid + 1; else if( k2 > k ) max = mid; else { + S_VALUE v2 = st->values[mid]; + if( S_CMP(v,v2) ) min = mid+1; else if( S_CMP(v2,v) ) max = mid; else return false; + } + } + S_NAME(check_size)(alloc,st); + pos = (min + max) >> 1; + memmove(st->keys + pos + 1, st->keys + pos, (st->cur - pos) * sizeof(S_KEY)); + memmove(st->values + pos + 1, st->values + pos, (st->cur - pos) * sizeof(S_VALUE)); + st->keys[pos] = k; + st->values[pos] = v; + st->cur++; + return true; +} #endif INLINE static bool S_NAME(exists)( S_TYPE st, S_KEY k ) { diff --git a/src/jit.h b/src/jit.h index 154eda309..2f444b256 100644 --- a/src/jit.h +++ b/src/jit.h @@ -70,7 +70,6 @@ typedef enum { M_UI8, M_UI16, M_I32, - M_I64, M_PTR, M_F64, M_F32, diff --git a/src/jit_dump.c b/src/jit_dump.c index c4e01fa93..66746eacc 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -163,7 +163,6 @@ static const char *emit_mode_str( emit_mode mode ) { case M_UI8: return "-ui8"; case M_UI16: return "-ui16"; case M_I32: return "-i32"; - case M_I64: return "-i64"; case M_F32: return "-f32"; case M_F64: return "-f64"; case M_PTR: return ""; diff --git a/src/jit_emit.c b/src/jit_emit.c index f55dd2fb4..2cca28808 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -31,7 +31,7 @@ # define emit_debug(...) #endif -int hl_emit_mode_sizes[] = {0,1,2,4,8,HL_WSIZE,8,4,0,0}; +int hl_emit_mode_sizes[] = {0,1,2,4,HL_WSIZE,8,4,0,0}; typedef struct { hl_type *t; @@ -65,7 +65,7 @@ typedef struct _tmp_phi tmp_phi; #define S_KEY ereg #define S_VALUE emit_block* #include "data_struct.c" -#define ereg_add(set,k,v) ereg_add_impl(DEF_ALLOC,&(set),k,v) +#define ereg_add(set,k,v) ereg_add_pair_impl(DEF_ALLOC,&(set),k,v) #define S_MAP @@ -248,18 +248,18 @@ static linked_inf *link_sort_remove( linked_inf *head, int id ) { } static emit_mode hl_type_mode( hl_type *t ) { - if( t->kind == HVOID ) - return M_VOID; - if( t->kind == HBOOL ) - return sizeof(bool) == 1 ? M_UI8 : M_I32; - if( t->kind == HGUID ) - return M_I64; - if( t->kind == HF32 ) - return M_F32; - if( t->kind == HF64 ) - return M_F64; - if( t->kind < HBOOL ) - return (emit_mode)t->kind; + static emit_mode CONV[] = { + M_VOID, + M_UI8, + M_UI16, + M_I32, + M_PTR, + M_F32, + M_F64, + sizeof(bool) == 1 ? M_UI8 : M_I32, + }; + if( t->kind <= HBOOL ) + return CONV[t->kind]; return M_PTR; } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index c82c3f734..4b476b4b3 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -413,7 +413,7 @@ void hl_jit_init_regs( regs_config *cfg ) { // extra cfg->req_bit_shifts = R(RCX); cfg->req_div_a = R(RAX); - cfg->req_div_b = R(RDX); + cfg->req_div_b = R(RCX); cfg->stack_reg = R(RSP); cfg->stack_pos = R(RBP); cfg->stack_align = 16; @@ -475,7 +475,7 @@ static preg make_reg( ereg r, emit_mode m ) { static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, int_val value ) { opform *f = &OP_FORMS[op]; - int mode64 = (mode == M_PTR || mode == M_I64) && (f->r_mem&FLAG_DEF64) == 0 ? 8 : 0; + int mode64 = mode == M_PTR && (f->r_mem&FLAG_DEF64) == 0 ? 8 : 0; int r64 = mode64; preg a = make_reg(_a,mode), b = make_reg(_b,mode); switch( ID2(a.kind,b.kind) ) { @@ -487,14 +487,14 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, case ID2(RFPU,RFPU): if( f->mem_r ) { // canonical form - if( a.reg > 7 ) r64 |= 1; - if( b.reg > 7 ) r64 |= 4; + if( a.reg & 8 ) r64 |= 1; + if( b.reg & 8 ) r64 |= 4; OP(f->mem_r); MOD_RM(3,b.reg,a.reg); } else { ERRIF( f->r_mem == 0 ); - if( a.reg > 7 ) r64 |= 4; - if( b.reg > 7 ) r64 |= 1; + if( a.reg & 8 ) r64 |= 4; + if( b.reg & 8 ) r64 |= 1; OP(f->r_mem); MOD_RM(3,a.reg,b.reg); } @@ -502,14 +502,14 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, case ID2(RCPU,RFPU): case ID2(RFPU,RCPU): ERRIF( (f->r_mem>>16) == 0 ); - if( a.reg > 7 ) r64 |= 4; - if( b.reg > 7 ) r64 |= 1; + if( a.reg & 8 ) r64 |= 4; + if( b.reg & 8 ) r64 |= 1; OP(f->r_mem); MOD_RM(3,a.reg,b.reg); break; case ID2(RCPU,RUNUSED): ERRIF( f->r_mem == 0 ); - if( a.reg > 7 ) r64 |= 1; + if( a.reg & 8 ) r64 |= 1; if( GET_RM(f->r_mem) > 0 ) { OP(f->r_mem); MOD_RM(3, GET_RM(f->r_mem)-1, a.reg); @@ -529,14 +529,14 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, break; case ID2(RCPU,RCONST): ERRIF( f->r_const == 0 && f->r_i8 == 0 ); - if( a.reg > 7 ) r64 |= 1; + if( a.reg & 8 ) r64 |= 1; if( f->r_i8 && IS_SBYTE(value) ) { - if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; + if( (f->r_i8&FLAG_DUAL) && (a.reg & 8) ) r64 |= 4; OP(f->r_i8); if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_i8)-1,a.reg); B(value); } else if( GET_RM(f->r_const) > 0 || (f->r_const&FLAG_DUAL) ) { - if( (f->r_i8&FLAG_DUAL) && a.reg > 7 ) r64 |= 4; + if( (f->r_i8&FLAG_DUAL) && (a.reg & 8) ) r64 |= 4; OP(f->r_const&0xFF); if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_const)-1,a.reg); if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); @@ -549,7 +549,7 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, case ID2(RSTACK,RCPU): case ID2(RSTACK,RFPU): ERRIF( f->mem_r == 0 ); - if( b.reg > 7 ) r64 |= 4; + if( b.reg & 8 ) r64 |= 4; OP(f->mem_r); if( IS_SBYTE(a.value) ) { MOD_RM(1,b.reg,RBP); @@ -562,7 +562,7 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, case ID2(RCPU,RSTACK): case ID2(RFPU,RSTACK): ERRIF( f->r_mem == 0 ); - if( a.reg > 7 ) r64 |= 4; + if( a.reg & 8 ) r64 |= 4; OP(f->r_mem); if( IS_SBYTE(b.value) ) { MOD_RM(1,a.reg,RBP); @@ -579,7 +579,7 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, break; case ID2(RMEM,RUNUSED): ERRIF( f->mem_r == 0 ); - if( a.reg > 7 ) r64 |= 1; + if( a.reg & 8 ) r64 |= 1; OP(f->mem_r); if( value == 0 && (a.reg&7) != RBP ) { MOD_RM(0,GET_RM(f->mem_r)-1,a.reg); @@ -597,8 +597,8 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, case ID2(RCPU, RMEM): case ID2(RFPU, RMEM): ERRIF( f->r_mem == 0 ); - if( a.reg > 7 ) r64 |= 4; - if( b.reg > 7 ) r64 |= 1; + if( a.reg & 8 ) r64 |= 4; + if( b.reg & 8 ) r64 |= 1; OP(f->r_mem); if( value == 0 && (b.reg&7) != RBP ) { MOD_RM(0,a.reg,b.reg); @@ -616,8 +616,8 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, case ID2(RMEM, RCPU): case ID2(RMEM, RFPU): ERRIF( f->mem_r == 0 ); - if( a.reg > 7 ) r64 |= 1; - if( b.reg > 7 ) r64 |= 4; + if( a.reg & 8 ) r64 |= 1; + if( b.reg & 8 ) r64 |= 4; OP(f->mem_r); if( value == 0 && (a.reg&7) != RBP ) { MOD_RM(0,b.reg,a.reg); @@ -682,11 +682,11 @@ static void emit_mov_ext( code_ctx *ctx, ereg out, int out_val, ereg val, int in emit_mov_ext(ctx, tmp, 0, val, in_val, mode); emit_mov_ext(ctx, out, out_val, tmp, 0, mode); } else { - static CpuOp MOV_OP[] = {_MOV,MOV8,MOV16,_MOV,_MOV,_MOV,MOVSD,MOVSS,_MOV,_MOV}; + static CpuOp MOV_OP[] = {_MOV,MOV8,MOV16,_MOV,_MOV,MOVSD,MOVSS,_MOV,_MOV}; CpuOp op = MOV_OP[mode]; if( (mode == M_UI8 || mode == M_UI16) && IS_PURE(out) ) { op++; // MOVZX - mode = M_I64; + mode = M_PTR; } emit_ext(ctx,op,out,val,mode,IS_PURE(out) ? in_val : out_val); } @@ -696,6 +696,72 @@ static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode ) { emit_mov_ext(ctx,out,0,val,0,mode); } +static int jump_near( code_ctx *ctx, int mode ) { + int pos = byte_count(ctx->code); + if( mode < 0 ) { + // backwards + int target = -mode; + B(JAlways_short); + B(target - (pos + 2)); + } else { + B(mode == JAlways ? JAlways_short : mode - 0x10); + B(0); + } + return pos; +} + +static void patch_jump_near( code_ctx *ctx, int jpos ) { + if( !jpos ) return; + ctx->code.values[jpos + 1] = (unsigned char)(byte_count(ctx->code) - (jpos + 2)); +} + +static void emit_div_mod( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_mode mode ) { + if( IS_FLOAT(mode) ) { + BREAK(); + return; + } + ereg bas = R(RAX), div = R(RDX); + if( out != bas ) EMIT(_PUSH,bas,UNUSED,M_PTR); + if( out != div ) EMIT(_PUSH,div,UNUSED,M_PTR); + if( b == bas || b == div || !IS_PURE(div) ) { + EMIT(_MOV,RTMP,b,mode); + b = RTMP; + } + if( a != bas ) EMIT(_MOV,bas,a,mode); + + // check for div = 0 + EMIT(_TEST,bas,bas,mode); + int jz = jump_near(ctx,JZero); + int jz1 = 0; + // Prevent MIN/-1 overflow exception + // OSMod: r = (b == 0 || b == -1) ? 0 : a % b + // OSDiv: r = (b == 0 || b == -1) ? a * b : a / b + if( op == OSMod || op == OSDiv ) { + emit_ext(ctx,_CMP,bas,VAL_CONST,mode,-1); + jz1 = jump_near(ctx,JZero); + } + bool unsign = op == OUDiv || op == OUMod; + if( unsign ) + EMIT(XOR,div,div,mode); + else + EMIT(CDQ, UNUSED, UNUSED, mode); + EMIT(unsign ? DIV : IDIV, b, UNUSED, mode); + ereg res = (op == OUDiv || op == OSDiv) ? bas : div; + int jn = jump_near(ctx,JAlways); + patch_jump_near(ctx,jz); + patch_jump_near(ctx,jz1); + if( op != OSDiv ) { + EMIT(XOR, res, res, mode); + } else { + if( res != bas ) EMIT(_MOV,res,bas,mode); + EMIT(IMUL,res,b,mode); + } + patch_jump_near(ctx,jn); + if( out != res ) EMIT(_MOV,out,res,mode); + if( out != div ) EMIT(_POP,div,UNUSED,M_PTR); + if( out != bas ) EMIT(_POP,bas,UNUSED,M_PTR); +} + static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_mode mode ) { CpuOp cop; # define F_OP(iop,f32,f64) cop = mode == M_F32 ? f32 : (mode == M_F64 ? f64 : iop); @@ -757,13 +823,12 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ cop = (op == OShl ? SHL : (op == OSShr ? SAR : SHR)); break; case OSDiv: - F_OP(IDIV,DIVSS,DIVSD); - if( !IS_FLOAT(mode) ) BREAK(); - break; + F_OP(0,DIVSS,DIVSD); + if( IS_FLOAT(mode) ) break; case OSMod: case OUMod: case OUDiv: - BREAK(); + emit_div_mod(ctx,op,out,a,b,mode); return; case ONot: if( IS_PURE(a) ) { @@ -987,9 +1052,21 @@ void hl_codegen_function( jit_ctx *jit ) { break; case STORE: if( !IS_PURE(e->a) && !IS_PURE(e->b) && (e->a & FL_NATMASK) != FL_STACKOFFS && (e->b & FL_NATMASK) != FL_STACKOFFS ) { - EMIT(_PUSH,e->b,UNUSED,e->mode); - emit_mov(ctx, RTMP, e->a, M_PTR); - emit_ext(ctx, _POP,VAL_MEM(RTMP), UNUSED, e->mode, e->size_offs); + if( e->mode != M_PTR ) { + // no push/pop 32 bit + ereg tmp2 = R(RAX); + emit_mode mode = e->mode == M_F64 ? M_PTR : e->mode == M_F32 ? M_I32 : e->mode; + EMIT(_PUSH,tmp2,UNUSED,M_PTR); + emit_mov(ctx, RTMP, e->a, M_PTR); + emit_mov(ctx, tmp2, e->b, mode); + emit_mov_ext(ctx, VAL_MEM(RTMP), e->size_offs, tmp2, 0, mode); + EMIT(_POP,tmp2,UNUSED,M_PTR); + } else { + if( IS_FLOAT(e->mode) ) BREAK(); + EMIT(_PUSH,e->b,UNUSED,e->mode); + emit_mov(ctx, RTMP, e->a, M_PTR); + emit_ext(ctx, _POP,VAL_MEM(RTMP), UNUSED, e->mode, e->size_offs); + } } else if( (e->a & FL_NATMASK) == FL_STACKREG ) { emit_mov(ctx, RTMP, e->a, M_PTR); emit_mov_ext(ctx, VAL_MEM(RTMP), e->size_offs, e->b, 0, e->mode); @@ -1031,7 +1108,7 @@ void hl_codegen_function( jit_ctx *jit ) { { emit_mode mode = e->mode; if( !IS_PURE(out) ) - mode = (mode == M_F32 ? M_I32 : mode == M_F64 ? M_I64 : mode); // don't use FP for stack ops + mode = (mode == M_F32 ? M_I32 : mode == M_F64 ? M_PTR : mode); // don't use FP for stack ops ereg w = IS_PURE(out) ? out : get_tmp(mode); if( e->value == 0 ) EMIT(mode == M_F32 ? XORPS : mode == M_F64 ? XORPD : XOR, w, w, mode); @@ -1044,7 +1121,7 @@ void hl_codegen_function( jit_ctx *jit ) { MOD_RM(0,out&7,5); W(0); alloc_const(ctx, e->value); - } else if( (mode == M_PTR || mode == M_I64) && (e->value&0xFFFFFFFF) == e->value ) + } else if( mode == M_PTR && (e->value&0xFFFFFFFF) == e->value ) emit_ext(ctx, _MOV, w, VAL_CONST, M_I32, e->value); else emit_ext(ctx, _MOV, w, VAL_CONST, mode, e->value); @@ -1219,10 +1296,61 @@ void hl_codegen_function( jit_ctx *jit ) { } } break; - case CONV: case CONV_UNSIGNED: BREAK(); break; + case CONV: + { + emit_mode in_mode = e->size_offs; + ereg r = IS_PURE(e->a) ? e->a : get_tmp(in_mode); + if( r != e->a ) emit_mov(ctx, r, e->a, in_mode); + CpuOp op = 0; + switch( ID2(e->mode,in_mode) ) { + case ID2(M_F32,M_UI8): + case ID2(M_F32,M_UI16): + case ID2(M_F32,M_I32): + case ID2(M_F32,M_PTR): + op = CVTSI2SS; + break; + case ID2(M_F64,M_UI8): + case ID2(M_F64,M_UI16): + case ID2(M_F64,M_I32): + case ID2(M_F64,M_PTR): + op = CVTSI2SD; + break; + case ID2(M_UI8,M_F32): + case ID2(M_UI16,M_F32): + case ID2(M_I32,M_F32): + case ID2(M_PTR,M_F32): + op = CVTSS2SI; + break; + case ID2(M_UI8,M_F64): + case ID2(M_UI16,M_F64): + case ID2(M_I32,M_F64): + case ID2(M_PTR,M_F64): + op = CVTSD2SI; + break; + case ID2(M_F32,M_F64): + op = CVTSD2SS; + break; + case ID2(M_F64,M_F32): + op = CVTSS2SD; + break; + default: + jit_assert(); + break; + } + if( op ) { + if( IS_PURE(out) ) + EMIT(op,out,r,0); + else { + ereg r2 = get_tmp(e->mode); + EMIT(op,r2,r,0); + emit_mov(ctx,out,r2,e->mode); + } + } + } + break; case BINOP: case UNOP: emit_anyop(ctx, e->size_offs, out, e->a, e->b, e->mode); @@ -1285,24 +1413,6 @@ static void flush_function( code_ctx *ctx, int start ) { if( byte_count(ctx->code) > ctx->code.max ) jit_assert(); } -static int jump_near( code_ctx *ctx, int mode ) { - int pos = byte_count(ctx->code); - if( mode < 0 ) { - // backwards - int target = -mode; - B(JAlways_short); - B(target - (pos + 2)); - } else { - B(mode == JAlways ? JAlways_short : mode - 0x10); - B(0); - } - return pos; -} - -static void patch_jump_near( code_ctx *ctx, int jpos ) { - ctx->code.values[jpos + 1] = (unsigned char)(byte_count(ctx->code) - (jpos + 2)); -} - void hl_codegen_init( jit_ctx *jit ) { code_ctx *ctx = jit->code; byte_reserve(ctx->code,1024); From e9a02f1f6d8030244dbd8a58a1c1eca72e0ebeb5 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 26 Apr 2026 17:39:00 +0200 Subject: [PATCH 46/83] added debug stack, added float % --- src/jit.c | 23 ++++++++++++++++++++++- src/jit.h | 1 + src/jit_dump.c | 2 ++ src/jit_emit.c | 11 +++++++++-- src/jit_x86_64.c | 1 - src/module.c | 6 ++---- 6 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/jit.c b/src/jit.c index 5dea0d44b..aec0e63e3 100644 --- a/src/jit.c +++ b/src/jit.c @@ -133,6 +133,10 @@ void hl_jit_init( jit_ctx *ctx, hl_module *m ) { #endif hl_codegen_init(ctx); jit_code_append(ctx); + if( m->code->hasdebug ) { + m->jit_debug = (hl_debug_infos*)malloc(sizeof(hl_debug_infos) * m->code->nfunctions); + memset(m->jit_debug, -1, sizeof(hl_debug_infos) * m->code->nfunctions); + } } void hl_jit_free( jit_ctx *ctx, h_bool can_reset ) { @@ -158,6 +162,23 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { hl_codegen_function(ctx); int pos = ctx->out_pos; hl_jit_define_function(ctx, pos, ctx->code_size); + if( m->jit_debug ) { + bool compact = ctx->code_size < 0xFFFF; + void *debug = malloc((compact ? sizeof(unsigned short) : sizeof(int)) * (f->nops + 1)); + for(int i=0;i<=f->nops;i++) { + int ipos = ctx->emit_pos_map[i]; + int rpos = ctx->reg_pos_map[ipos]; + int cpos = ctx->code_pos_map[rpos]; + if( compact ) + ((unsigned short*)debug)[i] = (unsigned short)cpos; + else + ((int*)debug)[i] = cpos; + } + int fid = (int)(f - m->code->functions); + m->jit_debug[fid].start = pos; + m->jit_debug[fid].offsets = debug; + m->jit_debug[fid].large = !compact; + } if( !jit_code_append(ctx) ) return -1; current_ctx = NULL; @@ -225,7 +246,7 @@ void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **d if( code == NULL ) return NULL; memcpy(code,ctx->output,size); *codesize = size; - *debug = NULL; + *debug = m->jit_debug; ctx->final_code = code; hl_emit_final(ctx); hl_codegen_final(ctx); diff --git a/src/jit.h b/src/jit.h index 2f444b256..28c92ab58 100644 --- a/src/jit.h +++ b/src/jit.h @@ -23,6 +23,7 @@ #define JIT_H #include +#include typedef enum { LOAD_ADDR, diff --git a/src/jit_dump.c b/src/jit_dump.c index 66746eacc..0948d2529 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -271,6 +271,8 @@ static void hl_dump_ptr_name( jit_ctx *ctx, void *ptr ) { N(setjmp); N(_setjmp); N2("assert",hl_jit_assert); + N(fmod); + N(fmodf); i = 0; } # undef N diff --git a/src/jit_emit.c b/src/jit_emit.c index 2cca28808..b88158346 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -860,7 +860,7 @@ void hl_emit_flush( jit_ctx *jit ) { emit_ctx *ctx = jit->emit; if( ctx->flushed ) return; ctx->flushed = true; - ctx->pos_map[ctx->fun->nops] = -1; + ctx->pos_map[ctx->fun->nops] = ctx->emit_pos; ctx->current_block->end_pos = ctx->emit_pos; hl_emit_remap_jumps(ctx,&ctx->jump_regs, ctx->instrs, ctx->pos_map); jit->instrs = ctx->instrs; @@ -1460,7 +1460,14 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { { ereg va = LOAD(ra); ereg vb = LOAD(rb); - STORE(dst, emit_gen_ext(ctx, BINOP, va, vb, hl_type_mode(dst->t), o->op)); + ereg r; + if( (dst->t->kind == HF32 || dst->t->kind == HF64) && o->op == OSMod ) { + ereg args[] = {va,vb}; + r = emit_native_call(ctx, dst->t->kind == HF32 ? (void*)fmodf : (void*)fmod, args, 2, dst->t); + } else { + r = emit_gen_ext(ctx, BINOP, va, vb, hl_type_mode(dst->t), o->op); + } + STORE(dst, r); } break; case ONeg: diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 4b476b4b3..072b082d1 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -851,7 +851,6 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ } return; } - BREAK(); cop = NEG; break; default: diff --git a/src/module.c b/src/module.c index 793fc1854..bf18b287d 100644 --- a/src/module.c +++ b/src/module.c @@ -73,7 +73,7 @@ static bool module_resolve_pos( hl_module *m, void *addr, int *fidx, int *fpos ) while( min < max ) { int mid = (min + max) >> 1; int offset = dbg->large ? ((int*)dbg->offsets)[mid] : ((unsigned short*)dbg->offsets)[mid]; - if( offset <= code_pos ) + if( offset < code_pos ) min = mid + 1; else max = mid; @@ -225,10 +225,8 @@ static int module_capture_stack( void **stack, int size ) { unsigned char *code = m->jit_code; int code_size = m->codesize; if( module_addr >= (void*)code && module_addr < (void*)(code + code_size) ) { - if( stack && count == size ) { + if( stack && count == size ) break; - } - if( stack ) stack[count++] = module_addr; else From 697aef2e9e7e6339a0700991d37b25b036c0dc57 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Mon, 27 Apr 2026 21:10:42 +0200 Subject: [PATCH 47/83] various fixes --- src/jit_emit.c | 6 +- src/jit_regs.c | 5 +- src/jit_x86_64.c | 167 ++++++++++++++++++++++++++++------------------- 3 files changed, 107 insertions(+), 71 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index b88158346..116088ec2 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -761,7 +761,7 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { ereg v1 = LOAD_CONST(0,dt); int jend = emit_jump(ctx, false); patch_jump(ctx, jnot); - ereg v2 = LOAD_MEM(v,0,dt); + ereg v2 = LOAD_MEM(v,HDYN_VALUE,dt); patch_jump(ctx, jend); return emit_phi(ctx, v1, v2); } @@ -1471,9 +1471,11 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case ONeg: - case ONot: STORE(dst, emit_gen_ext(ctx, UNOP, LOAD(ra), UNUSED, hl_type_mode(dst->t), o->op)); break; + case ONot: + STORE(dst, emit_gen_ext(ctx, UNOP, LOAD(ra), LOAD_CONST(1,&hlt_i32), hl_type_mode(dst->t), OXor)); + break; case OJFalse: case OJTrue: case OJNotNull: diff --git a/src/jit_regs.c b/src/jit_regs.c index c1ba483b1..4572c5c6b 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -87,8 +87,9 @@ static ereg get_call_reg( regs_ctx *ctx, call_regs regs, emit_mode m ) { ereg r; int mode = REG_MODE(m); reg_config *cfg = REG_CFG(mode); - if( regs[mode] < cfg->nargs ) - r = cfg->arg[regs[mode]++]; + int idx = IS_WINCALL64 ? 0 : mode; + if( regs[idx] < cfg->nargs ) + r = cfg->arg[regs[idx]++]; else r = UNUSED; return r; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 072b082d1..5f1e1cbc2 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -839,7 +839,6 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ return; case ONeg: if( IS_FLOAT(mode) ) { - BREAK(); if( out != a && IS_PURE(out) ) { EMIT(mode == M_F32 ? XORPS : XORPD, out, out, mode); EMIT(mode == M_F32 ? SUBSS : SUBSD, out, a, mode); @@ -911,6 +910,11 @@ static void emit_nop( code_ctx *ctx, int size ) { B(0x90); } +#define CALC_REX(w,a,b) (((w)&8) ? 4 : 0) | (((b)&8) ? 2 : 0) | (((a) & 8) ? 1 : 0) + +#define REX64(out,a,b) B(0x48 | CALC_REX(out,a,b)) +#define REX32(out,a,b) { int v = CALC_REX(out,a,b); if( v ) B(v|0x40); } + static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { einstr e = *_e; @@ -944,11 +948,10 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { return; } - int r64 = 8 | ((out&8) ? 4 : 0) | ((e.b&8) ? 2 : 0) | ((e.a & 8) ? 1 : 0); - REX(); + REX64(out,e.a,e.b); B(0x8D); - MOD_RM(offs == 0 ? 0 : 1,out&7,4); - SIB(mult,e.b&7,e.a&7); + MOD_RM(offs == 0 ? 0 : 1,out,4); + SIB(mult,e.b,e.a); if( offs != 0 ) { if( !IS_SBYTE(offs) ) jit_assert(); B(offs); @@ -993,6 +996,78 @@ static int emit_lea_rel( code_ctx *ctx, ereg out ) { return pos; } +static int get_cond_jump( code_ctx *ctx ) { + int prev = 0; + einstr *p; + do { + p = ctx->jit->reg_instrs + ctx->cur_op - (++prev); + } while( p->op == MOV || p->op == JCOND || p->op == CMOV || p->op == XCHG || p->op == CXCHG ); + int op; + switch( p->size_offs ) { + case OJFalse: + case OJNull: + op = JZero; + break; + case OJTrue: + case OJNotNull: + op = JNotZero; + break; + case OJSGte: + op = IS_FLOAT(p->mode) ? JUGte : JSGte; + break; + case OJSGt: + op = IS_FLOAT(p->mode) ? JUGt : JSGt; + break; + case OJUGte: + op = JUGte; + break; + case OJSLt: + op = IS_FLOAT(p->mode) ? JULt : JSLt; + break; + case OJSLte: + op = IS_FLOAT(p->mode) ? JULte : JSLte; + break; + case OJULt: + op = JULt; + break; + case OJEq: + op = JEq; + break; + case OJNotEq: + op = JNeq; + break; + case OJNotLt: + op = JUGte; + break; + case OJNotGte: + op = JULt; + break; + case 0: + if( p->op == DEBUG_BREAK ) { + // found a debug break ! + BREAK(); + op = JZero; + break; + } + // fallback + default: + jit_assert(); + break; + } + return op; +} + +static void emit_cmov( code_ctx *ctx, ereg out, ereg r, int cond, emit_mode m ) { + if( IS_FLOAT(m) ) jit_assert(); + if( hl_emit_mode_sizes[m] == 8 ) + REX64(out,r,UNUSED); + else + REX32(out,r,UNUSED); + B(0x0F); + B(cond - 0x40); + MOD_RM(3,out,r); +} + void hl_codegen_function( jit_ctx *jit ) { code_ctx *ctx = jit->code; ctx->flushed = false; @@ -1202,75 +1277,22 @@ void hl_codegen_function( jit_ctx *jit ) { case M_F64: op = COMISD; break; default: op = _CMP; break; } + ereg b = e->b; if( !IS_PURE(e->a) && !IS_PURE(e->b) ) { ereg tmp = get_tmp(e->mode); emit_mov(ctx, tmp, e->b, e->mode); - if( op == COMISS || op == COMISD ) BREAK(); - EMIT(op,e->a,tmp,e->mode); - } else - EMIT(op,e->a,e->b,e->mode); + b = tmp; + } + if( IS_FLOAT(e->mode) ) + EMIT(op,b,e->a,e->mode); + else + EMIT(op,e->a,b,e->mode); } break; case JCOND: { - int prev = 0; - einstr *p; - do { - p = jit->reg_instrs + ctx->cur_op - (++prev); - } while( p->op == MOV || p->op == JCOND || p->op == CMOV || p->op == XCHG || p->op == CXCHG ); - int op; - switch( p->size_offs ) { - case OJFalse: - case OJNull: - op = JZero; - break; - case OJTrue: - case OJNotNull: - op = JNotZero; - break; - case OJSGte: - op = IS_FLOAT(p->mode) ? JUGte : JSGte; - break; - case OJSGt: - op = IS_FLOAT(p->mode) ? JUGt : JSGt; - break; - case OJUGte: - op = JUGte; - break; - case OJSLt: - op = IS_FLOAT(p->mode) ? JULt : JSLt; - break; - case OJSLte: - op = IS_FLOAT(p->mode) ? JULte : JSLte; - break; - case OJULt: - op = JULt; - break; - case OJEq: - op = JEq; - break; - case OJNotEq: - op = JNeq; - break; - case OJNotLt: - op = JUGte; - break; - case OJNotGte: - op = JULt; - break; - case 0: - if( p->op == DEBUG_BREAK ) { - // found a debug break ! - BREAK(); - op = JZero; - break; - } - // fallback - default: - jit_assert(); - break; - } - emit_jump(ctx, op, e->size_offs); + int jump = get_cond_jump(ctx); + emit_jump(ctx, jump, e->size_offs); } break; case JUMP: @@ -1372,6 +1394,17 @@ void hl_codegen_function( jit_ctx *jit ) { BREAK(); break; case CMOV: + { + int cond = get_cond_jump(ctx); + if( !IS_PURE(out) ) jit_assert(); + if( IS_PURE(e->a) ) { + emit_cmov(ctx,out,e->a,cond,e->mode); + } else { + emit_mov(ctx,RTMP,e->a,e->mode); + emit_cmov(ctx,out,RTMP,cond,e->mode); + } + } + break; case CXCHG: BREAK(); break; From bfcf81305192e10d171386cadf419929ad1c3e9c Mon Sep 17 00:00:00 2001 From: ncannasse Date: Wed, 29 Apr 2026 22:09:56 +0200 Subject: [PATCH 48/83] review and cleanup reg encoding. added R_CONST for future use --- src/jit.c | 10 +- src/jit.h | 60 ++++++++---- src/jit_dump.c | 47 ++++++--- src/jit_regs.c | 40 ++++---- src/jit_x86_64.c | 248 ++++++++++++++++++++++------------------------- 5 files changed, 219 insertions(+), 186 deletions(-) diff --git a/src/jit.c b/src/jit.c index aec0e63e3..7ce357cfb 100644 --- a/src/jit.c +++ b/src/jit.c @@ -206,13 +206,15 @@ static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { hl_type *at = t->fun->args[i]; void *v = args[i]; if( at->kind == HF32 || at->kind == HF64 ) { - if( fp < arg_fp_count ) + if( fp < arg_fp_count ) { vargs.regs[arg_reg_count + fp++] = v; - else + if( IS_WINCALL64 ) rp++; + } else vargs.stack[--sp] = v; - } else if( rp < arg_reg_count ) + } else if( rp < arg_reg_count ) { vargs.regs[rp++] = v; - else + if( IS_WINCALL64 ) fp++; + } else vargs.stack[--sp] = v; } switch( t->fun->ret->kind ) { diff --git a/src/jit.h b/src/jit.h index 28c92ab58..158fe59bf 100644 --- a/src/jit.h +++ b/src/jit.h @@ -62,10 +62,6 @@ typedef enum { ADDRESS, } emit_op; -typedef enum { - REG_RBP, -} native_reg; - typedef enum { M_NONE, M_UI8, @@ -100,21 +96,45 @@ typedef struct { }; } einstr; -#define FL_NATMASK 0x70000000 -#define FL_NATREG 0x40000000 -#define FL_MEMPTR 0x20000000 -#define FL_STACK 0x10000000 -#define FL_STACKREG (FL_NATREG | FL_MEMPTR | FL_STACK) -#define FL_STACKOFFS (FL_NATREG | FL_STACK) -#define IS_NULL(e) ((e) == 0) -#define IS_NATREG(e) (((e) & (0x80000000 | FL_NATREG)) == FL_NATREG) -#define IS_PURE(e) ((e) != UNUSED && ((e)&(FL_MEMPTR | FL_STACK)) == 0) -#define MK_STACK_REG(v) (((v)&0xFFFFFFF) | FL_STACKREG) -#define MK_STACK_OFFS(v)(((v)&0xFFFFFFF) | FL_STACKOFFS) -#define GET_STACK_OFFS(v) ((int)(((v) & 0x8000000) ? ((v) | 0xF0000000) : ((v)&0xFFFFFFF))) +typedef enum { + R_VALUE = 0, + R_REG = 0x40000000, + R_REG_PTR = 0x50000000, + R_CONST = 0x60000000, + R_PHI = 0x70000000, +} rkind; + +// reg representation is : +// higher bits +// 0000 = positive value (for IR only VXXX) +// X100 = native register, lower 7 bits is the register, bits 8-28 are the offset (21 bits) +// X101 = same as above, but indirect address +// X110 = small constant value stored in offset +// 1111 = negative value (for IR phi PXXX) +// 10XX = unused + +#define STACK_REG 5 + +#define UNUSED ((ereg)0) +#define MK_REG(v,kind) (((v)&0x7F) | (kind)) +#define MK_REG_VAL(v,kind,val) (MK_REG(v,kind) | (((val) << 7)&0x8FFFFF80)) + +#define REG_KIND(r) ((r)&0x70000000) +#define REG_REG(r) ((r)&0x7F) +#define REG_VALUE(r) (((int)(((r) & 0x8000000) ? ((r) | 0xF0000000) : ((r)&0x0FFFFFFF)))>>7) +#define REG_PTR(r) _reg_chk(r,R_REG,(r)|R_REG_PTR) +#define REG_ADD_OFFSET(r,offs) _reg_chk(r,R_REG_PTR,MK_REG_VAL(r,REG_KIND(r),REG_VALUE(r)+(offs))) +#define REG_IS_VAL(r) (REG_KIND(r) == R_VALUE || REG_KIND(r) == R_PHI) + +#define IS_NULL(r) ((r) == 0) +#define IS_REG(r) (REG_KIND(r) == R_REG) +#define MK_STACK_REG(v) MK_REG_VAL(STACK_REG,R_REG_PTR,v) +#define MK_STACK_OFFS(v) MK_REG_VAL(STACK_REG,R_REG,v) +#define MK_CONST(v) MK_REG_VAL(0,R_CONST,v) + + #define IS_CALL(op) ((op) == CALL_PTR || (op) == CALL_REG || (op) == CALL_FUN) #define IS_FLOAT(mode) ((mode) == M_F64 || (mode) == M_F32) -#define UNUSED ((ereg)0) #define MAX_ARGS 16 @@ -264,4 +284,10 @@ void hl_jit_error( const char *msg, const char *func, int line ); void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ); void hl_jit_patch_method( void *old_fun, void **new_fun_table ); +static ereg _reg_chk( ereg r, rkind k, ereg ret ) { + if( REG_KIND(r) != k ) jit_assert(); + return ret; +} + + #endif diff --git a/src/jit_dump.c b/src/jit_dump.c index 0948d2529..bd2959177 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -67,26 +67,41 @@ const char *hl_emit_regstr( ereg v, emit_mode m ) { static int flip = 0; // allow up to four concurrent val_str char *fmt = fmts[flip++&3]; - if( IS_NULL(v) ) + if( IS_NULL(v) ) { sprintf(fmt,"NULL???"); - else if( v < 0 ) + return fmt; + } + int val = REG_VALUE(v); + switch( REG_KIND(v) ) { + case R_VALUE: + sprintf(fmt,"V%d",v); + break; + case R_PHI: sprintf(fmt,"P%d",-v); - else if( (v&FL_STACKREG) == FL_STACKREG ) { - int index = GET_STACK_OFFS(v); - if( index < 0 ) - sprintf(fmt,"[ST-%Xh]", -index); + break; + case R_CONST: + sprintf(fmt,"%d",val); + break; + case R_REG: + if( val == 0 ) + sprintf(fmt,"%s",hl_natreg_str(v,m)); + else if( val > 0 ) + sprintf(fmt,"%s+%Xh",hl_natreg_str(v,m),val); else - sprintf(fmt,"[ST+%Xh]", index); - } else if( (v&FL_STACKOFFS) == FL_STACKOFFS ) { - int index = GET_STACK_OFFS(v); - if( index < 0 ) - sprintf(fmt,"ST-%Xh", -index); + sprintf(fmt,"%s-%Xh",hl_natreg_str(v,m),-val); + break; + case R_REG_PTR: + if( val == 0 ) + sprintf(fmt,"[%s]",REG_REG(v) == STACK_REG ? "ST" : hl_natreg_str(v,M_PTR)); + else if( val > 0 ) + sprintf(fmt,"[%s+%Xh]",REG_REG(v) == STACK_REG ? "ST" : hl_natreg_str(v,M_PTR),val); else - sprintf(fmt,"ST+%Xh", index); - } else if( IS_NATREG(v) ) - sprintf(fmt,"%s", hl_natreg_str(v,m)); - else - sprintf(fmt,"V%d",v); + sprintf(fmt,"[%s-%Xh]",REG_REG(v) == STACK_REG ? "ST" : hl_natreg_str(v,M_PTR),-val); + break; + default: + jit_assert(); + break; + } return fmt; } diff --git a/src/jit_regs.c b/src/jit_regs.c index 4572c5c6b..26e435fc5 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -209,7 +209,7 @@ static void regs_assign( regs_ctx *ctx, value_info *v ) { static void regs_write_live( regs_ctx *ctx, ereg *r ) { if( IS_NULL(*r) ) jit_assert(); - if( IS_NATREG(*r) ) return; + if( !REG_IS_VAL(*r) ) return; // some are injections of native regs at emit value_info *v = VAL_REG(*r); v->last_read = ctx->loop_end && ctx->jit->values_writes[v->id] < ctx->loop_start ? ctx->loop_end : ctx->cur_op; v->tot_reads++; @@ -252,12 +252,15 @@ static void regs_compute_liveness( regs_ctx *ctx ) { call_regs regs = {0}; bool needs_push = false; for(int k=0;knargs;k++) { - value_info *v = VAL_REG(r[k]); - if( !IS_NULL(v->pref_reg) ) { + ereg arg = r[k]; + value_info *v = REG_IS_VAL(arg) ? VAL_REG(r[k]) : NULL; + ereg r = get_call_reg(ctx, regs, v ? v->mode : M_I32); + if( IS_NULL(r) ) { needs_push = true; continue; } - v->pref_reg = get_call_reg(ctx,regs,v->mode); + if( v && IS_NULL(v->pref_reg) ) + v->pref_reg = r; } if( !needs_push && e->mode != M_NORET ) ctx->has_direct_call = true; if( write && IS_NULL(write->pref_reg) ) @@ -371,7 +374,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { // try to preserve ops in the from A = A op B if( (e.op == UNOP || e.op == BINOP) && write->pref_reg == UNUSED ) { value_info *v = VAL_REG(e.a); - if( IS_PURE(v->reg) ) write->pref_reg = v->reg; + if( IS_REG(v->reg) ) write->pref_reg = v->reg; } } @@ -392,6 +395,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { } } for(int k=0;kmode); if( !IS_NULL(r) ) { @@ -417,7 +421,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { for(int n=0;nnvalues;n++) { value_info *vn = VAL_REG(p->values[n]); // ignore previously set pref_reg (minimize moves) - if( IS_PURE(vn->reg) && !regs_current(ctx,vn->reg) ) { + if( IS_REG(vn->reg) && !regs_current(ctx,vn->reg) ) { v->pref_reg = vn->reg; break; } @@ -440,6 +444,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { continue; case ADDRESS: { + if( REG_KIND(e.a) == R_CONST ) jit_assert(); value_info *v = VAL_REG(e.a); spill(ctx, v); break; @@ -477,7 +482,7 @@ static void flush_movs( regs_ctx *ctx, bool cond ) { if( !read ) { ereg from = int_arr_get(movs,k+1); int mode = int_arr_get(movs,k+2); - bool cmov = cond && IS_PURE(to); + bool cmov = cond && IS_REG(to); regs_emit(ctx,to,cmov?CMOV:MOV,from,UNUSED,mode,0); int_arr_remove_range(&movs,k,3); cycle = false; @@ -488,7 +493,7 @@ static void flush_movs( regs_ctx *ctx, bool cond ) { ereg to = int_arr_get(movs,0); ereg from = int_arr_get(movs,1); int mode = int_arr_get(movs,2); - bool cmov = cond && (IS_PURE(to) || IS_PURE(from)); + bool cmov = cond && (IS_REG(to) || IS_REG(from)); regs_emit(ctx,UNUSED,cmov?CXCHG:XCHG,to,from,mode,0); int_arr_remove_range(&movs,0,3); size -= 3; @@ -572,15 +577,16 @@ static void regs_emit_instrs( regs_ctx *ctx ) { int stack_args = 0; int stack_bits = 0; for(int k=0;kmode); + value_info *v = REG_IS_VAL(args[k]) ? VAL_REG(args[k]) : NULL; + emit_mode mode = v ? v->mode : M_I32; + ereg r = get_call_reg(ctx,regs,mode); if( IS_NULL(r) ) { - stack_args += get_stack_size(v->mode); + stack_args += get_stack_size(mode); stack_bits |= 1 << k; - } else if( r != v->reg ) { + } else if( !v || r != v->reg ) { int_arr_add(ctx->pack_movs,r); - int_arr_add(ctx->pack_movs,v->reg); - int_arr_add(ctx->pack_movs,v->mode); + int_arr_add(ctx->pack_movs,v ? v->reg : args[k]); + int_arr_add(ctx->pack_movs,mode); } } if( stack_args > 0 ) { @@ -615,7 +621,7 @@ static void regs_emit_instrs( regs_ctx *ctx ) { ereg **regs = hl_emit_get_regs(&e,&nread); for(int k=0;kreg; } @@ -681,8 +687,8 @@ static void regs_emit_instrs( regs_ctx *ctx ) { default: if( e.op == ADDRESS ) { e.op = LEA; - if( !(e.a & FL_MEMPTR) ) jit_assert(); - e.a &= ~FL_MEMPTR; + if( REG_KIND(e.a) != R_REG_PTR ) jit_assert(); + e.a = (e.a & ~R_REG_PTR) | R_REG; } if( ret_val && out ) { regs_write_instr(ctx, &e, *ret_val); diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 5f1e1cbc2..1545eebc8 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -71,9 +71,8 @@ typedef enum { _UNUSED = 0xFF } CpuReg; -#define R(id) ((id) | FL_NATREG) -#define X(id) R(id) -#define MMX(id) R(id+64) +#define R(id) MK_REG(id,R_REG) +#define MMX(id) MK_REG((id)+64,R_REG) typedef enum { _MOV, @@ -329,7 +328,7 @@ const char *hl_natreg_str( int reg, emit_mode m ) { static char out[16]; static const char *regs_str[] = { "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI" }; static const char *regs_str8[] = { "AL", "CL", "DL", "BL", "SPL", "BPL", "SIL", "DIL" }; - CpuReg r = (reg & 0xFF); + CpuReg r = REG_REG(reg); switch( m ) { case M_I32: if( r < 8 ) @@ -369,18 +368,18 @@ const char *hl_natreg_str( int reg, emit_mode m ) { static int scratch_float_reg = -1; -static ereg scratch_not_param[] = { X(RAX), X(R10), X(R11) }; +static ereg scratch_not_param[] = { R(RAX), R(R10), R(R11) }; void hl_jit_init_regs( regs_config *cfg ) { // exclude R11 at it's use as temporary for various ops # ifdef HL_WIN_CALL - static int scratch_regs[] = { X(RAX), X(RCX), X(RDX), X(R8), X(R9), X(R10), /*X(R11)*/ }; - static int free_regs[] = { X(RSI), X(RDI), X(RBX), X(R12), X(R13), X(R14), X(R15) }; - static int call_regs[] = { X(RCX), X(RDX), X(R8), X(R9) }; + static int scratch_regs[] = { R(RAX), R(RCX), R(RDX), R(R8), R(R9), R(R10), /*R(R11)*/ }; + static int free_regs[] = { R(RSI), R(RDI), R(RBX), R(R12), R(R13), R(R14), R(R15) }; + static int call_regs[] = { R(RCX), R(RDX), R(R8), R(R9) }; # else - static int scratch_regs[] = { X(RAX), X(RCX), X(RDX), X(RSI), X(RDI), X(R8), X(R9), X(R10), /*X(R11)*/ }; - static int free_regs[] = { X(RBX), X(R12), X(R13), X(R14), X(R15) }; - static int call_regs[] = { X(RDI), X(RSI), X(RDX), X(RCX), X(R8), X(R9) }; + static int scratch_regs[] = { R(RAX), R(RCX), R(RDX), R(RSI), R(RDI), R(R8), R(R9), R(R10), /*R(R11)*/ }; + static int free_regs[] = { R(RBX), R(R12), R(R13), R(R14), R(R15) }; + static int call_regs[] = { R(RDI), R(RSI), R(RDX), R(RCX), R(R8), R(R9) }; # endif cfg->regs.ret = scratch_regs[0]; cfg->regs.nscratchs = sizeof(scratch_regs) / sizeof(int); @@ -437,12 +436,12 @@ typedef enum { typedef struct { preg_kind kind; CpuReg reg; - int value; + int64 value; } preg; #define ERRIF(v) if( v ) jit_assert() -static preg make_reg( ereg r, emit_mode m ) { +static preg make_reg( ereg r, uint64 value ) { preg p; if( IS_NULL(r) ) { p.kind = RUNUSED; @@ -450,34 +449,41 @@ static preg make_reg( ereg r, emit_mode m ) { } if( r == VAL_CONST ) { p.kind = RCONST; + p.value = value; return p; } - if( (r & FL_NATMASK) == (FL_NATREG | FL_MEMPTR) ) { - p.kind = RMEM; - p.reg = (r&0xFFFF); - p.value = 0; - return p; - } - ERRIF(!IS_NATREG(r)); - ERRIF((r&FL_NATMASK) == FL_STACKOFFS); - if( (r & FL_NATMASK) == FL_STACKREG ) { - p.kind = RSTACK; - p.value = GET_STACK_OFFS(r); - } else if( IS_FLOAT(m) ) { - p.kind = RFPU; - p.reg = (r&0xFF) - 64; - } else { - p.kind = RCPU; - p.reg = (r&0xFF); + p.reg = REG_REG(r); + p.value = REG_VALUE(r); + switch( REG_KIND(r) ) { + case R_REG: + if( p.reg >= 64 ) { + p.kind = RFPU; + p.reg -= 64; + } else + p.kind = RCPU; + break; + case R_REG_PTR: + if( p.reg == RBP ) + p.kind = RSTACK; + else + p.kind = RMEM; + break; + case R_CONST: + p.kind = RCONST; + break; + default: + jit_assert(); + break; } + if( p.reg < 0 || p.reg > 15 ) jit_assert(); return p; } -static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, int_val value ) { +static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, int_val _value ) { opform *f = &OP_FORMS[op]; int mode64 = mode == M_PTR && (f->r_mem&FLAG_DEF64) == 0 ? 8 : 0; int r64 = mode64; - preg a = make_reg(_a,mode), b = make_reg(_b,mode); + preg a = make_reg(_a,_value), b = make_reg(_b,_value); switch( ID2(a.kind,b.kind) ) { case ID2(RUNUSED,RUNUSED): ERRIF(f->r_mem == 0); @@ -524,26 +530,26 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, B(a.value); } else { MOD_RM(2,GET_RM(f->mem_r)-1,RBP); - W(a.value); + W((int)a.value); } break; case ID2(RCPU,RCONST): ERRIF( f->r_const == 0 && f->r_i8 == 0 ); if( a.reg & 8 ) r64 |= 1; - if( f->r_i8 && IS_SBYTE(value) ) { + if( f->r_i8 && IS_SBYTE(b.value) ) { if( (f->r_i8&FLAG_DUAL) && (a.reg & 8) ) r64 |= 4; OP(f->r_i8); if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_i8)-1,a.reg); - B(value); + B(b.value); } else if( GET_RM(f->r_const) > 0 || (f->r_const&FLAG_DUAL) ) { if( (f->r_i8&FLAG_DUAL) && (a.reg & 8) ) r64 |= 4; OP(f->r_const&0xFF); if( (f->r_i8&FLAG_DUAL) ) MOD_RM(3,a.reg,a.reg); else MOD_RM(3,GET_RM(f->r_const)-1,a.reg); - if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); + if( mode64 && IS_64 && op == _MOV ) W64(b.value); else W((int)b.value); } else { ERRIF( f->r_const == 0); OP((f->r_const&0xFF) + (a.reg&7)); - if( mode64 && IS_64 && op == _MOV ) W64(value); else W((int)value); + if( mode64 && IS_64 && op == _MOV ) W64(b.value); else W((int)b.value); } break; case ID2(RSTACK,RCPU): @@ -556,7 +562,7 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, B(a.value); } else { MOD_RM(2,b.reg,RBP); - W(a.value); + W((int)a.value); } break; case ID2(RCPU,RSTACK): @@ -569,29 +575,29 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, B(b.value); } else { MOD_RM(2,a.reg,RBP); - W(b.value); + W((int)b.value); } break; case ID2(RCONST,RUNUSED): ERRIF( f->r_const == 0 ); OP(f->r_const); - if( f->r_const & FLAG_8B ) B(value); else W((int)value); + if( f->r_const & FLAG_8B ) B(a.value); else W((int)a.value); break; case ID2(RMEM,RUNUSED): ERRIF( f->mem_r == 0 ); if( a.reg & 8 ) r64 |= 1; OP(f->mem_r); - if( value == 0 && (a.reg&7) != RBP ) { + if( a.value == 0 && (a.reg&7) != RBP ) { MOD_RM(0,GET_RM(f->mem_r)-1,a.reg); if( (a.reg&7) == RSP ) B(0x24); - } else if( IS_SBYTE(value) ) { + } else if( IS_SBYTE(a.value) ) { MOD_RM(1,GET_RM(f->mem_r)-1,a.reg); if( (a.reg&7) == RSP ) B(0x24); - B(value); + B(a.value); } else { MOD_RM(2,GET_RM(f->mem_r)-1,a.reg); if( (a.reg&7) == RSP ) B(0x24); - W((int)value); + W((int)a.value); } break; case ID2(RCPU, RMEM): @@ -600,17 +606,17 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, if( a.reg & 8 ) r64 |= 4; if( b.reg & 8 ) r64 |= 1; OP(f->r_mem); - if( value == 0 && (b.reg&7) != RBP ) { + if( b.value == 0 && (b.reg&7) != RBP ) { MOD_RM(0,a.reg,b.reg); if( (b.reg&7) == RSP ) B(0x24); - } else if( IS_SBYTE(value) ) { + } else if( IS_SBYTE(b.value) ) { MOD_RM(1,a.reg,b.reg); if( (b.reg&7) == RSP ) B(0x24); - B(value); + B(b.value); } else { MOD_RM(2,a.reg,b.reg); if( (b.reg&7) == RSP ) B(0x24); - W((int)value); + W((int)b.value); } break; case ID2(RMEM, RCPU): @@ -619,17 +625,17 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, if( a.reg & 8 ) r64 |= 1; if( b.reg & 8 ) r64 |= 4; OP(f->mem_r); - if( value == 0 && (a.reg&7) != RBP ) { + if( a.value == 0 && (a.reg&7) != RBP ) { MOD_RM(0,b.reg,a.reg); if( (a.reg&7) == RSP ) B(0x24); - } else if( IS_SBYTE(value) ) { + } else if( IS_SBYTE(a.value) ) { MOD_RM(1,b.reg,a.reg); if( (a.reg&7) == RSP ) B(0x24); - B(value); + B(a.value); } else { MOD_RM(2,b.reg,a.reg); if( (a.reg&7) == RSP ) B(0x24); - W((int)value); + W((int)a.value); } break; default: @@ -666,36 +672,26 @@ static ereg get_tmp( emit_mode mode ) { return RTMP; } -static void emit_mov_ext( code_ctx *ctx, ereg out, int out_val, ereg val, int in_val, emit_mode mode ) { +static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode ) { if( out == val ) return; - if( (val & FL_NATMASK) == FL_STACKOFFS ) { - emit_mov_ext(ctx,out,out_val,VAL_MEM(R(RBP)),in_val + GET_STACK_OFFS(val),mode); - return; - } - if( (out & FL_NATMASK) == FL_STACKOFFS ) { - emit_mov_ext(ctx,VAL_MEM(R(RBP)),out_val + GET_STACK_OFFS(val),val,in_val,mode); - return; - } - if( !IS_PURE(out) && !IS_PURE(val) ) { + if( !IS_REG(out) && (!IS_REG(val) || REG_VALUE(val) != 0) ) { ereg tmp = get_tmp(mode); - emit_mov_ext(ctx, tmp, 0, val, in_val, mode); - emit_mov_ext(ctx, out, out_val, tmp, 0, mode); + emit_mov(ctx, tmp, val, mode); + emit_mov(ctx, out, tmp, mode); + } else if( IS_REG(val) && REG_VALUE(val) != 0 ) { + emit_ext(ctx,_LEA,out,REG_PTR(val),M_PTR,0); } else { static CpuOp MOV_OP[] = {_MOV,MOV8,MOV16,_MOV,_MOV,MOVSD,MOVSS,_MOV,_MOV}; CpuOp op = MOV_OP[mode]; - if( (mode == M_UI8 || mode == M_UI16) && IS_PURE(out) ) { + if( (mode == M_UI8 || mode == M_UI16) && IS_REG(out) ) { op++; // MOVZX mode = M_PTR; } - emit_ext(ctx,op,out,val,mode,IS_PURE(out) ? in_val : out_val); + emit_ext(ctx,op,out,val,mode,0); } } -static void emit_mov( code_ctx *ctx, ereg out, ereg val, emit_mode mode ) { - emit_mov_ext(ctx,out,0,val,0,mode); -} - static int jump_near( code_ctx *ctx, int mode ) { int pos = byte_count(ctx->code); if( mode < 0 ) { @@ -723,7 +719,7 @@ static void emit_div_mod( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emi ereg bas = R(RAX), div = R(RDX); if( out != bas ) EMIT(_PUSH,bas,UNUSED,M_PTR); if( out != div ) EMIT(_PUSH,div,UNUSED,M_PTR); - if( b == bas || b == div || !IS_PURE(div) ) { + if( b == bas || b == div || !IS_REG(div) ) { EMIT(_MOV,RTMP,b,mode); b = RTMP; } @@ -737,7 +733,7 @@ static void emit_div_mod( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emi // OSMod: r = (b == 0 || b == -1) ? 0 : a % b // OSDiv: r = (b == 0 || b == -1) ? a * b : a / b if( op == OSMod || op == OSDiv ) { - emit_ext(ctx,_CMP,bas,VAL_CONST,mode,-1); + EMIT(_CMP,bas,MK_CONST(-1),mode); jz1 = jump_near(ctx,JZero); } bool unsign = op == OUDiv || op == OUMod; @@ -831,15 +827,15 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ emit_div_mod(ctx,op,out,a,b,mode); return; case ONot: - if( IS_PURE(a) ) { - emit_ext(ctx,XOR,a,VAL_CONST,M_I32,1); + if( IS_REG(a) ) { + EMIT(XOR,a,MK_CONST(1),M_I32); } else { BREAK(); } return; case ONeg: if( IS_FLOAT(mode) ) { - if( out != a && IS_PURE(out) ) { + if( out != a && IS_REG(out) ) { EMIT(mode == M_F32 ? XORPS : XORPD, out, out, mode); EMIT(mode == M_F32 ? SUBSS : SUBSD, out, a, mode); } else { @@ -856,9 +852,9 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ jit_assert(); break; } - if( out == a && IS_PURE(a) ) { + if( out == a && IS_REG(a) ) { EMIT(cop,out,b,mode); - } else if( !IS_PURE(out) || out == b ) { + } else if( !IS_REG(out) || out == b ) { ereg tmp = get_tmp(mode); emit_mov(ctx, tmp, a, mode); EMIT(cop,tmp,b,mode); @@ -921,30 +917,28 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { int mult = e.size_offs & 0xFF; int offs = e.size_offs >> 8; - if( (e.a & FL_NATMASK) == FL_STACKOFFS ) { - offs += GET_STACK_OFFS(e.a); - e.a = R(RBP); - } + if( IS_REG(e.a) ) + offs += REG_VALUE(e.a); - if( !IS_PURE(e.a) ) { + if( !IS_REG(e.a) ) { // a is always a mem address ! emit_mov(ctx, RTMP, e.a, M_PTR); e.a = RTMP; - if( e.b && !IS_PURE(e.b) ) { - if( !IS_PURE(out) ) jit_assert(); + if( e.b && !IS_REG(e.b) ) { + if( !IS_REG(out) ) jit_assert(); emit_mov(ctx, out, e.b, M_I32); e.b = out; } - } else if( e.b && !IS_PURE(e.b) ) { + } else if( e.b && !IS_REG(e.b) ) { // b is always an int index ! emit_mov(ctx, RTMP, e.b, M_I32); e.b = RTMP; } if( mult == 0 ) { - if( e.a & FL_MEMPTR ) jit_assert(); + if( REG_KIND(e.a) != R_REG ) jit_assert(); // no index - emit_ext(ctx,_LEA,out,e.a | FL_MEMPTR,M_PTR,offs); + emit_ext(ctx,_LEA,out,MK_REG_VAL(e.a,R_REG_PTR,offs),M_PTR,0); return; } @@ -1107,17 +1101,12 @@ void hl_codegen_function( jit_ctx *jit ) { case LOAD_ARG: continue; // nop case MOV: - if( (e->a & FL_NATMASK) == FL_STACKOFFS ) { - ereg w = IS_PURE(out) ? out : RTMP; - emit_ext(ctx,_LEA,w,VAL_MEM(R(RBP)),M_PTR,GET_STACK_OFFS(e->a)); - if( w != out ) emit_mov(ctx, out, w, M_PTR); - } else - emit_mov(ctx, out, e->a, e->mode); + emit_mov(ctx, out, e->a, e->mode); break; case XCHG: { ereg tmp = get_tmp(e->mode); - if( !IS_PURE(e->a) && !IS_PURE(e->b) ) + if( !IS_REG(e->a) && !IS_REG(e->b) ) jit_assert(); emit_mov(ctx, tmp, e->a, e->mode); emit_mov(ctx, e->a, e->b, e->mode); @@ -1125,7 +1114,7 @@ void hl_codegen_function( jit_ctx *jit ) { } break; case STORE: - if( !IS_PURE(e->a) && !IS_PURE(e->b) && (e->a & FL_NATMASK) != FL_STACKOFFS && (e->b & FL_NATMASK) != FL_STACKOFFS ) { + if( !IS_REG(e->a) && !IS_REG(e->b) ) { if( e->mode != M_PTR ) { // no push/pop 32 bit ereg tmp2 = R(RAX); @@ -1133,23 +1122,23 @@ void hl_codegen_function( jit_ctx *jit ) { EMIT(_PUSH,tmp2,UNUSED,M_PTR); emit_mov(ctx, RTMP, e->a, M_PTR); emit_mov(ctx, tmp2, e->b, mode); - emit_mov_ext(ctx, VAL_MEM(RTMP), e->size_offs, tmp2, 0, mode); + emit_mov(ctx, MK_REG_VAL(RTMP,R_REG_PTR,e->size_offs), tmp2, mode); EMIT(_POP,tmp2,UNUSED,M_PTR); } else { if( IS_FLOAT(e->mode) ) BREAK(); EMIT(_PUSH,e->b,UNUSED,e->mode); emit_mov(ctx, RTMP, e->a, M_PTR); - emit_ext(ctx, _POP,VAL_MEM(RTMP), UNUSED, e->mode, e->size_offs); + emit_ext(ctx, _POP,REG_ADD_OFFSET(REG_PTR(RTMP),e->size_offs), UNUSED, e->mode, 0); } - } else if( (e->a & FL_NATMASK) == FL_STACKREG ) { + } else if( !IS_REG(e->a) ) { emit_mov(ctx, RTMP, e->a, M_PTR); - emit_mov_ext(ctx, VAL_MEM(RTMP), e->size_offs, e->b, 0, e->mode); + emit_mov(ctx, MK_REG_VAL(RTMP,R_REG_PTR,e->size_offs), e->b, e->mode); } else - emit_mov_ext(ctx, VAL_MEM(e->a), e->size_offs, e->b, 0, e->mode); + emit_mov(ctx, REG_ADD_OFFSET(REG_PTR(e->a),e->size_offs), e->b, e->mode); break; case PUSH: if( IS_FLOAT(e->mode) ) BREAK(); - if( (e->a&FL_NATMASK) == FL_STACKOFFS ) { + if( IS_REG(e->a) && REG_VALUE(e->a) != 0 ) { emit_mov(ctx, RTMP, e->a, e->mode); emit_ext(ctx, _PUSH, RTMP, UNUSED, e->mode, 0); } else @@ -1181,9 +1170,9 @@ void hl_codegen_function( jit_ctx *jit ) { case LOAD_CONST: { emit_mode mode = e->mode; - if( !IS_PURE(out) ) + if( !IS_REG(out) ) mode = (mode == M_F32 ? M_I32 : mode == M_F64 ? M_PTR : mode); // don't use FP for stack ops - ereg w = IS_PURE(out) ? out : get_tmp(mode); + ereg w = IS_REG(out) ? out : get_tmp(mode); if( e->value == 0 ) EMIT(mode == M_F32 ? XORPS : mode == M_F64 ? XORPD : XOR, w, w, mode); else if( IS_FLOAT(mode) ) { @@ -1204,20 +1193,16 @@ void hl_codegen_function( jit_ctx *jit ) { } break; case LOAD_ADDR: - { - ereg addr; - if( (e->a & FL_STACKREG) == FL_STACKREG ) { - emit_mov(ctx,RTMP,e->a,M_PTR); - addr = VAL_MEM(RTMP); - } else { - addr = VAL_MEM(e->a); - } - emit_mov_ext(ctx,out,0,addr,e->size_offs,e->mode); + if( IS_REG(e->a) ) { + emit_mov(ctx, out, REG_ADD_OFFSET(REG_PTR(e->a),e->size_offs), e->mode); + } else { + emit_mov(ctx, RTMP, e->a, M_PTR); + emit_mov(ctx, out, MK_REG_VAL(RTMP,R_REG_PTR,e->size_offs), e->mode); } break; case LOAD_FUN: { - ereg w = IS_PURE(out) ? out : RTMP; + ereg w = IS_REG(out) ? out : RTMP; int pos = emit_lea_rel(ctx,w); int fid = e->size_offs; int_arr_add_impl(&ctx->jit->galloc,&ctx->funs,pos); @@ -1260,7 +1245,7 @@ void hl_codegen_function( jit_ctx *jit ) { case TEST: if( IS_FLOAT(e->mode) ) jit_assert(); - if( !IS_PURE(e->a) ) { + if( !IS_REG(e->a) ) { ereg tmp = get_tmp(e->mode); emit_mov(ctx, tmp, e->a, e->mode); EMIT(_TEST,tmp,tmp,e->mode); @@ -1278,7 +1263,7 @@ void hl_codegen_function( jit_ctx *jit ) { default: op = _CMP; break; } ereg b = e->b; - if( !IS_PURE(e->a) && !IS_PURE(e->b) ) { + if( !IS_REG(e->a) && !IS_REG(e->b) ) { ereg tmp = get_tmp(e->mode); emit_mov(ctx, tmp, e->b, e->mode); b = tmp; @@ -1323,7 +1308,7 @@ void hl_codegen_function( jit_ctx *jit ) { case CONV: { emit_mode in_mode = e->size_offs; - ereg r = IS_PURE(e->a) ? e->a : get_tmp(in_mode); + ereg r = IS_REG(e->a) ? e->a : get_tmp(in_mode); if( r != e->a ) emit_mov(ctx, r, e->a, in_mode); CpuOp op = 0; switch( ID2(e->mode,in_mode) ) { @@ -1362,7 +1347,7 @@ void hl_codegen_function( jit_ctx *jit ) { break; } if( op ) { - if( IS_PURE(out) ) + if( IS_REG(out) ) EMIT(op,out,r,0); else { ereg r2 = get_tmp(e->mode); @@ -1377,7 +1362,7 @@ void hl_codegen_function( jit_ctx *jit ) { emit_anyop(ctx, e->size_offs, out, e->a, e->b, e->mode); break; case LEA: - if( !IS_PURE(out) ) { + if( !IS_REG(out) ) { ereg tmp = get_tmp(e->mode); emit_lea(ctx,tmp,e); emit_mov(ctx,out,tmp,e->mode); @@ -1386,9 +1371,9 @@ void hl_codegen_function( jit_ctx *jit ) { break; case STACK_OFFS: if( e->size_offs >= 0 ) - emit_ext(ctx,ADD,R(RSP),VAL_CONST,M_PTR,e->size_offs); + EMIT(ADD,R(RSP),MK_CONST(e->size_offs),M_PTR); else - emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,-e->size_offs); + EMIT(SUB,R(RSP),MK_CONST(-e->size_offs),M_PTR); break; case PREFETCH: BREAK(); @@ -1396,8 +1381,8 @@ void hl_codegen_function( jit_ctx *jit ) { case CMOV: { int cond = get_cond_jump(ctx); - if( !IS_PURE(out) ) jit_assert(); - if( IS_PURE(e->a) ) { + if( !IS_REG(out) ) jit_assert(); + if( IS_REG(e->a) ) { emit_cmov(ctx,out,e->a,cond,e->mode); } else { emit_mov(ctx,RTMP,e->a,e->mode); @@ -1431,7 +1416,6 @@ void hl_codegen_function( jit_ctx *jit ) { } } - void hl_codegen_alloc( jit_ctx *jit ) { code_ctx *ctx = (code_ctx*)malloc(sizeof(code_ctx)); memset(ctx,0,sizeof(code_ctx)); @@ -1454,7 +1438,7 @@ void hl_codegen_init( jit_ctx *jit ) { ctx->null_access_pos = jit->out_pos + byte_count(ctx->code); EMIT(_PUSH,R(RBP),UNUSED,M_PTR); EMIT(_MOV,R(RBP),R(RSP),M_PTR); - emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x20); + EMIT(SUB,R(RSP),MK_CONST(0x20),M_PTR); emit_ext(ctx,_MOV,R(RAX),VAL_CONST,M_PTR,(int_val)hl_null_access); EMIT(_CALL,R(RAX),UNUSED,M_PTR); BREAK(); @@ -1464,8 +1448,8 @@ void hl_codegen_init( jit_ctx *jit ) { ctx->null_field_pos = jit->out_pos + byte_count(ctx->code); EMIT(_PUSH,R(RBP),UNUSED,M_PTR); EMIT(_MOV,R(RBP),R(RSP),M_PTR); - emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x28); - emit_ext(ctx,_MOV,jit->cfg.regs.arg[0],VAL_MEM(R(RBP)),M_I32,HL_WSIZE*2); + EMIT(SUB,R(RSP),MK_CONST(0x28),M_PTR); + EMIT(_MOV,jit->cfg.regs.arg[0],MK_REG_VAL(RBP,R_REG_PTR,HL_WSIZE*2),M_I32); emit_ext(ctx,_MOV,R(RAX),VAL_CONST,M_PTR,(int_val)hl_jit_null_field_access); EMIT(_CALL,R(RAX),UNUSED,M_PTR); BREAK(); @@ -1485,20 +1469,20 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_MOV,nargs,cfg->regs.arg[2],M_I32); for(int i=0;iregs.nargs;i++) - emit_ext(ctx, _MOV, cfg->regs.arg[i], vargs|FL_MEMPTR, M_PTR, i * 8); + emit_ext(ctx, _MOV, cfg->regs.arg[i], MK_REG(vargs,R_REG_PTR), M_PTR, i * 8); for(int i=0;ifloats.nargs;i++) - emit_ext(ctx, MOVSD, cfg->floats.arg[i]-64, vargs|FL_MEMPTR, M_PTR, (i + cfg->regs.nargs) * 8); + emit_ext(ctx, MOVSD, cfg->floats.arg[i]-64, MK_REG(vargs,R_REG_PTR), M_PTR, (i + cfg->regs.nargs) * 8); - emit_ext(ctx,ADD,vargs,VAL_CONST,M_PTR,(MAX_ARGS - 1) * HL_WSIZE); + EMIT(ADD,vargs,MK_CONST((MAX_ARGS - 1) * HL_WSIZE),M_PTR); int begin = byte_count(ctx->code); EMIT(_TEST,nargs,nargs,M_I32); int pos = jump_near(ctx,JZero); - EMIT(_PUSH,vargs|FL_MEMPTR,UNUSED,M_PTR); + EMIT(_PUSH,MK_REG(vargs,R_REG_PTR),UNUSED,M_PTR); EMIT(DEC,nargs,UNUSED,M_I32); jump_near(ctx,-begin); patch_jump_near(ctx,pos); - if( IS_WINCALL64 ) emit_ext(ctx,SUB,R(RSP),VAL_CONST,M_PTR,0x20); + if( IS_WINCALL64 ) EMIT(SUB,R(RSP),MK_CONST(0x20),M_PTR); EMIT(_CALL, fptr, UNUSED, M_NONE); EMIT(_MOV,R(RSP),R(RBP),M_PTR); @@ -1552,4 +1536,4 @@ void hl_codegen_final( jit_ctx *jit ) { *(void**)(jit->final_code + ctx->const_table_pos + pos) = jit->final_code + offs; } int_arr_free(&ctx->const_addr); -} \ No newline at end of file +} From 9574dfe21d1aa0dd80be37bff3e12e52f3d85451 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 1 May 2026 11:19:18 +0200 Subject: [PATCH 49/83] various bug fixes --- src/jit.c | 2 +- src/jit_dump.c | 2 ++ src/jit_emit.c | 25 +++++++++++-------------- src/jit_regs.c | 5 ++--- src/jit_x86_64.c | 44 ++++++++++++++++++++++++++++++++++---------- 5 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/jit.c b/src/jit.c index 7ce357cfb..ec17700fc 100644 --- a/src/jit.c +++ b/src/jit.c @@ -162,7 +162,7 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { hl_codegen_function(ctx); int pos = ctx->out_pos; hl_jit_define_function(ctx, pos, ctx->code_size); - if( m->jit_debug ) { + if( m->jit_debug && ctx->code_pos_map ) { bool compact = ctx->code_size < 0xFFFF; void *debug = malloc((compact ? sizeof(unsigned short) : sizeof(int)) * (f->nops + 1)); for(int i=0;i<=f->nops;i++) { diff --git a/src/jit_dump.c b/src/jit_dump.c index bd2959177..d47ae78b9 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -390,6 +390,8 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { dump_value(ctx, e->value, e->mode); break; case LOAD_ADDR: + if( e->nargs != e->mode ) + printf("%s", emit_mode_str(e->nargs)); printf(" %s[%Xh]", val_str(e->a,M_PTR), e->size_offs); break; case STORE: diff --git a/src/jit_emit.c b/src/jit_emit.c index 116088ec2..4118a3923 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -102,7 +102,7 @@ struct _tmp_phi { int final_id; bool locked; bool opt; - unsigned char mode; + emit_mode mode; emit_block *b; ereg_map vals; phi_arr ref_phis; @@ -149,7 +149,7 @@ struct _emit_ctx { #define STORE(r, v) emit_store_reg(ctx, r, v) #define LOAD_CONST(v, t) emit_load_const(ctx, (uint64)(v), t) #define LOAD_CONST_PTR(v) LOAD_CONST(v,&hlt_bytes) -#define LOAD_MEM(v, offs, t) emit_load_mem(ctx, v, offs, t) +#define LOAD_MEM(v, offs, t) emit_load_mem(ctx, v, offs, t, t) #define LOAD_MEM_PTR(v, offs) LOAD_MEM(v, offs, &hlt_bytes) #define STORE_MEM(to, offs, v) emit_store_mem(ctx, to, offs, v) #define LOAD_OBJ_METHOD(obj,id) LOAD_MEM_PTR(LOAD_MEM_PTR(LOAD_MEM_PTR(obj,0),HL_WSIZE*2),HL_WSIZE*(id)) @@ -274,7 +274,7 @@ static ereg *get_tmp_args( emit_ctx *ctx, int count ) { return ctx->tmp_args; } -static unsigned char emit_get_mode( emit_ctx *ctx, ereg v ) { +static emit_mode emit_get_mode( emit_ctx *ctx, ereg v ) { if( IS_NULL(v) ) jit_assert(); if( v < 0 ) return GET_PHI(v)->mode; @@ -475,11 +475,11 @@ static ereg emit_load_const( emit_ctx *ctx, uint64 value, hl_type *size_t ) { return new_value(ctx); } -static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t ) { +static ereg emit_load_mem( emit_ctx *ctx, ereg v, int offset, hl_type *size_t, hl_type *to_t ) { einstr *e = emit_instr(ctx, LOAD_ADDR); - e->mode = hl_type_mode(size_t); + e->mode = hl_type_mode(to_t); e->a = v; - e->b = UNUSED; + e->nargs = hl_type_mode(size_t); e->size_offs = offset; return new_value(ctx); } @@ -993,6 +993,7 @@ void hl_emit_function( jit_ctx *jit ) { } emit_gen(ctx,ENTER,UNUSED,UNUSED,M_NONE); + emit_gen_size(ctx, BLOCK, 0); for(i=0;itype->fun->nargs;i++) { hl_type *t = f->type->fun->args[i]; STORE(R(i), emit_gen(ctx, LOAD_ARG, UNUSED, UNUSED, hl_type_mode(t))); @@ -1004,10 +1005,6 @@ void hl_emit_function( jit_ctx *jit ) { ctx->pos_map[op_pos] = ctx->emit_pos-1; else ctx->pos_map[op_pos] = ctx->emit_pos; - if( op_pos == 0 ) { - ctx->current_block->start_pos = ctx->emit_pos; - emit_gen_size(ctx, BLOCK, 0); - } if( ctx->arrival_points ) { if( ctx->arrival_points->id < op_pos ) jit_assert(); @@ -1811,14 +1808,14 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { int dyn_size = sizeof(vdynamic); ereg edyn = need_dyn ? emit_gen_size(ctx, ALLOC_STACK, dyn_size) : LOAD_CONST_PTR(NULL); - args = get_tmp_args(ctx, 4); + args = get_tmp_args(ctx, 5); args[0] = LOAD_MEM_PTR(obj,HL_WSIZE); args[1] = LOAD_CONST_PTR(_o->t->virt->fields[o->p2].t); args[2] = LOAD_CONST(_o->t->virt->fields[o->p2].hashed_name,&hlt_i32); args[3] = eargs; args[4] = edyn; - ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 4, dst->t); + ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 5, dst->t); patch_jump(ctx, jend); @@ -1846,9 +1843,9 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { case OGetI16: case OGetMem: { + hl_type *size_t = o->op == OGetI8 ? &hlt_ui8 : o->op == OGetI16 ? &hlt_ui16 : dst->t; ereg offs = OFFSET(LOAD(ra),LOAD(rb),1,0); - ereg val = LOAD_MEM(offs, 0, dst->t); - if( o->op != OGetMem ) patch_instr_mode(ctx, o->op == OGetI8 ? M_UI8 : M_UI16); + ereg val = emit_load_mem(ctx, offs, 0, size_t, dst->t); STORE(dst, val); } break; diff --git a/src/jit_regs.c b/src/jit_regs.c index 26e435fc5..87270f867 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -339,7 +339,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { jit_ctx *jit = ctx->jit; // assign args call_regs regs = {0}; - int args_size = 0; + int args_count = 0; for(int i=1;i<=ctx->jit->fun->type->fun->nargs;i++) { value_info *v = VAL(i); einstr *e = ctx->jit->instrs + ctx->jit->values_writes[i]; @@ -352,8 +352,7 @@ static void regs_assign_regs( regs_ctx *ctx ) { } if( IS_NULL(r) || IS_WINCALL64 ) { // use existing stack storage - v->stack_pos = args_size + HL_WSIZE*2; - args_size += size < 4 ? 4 : size; + v->stack_pos = (args_count++ + 2) * HL_WSIZE; if( IS_NULL(r) ) v->reg = MK_STACK_REG(v->stack_pos); } } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 1545eebc8..1df4e81de 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1071,6 +1071,7 @@ void hl_codegen_function( jit_ctx *jit ) { free(ctx->pos_map); ctx->pos_map = (int*)malloc((jit->reg_instr_count + 1) * sizeof(int)); ctx->pos_map[0] = 0; + int const_addr_prev = int_arr_count(ctx->const_addr); byte_reserve(ctx->code,64); ctx->code.cur -= 64; # ifdef GEN_DEBUG @@ -1194,10 +1195,10 @@ void hl_codegen_function( jit_ctx *jit ) { break; case LOAD_ADDR: if( IS_REG(e->a) ) { - emit_mov(ctx, out, REG_ADD_OFFSET(REG_PTR(e->a),e->size_offs), e->mode); + emit_mov(ctx, out, REG_ADD_OFFSET(REG_PTR(e->a),e->size_offs), e->nargs); } else { emit_mov(ctx, RTMP, e->a, M_PTR); - emit_mov(ctx, out, MK_REG_VAL(RTMP,R_REG_PTR,e->size_offs), e->mode); + emit_mov(ctx, out, MK_REG_VAL(RTMP,R_REG_PTR,e->size_offs), e->nargs); } break; case LOAD_FUN: @@ -1291,14 +1292,32 @@ void hl_codegen_function( jit_ctx *jit ) { int_arr_add_impl(&ctx->jit->galloc,&ctx->const_refs,start); ereg a = RTMP; ereg b = e->a; - B(0x40 | ((a&8)?1:0) | ((b&8)?2:0)); - B(0xFF); - B(0x24); - SIB(3,(b&7),(a&7)); - int here = jit->out_pos + byte_count(ctx->code); + if( IS_REG(b) ) { + // jump [a+b*8] + B(0x40 | ((a&8)?1:0) | ((b&8)?2:0)); + B(0xFF); + B(0x24); + SIB(3,(b&7),(a&7)); + } else { + ereg save = R(RAX); + EMIT(_PUSH,save,UNUSED,M_PTR); + EMIT(_MOV,save,b,M_I32); + // lea tmp, [tmp+save*8] + einstr etmp; + etmp.a = a; + etmp.b = save; + etmp.size_offs = 8; + emit_lea(ctx, RTMP, &etmp); + EMIT(_POP,save,UNUSED,M_PTR); + // jump [tmp] + B(0x40 | ((RTMP&8)?1:0)); + B(0xFF); + MOD_RM(0,4,RTMP&7); + } + ereg *args = hl_emit_get_args(jit->emit,e); for(int k=0;knargs;k++) { int_arr_add_impl(&jit->galloc,&ctx->const_addr,start + k * HL_WSIZE); - int_arr_add_impl(&jit->galloc,&ctx->const_addr,here); + int_arr_add_impl(&jit->galloc,&ctx->const_addr,ctx->cur_op + (int)args[k] + 1); } } break; @@ -1383,10 +1402,10 @@ void hl_codegen_function( jit_ctx *jit ) { int cond = get_cond_jump(ctx); if( !IS_REG(out) ) jit_assert(); if( IS_REG(e->a) ) { - emit_cmov(ctx,out,e->a,cond,e->mode); + emit_cmov(ctx,out,e->a,cond,M_PTR); } else { emit_mov(ctx,RTMP,e->a,e->mode); - emit_cmov(ctx,out,RTMP,cond,e->mode); + emit_cmov(ctx,out,RTMP,cond,M_PTR); } } break; @@ -1414,6 +1433,11 @@ void hl_codegen_function( jit_ctx *jit ) { int offset = ctx->pos_map[target] - (pos + 4); *(int*)&ctx->code.values[pos] = offset; } + for(int i=const_addr_prev;iconst_addr);i+=2) { + int target = int_arr_get(ctx->const_addr,i+1); + int offs = jit->out_pos + ctx->pos_map[target]; + ctx->const_addr.values[i+1] = offs; + } } void hl_codegen_alloc( jit_ctx *jit ) { From 1e781678f94184f140f910c957bb51aa4900711c Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 1 May 2026 16:19:47 +0200 Subject: [PATCH 50/83] added hl2c wrapper --- src/jit.c | 92 ++++++++++++++++++++++++++++++++++++++++-------- src/jit.h | 5 ++- src/jit_dump.c | 4 ++- src/jit_x86_64.c | 79 ++++++++++++++++++++++++++++++++++++----- 4 files changed, 154 insertions(+), 26 deletions(-) diff --git a/src/jit.c b/src/jit.c index ec17700fc..8f8cd016e 100644 --- a/src/jit.c +++ b/src/jit.c @@ -185,14 +185,32 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { return pos; } -static void *null_wrapper( hl_type *ft ) { - return hl_jit_assert; -} - static void *call_jit_c2hl = hl_jit_assert; +static void *call_jit_hl2c = hl_jit_assert; static int arg_reg_count = 0; static int arg_fp_count = 0; +static int get_next_reg( hl_type *t, int *rp, int *fp ) { + if( t->kind == HF32 || t->kind == HF64 ) { + if( *fp < arg_fp_count ) { + int r = (*fp)++; + if( IS_WINCALL64 ) (*rp)++; + return r; + } + return -1; + } + if( *rp < arg_fp_count ) { + int r = (*rp)++; + if( IS_WINCALL64 ) (*fp)++; + return r; + } + return -1; +} + +static void *default_wrapper( hl_type *ft ) { + return call_jit_hl2c; +} + static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { int nargs = t->fun->nargs; if( nargs > MAX_ARGS ) @@ -205,16 +223,10 @@ static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { for(int i=0;ifun->nargs;i++) { hl_type *at = t->fun->args[i]; void *v = args[i]; - if( at->kind == HF32 || at->kind == HF64 ) { - if( fp < arg_fp_count ) { - vargs.regs[arg_reg_count + fp++] = v; - if( IS_WINCALL64 ) rp++; - } else - vargs.stack[--sp] = v; - } else if( rp < arg_reg_count ) { - vargs.regs[rp++] = v; - if( IS_WINCALL64 ) fp++; - } else + int r = get_next_reg(at,&rp,&fp); + if( r >= 0 ) + vargs.regs[r + (at->kind == HF32 || at->kind == HF64 ? arg_reg_count : 0)] = v; + else vargs.stack[--sp] = v; } switch( t->fun->ret->kind ) { @@ -239,6 +251,55 @@ static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { } } +static vdynamic *callback_hl2c( vclosure_wrapper *c, char *stack_args, void **regs ) { + vdynamic *args[MAX_ARGS]; + int nargs = c->cl.t->fun->nargs; + if( nargs > MAX_ARGS ) + hl_error("Too many arguments for wrapped call"); + int rp = 0, fp = 0; + rp++; // skip fptr in HL64 - was passed as arg0 + if( IS_WINCALL64 ) fp++; + for(int i=0;icl.t->fun->args[i]; + int creg = get_next_reg(t,&rp,&fp); + if( creg < 0 ) { + args[i] = hl_is_dynamic(t) ? *(vdynamic**)stack_args : hl_make_dyn(stack_args,t); + stack_args += (t->kind == HF64 ? 8 : HL_WSIZE); + } else if( hl_is_dynamic(t) ) { + args[i] = *(vdynamic**)(regs + creg); + } else if( t->kind == HF32 || t->kind == HF64 ) { + args[i] = hl_make_dyn(regs + arg_reg_count + creg,&hlt_f64); + } else { + args[i] = hl_make_dyn(regs + creg,t); + } + } + return hl_dyn_call(c->wrappedFun,args,nargs); +} + +void *hl_jit_wrapper_ptr( vclosure_wrapper *c, char *stack_args, void **regs ) { + vdynamic *ret = callback_hl2c(c, stack_args, regs); + hl_type *tret = c->cl.t->fun->ret; + switch( tret->kind ) { + case HVOID: + return NULL; + case HUI8: + case HUI16: + case HI32: + case HBOOL: + return (void*)(int_val)hl_dyn_casti(&ret,&hlt_dyn,tret); + case HI64: + case HGUID: + return (void*)(int_val)hl_dyn_casti64(&ret,&hlt_dyn); + default: + return hl_dyn_castp(&ret,&hlt_dyn,tret); + } +} + +double hl_jit_wrapper_d( vclosure_wrapper *c, char *stack_args, void **regs ) { + vdynamic *ret = callback_hl2c(c, stack_args, regs); + return hl_dyn_castd(&ret,&hlt_dyn); +} + void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **debug, hl_module *previous ) { hl_codegen_flush_consts(ctx); jit_code_append(ctx); @@ -255,7 +316,8 @@ void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **d arg_reg_count = ctx->cfg.regs.nargs; arg_fp_count = ctx->cfg.floats.nargs; call_jit_c2hl = ctx->final_code + ctx->code_funs.c2hl; - hl_setup.get_wrapper = null_wrapper; + call_jit_hl2c = ctx->final_code + ctx->code_funs.hl2c; + hl_setup.get_wrapper = default_wrapper; hl_setup.static_call = callback_c2hl; return code; } diff --git a/src/jit.h b/src/jit.h index 158fe59bf..22f17f884 100644 --- a/src/jit.h +++ b/src/jit.h @@ -131,7 +131,7 @@ typedef enum { #define MK_STACK_REG(v) MK_REG_VAL(STACK_REG,R_REG_PTR,v) #define MK_STACK_OFFS(v) MK_REG_VAL(STACK_REG,R_REG,v) #define MK_CONST(v) MK_REG_VAL(0,R_CONST,v) - +#define MK_ADDR(reg,offs) MK_REG_VAL(reg,R_REG_PTR,offs) #define IS_CALL(op) ((op) == CALL_PTR || (op) == CALL_REG || (op) == CALL_FUN) #define IS_FLOAT(mode) ((mode) == M_F64 || (mode) == M_F32) @@ -200,6 +200,7 @@ typedef struct { typedef struct { int c2hl; + int hl2c; } jit_special_funs; struct _jit_ctx { @@ -247,6 +248,8 @@ void hl_jit_define_function( jit_ctx *ctx, int start, int size ); void hl_jit_null_field_access(); void hl_jit_assert(); +void *hl_jit_wrapper_ptr( vclosure_wrapper *c, char *stack_args, void **regs ); +double hl_jit_wrapper_d( vclosure_wrapper *c, char *stack_args, void **regs ); // emit & dump void hl_emit_dump( jit_ctx *ctx ); diff --git a/src/jit_dump.c b/src/jit_dump.c index d47ae78b9..7c3cd6ea4 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -515,8 +515,10 @@ void hl_emit_dump( jit_ctx *ctx ) { if( n > 0 ) printf(","); printf("%s:%d",val_str(p->values[n],p->mode),p->blocks[n]); } + if( p->nvalues == 0 ) + printf("unwritten"); printf(")"); - if( p->nvalues <= 1 ) + if( p->nvalues == 1 ) printf(" ???"); } cur_block = b; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 1df4e81de..a2f049d5b 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -938,7 +938,7 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { if( mult == 0 ) { if( REG_KIND(e.a) != R_REG ) jit_assert(); // no index - emit_ext(ctx,_LEA,out,MK_REG_VAL(e.a,R_REG_PTR,offs),M_PTR,0); + emit_ext(ctx,_LEA,out,MK_ADDR(e.a,offs),M_PTR,0); return; } @@ -1123,7 +1123,7 @@ void hl_codegen_function( jit_ctx *jit ) { EMIT(_PUSH,tmp2,UNUSED,M_PTR); emit_mov(ctx, RTMP, e->a, M_PTR); emit_mov(ctx, tmp2, e->b, mode); - emit_mov(ctx, MK_REG_VAL(RTMP,R_REG_PTR,e->size_offs), tmp2, mode); + emit_mov(ctx, MK_ADDR(RTMP,e->size_offs), tmp2, mode); EMIT(_POP,tmp2,UNUSED,M_PTR); } else { if( IS_FLOAT(e->mode) ) BREAK(); @@ -1133,7 +1133,7 @@ void hl_codegen_function( jit_ctx *jit ) { } } else if( !IS_REG(e->a) ) { emit_mov(ctx, RTMP, e->a, M_PTR); - emit_mov(ctx, MK_REG_VAL(RTMP,R_REG_PTR,e->size_offs), e->b, e->mode); + emit_mov(ctx, MK_ADDR(RTMP,e->size_offs), e->b, e->mode); } else emit_mov(ctx, REG_ADD_OFFSET(REG_PTR(e->a),e->size_offs), e->b, e->mode); break; @@ -1194,11 +1194,14 @@ void hl_codegen_function( jit_ctx *jit ) { } break; case LOAD_ADDR: - if( IS_REG(e->a) ) { + if( IS_REG(e->a) && e->nargs == e->mode ) { emit_mov(ctx, out, REG_ADD_OFFSET(REG_PTR(e->a),e->size_offs), e->nargs); } else { + ereg tmp = IS_REG(out) || (e->nargs == e->mode) ? out : RTMP; emit_mov(ctx, RTMP, e->a, M_PTR); - emit_mov(ctx, out, MK_REG_VAL(RTMP,R_REG_PTR,e->size_offs), e->nargs); + emit_mov(ctx, tmp, MK_ADDR(RTMP,e->size_offs), e->nargs); + if( out != tmp ) + emit_mov(ctx, out, tmp, e->mode); } break; case LOAD_FUN: @@ -1473,7 +1476,7 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_PUSH,R(RBP),UNUSED,M_PTR); EMIT(_MOV,R(RBP),R(RSP),M_PTR); EMIT(SUB,R(RSP),MK_CONST(0x28),M_PTR); - EMIT(_MOV,jit->cfg.regs.arg[0],MK_REG_VAL(RBP,R_REG_PTR,HL_WSIZE*2),M_I32); + EMIT(_MOV,jit->cfg.regs.arg[0],MK_ADDR(RBP,HL_WSIZE*2),M_I32); emit_ext(ctx,_MOV,R(RAX),VAL_CONST,M_PTR,(int_val)hl_jit_null_field_access); EMIT(_CALL,R(RAX),UNUSED,M_PTR); BREAK(); @@ -1493,15 +1496,15 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_MOV,nargs,cfg->regs.arg[2],M_I32); for(int i=0;iregs.nargs;i++) - emit_ext(ctx, _MOV, cfg->regs.arg[i], MK_REG(vargs,R_REG_PTR), M_PTR, i * 8); + emit_ext(ctx, _MOV, cfg->regs.arg[i], MK_ADDR(vargs,0), M_PTR, i * 8); for(int i=0;ifloats.nargs;i++) - emit_ext(ctx, MOVSD, cfg->floats.arg[i]-64, MK_REG(vargs,R_REG_PTR), M_PTR, (i + cfg->regs.nargs) * 8); + emit_ext(ctx, MOVSD, cfg->floats.arg[i]-64, MK_ADDR(vargs,0), M_PTR, (i + cfg->regs.nargs) * 8); EMIT(ADD,vargs,MK_CONST((MAX_ARGS - 1) * HL_WSIZE),M_PTR); int begin = byte_count(ctx->code); EMIT(_TEST,nargs,nargs,M_I32); int pos = jump_near(ctx,JZero); - EMIT(_PUSH,MK_REG(vargs,R_REG_PTR),UNUSED,M_PTR); + EMIT(_PUSH,MK_ADDR(vargs,0),UNUSED,M_PTR); EMIT(DEC,nargs,UNUSED,M_I32); jump_near(ctx,-begin); patch_jump_near(ctx,pos); @@ -1514,6 +1517,64 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_RET,UNUSED,UNUSED,M_NONE); flush_function(ctx, ctx->null_field_pos); + + // generate hl2c stub + jit->code_funs.hl2c = jit->out_pos + byte_count(ctx->code); + ereg cl = cfg->regs.arg[0]; + ereg tmp = cfg->regs.arg[1]; + EMIT(_PUSH,R(RBP),UNUSED,M_PTR); + EMIT(_MOV,R(RBP),R(RSP),M_PTR); + EMIT(SUB,R(RSP),MK_CONST(cfg->floats.nargs*8),M_PTR); + + // push all possible call registers + for(int i=0;ifloats.nargs;i++) + EMIT(MOVSD,MK_ADDR(RSP,i*8),cfg->floats.arg[cfg->floats.nargs - 1 - i],M_F64); + for(int i=0;iregs.nargs;i++) + EMIT(_PUSH,cfg->regs.arg[cfg->regs.nargs - 1 - i],UNUSED,M_PTR); + + // opcodes for: + // switch( arg0->t->fun->ret->kind ) { + // case HF32: case HF64: return jit_wrapper_d(arg0,&args); + // default: return jit_wrapper_ptr(arg0,&args); + // } + hl_type_fun *ft = NULL; + ereg fun_ptr = scratch_not_param[0]; + + EMIT(_MOV,tmp,MK_ADDR(cl,0),M_PTR); // ->t + EMIT(_MOV,tmp,MK_ADDR(tmp,HL_WSIZE),M_PTR); // ->fun + EMIT(_MOV,tmp,MK_ADDR(tmp,(int)(int_val)&ft->ret),M_PTR); // ->rets + EMIT(_MOV,tmp,MK_ADDR(tmp,0),M_I32); // ->kind + + EMIT(_CMP,tmp,MK_CONST(HF64),M_I32); + int float1 = jump_near(ctx,JEq); + EMIT(_CMP,tmp,MK_CONST(HF32),M_I32); + int float2 = jump_near(ctx,JEq); + emit_ext(ctx,_MOV,fun_ptr,VAL_CONST,M_PTR,(int_val)hl_jit_wrapper_ptr); + + int jexit = jump_near(ctx, JAlways); + patch_jump_near(ctx, float1); + patch_jump_near(ctx, float2); + emit_ext(ctx,_MOV,fun_ptr,VAL_CONST,M_PTR,(int_val)hl_jit_wrapper_d); + patch_jump_near(ctx, jexit); + + int stack_args_pos = HL_WSIZE * (IS_64?2:3); + if( IS_WINCALL64 ) { + stack_args_pos += 0x20; + EMIT(SUB,R(RSP),MK_CONST(0x20),M_PTR); + } + EMIT(_LEA,cfg->regs.arg[1],MK_ADDR(R(RBP),stack_args_pos),M_PTR); + EMIT(_LEA,cfg->regs.arg[2],MK_ADDR(R(RBP),-(cfg->floats.nargs * 8 + cfg->regs.nargs * HL_WSIZE)),M_PTR); + EMIT(_CALL,fun_ptr,UNUSED,M_PTR); + + if( IS_WINCALL64 ) + EMIT(ADD,R(RSP),MK_CONST(0x20),M_PTR); + + EMIT(_MOV,R(RSP),R(RBP),M_PTR); + EMIT(_POP,R(RBP),UNUSED,M_PTR); + EMIT(_RET,UNUSED,UNUSED,M_NONE); + + flush_function(ctx, jit->code_funs.hl2c); + hl_codegen_flush(jit); } From 9be0ec340828b65c1fe91506681edbeb40d2a269 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 2 May 2026 00:54:52 +0200 Subject: [PATCH 51/83] more work on conv and types sizes --- src/jit.c | 4 +- src/jit_dump.c | 10 ++- src/jit_emit.c | 161 ++++++++++++++++++++--------------------------- src/jit_x86_64.c | 88 +++++++++++++++++++------- 4 files changed, 144 insertions(+), 119 deletions(-) diff --git a/src/jit.c b/src/jit.c index 8f8cd016e..a374f35db 100644 --- a/src/jit.c +++ b/src/jit.c @@ -225,9 +225,9 @@ static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { void *v = args[i]; int r = get_next_reg(at,&rp,&fp); if( r >= 0 ) - vargs.regs[r + (at->kind == HF32 || at->kind == HF64 ? arg_reg_count : 0)] = v; + vargs.regs[r + (at->kind == HF32 || at->kind == HF64 ? arg_reg_count : 0)] = *(void**)v; else - vargs.stack[--sp] = v; + vargs.stack[--sp] = *(void**)v; } switch( t->fun->ret->kind ) { case HUI8: diff --git a/src/jit_dump.c b/src/jit_dump.c index 7c3cd6ea4..d973a6b55 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -390,8 +390,10 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { dump_value(ctx, e->value, e->mode); break; case LOAD_ADDR: - if( e->nargs != e->mode ) - printf("%s", emit_mode_str(e->nargs)); + if( e->nargs != e->mode ) { + if( e->mode == M_PTR ) printf("-ptr"); + printf("%s", e->nargs == M_PTR ? "-ptr" : emit_mode_str(e->nargs)); + } printf(" %s[%Xh]", val_str(e->a,M_PTR), e->size_offs); break; case STORE: @@ -406,7 +408,8 @@ static void dump_instr( jit_ctx *ctx, einstr *e, int cur_pos ) { break; case CONV: case CONV_UNSIGNED: - printf("%s %s", emit_mode_str(e->size_offs), val_str(e->a,(emit_mode)e->size_offs)); + if( e->mode == M_PTR ) printf("-i64"); + printf("%s %s", e->size_offs == M_PTR ? "-i64" : emit_mode_str(e->size_offs), val_str(e->a,(emit_mode)e->size_offs)); break; case LEA: printf(" [%s", reg_str(e->a)); @@ -540,6 +543,7 @@ void hl_emit_dump( jit_ctx *ctx ) { if( new_op ) { new_op = false; cpos += ctx->cfg.debug_prefix_size; + if( cpos == ctx->code_pos_map[rpos+1] ) break; } } printf("%.2X",ctx->code_instrs[cpos++]); diff --git a/src/jit_emit.c b/src/jit_emit.c index 4118a3923..28959c1f3 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -742,12 +742,11 @@ static void emit_store_size( emit_ctx *ctx, ereg dst, int dst_offset, ereg src, static ereg emit_conv( emit_ctx *ctx, ereg v, emit_mode from, emit_mode to, bool _unsigned ) { + if( from == to && !_unsigned ) + return emit_gen(ctx,MOV,v,UNUSED,to); if( IS_FLOAT(from) != IS_FLOAT(to) ) return emit_gen_ext(ctx, _unsigned ? CONV_UNSIGNED : CONV, v, UNUSED, to, from); - if( from != to && IS_FLOAT(from) && IS_FLOAT(to) ) - return emit_gen_ext(ctx, CONV, v, UNUSED, to, from); - // no-op - return emit_gen(ctx,MOV,v,UNUSED,to); + return emit_gen_ext(ctx, CONV, v, UNUSED, to, from); } static bool dyn_need_type( hl_type *t ) { @@ -1190,102 +1189,80 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type patch_jump(ctx,jeq); } else jit_assert(); - return; } + return; case HVIRTUAL: - { - BREAK(); - /* - preg p; - preg *pa = alloc_cpu(ctx,a,true); - preg *pb = alloc_cpu(ctx,b,true); - int ja,jb,jav,jbv,jvalue; - if( b->t->kind == HOBJ ) { - if( op->op == OJEq ) { - // if( a ? (b && a->value == b) : (b == NULL) ) goto - op64(ctx,TEST,pa,pa); - XJump_small(JZero,ja); - op64(ctx,TEST,pb,pb); - XJump_small(JZero,jb); - op64(ctx,MOV,pa,pmem(&p,pa->id,HL_WSIZE)); - op64(ctx,CMP,pa,pb); - XJump_small(JAlways,jvalue); - patch_jump(ctx,ja); - op64(ctx,TEST,pb,pb); - patch_jump(ctx,jvalue); - register_jump(ctx,do_jump(ctx,OJEq,false),targetPos); - patch_jump(ctx,jb); - } else if( op->op == OJNotEq ) { - // if( a ? (b == NULL || a->value != b) : (b != NULL) ) goto - op64(ctx,TEST,pa,pa); - XJump_small(JZero,ja); - op64(ctx,TEST,pb,pb); - register_jump(ctx,do_jump(ctx,OJEq,false),targetPos); - op64(ctx,MOV,pa,pmem(&p,pa->id,HL_WSIZE)); - op64(ctx,CMP,pa,pb); - XJump_small(JAlways,jvalue); - patch_jump(ctx,ja); - op64(ctx,TEST,pb,pb); - patch_jump(ctx,jvalue); - register_jump(ctx,do_jump(ctx,OJNotEq,false),targetPos); - } else - ASSERT(op->op); - scratch(pa); - return; - } - op64(ctx,CMP,pa,pb); - if( op->op == OJEq ) { - // if( a == b || (a && b && a->value && b->value && a->value == b->value) ) goto - register_jump(ctx,do_jump(ctx,OJEq, false),targetPos); - op64(ctx,TEST,pa,pa); - XJump_small(JZero,ja); - op64(ctx,TEST,pb,pb); - XJump_small(JZero,jb); - op64(ctx,MOV,pa,pmem(&p,pa->id,HL_WSIZE)); - op64(ctx,TEST,pa,pa); - XJump_small(JZero,jav); - op64(ctx,MOV,pb,pmem(&p,pb->id,HL_WSIZE)); - op64(ctx,TEST,pb,pb); - XJump_small(JZero,jbv); - op64(ctx,CMP,pa,pb); - XJump_small(JNeq,jvalue); - register_jump(ctx,do_jump(ctx,OJEq, false),targetPos); + if( bt->kind == HOBJ ) { + if( op == OJEq ) { + // if( a == b || (a && a->value == b) ) goto + emit_cmp(ctx, a, b, OJEq); + register_block_jump(ctx,offset,true); + split_block(ctx); + emit_test(ctx, a, OJNull); + int jnot = emit_jump(ctx, true); + emit_cmp(ctx, LOAD_MEM_PTR(a,HL_WSIZE), b, OJEq); + register_block_jump(ctx,offset,true); + split_block(ctx); + patch_jump(ctx, jnot); + } else if( op == OJNotEq ) { + // if( a != b && (!a || a->value != b) ) goto + emit_cmp(ctx, a, b, OJEq); + int jsame = emit_jump(ctx, true); + emit_test(ctx, a, OJNull); + register_block_jump(ctx,offset,true); + split_block(ctx); + emit_cmp(ctx, LOAD_MEM_PTR(a,HL_WSIZE), b, OJNotEq); + register_block_jump(ctx,offset,true); + split_block(ctx); + patch_jump(ctx,jsame); + } else + jit_assert(); + } else { + if( op == OJEq ) { + // if( a == b || (a && b && a->value && a->value == b->value) ) goto + emit_cmp(ctx, a, b, OJEq); + register_block_jump(ctx,offset,true); + split_block(ctx); + emit_test(ctx, a, OJNull); + int ja = emit_jump(ctx, true); + emit_test(ctx, b, OJNull); + int jb = emit_jump(ctx, true); + ereg va = LOAD_MEM_PTR(a,HL_WSIZE); + emit_test(ctx, va, OJNull); + int jva = emit_jump(ctx, true); + ereg vb = LOAD_MEM_PTR(a,HL_WSIZE); + emit_cmp(ctx, va, vb, OJEq); + register_block_jump(ctx,offset,true); + split_block(ctx); patch_jump(ctx,ja); patch_jump(ctx,jb); - patch_jump(ctx,jav); - patch_jump(ctx,jbv); - patch_jump(ctx,jvalue); - } else if( op->op == OJNotEq ) { - int jnext; - // if( a != b && (!a || !b || !a->value || !b->value || a->value != b->value) ) goto - XJump_small(JEq,jnext); - op64(ctx,TEST,pa,pa); - XJump_small(JZero,ja); - op64(ctx,TEST,pb,pb); - XJump_small(JZero,jb); - op64(ctx,MOV,pa,pmem(&p,pa->id,HL_WSIZE)); - op64(ctx,TEST,pa,pa); - XJump_small(JZero,jav); - op64(ctx,MOV,pb,pmem(&p,pb->id,HL_WSIZE)); - op64(ctx,TEST,pb,pb); - XJump_small(JZero,jbv); - op64(ctx,CMP,pa,pb); - XJump_small(JEq,jvalue); + patch_jump(ctx,jva); + } else if( op == OJNotEq ) { + // if( a != b && (!a || !b || !a->value || a->value != b->value) ) goto + emit_cmp(ctx, a, b, OJNotEq); + register_block_jump(ctx,offset,true); + split_block(ctx); + emit_test(ctx, a, OJNull); + int ja = emit_jump(ctx, true); + emit_test(ctx, b, OJNull); + int jb = emit_jump(ctx, true); + ereg va = LOAD_MEM_PTR(a,HL_WSIZE); + emit_test(ctx, va, OJNull); + int jva = emit_jump(ctx, true); + ereg vb = LOAD_MEM_PTR(a,HL_WSIZE); + emit_cmp(ctx, va, vb, OJEq); + int jeq = emit_jump(ctx, true); + split_block(ctx); patch_jump(ctx,ja); patch_jump(ctx,jb); - patch_jump(ctx,jav); - patch_jump(ctx,jbv); - register_jump(ctx,do_jump(ctx,OJAlways, false),targetPos); - patch_jump(ctx,jnext); - patch_jump(ctx,jvalue); + patch_jump(ctx,jva); + register_block_jump(ctx,offset,true); + split_block(ctx); + patch_jump(ctx,jeq); } else - ASSERT(op->op); - scratch(pa); - scratch(pb); - return; - */ + jit_assert(); } - break; + return; case HOBJ: case HSTRUCT: if( bt->kind == HVIRTUAL ) { diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index a2f049d5b..1ceef0392 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -101,6 +101,7 @@ typedef enum { INC, DEC, JMP, + MOVSXD, // FPU FSTP, FSTP32, @@ -131,13 +132,20 @@ typedef enum { STMXCSR, LDMXCSR, // 8-16 bits + ADD8, + SUB8, MOV8, MOVZX8, + MOVSX8, CMP8, TEST8, PUSH8, + ADD16, + SUB16, + IMUL16, MOV16, MOVZX16, + MOVSX16, CMP16, TEST16, // prefetchs @@ -180,7 +188,7 @@ typedef enum { #define GET_RM(op) (((op) >> ((op) < 0 ? 24 : 8)) & 15) #define SBYTE(op) ((op) << 16) #define LONG_OP(op) ((op) | FLAG_LONGOP) -#define OP16(op) LONG_OP((op) | FLAG_16B) +#define OP16(op) ((op) | FLAG_16B) #define LONG_RM(op,id) LONG_OP(op | (((id) + 1) << 24)) typedef struct { @@ -218,6 +226,7 @@ static opform OP_FORMS[] = { { "INC", IS_64 ? RM(0xFF,0) : 0x40, RM(0xFF,0) }, { "DEC", IS_64 ? RM(0xFF,1) : 0x48, RM(0xFF,1) }, { "JMP", RM(0xFF,4) }, + { "MOVSXD", 0x63 }, // FPU { "FSTP", 0, RM(0xDD,3) }, { "FSTP32", 0, RM(0xD9,3) }, @@ -248,13 +257,20 @@ static opform OP_FORMS[] = { { "STMXCSR", 0, LONG_RM(0x0FAE,3) }, { "LDMXCSR", 0, LONG_RM(0x0FAE,2) }, // 8 bits, + { "ADD8", 0, RM(0x00,3) }, + { "SUB8", 0, 0x28 }, { "MOV8", 0x8A, 0x88, 0, RM(0xC6,0) }, { "MOVZX8", LONG_OP(0x0FB6) }, + { "MOVSX8", LONG_OP(0x0FBE) }, { "CMP8", 0x3A, 0x38, 0, RM(0x80,7) }, { "TEST8", 0x84, 0x84, RM(0xF6,0) }, { "PUSH8", FLAG_DEF64, 0, 0x6A | FLAG_8B }, + { "ADD16", 0, OP16(0x01) }, + { "SUB16", 0, OP16(0x29) }, + { "IMUL16", OP16(LONG_OP(0x0FAF)) }, { "MOV16", OP16(0x8B), OP16(0x89), OP16(0xB8) }, { "MOVZX16", LONG_OP(0x0FB7) }, + { "MOVSX16", LONG_OP(0x0FBF) }, { "CMP16", OP16(0x3B), OP16(0x39) }, { "TEST16", OP16(0x85) }, // prefetchs @@ -295,8 +311,8 @@ static const int SIB_MULT[] = {-1, 0, 1, -1, 2, -1, -1, -1, 3}; REX(); \ } else {\ REX(); \ - if( (b) & FLAG_LONGOP ) B((b)>>8); \ }\ + if( (b) & FLAG_LONGOP ) B((b)>>8); \ B(b); \ } @@ -760,16 +776,19 @@ static void emit_div_mod( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emi static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_mode mode ) { CpuOp cop; + int mask = 0; # define F_OP(iop,f32,f64) cop = mode == M_F32 ? f32 : (mode == M_F64 ? f64 : iop); +# define DECL_OP(i8,i16,iop,f32,f64) static CpuOp ops_##iop[] = {-1,i8,i16,iop,iop,f64,f32,-1,-1}; cop = ops_##iop[mode] switch( op ) { case OAdd: - F_OP(ADD,ADDSS,ADDSD); + DECL_OP(ADD8,ADD16,ADD,ADDSS,ADDSD); break; case OSub: - F_OP(SUB,SUBSS,SUBSD); + DECL_OP(SUB8,SUB16,SUB,SUBSS,SUBSD); break; case OMul: - F_OP(IMUL,MULSS,MULSD); + DECL_OP(IMUL16/*NO IMUL8*/,IMUL16,IMUL,MULSS,MULSD); + if( mode == M_UI8 ) mask = 0xFF; break; case OIncr: cop = INC; @@ -793,18 +812,18 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ ereg f = R(RCX); if( b != f ) { if( a == f || out == f ) { - EMIT(_MOV,RTMP,a,M_I32); + EMIT(_MOV,RTMP,a,mode); a = RTMP; } if( out == f ) { - EMIT(_MOV,f,b,M_I32); + EMIT(_MOV,f,b,mode); emit_anyop(ctx, op, RTMP, RTMP, f, mode); - EMIT(_MOV,f,RTMP,M_I32); + EMIT(_MOV,f,RTMP,mode); } else { - EMIT(_PUSH,f,UNUSED,M_I32); - EMIT(_MOV,f,b,M_I32); + EMIT(_PUSH,f,UNUSED,M_PTR); + EMIT(_MOV,f,b,mode); emit_anyop(ctx, op, out, a, f, mode); - EMIT(_POP,f,UNUSED,M_I32); + EMIT(_POP,f,UNUSED,M_PTR); } return; } @@ -852,17 +871,23 @@ static void emit_anyop( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emit_ jit_assert(); break; } + if( out == a && IS_REG(a) ) { EMIT(cop,out,b,mode); } else if( !IS_REG(out) || out == b ) { ereg tmp = get_tmp(mode); emit_mov(ctx, tmp, a, mode); EMIT(cop,tmp,b,mode); + if( mask ) { + EMIT(AND,tmp,MK_CONST(mask),M_I32); + mask = 0; + } emit_mov(ctx, out, tmp, mode); } else { emit_mov(ctx, out, a, mode); EMIT(cop,out,b,mode); } + if( mask ) EMIT(AND,out,MK_CONST(mask),M_I32); } void hl_codegen_flush( jit_ctx *jit ) { @@ -1332,7 +1357,7 @@ void hl_codegen_function( jit_ctx *jit ) { emit_mode in_mode = e->size_offs; ereg r = IS_REG(e->a) ? e->a : get_tmp(in_mode); if( r != e->a ) emit_mov(ctx, r, e->a, in_mode); - CpuOp op = 0; + CpuOp op = -1; switch( ID2(e->mode,in_mode) ) { case ID2(M_F32,M_UI8): case ID2(M_F32,M_UI16): @@ -1364,18 +1389,37 @@ void hl_codegen_function( jit_ctx *jit ) { case ID2(M_F64,M_F32): op = CVTSS2SD; break; + case ID2(M_PTR,M_I32): + // sign extend 32-64 bit conv + op = MOVSXD; + break; + case ID2(M_UI16,M_UI8): + case ID2(M_I32,M_UI8): + case ID2(M_PTR,M_UI8): + case ID2(M_UI8, M_UI16): + case ID2(M_UI8, M_I32): + case ID2(M_UI8, M_PTR): + op = MOVZX8; + break; + case ID2(M_I32,M_UI16): + case ID2(M_PTR,M_UI16): + case ID2(M_UI16, M_I32): + case ID2(M_UI16, M_PTR): + op = MOVZX16; + break; + case ID2(M_I32,M_PTR): + op = _MOV; + break; default: jit_assert(); break; } - if( op ) { - if( IS_REG(out) ) - EMIT(op,out,r,0); - else { - ereg r2 = get_tmp(e->mode); - EMIT(op,r2,r,0); - emit_mov(ctx,out,r2,e->mode); - } + if( IS_REG(out) || op == _MOV ) + EMIT(op,out,r,e->mode); + else { + ereg r2 = get_tmp(e->mode); + EMIT(op,r2,r,e->mode); + emit_mov(ctx,out,r2,e->mode); } } break; @@ -1496,9 +1540,9 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_MOV,nargs,cfg->regs.arg[2],M_I32); for(int i=0;iregs.nargs;i++) - emit_ext(ctx, _MOV, cfg->regs.arg[i], MK_ADDR(vargs,0), M_PTR, i * 8); + EMIT(_MOV, cfg->regs.arg[i], MK_ADDR(vargs,i*8), M_PTR); for(int i=0;ifloats.nargs;i++) - emit_ext(ctx, MOVSD, cfg->floats.arg[i]-64, MK_ADDR(vargs,0), M_PTR, (i + cfg->regs.nargs) * 8); + EMIT(MOVSD, cfg->floats.arg[i]-64, MK_ADDR(vargs,(i + cfg->regs.nargs) * 8), M_PTR); EMIT(ADD,vargs,MK_CONST((MAX_ARGS - 1) * HL_WSIZE),M_PTR); int begin = byte_count(ctx->code); From 18127b21247768e4db9f03ec3e4e79bef4632923 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 2 May 2026 00:55:09 +0200 Subject: [PATCH 52/83] i8/i16 -> ui8/ui16 --- src/std/types.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/std/types.c b/src/std/types.c index eaf228db6..8db708185 100644 --- a/src/std/types.c +++ b/src/std/types.c @@ -35,7 +35,7 @@ HL_PRIM hl_type hlt_bool = { HBOOL }; HL_PRIM hl_type hlt_abstract = { HABSTRACT, {USTR("")} }; static const uchar *TSTR[] = { - USTR("void"), USTR("i8"), USTR("i16"), USTR("i32"), USTR("i64"), USTR("f32"), USTR("f64"), + USTR("void"), USTR("ui8"), USTR("ui16"), USTR("i32"), USTR("i64"), USTR("f32"), USTR("f64"), USTR("bool"), USTR("bytes"), USTR("dynamic"), NULL, NULL, USTR("array"), USTR("type"), NULL, NULL, USTR("dynobj"), NULL, NULL, NULL, NULL, NULL, NULL, USTR("guid") @@ -43,8 +43,8 @@ static const uchar *TSTR[] = { static int T_SIZES[] = { 0, // VOID - 1, // I8 - 2, // I16 + 1, // UI8 + 2, // UI16 4, // I32 8, // I64 4, // F32 @@ -160,8 +160,8 @@ HL_PRIM bool hl_same_type( hl_type *a, hl_type *b ) { HL_PRIM bool hl_is_dynamic( hl_type *t ) { static bool T_IS_DYNAMIC[] = { false, // HVOID, - false, // HI8 - false, // HI16 + false, // HUI8 + false, // HUI16 false, // HI32 false, // HI64 false, // HF32 @@ -190,8 +190,8 @@ HL_PRIM bool hl_is_dynamic( hl_type *t ) { HL_PRIM bool hl_is_ptr( hl_type *t ) { static bool T_IS_PTR[] = { false, // HVOID, - false, // HI8 - false, // HI16 + false, // HUI8 + false, // HUI16 false, // HI32 false, // HI64 false, // HF32 From 181562f191e9cb7b46876d6c3ef1a9930ca195f2 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 2 May 2026 21:25:19 +0200 Subject: [PATCH 53/83] more fixes --- src/jit.c | 22 ++++++++++++++++++++-- src/jit_regs.c | 3 ++- src/jit_x86_64.c | 17 +++++++---------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/jit.c b/src/jit.c index a374f35db..767c60a99 100644 --- a/src/jit.c +++ b/src/jit.c @@ -224,10 +224,28 @@ static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { hl_type *at = t->fun->args[i]; void *v = args[i]; int r = get_next_reg(at,&rp,&fp); + int_val iv; + switch( at->kind ) { + case HBOOL: + case HUI8: + case HUI16: + case HI32: + case HF32: + iv = *(int*)v; + break; + case HI64: + case HGUID: + case HF64: + iv = *(int_val*)v; + break; + default: + iv = (int_val)v; + break; + } if( r >= 0 ) - vargs.regs[r + (at->kind == HF32 || at->kind == HF64 ? arg_reg_count : 0)] = *(void**)v; + vargs.regs[r + (at->kind == HF32 || at->kind == HF64 ? arg_reg_count : 0)] = (void*)iv; else - vargs.stack[--sp] = *(void**)v; + vargs.stack[--sp] = (void*)iv; } switch( t->fun->ret->kind ) { case HUI8: diff --git a/src/jit_regs.c b/src/jit_regs.c index 87270f867..e30e2d7c1 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -211,7 +211,8 @@ static void regs_write_live( regs_ctx *ctx, ereg *r ) { if( IS_NULL(*r) ) jit_assert(); if( !REG_IS_VAL(*r) ) return; // some are injections of native regs at emit value_info *v = VAL_REG(*r); - v->last_read = ctx->loop_end && ctx->jit->values_writes[v->id] < ctx->loop_start ? ctx->loop_end : ctx->cur_op; + int write = v->id >= 0 ? ctx->jit->values_writes[v->id] : -1; + v->last_read = ctx->loop_end && write < ctx->loop_start ? ctx->loop_end : ctx->cur_op; v->tot_reads++; } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 1ceef0392..312d3ad9b 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -236,8 +236,8 @@ static opform OP_FORMS[] = { // SSE { "MOVSD", 0xF20F10, 0xF20F11 }, { "MOVSS", 0xF30F10, 0xF30F11 }, - { "COMISD", 0x660F2F, LONG_RM(0x660F2F,1) }, - { "COMISS", LONG_OP(0x0F2F), LONG_RM(0x0F2F,1) }, + { "COMISD", LONG_RM(0x660F2F,1) }, + { "COMISS", LONG_RM(0x0F2F,1) }, { "ADDSD", 0xF20F58 }, { "SUBSD", 0xF20F5C }, { "MULSD", 0xF20F59 }, @@ -1291,16 +1291,13 @@ void hl_codegen_function( jit_ctx *jit ) { case M_F64: op = COMISD; break; default: op = _CMP; break; } - ereg b = e->b; - if( !IS_REG(e->a) && !IS_REG(e->b) ) { + ereg a = e->a; + if( !IS_REG(e->a) && (IS_FLOAT(e->mode) || !IS_REG(e->b)) ) { ereg tmp = get_tmp(e->mode); - emit_mov(ctx, tmp, e->b, e->mode); - b = tmp; + emit_mov(ctx, tmp, e->a, e->mode); + a = tmp; } - if( IS_FLOAT(e->mode) ) - EMIT(op,b,e->a,e->mode); - else - EMIT(op,e->a,b,e->mode); + EMIT(op,a,e->b,e->mode); } break; case JCOND: From e98c86f138c2d9a82a6566a2f52f45ff6cf2f2cd Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 3 May 2026 08:45:25 +0200 Subject: [PATCH 54/83] handle nans, split phi assigns before/after jcond & other fixes --- src/jit_regs.c | 29 ++++++++++++++++++++++------- src/jit_x86_64.c | 48 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/src/jit_regs.c b/src/jit_regs.c index e30e2d7c1..447ee2053 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -170,6 +170,13 @@ static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { break; } } + if( free ) { + for(int i=0;ipersists_uses[mode];i++) + if( cfg->persist[i] == v->pref_reg ) { + free = false; + break; + } + } if( free ) { v->reg = v->pref_reg; return true; @@ -328,6 +335,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { regs_debug("ADD PHI %s:=%s to #%d@%X\n",val_str(ph->value,ph->mode),val_str(v,ph->mode),(int)(b2 - jit->blocks),b2->end_pos-1); int_arr_add(*arr,v); int_arr_add(*arr,ph->value); + int_arr_add(*arr,(bl - b2) == 1); val->tot_reads++; if( val->last_read < b2->end_pos ) val->last_read = b2->end_pos; @@ -482,7 +490,7 @@ static void flush_movs( regs_ctx *ctx, bool cond ) { if( !read ) { ereg from = int_arr_get(movs,k+1); int mode = int_arr_get(movs,k+2); - bool cmov = cond && IS_REG(to); + bool cmov = cond && IS_REG(to); regs_emit(ctx,to,cmov?CMOV:MOV,from,UNUSED,mode,0); int_arr_remove_range(&movs,k,3); cycle = false; @@ -509,7 +517,7 @@ static void flush_movs( regs_ctx *ctx, bool cond ) { int_arr_reset(&ctx->pack_movs); } -static void flush_phis( regs_ctx *ctx, eblock *b, bool cond ) { +static void flush_phis( regs_ctx *ctx, eblock *b, bool cond, bool after ) { if( !b ) return; jit_ctx *jit = ctx->jit; int bid = (int)(b - jit->blocks); @@ -520,6 +528,9 @@ static void flush_phis( regs_ctx *ctx, eblock *b, bool cond ) { while( idx < int_arr_count(arr) ) { ereg a = int_arr_get(arr,idx++); ereg b = int_arr_get(arr,idx++); + int bcount = int_arr_get(arr,idx++); + if( after != (bcount == 1) ) + continue; value_info *from = VAL_REG(a); value_info *to = VAL_REG(b); if( from->reg == to->reg ) continue; @@ -538,7 +549,8 @@ static void flush_phis( regs_ctx *ctx, eblock *b, bool cond ) { } } ctx->pack_movs = movs; - int_arr_free(&ctx->blocks_phis[bid]); + if( !cond ) + int_arr_free(&ctx->blocks_phis[bid]); flush_movs(ctx, cond); } @@ -598,8 +610,10 @@ static void regs_emit_instrs( regs_ctx *ctx ) { if( offset ) regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,0,-offset); for(int k=e.nargs-1;k>=0;k--) { - if( stack_bits & (1 << k) ) - EMIT(PUSH,VAL_REG(args[k])->reg,UNUSED,M_PTR); + if( stack_bits & (1 << k) ) { + value_info *v = REG_IS_VAL(args[k]) ? VAL_REG(args[k]) : NULL; + EMIT(PUSH,VAL_REG(args[k])->reg,UNUSED,v && IS_FLOAT(v->mode) ? v->mode : M_PTR); + } } if( IS_WINCALL64 ) { regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,0,-0x20); @@ -657,7 +671,7 @@ static void regs_emit_instrs( regs_ctx *ctx ) { case JCOND: case JUMP: case JUMP_TABLE: - flush_phis(ctx,cur_block, e.op == JCOND); + flush_phis(ctx,cur_block, e.op == JCOND, false); if( e.op == JUMP_TABLE ) { // copy args (remap later) hl_emit_store_args(jit->emit,&e,hl_emit_get_args(jit->emit,&e),e.nargs); @@ -665,6 +679,7 @@ static void regs_emit_instrs( regs_ctx *ctx ) { regs_write_instr(ctx, &e, out); int_arr_add(ctx->jump_regs, ctx->emit_pos - 1); int_arr_add(ctx->jump_regs, cur_op + 1 + (e.op == JUMP_TABLE ? 0 : e.size_offs)); + if( e.op == JCOND ) flush_phis(ctx,cur_block, false, true); break; case RET: if( e.a ) { @@ -700,7 +715,7 @@ static void regs_emit_instrs( regs_ctx *ctx ) { if( instr_stack_offset ) regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,instr_stack_offset); if( cur_block && cur_block->end_pos == cur_op+1 ) - flush_phis(ctx,cur_block,false); + flush_phis(ctx,cur_block,false,true); ctx->pos_map[cur_op+1] = ctx->emit_pos; } } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 312d3ad9b..34fd25e25 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -125,12 +125,14 @@ typedef enum { XORPD, CVTSI2SD, CVTSI2SS, - CVTSD2SI, + CVTTSD2SI, CVTSD2SS, CVTSS2SD, - CVTSS2SI, + CVTTSS2SI, STMXCSR, LDMXCSR, + STC, + CLC, // 8-16 bits ADD8, SUB8, @@ -250,12 +252,14 @@ static opform OP_FORMS[] = { { "XORPD", 0x660F57 }, { "CVTSI2SD", 0xF20F2A }, { "CVTSI2SS", 0xF30F2A }, - { "CVTSD2SI", 0xF20F2D }, + { "CVTTSD2SI", 0xF20F2C }, { "CVTSD2SS", 0xF20F5A }, { "CVTSS2SD", 0xF30F5A }, - { "CVTSS2SI", 0xF30F2D }, + { "CVTTSS2SI", 0xF30F2C }, { "STMXCSR", 0, LONG_RM(0x0FAE,3) }, { "LDMXCSR", 0, LONG_RM(0x0FAE,2) }, + { "STC", 0xF9 }, + { "CLC", 0xF8 }, // 8 bits, { "ADD8", 0, RM(0x00,3) }, { "SUB8", 0, 0x28 }, @@ -1163,7 +1167,10 @@ void hl_codegen_function( jit_ctx *jit ) { emit_mov(ctx, REG_ADD_OFFSET(REG_PTR(e->a),e->size_offs), e->b, e->mode); break; case PUSH: - if( IS_FLOAT(e->mode) ) BREAK(); + if( IS_FLOAT(e->mode) ) { + BREAK(); + break; + } if( IS_REG(e->a) && REG_VALUE(e->a) != 0 ) { emit_mov(ctx, RTMP, e->a, e->mode); emit_ext(ctx, _PUSH, RTMP, UNUSED, e->mode, 0); @@ -1298,6 +1305,33 @@ void hl_codegen_function( jit_ctx *jit ) { a = tmp; } EMIT(op,a,e->b,e->mode); + if( IS_FLOAT(e->mode) && e->size_offs != OJSGt && e->size_offs != OJNull && e->size_offs != OJNotNull ) { + // handle NaNs + int jnotnan = jump_near(ctx,JNParity); + switch( e->size_offs ) { + case OJSLt: + case OJNotLt: + // set CF=0, ZF=1 + EMIT(XOR,RTMP,RTMP,M_I32); + break; + case OJSGte: + case OJNotGte: + // set ZF=0, CF=1 + EMIT(XOR,RTMP,RTMP,M_I32); + EMIT(STC,UNUSED,UNUSED,0); + break; + case OJNotEq: + case OJEq: + // set ZF=0, CF=? + case OJSLte: + // set ZF=0, CF=0 + EMIT(TEST,R(RSP),R(RSP),M_PTR); + break; + default: + jit_assert(); + } + patch_jump_near(ctx,jnotnan); + } } break; case JCOND: @@ -1372,13 +1406,13 @@ void hl_codegen_function( jit_ctx *jit ) { case ID2(M_UI16,M_F32): case ID2(M_I32,M_F32): case ID2(M_PTR,M_F32): - op = CVTSS2SI; + op = CVTTSS2SI; break; case ID2(M_UI8,M_F64): case ID2(M_UI16,M_F64): case ID2(M_I32,M_F64): case ID2(M_PTR,M_F64): - op = CVTSD2SI; + op = CVTTSD2SI; break; case ID2(M_F32,M_F64): op = CVTSD2SS; From c6ea9162d40481820457d4022e204a34955e1cc3 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 3 May 2026 11:19:00 +0200 Subject: [PATCH 55/83] more fixes --- src/jit_emit.c | 45 ++++++++++++++++++++------------------------- src/jit_regs.c | 13 +++++++------ src/jit_x86_64.c | 6 ++---- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 28959c1f3..3d9a3912f 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -146,7 +146,7 @@ struct _emit_ctx { #define R(i) (ctx->vregs + (i)) #define LOAD(r) emit_load_reg(ctx, r) -#define STORE(r, v) emit_store_reg(ctx, r, v) +#define STORE(r, v) { ereg __v = (v); if( (r)->t->kind != HVOID ) emit_store_reg(ctx, r, __v); } #define LOAD_CONST(v, t) emit_load_const(ctx, (uint64)(v), t) #define LOAD_CONST_PTR(v) LOAD_CONST(v,&hlt_bytes) #define LOAD_MEM(v, offs, t) emit_load_mem(ctx, v, offs, t, t) @@ -621,16 +621,6 @@ static void seal_block( emit_ctx *ctx, emit_block *b ) { b->sealed = true; } -static ereg emit_phi( emit_ctx *ctx, ereg v1, ereg v2 ) { - unsigned char mode = GET_MODE(v1); - if( mode != GET_MODE(v2) ) jit_assert(); - tmp_phi *p = alloc_phi(ctx, ctx->current_block, NULL); - p->mode = mode; - phi_add_val(ctx, p, v1, blocks_get(ctx->blocks,ctx->current_block->id - 2)); - phi_add_val(ctx, p, v2, blocks_get(ctx->blocks,ctx->current_block->id - 1)); - return p->value; -} - static ereg emit_call_fid( emit_ctx *ctx, int findex, ereg *args, int nargs, emit_mode mode ) { einstr *e = emit_instr(ctx, CALL_FUN); e->mode = mode; @@ -647,7 +637,7 @@ static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int for(int i=0;ifunctions_ptrs[findex], args, count, dst->t)); + STORE(dst, emit_native_call(ctx, m->functions_ptrs[findex], args, count, dst->t)) else { ereg out = emit_call_fid(ctx,findex,args,count,hl_type_mode(dst->t)); if( out ) STORE(dst, out); @@ -753,16 +743,19 @@ static bool dyn_need_type( hl_type *t ) { return !(t->kind == HF32 || t->kind == HF64 || t->kind == HI64 || t->kind == HGUID); } -static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { +static void emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, vreg *dst ) { + hl_type *dt = dst->t; if( t->kind == HNULL && t->tparam->kind == dt->kind ) { emit_test(ctx, v, OJNotNull); int jnot = emit_jump(ctx, true); ereg v1 = LOAD_CONST(0,dt); + STORE(dst, v1); int jend = emit_jump(ctx, false); patch_jump(ctx, jnot); ereg v2 = LOAD_MEM(v,HDYN_VALUE,dt); + STORE(dst, v2); patch_jump(ctx, jend); - return emit_phi(ctx, v1, v2); + return; } bool need_dyn = dyn_need_type(dt); ereg st = emit_gen_size(ctx, ALLOC_STACK, HL_WSIZE); @@ -772,7 +765,7 @@ static ereg emit_dyn_cast( emit_ctx *ctx, ereg v, hl_type *t, hl_type *dt ) { args[1] = LOAD_CONST_PTR(t); if( need_dyn ) args[2] = LOAD_CONST_PTR(dt); ereg r = emit_native_call(ctx, get_dyncast(dt), args, need_dyn ? 3 : 2, dt); - return r; + STORE(dst, r); } static void emit_opcode( emit_ctx *ctx, hl_opcode *o ); @@ -991,8 +984,8 @@ void hl_emit_function( jit_ctx *jit ) { r->t = f->regs[i]; } - emit_gen(ctx,ENTER,UNUSED,UNUSED,M_NONE); emit_gen_size(ctx, BLOCK, 0); + emit_gen(ctx,ENTER,UNUSED,UNUSED,M_NONE); for(i=0;itype->fun->nargs;i++) { hl_type *t = f->type->fun->args[i]; STORE(R(i), emit_gen(ctx, LOAD_ARG, UNUSED, UNUSED, hl_type_mode(t))); @@ -1572,7 +1565,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[0] = LOAD(ra); args[1] = st; args[2] = LOAD_CONST(o->p3,&hlt_i32); - STORE(dst, emit_dyn_cast(ctx,emit_native_call(ctx,hl_dyn_call,args,3,dst->t),ra->t,dst->t)); + emit_dyn_cast(ctx,emit_native_call(ctx,hl_dyn_call,args,3,dst->t),ra->t,dst); } else { ereg r = LOAD(ra); ereg *args = get_tmp_args(ctx,o->p3+1); @@ -1585,13 +1578,14 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { for(i=0;ip3;i++) args[i+1] = LOAD(R(o->extra[i])); ereg v1 = emit_dyn_call(ctx,LOAD_MEM_PTR(r,HL_WSIZE),args,o->p3 + 1,dst->t); + STORE(dst, v1); int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); for(i=0;ip3;i++) args[i] = LOAD(R(o->extra[i])); ereg v2 = emit_dyn_call(ctx,LOAD_MEM_PTR(r,HL_WSIZE),args,o->p3,dst->t); + STORE(dst, v2); patch_jump(ctx, jend); - if( dst->t->kind != HVOID ) STORE(dst, emit_phi(ctx,v1,v2)); } break; case OStaticClosure: @@ -1626,6 +1620,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { emit_test(ctx, field, OJNull); int jidx = emit_jump(ctx, true); ereg v1 = LOAD_MEM(field,0,dst->t); + STORE(dst, v1); int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); bool need_type = dyn_need_type(dst->t); @@ -1634,8 +1629,8 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[1] = LOAD_CONST(ra->t->virt->fields[o->p3].hashed_name,&hlt_i32); if( need_type ) args[2] = LOAD_CONST_PTR(dst->t); ereg v2 = emit_native_call(ctx,get_dynget(dst->t),args,need_type?3:2,dst->t); + STORE(dst, v2); patch_jump(ctx, jend); - STORE(dst, emit_phi(ctx, v1, v2)); } break; default: @@ -1773,6 +1768,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { for(i=1;iextra[i])); ereg v1 = emit_dyn_call(ctx,fun,args,nargs,dst->t); + STORE(dst, v1); int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); @@ -1793,10 +1789,8 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[4] = edyn; ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 5, dst->t); - + STORE(dst, v2); patch_jump(ctx, jend); - - if( dst->t->kind != HVOID ) STORE(dst, emit_phi(ctx, v1, v2)); } break; default: @@ -1845,11 +1839,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { emit_test(ctx, r, OJNotNull); int jidx = emit_jump(ctx, true); ereg v1 = LOAD_CONST_PTR(&hlt_void); + STORE(dst,v1); int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); ereg v2 = LOAD_MEM_PTR(r,0); + STORE(dst,v2); patch_jump(ctx, jend); - STORE(dst, emit_phi(ctx, v1, v2)); } break; case OGetArray: @@ -1929,10 +1924,10 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[0] = LOAD_CONST_PTR(dst->t); args[1] = LOAD_CONST(o->p2,&hlt_i32); ereg en = emit_native_call(ctx, hl_alloc_enum, args, 2, dst->t); - STORE(dst, en); hl_enum_construct *c = &dst->t->tenum->constructs[o->p2]; for(int i=0;inparams;i++) STORE_MEM(en, c->offsets[i], LOAD(R(o->extra[i]))); + STORE(dst, en); } break; case OEnumAlloc: @@ -2002,7 +1997,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } break; case OSafeCast: - STORE(dst, emit_dyn_cast(ctx, LOAD(ra), ra->t, dst->t)); + emit_dyn_cast(ctx, LOAD(ra), ra->t, dst); break; case ODynGet: { diff --git a/src/jit_regs.c b/src/jit_regs.c index 447ee2053..bd434e0cf 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -182,10 +182,12 @@ static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { return true; } } + value_info *first = NULL; for(int i=0;inscratchs;i++) { ereg r = cfg->scratch[i]; for_iter(values,v2,ctx->scratch) { if( v2->reg == r ) { + if( first == NULL ) first = v2; r = UNUSED; break; } @@ -200,10 +202,9 @@ static bool regs_alloc_reg( regs_ctx *ctx, value_info *v ) { return false; } // free the oldest scratch reg - value_info *v2 = values_first(ctx->scratch); - if( !v2 ) jit_assert(); - v->reg = v2->reg; - spill(ctx, v2); + if( !first ) jit_assert(); + v->reg = first->reg; + spill(ctx, first); return true; } @@ -371,8 +372,8 @@ static void regs_assign_regs( regs_ctx *ctx ) { einstr e = jit->instrs[cur_op]; value_info *write = NULL; # ifdef HL_DEBUG - int uid = (jit->fun->findex << 16) | cur_op; - __ignore(&uid); + int eid = (jit->fun->findex << 16) | cur_op; + __ignore(&eid); # endif ctx->cur_op = cur_op; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 34fd25e25..792b2a49d 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1381,8 +1381,6 @@ void hl_codegen_function( jit_ctx *jit ) { } break; case CONV_UNSIGNED: - BREAK(); - break; case CONV: { emit_mode in_mode = e->size_offs; @@ -1446,10 +1444,10 @@ void hl_codegen_function( jit_ctx *jit ) { break; } if( IS_REG(out) || op == _MOV ) - EMIT(op,out,r,e->mode); + EMIT(op,out,r,e->op == CONV_UNSIGNED ? M_PTR : e->mode); else { ereg r2 = get_tmp(e->mode); - EMIT(op,r2,r,e->mode); + EMIT(op,r2,r,e->op == CONV_UNSIGNED ? M_PTR : e->mode); emit_mov(ctx,out,r2,e->mode); } } From 9b7e452cfa8bf401b429946ac95c64a84ba2bc5c Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 8 May 2026 09:24:20 +0200 Subject: [PATCH 56/83] more fixes --- src/jit_emit.c | 32 ++++++++++++++++---------------- src/jit_x86_64.c | 30 +++++++++++++++++------------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 3d9a3912f..40e665a89 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -1124,8 +1124,6 @@ static void prepare_loop_block( emit_ctx *ctx ) { } } -#define HL_DYN_VALUE 8 - static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type *bt, ereg b, int offset ) { if( at->kind == HDYN || bt->kind == HDYN || at->kind == HFUN || bt->kind == HFUN ) { ereg args[2] = { a, b }; @@ -1159,7 +1157,7 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type emit_test(ctx,b,OJNull); int jb = emit_jump(ctx,true); hl_type *vt = at->tparam; - emit_cmp(ctx, LOAD_MEM(a,HL_DYN_VALUE,vt), LOAD_MEM(b,HL_DYN_VALUE,vt), OJEq); + emit_cmp(ctx, LOAD_MEM(a,HDYN_VALUE,vt), LOAD_MEM(b,HDYN_VALUE,vt), OJEq); register_block_jump(ctx,offset,true); patch_jump(ctx,ja); patch_jump(ctx,jb); @@ -1174,7 +1172,7 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type register_block_jump(ctx,offset,true); split_block(ctx); hl_type *vt = at->tparam; - emit_cmp(ctx, LOAD_MEM(a,HL_DYN_VALUE,vt), LOAD_MEM(b,HL_DYN_VALUE,vt), OJNull); + emit_cmp(ctx, LOAD_MEM(a,HDYN_VALUE,vt), LOAD_MEM(b,HDYN_VALUE,vt), OJNull); add_jump_target(ctx, 0); int jcmp = emit_jump(ctx,true); register_block_jump(ctx,offset,true); @@ -1223,7 +1221,7 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type ereg va = LOAD_MEM_PTR(a,HL_WSIZE); emit_test(ctx, va, OJNull); int jva = emit_jump(ctx, true); - ereg vb = LOAD_MEM_PTR(a,HL_WSIZE); + ereg vb = LOAD_MEM_PTR(b,HL_WSIZE); emit_cmp(ctx, va, vb, OJEq); register_block_jump(ctx,offset,true); split_block(ctx); @@ -1232,9 +1230,8 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type patch_jump(ctx,jva); } else if( op == OJNotEq ) { // if( a != b && (!a || !b || !a->value || a->value != b->value) ) goto - emit_cmp(ctx, a, b, OJNotEq); - register_block_jump(ctx,offset,true); - split_block(ctx); + emit_cmp(ctx, a, b, OJEq); + int jeq1 = emit_jump(ctx, true); emit_test(ctx, a, OJNull); int ja = emit_jump(ctx, true); emit_test(ctx, b, OJNull); @@ -1242,16 +1239,17 @@ static void emit_jump_dyn( emit_ctx *ctx, hl_op op, hl_type *at, ereg a, hl_type ereg va = LOAD_MEM_PTR(a,HL_WSIZE); emit_test(ctx, va, OJNull); int jva = emit_jump(ctx, true); - ereg vb = LOAD_MEM_PTR(a,HL_WSIZE); + ereg vb = LOAD_MEM_PTR(b,HL_WSIZE); emit_cmp(ctx, va, vb, OJEq); - int jeq = emit_jump(ctx, true); + int jeq2 = emit_jump(ctx, true); split_block(ctx); patch_jump(ctx,ja); patch_jump(ctx,jb); patch_jump(ctx,jva); - register_block_jump(ctx,offset,true); + register_block_jump(ctx,offset,false); split_block(ctx); - patch_jump(ctx,jeq); + patch_jump(ctx,jeq1); + patch_jump(ctx,jeq2); } else jit_assert(); } @@ -1778,8 +1776,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { for(i=0;iextra[i+1]))); bool need_dyn = !hl_is_ptr(dst->t) && dst->t->kind != HVOID; - int dyn_size = sizeof(vdynamic); - ereg edyn = need_dyn ? emit_gen_size(ctx, ALLOC_STACK, dyn_size) : LOAD_CONST_PTR(NULL); + ereg edyn = need_dyn ? emit_gen_size(ctx, ALLOC_STACK, sizeof(vdynamic)) : LOAD_CONST_PTR(NULL); args = get_tmp_args(ctx, 5); args[0] = LOAD_MEM_PTR(obj,HL_WSIZE); @@ -1788,8 +1785,11 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { args[3] = eargs; args[4] = edyn; - ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 5, dst->t); - STORE(dst, v2); + ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 5, &hlt_bytes); + if( need_dyn ) + STORE(dst, LOAD_MEM(edyn,HDYN_VALUE,dst->t)) + else + STORE(dst, v2); patch_jump(ctx, jend); } break; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 792b2a49d..324d09061 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -739,21 +739,21 @@ static void emit_div_mod( code_ctx *ctx, hl_op op, ereg out, ereg a, ereg b, emi ereg bas = R(RAX), div = R(RDX); if( out != bas ) EMIT(_PUSH,bas,UNUSED,M_PTR); if( out != div ) EMIT(_PUSH,div,UNUSED,M_PTR); - if( b == bas || b == div || !IS_REG(div) ) { + if( b == bas || b == div || !IS_REG(b) ) { EMIT(_MOV,RTMP,b,mode); b = RTMP; } if( a != bas ) EMIT(_MOV,bas,a,mode); // check for div = 0 - EMIT(_TEST,bas,bas,mode); + EMIT(_TEST,b,b,mode); int jz = jump_near(ctx,JZero); int jz1 = 0; // Prevent MIN/-1 overflow exception // OSMod: r = (b == 0 || b == -1) ? 0 : a % b // OSDiv: r = (b == 0 || b == -1) ? a * b : a / b if( op == OSMod || op == OSDiv ) { - EMIT(_CMP,bas,MK_CONST(-1),mode); + EMIT(_CMP,b,MK_CONST(-1),mode); jz1 = jump_near(ctx,JZero); } bool unsign = op == OUDiv || op == OUMod; @@ -971,11 +971,12 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { return; } + bool use_offs = offs != 0 || (e.a&7) == RBP; REX64(out,e.a,e.b); B(0x8D); - MOD_RM(offs == 0 ? 0 : 1,out,4); + MOD_RM(use_offs ? 1 : 0,out,4); SIB(mult,e.b,e.a); - if( offs != 0 ) { + if( use_offs ) { if( !IS_SBYTE(offs) ) jit_assert(); B(offs); } @@ -1168,18 +1169,21 @@ void hl_codegen_function( jit_ctx *jit ) { break; case PUSH: if( IS_FLOAT(e->mode) ) { - BREAK(); - break; - } - if( IS_REG(e->a) && REG_VALUE(e->a) != 0 ) { + if( !IS_REG(e->a) ) + EMIT(_PUSH,e->a,UNUSED,M_PTR); + else { + EMIT(SUB,R(RSP),MK_CONST(8),M_PTR); + EMIT(e->mode == M_F32 ? MOVSS : MOVSD,REG_PTR(R(RSP)),e->a,e->mode); + } + } else if( IS_REG(e->a) && REG_VALUE(e->a) != 0 ) { emit_mov(ctx, RTMP, e->a, e->mode); - emit_ext(ctx, _PUSH, RTMP, UNUSED, e->mode, 0); + EMIT(_PUSH, RTMP, UNUSED, M_PTR); } else - emit_ext(ctx, _PUSH, e->a, UNUSED, e->mode, 0); + EMIT(_PUSH, e->a, UNUSED, M_PTR); break; case POP: - if( IS_FLOAT(e->mode) ) BREAK(); - emit_ext(ctx, _POP, e->a, UNUSED, e->mode, 0); + if( IS_FLOAT(e->mode) ) jit_assert(); + EMIT(_POP, e->a, UNUSED, M_PTR); break; case PUSH_CONST: if( e->mode != M_PTR ) jit_assert(); From 00c66c869cb116740694459e921954583431afca Mon Sep 17 00:00:00 2001 From: ncannasse Date: Fri, 8 May 2026 15:44:00 +0200 Subject: [PATCH 57/83] more fixes & started trap phi handling --- src/jit_emit.c | 62 ++++++++++++++++++++++++++++++++---------------- src/jit_x86_64.c | 20 ++++++++++++++-- 2 files changed, 60 insertions(+), 22 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 40e665a89..fba9b2a04 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -109,6 +109,12 @@ struct _tmp_phi { linked_inf *ref_blocks; }; +typedef struct { + ereg stack; + int calls_count; + int target; +} trap_inf; + struct _emit_ctx { hl_module *mod; hl_function *fun; @@ -127,7 +133,7 @@ struct _emit_ctx { bool flushed; ereg tmp_args[MAX_TMP_ARGS]; - ereg traps[MAX_TRAPS]; + trap_inf traps[MAX_TRAPS]; int *pos_map; int pos_map_size; int trap_count; @@ -135,6 +141,7 @@ struct _emit_ctx { int_arr args_data; int_arr jump_regs; int_arr values; + int_arr trap_calls; blocks blocks; emit_block *current_block; @@ -490,7 +497,16 @@ static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { store_block_var(ctx,ctx->current_block,to,v); } +static void before_call( emit_ctx *ctx ) { + if( ctx->trap_count > 0 ) { + split_block(ctx); + int_arr_add(ctx->trap_calls,ctx->current_block->id); + } +} + static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int nargs, hl_type *ret ) { + if( native_ptr != hl_get_thread ) + before_call(ctx); einstr *e = emit_instr(ctx, CALL_PTR); e->mode = (unsigned char)(ret ? hl_type_mode(ret) : M_NORET); e->value = (int_val)native_ptr; @@ -499,6 +515,7 @@ static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int } static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_type *ret ) { + before_call(ctx); einstr *e = emit_instr(ctx, CALL_REG); e->mode = hl_type_mode(ret); e->a = f; @@ -564,8 +581,8 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { store_block_var(ctx,b,p->r,same); l = l->next; } - emit_block *bsame = ereg_find(p->vals,same); for_iter(phi,p2,p->ref_phis) { + emit_block *bsame = ereg_find(p2->vals,p->value); phi_remove_val(ctx,p2,p->value); phi_add_val(ctx,p2,same,bsame); } @@ -575,7 +592,7 @@ static ereg optimize_phi_rec( emit_ctx *ctx, tmp_phi *p ) { for(int i=0;iphi_depth--; - emit_debug("%sPHI-OPT-DONE %s = %s:#%d\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(same,p->mode), bsame->id); + emit_debug("%sPHI-OPT-DONE %s = %s\n", phi_prefix(ctx), val_str(p->value,p->mode), val_str(same,p->mode)); return optimize_phi_rec(ctx,p); } @@ -622,6 +639,7 @@ static void seal_block( emit_ctx *ctx, emit_block *b ) { } static ereg emit_call_fid( emit_ctx *ctx, int findex, ereg *args, int nargs, emit_mode mode ) { + before_call(ctx); einstr *e = emit_instr(ctx, CALL_FUN); e->mode = mode; e->a = findex; @@ -954,6 +972,7 @@ void hl_emit_function( jit_ctx *jit ) { ctx->trap_count = 0; ctx->phi_count = 0; ctx->flushed = false; + int_arr_free(&ctx->trap_calls); int_arr_free(&ctx->args_data); int_arr_free(&ctx->jump_regs); int_arr_free(&ctx->values); @@ -1008,6 +1027,13 @@ void hl_emit_function( jit_ctx *jit ) { if( b ) block_add_pred(ctx, ctx->current_block, b); ctx->arrival_points = ctx->arrival_points->next; } + if( ctx->trap_count && ctx->traps[ctx->trap_count-1].target == ctx->op_pos ) { + ctx->trap_count--; + trap_inf *inf = &ctx->traps[ctx->trap_count]; + //for(int i=inf->calls_count;itrap_calls);i++) + // block_add_pred(ctx, ctx->current_block, ctx->blocks.values[int_arr_get(ctx->trap_calls,i)]); + ctx->trap_calls.cur = inf->calls_count; + } } emit_opcode(ctx,f->ops + op_pos); } @@ -1773,8 +1799,13 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { nargs = o->p3 - 1; ereg eargs = nargs == 0 ? LOAD_CONST_PTR(NULL) : emit_gen_size(ctx, ALLOC_STACK, nargs * HL_WSIZE); - for(i=0;iextra[i+1]))); + for(i=0;iextra[i+1]); + if( hl_is_ptr(r->t) ) + STORE_MEM(eargs,i*HL_WSIZE,LOAD(r)); + else + STORE_MEM(eargs,i*HL_WSIZE,emit_gen(ctx, ADDRESS, LOAD(r), UNUSED, M_PTR)); + } bool need_dyn = !hl_is_ptr(dst->t) && dst->t->kind != HVOID; ereg edyn = need_dyn ? emit_gen_size(ctx, ALLOC_STACK, sizeof(vdynamic)) : LOAD_CONST_PTR(NULL); @@ -2102,13 +2133,16 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { patch_jump(ctx, jskip); if( ctx->trap_count == MAX_TRAPS ) jit_error("Too many try/catch depth"); - ctx->traps[ctx->trap_count++] = st; + trap_inf *inf = &ctx->traps[ctx->trap_count++]; + inf->stack = st; + inf->target = o->p2 + 1 + ctx->op_pos; + inf->calls_count = int_arr_count(ctx->trap_calls); } break; case OEndTrap: { if( ctx->trap_count == 0 ) jit_assert(); - ereg st = ctx->traps[ctx->trap_count - 1]; + ereg st = ctx->traps[ctx->trap_count - 1].stack; ereg thread, current_addr; static hl_thread_info *tinf = NULL; @@ -2123,18 +2157,6 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE_MEM(current_addr, 0, LOAD_MEM_PTR(st,(int)(int_val)&trap->prev)); -# ifdef HL_WIN - // erase eip (prevent false positive in exception stack) - { - _JUMP_BUFFER *b = NULL; -# ifdef HL_64 - int offset = (int)(int_val)&(b->Rip); -# else - int offset = (int)(int_val)&(b->Eip); -# endif - STORE_MEM(st, offset, LOAD_CONST_PTR(NULL)); - } -# endif emit_instr(ctx, CATCH); } break; @@ -2185,7 +2207,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; } } - emit_gen(ctx, PREFETCH, r, UNUSED, o->p3); + emit_gen_ext(ctx, PREFETCH, r, UNUSED, M_NONE, o->p3); } break; case OAsm: diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 324d09061..7097b80a4 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -248,7 +248,7 @@ static opform OP_FORMS[] = { { "SUBSS", 0xF30F5C }, { "MULSS", 0xF30F59 }, { "DIVSS", 0xF30F5E }, - { "XORPS", 0x0F57 }, + { "XORPS", LONG_OP(0x0F57) }, { "XORPD", 0x660F57 }, { "CVTSI2SD", 0xF20F2A }, { "CVTSI2SS", 0xF30F2A }, @@ -1475,7 +1475,23 @@ void hl_codegen_function( jit_ctx *jit ) { EMIT(SUB,R(RSP),MK_CONST(-e->size_offs),M_PTR); break; case PREFETCH: - BREAK(); + { + CpuOp op; + switch( e->size_offs ) { + case 0: op = PREFETCHT0; break; + case 1: op = PREFETCHT1; break; + case 2: op = PREFETCHT2; break; + case 3: op = PREFETCHNTA; break; + case 4: op = PREFETCHW; break; + default: jit_assert(); + } + ereg a = e->a; + if( !IS_REG(e->a) ) { + emit_mov(ctx,RTMP,e->a,M_PTR); + a = RTMP; + } + EMIT(op,REG_PTR(a),UNUSED,M_PTR); + } break; case CMOV: { From 33abaf80e7b55eb5116bfedbce79409880b837c5 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 11:26:51 +0200 Subject: [PATCH 58/83] more fixes --- src/jit_emit.c | 6 +++--- src/jit_regs.c | 8 ++++---- src/jit_x86_64.c | 3 ++- src/std/fun.c | 4 ---- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index fba9b2a04..6a80a8eff 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -1889,12 +1889,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hl_runtime_obj *rt = hl_get_obj_rt(dst->t); osize = rt->size; // a mem offset into it } - ereg pos = OFFSET(LOAD(ra), LOAD(rb), osize, 0); + ereg pos = (osize <= 8 && ((osize - 1) & osize) == 0) ? OFFSET(LOAD(ra), LOAD(rb), osize, 0) : OFFSET(LOAD(ra), emit_gen_ext(ctx,BINOP,LOAD(rb),MK_CONST(osize),M_I32,OMul),1,0); ereg val = isPtr ? LOAD_MEM_PTR(pos,0) : pos; STORE(dst, val); } else { ereg pos = OFFSET(LOAD(ra), LOAD(rb), hl_type_size(dst->t), sizeof(varray)); - STORE(dst, LOAD_MEM_PTR(pos,0)); + STORE(dst, LOAD_MEM(pos,0,dst->t)); } } break; @@ -1909,7 +1909,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hl_runtime_obj *rt = hl_get_obj_rt(rb->t); osize = rt->size; } - ereg pos = OFFSET(LOAD(dst), LOAD(ra), osize, 0); + ereg pos = (osize <= 8 && ((osize - 1) & osize) == 0) ? OFFSET(LOAD(dst), LOAD(ra), osize, 0) : OFFSET(LOAD(dst), emit_gen_ext(ctx,BINOP,LOAD(ra),MK_CONST(osize),M_I32,OMul),1,0); emit_store_size(ctx, pos, 0, LOAD(rb), 0, osize); } else { ereg pos = OFFSET(LOAD(dst), LOAD(ra), hl_type_size(dst->t), sizeof(varray)); diff --git a/src/jit_regs.c b/src/jit_regs.c index bd434e0cf..f281bd2be 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -661,12 +661,12 @@ static void regs_emit_instrs( regs_ctx *ctx ) { { EMIT(PUSH,jit->cfg.stack_pos,UNUSED,M_PTR); regs_emit_mov(ctx,jit->cfg.stack_pos,jit->cfg.stack_reg,M_PTR); + if( stack_offset ) + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-stack_offset); for(int i=0;ipersists_uses[0];i++) EMIT(PUSH,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR); for(int i=0;ipersists_uses[1];i++) EMIT(PUSH,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); - if( stack_offset ) - regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-stack_offset); } break; case JCOND: @@ -688,12 +688,12 @@ static void regs_emit_instrs( regs_ctx *ctx ) { if( e.a != ret ) regs_emit_mov(ctx, ret, e.a, e.mode); } - if( stack_offset ) - regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); for(int i=ctx->persists_uses[1]-1;i>=0;i--) EMIT(POP,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); for(int i=ctx->persists_uses[0]-1;i>=0;i--) EMIT(POP,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR); + if( stack_offset ) + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); EMIT(POP,jit->cfg.stack_pos,UNUSED,M_PTR); EMIT(RET,UNUSED,UNUSED,M_NONE); break; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 7097b80a4..bf172b690 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -668,7 +668,7 @@ static void emit_jump( code_ctx *ctx, int mode, int offset ) { # ifdef GEN_DEBUG op_mult = 16; // additional debug info per op # else - op_mult = 8; + op_mult = 10; # endif if( IS_SBYTE(offset*op_mult) ) { // assume it's ok to use short jump @@ -945,6 +945,7 @@ static void emit_lea( code_ctx *ctx, ereg out, einstr *_e ) { int mult = e.size_offs & 0xFF; int offs = e.size_offs >> 8; + if( mult != 0 && (mult < 0 || mult > 8 || (mult & (mult - 1)) != 0) ) jit_assert(); if( IS_REG(e.a) ) offs += REG_VALUE(e.a); diff --git a/src/std/fun.c b/src/std/fun.c index ee611e8a8..fafa8177e 100644 --- a/src/std/fun.c +++ b/src/std/fun.c @@ -221,10 +221,6 @@ HL_PRIM vdynamic *hl_dyn_call( vclosure *c, vdynamic **args, int nargs ) { tmp.args[i+1] = args[i]; c = &ctmp; } else { - if( nargs == 0 && c->t->fun->ret->kind == HVOID ) { - ((void (*)(void))c->fun)(); - return NULL; - } for(i=0;i Date: Sat, 9 May 2026 13:06:35 +0200 Subject: [PATCH 59/83] minor fixes, passing haxe unit tests --- src/jit.c | 3 +++ src/jit_emit.c | 36 ++++++++++++++---------------------- src/jit_x86_64.c | 2 +- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/jit.c b/src/jit.c index 767c60a99..ddf9a187d 100644 --- a/src/jit.c +++ b/src/jit.c @@ -335,6 +335,9 @@ void *hl_jit_code( jit_ctx *ctx, hl_module *m, int *codesize, hl_debug_infos **d arg_fp_count = ctx->cfg.floats.nargs; call_jit_c2hl = ctx->final_code + ctx->code_funs.c2hl; call_jit_hl2c = ctx->final_code + ctx->code_funs.hl2c; +# ifdef WIN64_UNWIND_TABLES + ctx->mod->unwind_table_size = ctx->fdef_index; +# endif hl_setup.get_wrapper = default_wrapper; hl_setup.static_call = callback_c2hl; return code; diff --git a/src/jit_emit.c b/src/jit_emit.c index 6a80a8eff..48f125cc9 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -36,6 +36,7 @@ int hl_emit_mode_sizes[] = {0,1,2,4,HL_WSIZE,8,4,0,0}; typedef struct { hl_type *t; int id; + ereg stored; } vreg; #define MAX_TMP_ARGS 32 @@ -111,7 +112,6 @@ struct _tmp_phi { typedef struct { ereg stack; - int calls_count; int target; } trap_inf; @@ -141,7 +141,6 @@ struct _emit_ctx { int_arr args_data; int_arr jump_regs; int_arr values; - int_arr trap_calls; blocks blocks; emit_block *current_block; @@ -153,7 +152,7 @@ struct _emit_ctx { #define R(i) (ctx->vregs + (i)) #define LOAD(r) emit_load_reg(ctx, r) -#define STORE(r, v) { ereg __v = (v); if( (r)->t->kind != HVOID ) emit_store_reg(ctx, r, __v); } +#define STORE(r, v) emit_store_reg(ctx, r, v) #define LOAD_CONST(v, t) emit_load_const(ctx, (uint64)(v), t) #define LOAD_CONST_PTR(v) LOAD_CONST(v,&hlt_bytes) #define LOAD_MEM(v, offs, t) emit_load_mem(ctx, v, offs, t, t) @@ -495,18 +494,16 @@ static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { if( to->t->kind == HVOID ) return; if( IS_NULL(v) ) jit_assert(); store_block_var(ctx,ctx->current_block,to,v); -} - -static void before_call( emit_ctx *ctx ) { if( ctx->trap_count > 0 ) { - split_block(ctx); - int_arr_add(ctx->trap_calls,ctx->current_block->id); + // if the value was written before the trap, let's update it + if( !IS_NULL(to->stored) ) + STORE_MEM(emit_gen(ctx,ADDRESS,to->stored,UNUSED,M_PTR), 0, v); + } else { + to->stored = v; } } static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int nargs, hl_type *ret ) { - if( native_ptr != hl_get_thread ) - before_call(ctx); einstr *e = emit_instr(ctx, CALL_PTR); e->mode = (unsigned char)(ret ? hl_type_mode(ret) : M_NORET); e->value = (int_val)native_ptr; @@ -515,7 +512,6 @@ static ereg emit_native_call( emit_ctx *ctx, void *native_ptr, ereg args[], int } static ereg emit_dyn_call( emit_ctx *ctx, ereg f, ereg args[], int nargs, hl_type *ret ) { - before_call(ctx); einstr *e = emit_instr(ctx, CALL_REG); e->mode = hl_type_mode(ret); e->a = f; @@ -639,7 +635,6 @@ static void seal_block( emit_ctx *ctx, emit_block *b ) { } static ereg emit_call_fid( emit_ctx *ctx, int findex, ereg *args, int nargs, emit_mode mode ) { - before_call(ctx); einstr *e = emit_instr(ctx, CALL_FUN); e->mode = mode; e->a = findex; @@ -655,7 +650,7 @@ static void emit_call_fun( emit_ctx *ctx, vreg *dst, int findex, int count, int for(int i=0;ifunctions_ptrs[findex], args, count, dst->t)) + STORE(dst, emit_native_call(ctx, m->functions_ptrs[findex], args, count, dst->t)); else { ereg out = emit_call_fid(ctx,findex,args,count,hl_type_mode(dst->t)); if( out ) STORE(dst, out); @@ -972,7 +967,6 @@ void hl_emit_function( jit_ctx *jit ) { ctx->trap_count = 0; ctx->phi_count = 0; ctx->flushed = false; - int_arr_free(&ctx->trap_calls); int_arr_free(&ctx->args_data); int_arr_free(&ctx->jump_regs); int_arr_free(&ctx->values); @@ -1001,6 +995,7 @@ void hl_emit_function( jit_ctx *jit ) { for(i=0;inregs;i++) { vreg *r = R(i); r->t = f->regs[i]; + r->stored = UNUSED; } emit_gen_size(ctx, BLOCK, 0); @@ -1027,16 +1022,14 @@ void hl_emit_function( jit_ctx *jit ) { if( b ) block_add_pred(ctx, ctx->current_block, b); ctx->arrival_points = ctx->arrival_points->next; } - if( ctx->trap_count && ctx->traps[ctx->trap_count-1].target == ctx->op_pos ) { + if( ctx->trap_count && ctx->traps[ctx->trap_count-1].target == ctx->op_pos ) ctx->trap_count--; - trap_inf *inf = &ctx->traps[ctx->trap_count]; - //for(int i=inf->calls_count;itrap_calls);i++) - // block_add_pred(ctx, ctx->current_block, ctx->blocks.values[int_arr_get(ctx->trap_calls,i)]); - ctx->trap_calls.cur = inf->calls_count; - } } emit_opcode(ctx,f->ops + op_pos); } + // emit a break if we're not supposed to reach here : will fix RtlUnwind on windows too. + if( f->nops == 0 || f->ops[f->nops-1].op != ORet ) + BREAK(); if( ctx->arrival_points ) jit_assert(); @@ -1818,7 +1811,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { ereg v2 = emit_native_call(ctx, hl_dyn_call_obj, args, 5, &hlt_bytes); if( need_dyn ) - STORE(dst, LOAD_MEM(edyn,HDYN_VALUE,dst->t)) + STORE(dst, LOAD_MEM(edyn,HDYN_VALUE,dst->t)); else STORE(dst, v2); patch_jump(ctx, jend); @@ -2136,7 +2129,6 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { trap_inf *inf = &ctx->traps[ctx->trap_count++]; inf->stack = st; inf->target = o->p2 + 1 + ctx->op_pos; - inf->calls_count = int_arr_count(ctx->trap_calls); } break; case OEndTrap: diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index bf172b690..72ba7e686 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1610,7 +1610,7 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_POP,R(RBP),UNUSED,M_PTR); EMIT(_RET,UNUSED,UNUSED,M_NONE); - flush_function(ctx, ctx->null_field_pos); + flush_function(ctx, jit->code_funs.c2hl); // generate hl2c stub jit->code_funs.hl2c = jit->out_pos + byte_count(ctx->code); From ea75b910db23a3e3d7ff6f289958a0616ce264b6 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 14:43:54 +0200 Subject: [PATCH 60/83] bump version --- src/hl.h | 2 +- src/main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hl.h b/src/hl.h index d75b98fe9..ae816580a 100644 --- a/src/hl.h +++ b/src/hl.h @@ -27,7 +27,7 @@ https://github.com/HaxeFoundation/hashlink/wiki/ **/ -#define HL_VERSION 0x011000 +#define HL_VERSION 0x020000 #if defined(_WIN32) # define HL_WIN diff --git a/src/main.c b/src/main.c index 60c4db77e..70f66479c 100644 --- a/src/main.c +++ b/src/main.c @@ -259,7 +259,7 @@ int main(int argc, pchar *argv[]) { file = PSTR("hlboot.dat"); fchk = pfopen(file,"rb"); if( fchk == NULL ) { - printf("HL/JIT %d.%d.%d (c)2015-2025 Haxe Foundation\n Usage : hl [--debug ] [--debug-wait] \n",HL_VERSION>>16,(HL_VERSION>>8)&0xFF,HL_VERSION&0xFF); + printf("HL/JIT %d.%d.%d (c)2015-2026 Haxe Foundation\n Usage : hl [--debug ] [--debug-wait] \n",HL_VERSION>>16,(HL_VERSION>>8)&0xFF,HL_VERSION&0xFF); return 1; } fclose(fchk); From 1b79402e49a3b7cc0a12ba2a8996df482cb8caa1 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 14:44:28 +0200 Subject: [PATCH 61/83] fix for windows rtl & some other fixes --- src/jit.h | 1 + src/jit_dump.c | 1 + src/jit_regs.c | 11 ++++++++++- src/jit_x86_64.c | 6 ++++-- src/module.c | 12 ++++++++---- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/jit.h b/src/jit.h index 22f17f884..78a8d3de0 100644 --- a/src/jit.h +++ b/src/jit.h @@ -60,6 +60,7 @@ typedef enum { STACK_OFFS, CATCH, ADDRESS, + NOP, } emit_op; typedef enum { diff --git a/src/jit_dump.c b/src/jit_dump.c index d973a6b55..8ce4ffa24 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -56,6 +56,7 @@ static const char *op_names[] = { "stack", "catch", "address", + "nop" }; bool hl_jit_dump_bin = false; diff --git a/src/jit_regs.c b/src/jit_regs.c index f281bd2be..016975280 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -692,8 +692,17 @@ static void regs_emit_instrs( regs_ctx *ctx ) { EMIT(POP,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); for(int i=ctx->persists_uses[0]-1;i>=0;i--) EMIT(POP,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR); - if( stack_offset ) + if( stack_offset ) { +# ifdef WIN64_UNWIND_TABLES + // if we have our stack offset just after a call, the unwind algorithm + // will subtract and create invalid stack frame. this is because we do + // not register the stack offset in our unwind table so all functions + // can share the same definition + if( cur_op && IS_CALL(ctx->instrs[cur_op-1].op) ) + EMIT(NOP,UNUSED,UNUSED,M_NONE); +# endif regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); + } EMIT(POP,jit->cfg.stack_pos,UNUSED,M_PTR); EMIT(RET,UNUSED,UNUSED,M_NONE); break; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 72ba7e686..ee53bc7e1 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -94,7 +94,6 @@ typedef enum { XOR, _CMP, _TEST, - NOP, SHL, SHR, SAR, @@ -221,7 +220,6 @@ static opform OP_FORMS[] = { { "XOR", 0x33, 0x31, RM(0x81,6), RM(0x83,6) }, { "CMP", 0x3B, 0x39, RM(0x81,7), RM(0x83,7) }, { "TEST", 0x85, 0x85/*SWP?*/, RM(0xF7,0) }, - { "NOP", 0x90 }, { "SHL", RM(0xD3,4), 0, 0, RM(0xC1,4) }, { "SHR", RM(0xD3,5), 0, 0, RM(0xC1,5) }, { "SAR", RM(0xD3,7), 0, 0, RM(0xC1,7) }, @@ -1509,6 +1507,9 @@ void hl_codegen_function( jit_ctx *jit ) { case CXCHG: BREAK(); break; + case NOP: + emit_nop(ctx,1); + break; default: jit_assert(); break; @@ -1599,6 +1600,7 @@ void hl_codegen_init( jit_ctx *jit ) { EMIT(_TEST,nargs,nargs,M_I32); int pos = jump_near(ctx,JZero); EMIT(_PUSH,MK_ADDR(vargs,0),UNUSED,M_PTR); + EMIT(SUB,vargs,MK_CONST(HL_WSIZE),M_PTR); EMIT(DEC,nargs,UNUSED,M_I32); jump_near(ctx,-begin); patch_jump_near(ctx,pos); diff --git a/src/module.c b/src/module.c index bf18b287d..549c4c41d 100644 --- a/src/module.c +++ b/src/module.c @@ -35,6 +35,10 @@ EXTERN_C IMAGE_DOS_HEADER __ImageBase; #define HOT_RELOAD_EXTRA_GLOBALS 4096 +#ifdef HL_DEBUG +# define ALLOW_DUMP +#endif + HL_API void hl_prim_not_loaded( const uchar *err ); static hl_module **cur_modules = NULL; @@ -714,7 +718,7 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { if( ctx == NULL ) return 0; hl_jit_init(ctx, m); -# ifdef HL_DEBUG +# ifdef ALLOW_DUMP bool dump = false; int filter = -1; for(i=0;icode->nfunctions;i++) { hl_function *f = m->code->functions + i; -# ifdef HL_DEBUG +# ifdef ALLOW_DUMP if( filter >= 0 && filter != f->findex ) continue; # endif int fpos = hl_jit_function(ctx, m, f); @@ -747,12 +751,12 @@ int hl_module_init( hl_module *m, h_bool hot_reload ) { return 0; } m->functions_ptrs[f->findex] = (void*)(int_val)fpos; -# ifdef HL_DEBUG +# ifdef ALLOW_DUMP if( dump ) hl_emit_dump(ctx); # endif } m->jit_code = hl_jit_code(ctx, m, &m->codesize, &m->jit_debug, NULL); -# ifdef HL_DEBUG +# ifdef ALLOW_DUMP if( filter >= 0 ) exit(0); # endif for(i=0;icode->nfunctions;i++) { From ddd0cb8d28824788fe95735244b3f861edc296a5 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 16:13:41 +0200 Subject: [PATCH 62/83] fixes stack windows --- src/jit_regs.c | 27 ++++++++++++++++++--------- src/jit_x86_64.c | 6 +++--- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/jit_regs.c b/src/jit_regs.c index 016975280..210cb7a58 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -480,6 +480,12 @@ static void flush_movs( regs_ctx *ctx, bool cond ) { bool cycle = true; for(int k=0;kstack_size; int push_size = HL_WSIZE * 2 + ctx->stack_offset; // RIP + RBP save - if( IS_WINCALL64 && ctx->has_direct_call ) stack_offset += 0x20; // reserve if( jit->cfg.stack_align ) { int align = (stack_offset + push_size) % jit->cfg.stack_align; if( align ) stack_offset += jit->cfg.stack_align - align; @@ -667,6 +672,8 @@ static void regs_emit_instrs( regs_ctx *ctx ) { EMIT(PUSH,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR); for(int i=0;ipersists_uses[1];i++) EMIT(PUSH,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); + if( IS_WINCALL64 && ctx->has_direct_call ) + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,-0x20); } break; case JCOND: @@ -688,19 +695,21 @@ static void regs_emit_instrs( regs_ctx *ctx ) { if( e.a != ret ) regs_emit_mov(ctx, ret, e.a, e.mode); } +# ifdef WIN64_UNWIND_TABLES + // if we have our stack offset just after a call, the unwind algorithm + // will subtract and create invalid stack frame. this is because we do + // not register the stack offset in our unwind table so all functions + // can share the same definition + if( cur_op && IS_CALL(jit->instrs[cur_op-1].op) ) + EMIT(NOP,UNUSED,UNUSED,M_NONE); +# endif + if( IS_WINCALL64 && ctx->has_direct_call ) + regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,0x20); for(int i=ctx->persists_uses[1]-1;i>=0;i--) EMIT(POP,ctx->jit->cfg.floats.persist[i],UNUSED,M_F64); for(int i=ctx->persists_uses[0]-1;i>=0;i--) EMIT(POP,ctx->jit->cfg.regs.persist[i],UNUSED,M_PTR); if( stack_offset ) { -# ifdef WIN64_UNWIND_TABLES - // if we have our stack offset just after a call, the unwind algorithm - // will subtract and create invalid stack frame. this is because we do - // not register the stack offset in our unwind table so all functions - // can share the same definition - if( cur_op && IS_CALL(ctx->instrs[cur_op-1].op) ) - EMIT(NOP,UNUSED,UNUSED,M_NONE); -# endif regs_emit(ctx,UNUSED,STACK_OFFS,UNUSED,UNUSED,M_PTR,stack_offset); } EMIT(POP,jit->cfg.stack_pos,UNUSED,M_PTR); diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index ee53bc7e1..15bb08f69 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1138,9 +1138,9 @@ void hl_codegen_function( jit_ctx *jit ) { ereg tmp = get_tmp(e->mode); if( !IS_REG(e->a) && !IS_REG(e->b) ) jit_assert(); - emit_mov(ctx, tmp, e->a, e->mode); - emit_mov(ctx, e->a, e->b, e->mode); - emit_mov(ctx, e->b, tmp, e->mode); + emit_mov(ctx, tmp, e->a, M_PTR); + emit_mov(ctx, e->a, e->b, M_PTR); + emit_mov(ctx, e->b, tmp, M_PTR); } break; case STORE: From 5be440b81388bfda231b1f30df067fa96adad7ad Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 16:39:32 +0200 Subject: [PATCH 63/83] minor fixes --- hl.vcxproj | 1 + src/jit_x86_64.c | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hl.vcxproj b/hl.vcxproj index df1736b29..fef4a909e 100644 --- a/hl.vcxproj +++ b/hl.vcxproj @@ -196,6 +196,7 @@ libhl.lib;user32.lib;include/vtune/jitprofiling.lib false false + 4194304 PerMonitorHighDPIAware diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 15bb08f69..8aac604d3 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -662,11 +662,9 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, } static void emit_jump( code_ctx *ctx, int mode, int offset ) { - int op_mult; + int op_mult = 10; # ifdef GEN_DEBUG - op_mult = 16; // additional debug info per op -# else - op_mult = 10; + op_mult += 6; // additional debug info per op # endif if( IS_SBYTE(offset*op_mult) ) { // assume it's ok to use short jump @@ -1181,8 +1179,12 @@ void hl_codegen_function( jit_ctx *jit ) { EMIT(_PUSH, e->a, UNUSED, M_PTR); break; case POP: - if( IS_FLOAT(e->mode) ) jit_assert(); - EMIT(_POP, e->a, UNUSED, M_PTR); + if( IS_FLOAT(e->mode) ) { + EMIT(e->mode == M_F32 ? MOVSS : MOVSD,REG_PTR(R(RSP)),e->a,e->mode); + EMIT(ADD,R(RSP),MK_CONST(8),M_PTR); + } else { + EMIT(_POP, e->a, UNUSED, M_PTR); + } break; case PUSH_CONST: if( e->mode != M_PTR ) jit_assert(); From b845d5c4bd44517cf42a841b4a94ae2ee76b56ae Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 17:08:20 +0200 Subject: [PATCH 64/83] fix --- src/jit_x86_64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 8aac604d3..071e042a4 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -662,7 +662,7 @@ static void emit_ext( code_ctx *ctx, CpuOp op, ereg _a, ereg _b, emit_mode mode, } static void emit_jump( code_ctx *ctx, int mode, int offset ) { - int op_mult = 10; + int op_mult = 16; # ifdef GEN_DEBUG op_mult += 6; // additional debug info per op # endif From ecaa27f094b167eb5222a6313980d0938c0c114c Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 17:16:10 +0200 Subject: [PATCH 65/83] added new files --- CMakeLists.txt | 4 ++++ Makefile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ddd2fd260..03ac5b6b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -228,6 +228,10 @@ if (WITH_VM) add_executable(hl src/code.c src/jit.c + src/jit_emit.c + src/jit_regs.c + src/jit_x86_64.c + src/jit_dump.c src/main.c src/module.c src/debugger.c diff --git a/Makefile b/Makefile index aded6c272..47307277a 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ STD = src/std/array.o src/std/buffer.o src/std/bytes.o src/std/cast.o src/std/da src/std/socket.o src/std/string.o src/std/sys.o src/std/types.o src/std/ucs2.o src/std/thread.o src/std/process.o \ src/std/track.o -HL_OBJ = src/code.o src/jit.o src/main.o src/module.o src/debugger.o src/profile.o +HL_OBJ = src/code.o src/jit.o src/jit_emit.o src/jit_regs.o src/jit_x86_64.o src/jit_dump.o src/main.o src/module.o src/debugger.o src/profile.o FMT_CPPFLAGS = -I include/mikktspace -I include/minimp3 From d27b30dedcd0d23a28416f06761c04f5bca8d034 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 17:20:50 +0200 Subject: [PATCH 66/83] more minor fixes --- src/jit_emit.c | 4 +--- src/jit_x86_64.c | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 48f125cc9..c96a0125b 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -1099,10 +1099,9 @@ static void register_block_jump( emit_ctx *ctx, int offs, bool cond ) { } static void prepare_loop_block( emit_ctx *ctx ) { - int i, last_jump = -1; emit_block *b = ctx->current_block; // gather all backward jumps to know when the block will be finished - for(i=ctx->op_pos+1;ifun->nops;i++) { + for(int i=ctx->op_pos+1;ifun->nops;i++) { hl_opcode *op = &ctx->fun->ops[i]; int offs = 0; switch( op->op ) { @@ -1138,7 +1137,6 @@ static void prepare_loop_block( emit_ctx *ctx ) { b->wait_seal_next = ctx->wait_seal; ctx->wait_seal = b; } - last_jump = i; } } } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 071e042a4..a2b6185c3 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -417,8 +417,8 @@ void hl_jit_init_regs( regs_config *cfg ) { cfg->floats.nargs = 4; cfg->floats.nscratchs = 6; # else - cfg.floats.nargs = 8; - cfg.floats.nscratchs = 16; + cfg->floats.nargs = 8; + cfg->floats.nscratchs = 16; # endif scratch_float_reg = cfg->floats.nscratchs - 1; cfg->floats.nscratchs--; From d1e36f632ae90cf741253fc226b690230eb75cc4 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 17:30:49 +0200 Subject: [PATCH 67/83] minor compilation fixes --- src/jit.h | 2 +- src/jit_emit.c | 5 +++-- src/jit_regs.c | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/jit.h b/src/jit.h index 78a8d3de0..b7307d6e2 100644 --- a/src/jit.h +++ b/src/jit.h @@ -247,7 +247,7 @@ void hl_jit_init( jit_ctx *ctx, hl_module *m ); int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ); void hl_jit_define_function( jit_ctx *ctx, int start, int size ); -void hl_jit_null_field_access(); +void hl_jit_null_field_access( int fhash ); void hl_jit_assert(); void *hl_jit_wrapper_ptr( vclosure_wrapper *c, char *stack_args, void **regs ); double hl_jit_wrapper_d( vclosure_wrapper *c, char *stack_args, void **regs ); diff --git a/src/jit_emit.c b/src/jit_emit.c index c96a0125b..7524c5483 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -21,6 +21,7 @@ */ #include #include +#include #include "data_struct.h" //#define EMIT_DEBUG @@ -954,7 +955,7 @@ static void hl_emit_clean_phis( emit_ctx *ctx ) { p->final_id = -1; } for(int i=0;iemit_pos;i++) - hl_emit_reg_iter(ctx->jit, ctx->instrs + i, ctx, remap_phi_reg); + hl_emit_reg_iter(ctx->jit, ctx->instrs + i, ctx, (void*)remap_phi_reg); } void hl_emit_function( jit_ctx *jit ) { @@ -2014,7 +2015,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { e->mode = M_PTR; e->value = hashed_name; } - emit_native_call(ctx, null_field_access ? hl_jit_null_field_access : hl_null_access, NULL, 0, NULL); + emit_native_call(ctx, null_field_access ? (void*)hl_jit_null_field_access : (void*)hl_null_access, NULL, 0, NULL); patch_jump(ctx, jok); } break; diff --git a/src/jit_regs.c b/src/jit_regs.c index 210cb7a58..0a4974f28 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -254,7 +254,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { write = VAL(write_index++); ctx->cur_op = cur_op; - hl_emit_reg_iter(jit,e,ctx,regs_write_live); + hl_emit_reg_iter(jit,e,ctx,(void*)regs_write_live); if( IS_CALL(e->op) ) { // anticipate register usage in call so we can previlege this assign ereg *r = hl_emit_get_args(jit->emit, e); From f97f6ac51a81fd40523604a3a684354ec11ec331 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 9 May 2026 19:40:56 +0200 Subject: [PATCH 68/83] disable 32 bits builds & fix cmake version --- .github/workflows/build.yml | 4 ++-- CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0528289f3..60fff3b23 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: fail-fast: false matrix: target: [linux, darwin, windows] - architecture: [32, 64, arm64] + architecture: [64, arm64] build_system: [make, cmake, cmake-mingw, cmake-clang-cl, vs2019, makegcc14] include: @@ -429,7 +429,7 @@ jobs: fail-fast: false matrix: os: [darwin, linux, windows] - architecture: [x86_32, x86_64, arm64] + architecture: [x86_64, arm64] include: - architecture: arm64 test-flags: --skip-hl-jit # not yet supported diff --git a/CMakeLists.txt b/CMakeLists.txt index 03ac5b6b9..2228b2fc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13) -set(HL_VERSION_MAJOR 1) -set(HL_VERSION_MINOR 16) +set(HL_VERSION_MAJOR 2) +set(HL_VERSION_MINOR 0) set(HL_VERSION_PATCH 0) set(HL_VERSION ${HL_VERSION_MAJOR}.${HL_VERSION_MINOR}.${HL_VERSION_PATCH}) From 61f4f6216c279d8524f4174278b308cf1bd19933 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Thu, 14 May 2026 20:54:59 +0200 Subject: [PATCH 69/83] fix for xchg --- src/jit_regs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jit_regs.c b/src/jit_regs.c index 0a4974f28..d36ed03ff 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -512,7 +512,7 @@ static void flush_movs( regs_ctx *ctx, bool cond ) { regs_emit(ctx,UNUSED,cmov?CXCHG:XCHG,to,from,mode,0); int_arr_remove_range(&movs,0,3); size -= 3; - for(int k=0;k Date: Sun, 17 May 2026 06:14:01 +0200 Subject: [PATCH 70/83] fixed liveness of phi vars & some minor fixes --- src/jit_emit.c | 12 +++++++----- src/jit_regs.c | 24 ++++++++++++++++++------ src/jit_x86_64.c | 2 ++ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index 7524c5483..6d2e2ea7f 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -990,7 +990,7 @@ void hl_emit_function( jit_ctx *jit ) { free(ctx->pos_map); ctx->pos_map = (int*)malloc(sizeof(int) * (f->nops+1)); if( ctx->pos_map == NULL ) jit_assert(); - ctx->pos_map_size = f->nops; + ctx->pos_map_size = f->nops + 1; } for(i=0;inregs;i++) { @@ -2004,10 +2004,12 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { hashed_name = f->hashed_name; } else if( (next->op >= OCall1 && next->op <= OCallN) && next->p3 == o->p1 ) { int fid = next->p2 < 0 ? -1 : m->functions_indexes[next->p2]; - hl_function *cf = m->code->functions + fid; - const uchar *name = fun_field_name(cf); - null_field_access = true; - hashed_name = hl_hash_gen(name, true); + if( fid >= 0 && fid < m->code->nfunctions ) { + hl_function *cf = m->code->functions + fid; + const uchar *name = fun_field_name(cf); + null_field_access = true; + hashed_name = hl_hash_gen(name, true); + } } // ----------------------------------------- if( null_field_access ) { diff --git a/src/jit_regs.c b/src/jit_regs.c index d36ed03ff..4c5126f74 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -232,6 +232,16 @@ static value_info *regs_current( regs_ctx *ctx, ereg r ) { return NULL; } +static int get_loop_end( regs_ctx *ctx, eblock *b ) { + int loop_end = -1; + for(int k=0;kpred_count;k++) { + eblock *b2 = ctx->jit->blocks + b->preds[k]; + if( b2->start_pos > b->start_pos && b2->end_pos >= loop_end ) + loop_end = b2->end_pos - 1; + } + return loop_end; +} + static void regs_compute_liveness( regs_ctx *ctx ) { # define MAX_LOOP_DEPTH 256 int loop_saves[MAX_LOOP_DEPTH]; @@ -303,12 +313,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { { // are we in loop ? eblock *bl = jit->blocks + e->size_offs; - int loop_end = -1; - for(int k=0;kpred_count;k++) { - eblock *b2 = jit->blocks + bl->preds[k]; - if( b2->start_pos > bl->start_pos && b2->end_pos >= loop_end ) - loop_end = b2->end_pos - 1; - } + int loop_end = get_loop_end(ctx,bl); if( loop_end > 0 ) { loop_saves[loop_count++] = ctx->loop_start; loop_saves[loop_count++] = ctx->loop_end; @@ -340,6 +345,13 @@ static void regs_compute_liveness( regs_ctx *ctx ) { val->tot_reads++; if( val->last_read < b2->end_pos ) val->last_read = b2->end_pos; + // make sure our merged values are preserved if they are in a loop + int write_pos = val->id >= 0 ? jit->values_writes[val->id] : -1; + if( write_pos < b2->start_pos ) { + int loop_end = get_loop_end(ctx,b2); + if( loop_end > 0 && val->last_read < loop_end ) + val->last_read = loop_end; + } } } } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index a2b6185c3..0c320c6eb 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1133,6 +1133,8 @@ void hl_codegen_function( jit_ctx *jit ) { break; case XCHG: { + if( IS_FLOAT(e->mode) ) + BREAK(); ereg tmp = get_tmp(e->mode); if( !IS_REG(e->a) && !IS_REG(e->b) ) jit_assert(); From 3c4b5ee661f870875a209f0f5b7e8da7ec01f9b4 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 17 May 2026 06:54:08 +0200 Subject: [PATCH 71/83] minor --- src/gc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/gc.c b/src/gc.c index d4da96fd8..db94c6655 100644 --- a/src/gc.c +++ b/src/gc.c @@ -983,12 +983,14 @@ static void hl_gc_init() { # endif gc_stats.mark_bytes = 4; // prevent reading out of bmp memset(&gc_threads,0,sizeof(gc_threads)); - gc_threads.global_lock = hl_mutex_alloc(false); - gc_threads.exclusive_lock = hl_mutex_alloc(false); # ifdef HL_THREADS hl_add_root(&gc_threads.global_lock); hl_add_root(&gc_threads.exclusive_lock); hl_add_root(&mark_threads_done); +# endif + gc_threads.global_lock = hl_mutex_alloc(false); + gc_threads.exclusive_lock = hl_mutex_alloc(false); +# ifdef HL_THREADS mark_threads_done = hl_semaphore_alloc(0); char *nthreads = getenv("HL_GC_THREADS"); if( nthreads ) { From 9bcc4745d437ceac99dc27e6cb932b3c234eff2f Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 17 May 2026 13:38:49 +0200 Subject: [PATCH 72/83] compute block loops at emit and chain them, fix regs liveness by finding the good loop --- src/jit.h | 8 +++++-- src/jit_emit.c | 22 ++++++++++++++++++ src/jit_regs.c | 60 ++++++++++++++------------------------------------ 3 files changed, 44 insertions(+), 46 deletions(-) diff --git a/src/jit.h b/src/jit.h index b7307d6e2..ec9567ca8 100644 --- a/src/jit.h +++ b/src/jit.h @@ -161,16 +161,20 @@ struct _ephi { int *blocks; }; -typedef struct _eblock { +typedef struct _eblock eblock; + +struct _eblock { int start_pos; int end_pos; int next_count; int pred_count; int phi_count; + int loop_end; int *nexts; int *preds; ephi *phis; -} eblock; + eblock *loop_parent; +}; typedef struct _emit_ctx emit_ctx; typedef struct _regs_ctx regs_ctx; diff --git a/src/jit_emit.c b/src/jit_emit.c index 6d2e2ea7f..ce50e2a43 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -879,6 +879,28 @@ void hl_emit_flush( jit_ctx *jit ) { jit->values_writes = ctx->values.values; for_iter(blocks,b,ctx->blocks) emit_write_block(ctx,b); + { + eblock *cur_loop = NULL; + for(int b=0;bblock_count;b++) { + eblock *bl = jit->blocks + b; + while( cur_loop != NULL && bl->start_pos > cur_loop->loop_end ) + cur_loop = cur_loop->loop_parent; + int new_loop_end = -1; + for(int k=0;kpred_count;k++) { + eblock *pred = jit->blocks + bl->preds[k]; + if( pred->start_pos > bl->start_pos && pred->end_pos - 1 > new_loop_end ) + new_loop_end = pred->end_pos - 1; + } + if( new_loop_end > 0 ) { + bl->loop_end = new_loop_end; + bl->loop_parent = cur_loop; + cur_loop = bl; + } else { + bl->loop_end = 0; + bl->loop_parent = cur_loop; + } + } + } } void hl_emit_reg_iter( jit_ctx *jit, einstr *e, void *ctx, void (*iter_reg)( void *, ereg * ) ) { diff --git a/src/jit_regs.c b/src/jit_regs.c index 4c5126f74..ead3b54ec 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -71,10 +71,9 @@ struct _regs_ctx { int emit_pos; int stack_size; int stack_offset; - int loop_start; - int loop_end; einstr *instrs; ereg *out_write; + eblock *cur_block; int *pos_map; bool flushed; bool has_direct_call; @@ -215,13 +214,23 @@ static void regs_assign( regs_ctx *ctx, value_info *v ) { regs_debug("REG ASSIGN %s @%X-@%X\n",value_str(v),ctx->cur_op,v->last_read); } +static void regs_loop_liveness( regs_ctx *ctx, eblock *block, value_info *v, int pos ) { + int write = v->id >= 0 ? ctx->jit->values_writes[v->id] : -1; + eblock *b = (block->loop_end > 0) ? block : block->loop_parent; + while( b ) { + if( write >= b->start_pos ) break; + if( pos < b->loop_end ) pos = b->loop_end; + b = b->loop_parent; + } + if( v->last_read < pos ) v->last_read = pos; + v->tot_reads++; +} + static void regs_write_live( regs_ctx *ctx, ereg *r ) { if( IS_NULL(*r) ) jit_assert(); if( !REG_IS_VAL(*r) ) return; // some are injections of native regs at emit value_info *v = VAL_REG(*r); - int write = v->id >= 0 ? ctx->jit->values_writes[v->id] : -1; - v->last_read = ctx->loop_end && write < ctx->loop_start ? ctx->loop_end : ctx->cur_op; - v->tot_reads++; + regs_loop_liveness(ctx, ctx->cur_block, v, ctx->cur_op); } static value_info *regs_current( regs_ctx *ctx, ereg r ) { @@ -232,20 +241,8 @@ static value_info *regs_current( regs_ctx *ctx, ereg r ) { return NULL; } -static int get_loop_end( regs_ctx *ctx, eblock *b ) { - int loop_end = -1; - for(int k=0;kpred_count;k++) { - eblock *b2 = ctx->jit->blocks + b->preds[k]; - if( b2->start_pos > b->start_pos && b2->end_pos >= loop_end ) - loop_end = b2->end_pos - 1; - } - return loop_end; -} static void regs_compute_liveness( regs_ctx *ctx ) { -# define MAX_LOOP_DEPTH 256 - int loop_saves[MAX_LOOP_DEPTH]; - int loop_count = 0; int write_index = 1; jit_ctx *jit = ctx->jit; hl_type *tret = ctx->jit->fun->type->fun->ret; @@ -255,11 +252,6 @@ static void regs_compute_liveness( regs_ctx *ctx ) { einstr *e = jit->instrs + cur_op; value_info *write = NULL; - while( ctx->loop_end == cur_op && cur_op ) { - ctx->loop_end = loop_saves[--loop_count]; - ctx->loop_start = loop_saves[--loop_count]; - } - if( write_index < jit->value_count && jit->values_writes[write_index] == cur_op ) write = VAL(write_index++); @@ -310,23 +302,12 @@ static void regs_compute_liveness( regs_ctx *ctx ) { } break; case BLOCK: - { - // are we in loop ? - eblock *bl = jit->blocks + e->size_offs; - int loop_end = get_loop_end(ctx,bl); - if( loop_end > 0 ) { - loop_saves[loop_count++] = ctx->loop_start; - loop_saves[loop_count++] = ctx->loop_end; - ctx->loop_start = cur_op; - ctx->loop_end = loop_end; - } - } + ctx->cur_block = jit->blocks + e->size_offs; break; default: break; } } - if( loop_count != 0 ) jit_assert(); // compute reverse phis for(int b=0;bblock_count;b++) { eblock *bl = jit->blocks + b; @@ -342,16 +323,7 @@ static void regs_compute_liveness( regs_ctx *ctx ) { int_arr_add(*arr,v); int_arr_add(*arr,ph->value); int_arr_add(*arr,(bl - b2) == 1); - val->tot_reads++; - if( val->last_read < b2->end_pos ) - val->last_read = b2->end_pos; - // make sure our merged values are preserved if they are in a loop - int write_pos = val->id >= 0 ? jit->values_writes[val->id] : -1; - if( write_pos < b2->start_pos ) { - int loop_end = get_loop_end(ctx,b2); - if( loop_end > 0 && val->last_read < loop_end ) - val->last_read = loop_end; - } + regs_loop_liveness(ctx, b2, val, b2->end_pos); } } } From b4068e3c0bdbc93f8968fdf501fddc9ded6807e2 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 17 May 2026 17:43:33 +0200 Subject: [PATCH 73/83] implemented CMOV for XMM registers --- src/jit_x86_64.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 0c320c6eb..f25271771 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -101,6 +101,8 @@ typedef enum { DEC, JMP, MOVSXD, + PUSHFQ, + POPFQ, // FPU FSTP, FSTP32, @@ -108,6 +110,7 @@ typedef enum { FLD32, FLDCW, // SSE + MOVQ, MOVSD, MOVSS, COMISD, @@ -121,6 +124,9 @@ typedef enum { MULSS, DIVSS, XORPS, + ANDPD, + ANDNPD, + ORPD, XORPD, CVTSI2SD, CVTSI2SS, @@ -227,6 +233,8 @@ static opform OP_FORMS[] = { { "DEC", IS_64 ? RM(0xFF,1) : 0x48, RM(0xFF,1) }, { "JMP", RM(0xFF,4) }, { "MOVSXD", 0x63 }, + { "PUSHFQ", 0x9C }, + { "POPFQ", 0x9D }, // FPU { "FSTP", 0, RM(0xDD,3) }, { "FSTP32", 0, RM(0xD9,3) }, @@ -234,6 +242,7 @@ static opform OP_FORMS[] = { { "FLD32", 0, RM(0xD9,0) }, { "FLDCW", 0, RM(0xD9, 5) }, // SSE + { "MOVQ", 0x660F6E }, { "MOVSD", 0xF20F10, 0xF20F11 }, { "MOVSS", 0xF30F10, 0xF30F11 }, { "COMISD", LONG_RM(0x660F2F,1) }, @@ -247,6 +256,9 @@ static opform OP_FORMS[] = { { "MULSS", 0xF30F59 }, { "DIVSS", 0xF30F5E }, { "XORPS", LONG_OP(0x0F57) }, + { "ANDPD", 0x660F54 }, + { "ANDNPD", 0x660F55 }, + { "ORPD", 0x660F56 }, { "XORPD", 0x660F57 }, { "CVTSI2SD", 0xF20F2A }, { "CVTSI2SS", 0xF30F2A }, @@ -1089,6 +1101,13 @@ static void emit_cmov( code_ctx *ctx, ereg out, ereg r, int cond, emit_mode m ) MOD_RM(3,out,r); } +static void emit_cset( code_ctx *ctx, ereg out, int cond ) { + if( (out&8) ) B(0x41); + B(0x0F); + B(cond + 0x10); + MOD_RM(3,0,out); +} + void hl_codegen_function( jit_ctx *jit ) { code_ctx *ctx = jit->code; ctx->flushed = false; @@ -1500,7 +1519,33 @@ void hl_codegen_function( jit_ctx *jit ) { { int cond = get_cond_jump(ctx); if( !IS_REG(out) ) jit_assert(); - if( IS_REG(e->a) ) { + if( IS_FLOAT(e->mode) ) { + EMIT(PUSHFQ,UNUSED,UNUSED,M_PTR); + // create a mask in RTMP to be used for xmm + emit_cset(ctx,RTMP,cond); + EMIT(MOVZX8,RTMP,RTMP,M_PTR); + EMIT(NEG,RTMP,UNUSED,M_PTR); + // do dst := (mask & src) | (dst & ~mask) + ereg tmp = get_tmp(M_F64); + EMIT(MOVQ,tmp,RTMP,M_PTR); + EMIT(ANDNPD,tmp,out,M_F64); + EMIT(MOVQ,out,RTMP,M_PTR); + if( !IS_REG(e->a) ) { + // ANDNPD requires aligned address ! + ereg tmp2 = out == MMX(0) ? MMX(1) : MMX(0); + EMIT(SUB,R(RSP),MK_CONST(8),M_PTR); + EMIT(MOVSD,REG_PTR(R(RSP)),tmp2,M_F64); + EMIT(e->mode == M_F32 ? MOVSS : MOVSD,tmp2,e->a,e->mode); + EMIT(ANDPD,out,tmp2,M_F64); + EMIT(ORPD,out,tmp,M_F64); + EMIT(MOVSD,tmp2,REG_PTR(R(RSP)),M_PTR); + EMIT(ADD,R(RSP),MK_CONST(8),M_PTR); + } else { + EMIT(ANDPD,out,e->a,M_F64); + EMIT(ORPD,out,tmp,M_F64); + } + EMIT(POPFQ,UNUSED,UNUSED,M_PTR); + } else if( IS_REG(e->a) ) { emit_cmov(ctx,out,e->a,cond,M_PTR); } else { emit_mov(ctx,RTMP,e->a,e->mode); From 0a311034bda7618a0e34d2c12e5bbe99f5219e35 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 17 May 2026 19:18:18 +0200 Subject: [PATCH 74/83] fix pop float --- src/jit_x86_64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index f25271771..d2943df3c 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1201,7 +1201,7 @@ void hl_codegen_function( jit_ctx *jit ) { break; case POP: if( IS_FLOAT(e->mode) ) { - EMIT(e->mode == M_F32 ? MOVSS : MOVSD,REG_PTR(R(RSP)),e->a,e->mode); + EMIT(e->mode == M_F32 ? MOVSS : MOVSD,e->a,REG_PTR(R(RSP)),e->mode); EMIT(ADD,R(RSP),MK_CONST(8),M_PTR); } else { EMIT(_POP, e->a, UNUSED, M_PTR); From 6749f9ecfae168d31796cf7020184084a45107eb Mon Sep 17 00:00:00 2001 From: ncannasse Date: Mon, 18 May 2026 23:16:20 +0200 Subject: [PATCH 75/83] various fixes --- src/jit_dump.c | 2 +- src/jit_emit.c | 4 ++-- src/jit_regs.c | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/jit_dump.c b/src/jit_dump.c index c1b16a073..74061f2c3 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -222,7 +222,7 @@ static void dump_value( jit_ctx *ctx, uint64 value, emit_mode mode ) { default: if( value == 0 ) printf("NULL"); - else if( mode == M_PTR && value >= (uint64)code->types && value < (uint64)(code->types + code->ntypes) ) + else if( mode == M_PTR && value >= (uint64)code->types && value < (uint64)(code->types + code->ntypes) && (((char*)value-(char*)code->types) % sizeof(hl_type)) == 0 ) uprintf(USTR("<%s>"),hl_type_str((hl_type*)value)); else if( mode == M_PTR && value == (uint64)mod->globals_data ) printf(""); diff --git a/src/jit_emit.c b/src/jit_emit.c index ce50e2a43..abc4dfe35 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -1708,7 +1708,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { STORE_MEM(field, 0, val); int jend = emit_jump(ctx, false); patch_jump(ctx, jidx); - bool need_type = dyn_need_type(dst->t); + bool need_type = dyn_need_type(rb->t); ereg args[4]; args[0] = obj; args[1] = LOAD_CONST(dst->t->virt->fields[o->p2].hashed_name,&hlt_i32); @@ -1718,7 +1718,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { } else { args[2] = val; } - emit_native_call(ctx,get_dynset(dst->t),args,need_type?4:3,dst->t); + emit_native_call(ctx,get_dynset(rb->t),args,need_type?4:3,dst->t); patch_jump(ctx, jend); } break; diff --git a/src/jit_regs.c b/src/jit_regs.c index ead3b54ec..25f90218d 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -766,6 +766,13 @@ void hl_regs_function( jit_ctx *jit ) { v->mode = M_NONE; } } + for(int b=0;bblock_count;b++) { + eblock *bl = jit->blocks + b; + for(int p=0;pphi_count;p++) { + ephi *ph = bl->phis + p; + VAL_REG(ph->value)->mode = ph->mode; + } + } regs_compute_liveness(ctx); regs_assign_regs(ctx); regs_emit_instrs(ctx); From a43957b5aad873daf88fe8df2453404818cef683 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Tue, 19 May 2026 19:56:07 +0200 Subject: [PATCH 76/83] minor fixes --- src/jit.h | 3 +-- src/jit_emit.c | 2 +- src/jit_x86_64.c | 16 ++++++++++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/jit.h b/src/jit.h index ec9567ca8..805cbbf32 100644 --- a/src/jit.h +++ b/src/jit.h @@ -82,8 +82,7 @@ typedef struct { struct { unsigned char op; unsigned char mode; - unsigned char nargs; - unsigned char _unused; + unsigned short nargs; }; int header; }; diff --git a/src/jit_emit.c b/src/jit_emit.c index abc4dfe35..b613c3387 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -327,7 +327,7 @@ static void emit_store_mem( emit_ctx *ctx, ereg to, int offs, ereg from ) { void hl_emit_store_args( emit_ctx *ctx, einstr *e, ereg *args, int count ) { if( count < 0 ) jit_assert(); if( count > 256 ) jit_error("Too many arguments"); - e->nargs = (unsigned char)count; + e->nargs = (unsigned short)count; if( count == 0 ) return; if( count == 1 ) { e->size_offs = args[0]; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index d2943df3c..5afbf828e 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1157,9 +1157,9 @@ void hl_codegen_function( jit_ctx *jit ) { ereg tmp = get_tmp(e->mode); if( !IS_REG(e->a) && !IS_REG(e->b) ) jit_assert(); - emit_mov(ctx, tmp, e->a, M_PTR); - emit_mov(ctx, e->a, e->b, M_PTR); - emit_mov(ctx, e->b, tmp, M_PTR); + emit_mov(ctx, tmp, e->a, e->mode); + emit_mov(ctx, e->a, e->b, e->mode); + emit_mov(ctx, e->b, tmp, e->mode); } break; case STORE: @@ -1554,7 +1554,15 @@ void hl_codegen_function( jit_ctx *jit ) { } break; case CXCHG: - BREAK(); + { + if( IS_FLOAT(e->mode) ) BREAK(); + if( !IS_REG(e->a) || !IS_REG(e->b) ) BREAK(); + int cond = get_cond_jump(ctx); + ereg tmp = get_tmp(e->mode); + emit_mov(ctx, tmp, e->a, e->mode); + emit_cmov(ctx, e->a, e->b, cond, M_PTR); + emit_cmov(ctx, e->b, tmp, cond, M_PTR); + } break; case NOP: emit_nop(ctx,1); From d5ea08f68574f0e155aaa2fbe1d206070f2a6e63 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Thu, 28 May 2026 22:08:11 +0200 Subject: [PATCH 77/83] minor fixes --- src/jit_dump.c | 1 - src/jit_x86_64.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/jit_dump.c b/src/jit_dump.c index 74061f2c3..a85f918f1 100644 --- a/src/jit_dump.c +++ b/src/jit_dump.c @@ -154,7 +154,6 @@ static void hl_dump_op( hl_function *fun, hl_opcode *op ) { if( i != 0 ) printf(","); printf("@%X", (op->extra[i] + pos + 1)); } - printf(",def=@%X", op->p3 + pos + 1); } else { if( count == 0xFF ) count = op->p3; diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 5afbf828e..126fc6f33 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1652,7 +1652,7 @@ void hl_codegen_init( jit_ctx *jit ) { for(int i=0;ifloats.nargs;i++) EMIT(MOVSD, cfg->floats.arg[i]-64, MK_ADDR(vargs,(i + cfg->regs.nargs) * 8), M_PTR); - EMIT(ADD,vargs,MK_CONST((MAX_ARGS - 1) * HL_WSIZE),M_PTR); + EMIT(ADD,vargs,MK_CONST((2 * MAX_ARGS - 1) * HL_WSIZE),M_PTR); int begin = byte_count(ctx->code); EMIT(_TEST,nargs,nargs,M_I32); int pos = jump_near(ctx,JZero); From c2324714ca8cf9aafee29b586f9a4490b0087177 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 13 Jun 2026 21:08:59 +0200 Subject: [PATCH 78/83] fix XCHG with different sizes --- src/jit_regs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jit_regs.c b/src/jit_regs.c index 25f90218d..2dccdd67e 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -493,7 +493,7 @@ static void flush_movs( regs_ctx *ctx, bool cond ) { ereg from = int_arr_get(movs,1); int mode = int_arr_get(movs,2); bool cmov = cond && (IS_REG(to) || IS_REG(from)); - regs_emit(ctx,UNUSED,cmov?CXCHG:XCHG,to,from,mode,0); + regs_emit(ctx,UNUSED,cmov?CXCHG:XCHG,to,from,IS_FLOAT(mode)?M_F64:M_PTR,0); int_arr_remove_range(&movs,0,3); size -= 3; for(int k=1;k Date: Sun, 14 Jun 2026 09:31:27 +0200 Subject: [PATCH 79/83] fixed stack alignment and args invert in c2hl --- src/jit.c | 15 ++++++++------- src/jit_x86_64.c | 10 ++++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/jit.c b/src/jit.c index ddf9a187d..b101db0c8 100644 --- a/src/jit.c +++ b/src/jit.c @@ -219,7 +219,7 @@ static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { void *regs[MAX_ARGS]; void *stack[MAX_ARGS]; } vargs; - int rp = 0, fp = 0, sp = MAX_ARGS; + int rp = 0, fp = 0, sp = 0; for(int i=0;ifun->nargs;i++) { hl_type *at = t->fun->args[i]; void *v = args[i]; @@ -245,27 +245,28 @@ static void *callback_c2hl( void *f, hl_type *t, void **args, vdynamic *ret ) { if( r >= 0 ) vargs.regs[r + (at->kind == HF32 || at->kind == HF64 ? arg_reg_count : 0)] = (void*)iv; else - vargs.stack[--sp] = (void*)iv; + vargs.stack[sp++] = (void*)iv; } + if( sp & 1 ) sp++; // align stack switch( t->fun->ret->kind ) { case HUI8: case HUI16: case HI32: case HBOOL: - ret->v.i = ((int (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + ret->v.i = ((int (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, sp); return &ret->v.i; case HI64: case HGUID: - ret->v.i64 = ((int64 (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + ret->v.i64 = ((int64 (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, sp); return &ret->v.i64; case HF32: - ret->v.f = ((float (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + ret->v.f = ((float (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, sp); return &ret->v.f; case HF64: - ret->v.d = ((double (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + ret->v.d = ((double (*)(void *, void *, int))call_jit_c2hl)(f, &vargs, sp); return &ret->v.d; default: - return ((void *(*)(void *, void *, int))call_jit_c2hl)(f, &vargs, MAX_ARGS - sp); + return ((void *(*)(void *, void *, int))call_jit_c2hl)(f, &vargs, sp); } } diff --git a/src/jit_x86_64.c b/src/jit_x86_64.c index 126fc6f33..6a009ca6a 100644 --- a/src/jit_x86_64.c +++ b/src/jit_x86_64.c @@ -1652,12 +1652,18 @@ void hl_codegen_init( jit_ctx *jit ) { for(int i=0;ifloats.nargs;i++) EMIT(MOVSD, cfg->floats.arg[i]-64, MK_ADDR(vargs,(i + cfg->regs.nargs) * 8), M_PTR); - EMIT(ADD,vargs,MK_CONST((2 * MAX_ARGS - 1) * HL_WSIZE),M_PTR); + EMIT(ADD,vargs,MK_CONST(HL_WSIZE * MAX_ARGS),M_PTR); + einstr lea; + lea.a = vargs; + lea.b = nargs; + lea.size_offs = HL_WSIZE; + emit_lea(ctx,vargs,&lea); + int begin = byte_count(ctx->code); EMIT(_TEST,nargs,nargs,M_I32); int pos = jump_near(ctx,JZero); - EMIT(_PUSH,MK_ADDR(vargs,0),UNUSED,M_PTR); EMIT(SUB,vargs,MK_CONST(HL_WSIZE),M_PTR); + EMIT(_PUSH,MK_ADDR(vargs,0),UNUSED,M_PTR); EMIT(DEC,nargs,UNUSED,M_I32); jump_near(ctx,-begin); patch_jump_near(ctx,pos); From 018346bb5f431f8aed0de6454daf6787667696fb Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sun, 14 Jun 2026 15:53:08 +0200 Subject: [PATCH 80/83] fixed dyn_set using wrong type --- src/jit_emit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jit_emit.c b/src/jit_emit.c index b613c3387..d870dbace 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -2058,7 +2058,7 @@ static void emit_opcode( emit_ctx *ctx, hl_opcode *o ) { break; case ODynSet: { - bool need_type = dyn_need_type(dst->t); + bool need_type = dyn_need_type(rb->t); ereg args[4]; args[0] = LOAD(dst); args[1] = LOAD_CONST(hl_hash_utf8(m->code->strings[o->p2]),&hlt_i32); From 7276e3b4367c96fe4d7d3e105c5fda7144c25eaa Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 27 Jun 2026 18:25:44 +0200 Subject: [PATCH 81/83] added wip variable tracking --- src/code.c | 8 +++++--- src/debugger.c | 52 +++++++++++++++++++++++++++++--------------------- src/hlmodule.h | 5 +++++ src/jit.c | 15 +++++++++++---- src/jit.h | 4 ++++ src/jit_emit.c | 22 ++++++++++++++++++++- src/jit_regs.c | 32 +++++++++++++++++++++++++++++-- src/module.c | 5 +++++ 8 files changed, 111 insertions(+), 32 deletions(-) diff --git a/src/code.c b/src/code.c index b7b67dc5c..a5c855c71 100644 --- a/src/code.c +++ b/src/code.c @@ -506,13 +506,15 @@ hl_code *hl_code_read( const unsigned char *data, int size, char **error_msg ) { if( c->hasdebug ) { c->functions[i].debug = hl_read_debug_infos(r,c->functions[i].nops); if( c->version >= 3 ) { - // skip assigns (no need here) int nassigns = UINDEX(); + int *assigns = (int*)malloc(sizeof(int)*nassigns*2); int j; for(j=0;jfunctions[i].nassigns = nassigns; + c->functions[i].assigns = assigns; } } } diff --git a/src/debugger.c b/src/debugger.c index 68eb88485..98cd3a2a0 100644 --- a/src/debugger.c +++ b/src/debugger.c @@ -45,7 +45,7 @@ static void send( void *ptr, int size ) { hl_socket_send(client_socket, ptr, 0, size); } -static void hl_debug_loop( hl_module *m ) { +static void hl_debug_loop() { void *inf_addr = hl_gc_threads_info(); int flags = 0; int hl_ver = HL_VERSION; @@ -64,41 +64,49 @@ static void hl_debug_loop( hl_module *m ) { # endif hl_get_thread()->flags |= HL_THREAD_INVISIBLE; do { - int i; vbyte cmd; hl_socket *s = hl_socket_accept(debug_socket); if( s == NULL ) break; client_socket = s; - send("HLD1",4); + send("HLD2",4); send(&flags,4); send(&hl_ver, 4); send(&pid,4); send(&inf_addr, sizeof(void*)); - send(&m->globals_data,sizeof(void*)); - send(&m->jit_code,sizeof(void*)); - send(&m->codesize,4); - send(&m->code->types,sizeof(void*)); - for(i=1;i<=HBYTES;i++) { + for(int i=1;i<=HBYTES;i++) { hl_type t = {(hl_type_kind)i}; int k = 1 + hl_pad_struct(1,&t); send(&k,4); } - send(&m->code->nfunctions,4); - for(i=0;icode->nfunctions;i++) { - hl_function *f = m->code->functions + i; - hl_debug_infos *d = m->jit_debug + i; - struct { - int nops; - int start; - unsigned char large; - } fdata; - fdata.nops = f->nops; - fdata.start = d->start; - fdata.large = (unsigned char)d->large; - send(&fdata,9); - send(d->offsets,(d->large ? sizeof(int) : sizeof(unsigned short)) * (f->nops + 1)); + int nmodules; + hl_module **mods = hl_get_modules(&nmodules); + send(&nmodules,4); + for(int i=0;iglobals_data,sizeof(void*)); + send(&m->jit_code,sizeof(void*)); + send(&m->codesize,4); + send(&m->code->types,sizeof(void*)); + send(&m->code->nfunctions,4); + for(int j=0;jcode->nfunctions;j++) { + hl_function *f = m->code->functions + j; + hl_debug_infos *d = m->jit_debug + j; + struct { + int nops; + int start; + int nvars; + unsigned char large; + } fdata; + fdata.nops = f->nops; + fdata.start = d->start; + fdata.nvars = d->nvars; + fdata.large = (unsigned char)d->large; + send(&fdata,13); + send(d->offsets,(d->large ? sizeof(int) : sizeof(unsigned short)) * (f->nops + 1)); + send(d->vars,d->nvars * 4); + } } hl_setup.closure_stack_capture = 8; diff --git a/src/hlmodule.h b/src/hlmodule.h index adf29f9bd..853fb25ac 100644 --- a/src/hlmodule.h +++ b/src/hlmodule.h @@ -48,10 +48,12 @@ struct hl_function { int nregs; int nops; int ref; + int nassigns; hl_type *type; hl_type **regs; hl_opcode *ops; int *debug; + int *assigns; hl_type_obj *obj; union { @@ -103,7 +105,9 @@ typedef struct { typedef struct { void *offsets; + int *vars; int start; + int nvars; bool large; } hl_debug_infos; @@ -164,6 +168,7 @@ h_bool hl_module_patch( hl_module *m, hl_code *code ); void hl_module_free( hl_module *m ); h_bool hl_module_debug( hl_module *m, int port, h_bool wait ); hl_type *hl_module_resolve_type( hl_module *m, hl_type *t, bool err ); +hl_module **hl_get_modules( int *count ); void hl_profile_setup( int sample_count ); void hl_profile_end(); diff --git a/src/jit.c b/src/jit.c index b101db0c8..057f8cf65 100644 --- a/src/jit.c +++ b/src/jit.c @@ -135,7 +135,7 @@ void hl_jit_init( jit_ctx *ctx, hl_module *m ) { jit_code_append(ctx); if( m->code->hasdebug ) { m->jit_debug = (hl_debug_infos*)malloc(sizeof(hl_debug_infos) * m->code->nfunctions); - memset(m->jit_debug, -1, sizeof(hl_debug_infos) * m->code->nfunctions); + memset(m->jit_debug, 0, sizeof(hl_debug_infos) * m->code->nfunctions); } } @@ -175,9 +175,16 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { ((int*)debug)[i] = cpos; } int fid = (int)(f - m->code->functions); - m->jit_debug[fid].start = pos; - m->jit_debug[fid].offsets = debug; - m->jit_debug[fid].large = !compact; + hl_debug_infos *dbg = &m->jit_debug[fid]; + dbg->start = pos; + dbg->offsets = debug; + dbg->large = !compact; + dbg->nvars = ctx->regs_track_count; + dbg->vars = ctx->regs_track; + for(int i=0;iregs_track_count;i++) { + dbg->vars[(i<<2)|1] = ctx->code_pos_map[dbg->vars[(i<<2)|1]]; + dbg->vars[(i<<2)|2] = ctx->code_pos_map[dbg->vars[(i<<2)|2]]; + } } if( !jit_code_append(ctx) ) return -1; diff --git a/src/jit.h b/src/jit.h index 805cbbf32..04608e30c 100644 --- a/src/jit.h +++ b/src/jit.h @@ -221,15 +221,19 @@ struct _jit_ctx { int block_count; int value_count; int phi_count; + int track_count; einstr *instrs; eblock *blocks; int *values_writes; + int *values_track; int *emit_pos_map; // regs output int reg_instr_count; + int regs_track_count; einstr *reg_instrs; ereg *reg_writes; int *reg_pos_map; + int *regs_track; // codegen output int code_size; unsigned char *code_instrs; diff --git a/src/jit_emit.c b/src/jit_emit.c index d870dbace..31842f2eb 100644 --- a/src/jit_emit.c +++ b/src/jit_emit.c @@ -143,6 +143,9 @@ struct _emit_ctx { int_arr jump_regs; int_arr values; + int_arr values_track; + int current_assign; + blocks blocks; emit_block *current_block; emit_block *wait_seal; @@ -501,6 +504,12 @@ static void emit_store_reg( emit_ctx *ctx, vreg *to, ereg v ) { STORE_MEM(emit_gen(ctx,ADDRESS,to->stored,UNUSED,M_PTR), 0, v); } else { to->stored = v; + if( ctx->current_assign < ctx->fun->nassigns && ctx->fun->assigns[(ctx->current_assign<<1)|1] == ctx->op_pos ) { + int_arr_add(ctx->values_track,ctx->current_assign); + int_arr_add(ctx->values_track,v); + //printf("@%X R%d[%s] := V%d\n",ctx->emit_pos - 1, to->id, ctx->jit->mod->code->strings[ctx->fun->assigns[(ctx->current_assign<<1)]], v); + ctx->current_assign++; + } } } @@ -877,6 +886,8 @@ void hl_emit_flush( jit_ctx *jit ) { jit->blocks = hl_zalloc(&jit->falloc,sizeof(eblock) * jit->block_count); jit->value_count = int_arr_count(ctx->values); jit->values_writes = ctx->values.values; + jit->track_count = int_arr_count(ctx->values_track) >> 1; + jit->values_track = ctx->values_track.values; for_iter(blocks,b,ctx->blocks) emit_write_block(ctx,b); { @@ -989,10 +1000,12 @@ void hl_emit_function( jit_ctx *jit ) { ctx->emit_pos = 0; ctx->trap_count = 0; ctx->phi_count = 0; + ctx->current_assign = 0; ctx->flushed = false; int_arr_free(&ctx->args_data); int_arr_free(&ctx->jump_regs); int_arr_free(&ctx->values); + int_arr_free(&ctx->values_track); blocks_free(&ctx->blocks); int_arr_add(ctx->values,-1); ctx->current_block = alloc_block(ctx); @@ -1025,7 +1038,14 @@ void hl_emit_function( jit_ctx *jit ) { emit_gen(ctx,ENTER,UNUSED,UNUSED,M_NONE); for(i=0;itype->fun->nargs;i++) { hl_type *t = f->type->fun->args[i]; - STORE(R(i), emit_gen(ctx, LOAD_ARG, UNUSED, UNUSED, hl_type_mode(t))); + ereg r = emit_gen(ctx, LOAD_ARG, UNUSED, UNUSED, hl_type_mode(t)); + STORE(R(i), r); + if( ctx->current_assign < f->nassigns && f->assigns[(ctx->current_assign<<1)|1] < 0 ) { + //printf("@%X R%d[%s] := V%d\n",ctx->emit_pos - 1, i, ctx->jit->mod->code->strings[f->assigns[(ctx->current_assign<<1)]], r); + int_arr_add(ctx->values_track,ctx->current_assign); + int_arr_add(ctx->values_track,r); + ctx->current_assign++; + } } for(int op_pos=0;op_posnops;op_pos++) { diff --git a/src/jit_regs.c b/src/jit_regs.c index 2dccdd67e..d855ad4d6 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -45,9 +45,11 @@ typedef struct { int id; + int start; int stack_pos; int last_read; int tot_reads; + int tracked; emit_mode mode; ereg pref_reg; ereg reg; @@ -733,6 +735,23 @@ void hl_regs_flush( jit_ctx *jit ) { jit->reg_pos_map = ctx->pos_map; if( ctx->pos_map ) ctx->pos_map[ctx->cur_op+1] = ctx->emit_pos; hl_emit_remap_jumps(jit->emit, &ctx->jump_regs, ctx->instrs, ctx->pos_map); + + int_arr regs_track; + int_arr_free(®s_track); + for(int i=0;ivalue_count + jit->phi_count;i++) { + value_info *v = VAL(i); + if( v->tracked && v->reg ) { + int start = ctx->pos_map[v->start]; + int end = v->last_read < 0 ? start : ctx->pos_map[v->last_read]; + //printf(" @%X %s := %s\n", ctx->pos_map[v->start], jit->mod->code->strings[jit->fun->assigns[(v->tracked - 1) << 1]], val_str(v->reg,v->mode)); + int_arr_add(regs_track, v->tracked - 1); + int_arr_add(regs_track, start); + int_arr_add(regs_track, end); + int_arr_add(regs_track, v->reg); + } + } + jit->regs_track = regs_track.values; + jit->regs_track_count = regs_track.cur >> 2; } void hl_regs_function( jit_ctx *jit ) { @@ -760,17 +779,26 @@ void hl_regs_function( jit_ctx *jit ) { v->last_read = -1; if( i < jit->value_count ) { v->id = i; - v->mode = jit->instrs[jit->values_writes[i]].mode; + v->start = jit->values_writes[i]; + v->mode = jit->instrs[v->start].mode; } else { v->id = -(i-jit->value_count) - 1; v->mode = M_NONE; } } + for(int i=0;itrack_count;i++) { + int v = jit->values_track[(i<<1)|1]; + VAL(v)->tracked = jit->values_track[i<<1] + 1; + } for(int b=0;bblock_count;b++) { eblock *bl = jit->blocks + b; for(int p=0;pphi_count;p++) { ephi *ph = bl->phis + p; - VAL_REG(ph->value)->mode = ph->mode; + value_info *v = VAL_REG(ph->value); + v->start = bl->start_pos; + v->mode = ph->mode; + if( ph->nvalues ) + v->tracked = VAL_REG(ph->values[0])->tracked; } } regs_compute_liveness(ctx); diff --git a/src/module.c b/src/module.c index 549c4c41d..69833f909 100644 --- a/src/module.c +++ b/src/module.c @@ -88,6 +88,11 @@ static bool module_resolve_pos( hl_module *m, void *addr, int *fidx, int *fpos ) return true; } +hl_module **hl_get_modules( int *count ) { + *count = modules_count; + return cur_modules; +} + uchar *hl_module_resolve_symbol_full( void *addr, uchar *out, int *outSize, int **r_debug_addr ) { int *debug_addr; int file, line; From be1ae7c93f6799ee61adae6266b3a364183feaed Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 4 Jul 2026 12:40:41 +0200 Subject: [PATCH 82/83] more work on debugger vars tracking --- src/debugger.c | 6 +++--- src/hlmodule.h | 4 ++-- src/jit.c | 11 ++++++----- src/jit_regs.c | 15 +++++++++++---- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/debugger.c b/src/debugger.c index 98cd3a2a0..e59814b07 100644 --- a/src/debugger.c +++ b/src/debugger.c @@ -96,16 +96,16 @@ static void hl_debug_loop() { struct { int nops; int start; - int nvars; + int vars_size; unsigned char large; } fdata; fdata.nops = f->nops; fdata.start = d->start; - fdata.nvars = d->nvars; + fdata.vars_size = d->vars_size; fdata.large = (unsigned char)d->large; send(&fdata,13); send(d->offsets,(d->large ? sizeof(int) : sizeof(unsigned short)) * (f->nops + 1)); - send(d->vars,d->nvars * 4); + send(d->vars,d->vars_size); } } diff --git a/src/hlmodule.h b/src/hlmodule.h index 853fb25ac..8d237c618 100644 --- a/src/hlmodule.h +++ b/src/hlmodule.h @@ -105,9 +105,9 @@ typedef struct { typedef struct { void *offsets; - int *vars; + void *vars; int start; - int nvars; + int vars_size; bool large; } hl_debug_infos; diff --git a/src/jit.c b/src/jit.c index 057f8cf65..02b8e9794 100644 --- a/src/jit.c +++ b/src/jit.c @@ -179,12 +179,13 @@ int hl_jit_function( jit_ctx *ctx, hl_module *m, hl_function *f ) { dbg->start = pos; dbg->offsets = debug; dbg->large = !compact; - dbg->nvars = ctx->regs_track_count; - dbg->vars = ctx->regs_track; - for(int i=0;iregs_track_count;i++) { - dbg->vars[(i<<2)|1] = ctx->code_pos_map[dbg->vars[(i<<2)|1]]; - dbg->vars[(i<<2)|2] = ctx->code_pos_map[dbg->vars[(i<<2)|2]]; + dbg->vars_size = ctx->regs_track_count * sizeof(int); + dbg->vars = malloc(dbg->vars_size); + for(int i=0;iregs_track_count>>2;i++) { + ctx->regs_track[(i<<2)|1] = ctx->code_pos_map[ctx->regs_track[(i<<2)|1] + 1]; + ctx->regs_track[(i<<2)|2] = ctx->code_pos_map[ctx->regs_track[(i<<2)|2]]; } + memcpy(dbg->vars,ctx->regs_track,dbg->vars_size); } if( !jit_code_append(ctx) ) return -1; diff --git a/src/jit_regs.c b/src/jit_regs.c index d855ad4d6..1c54189b3 100644 --- a/src/jit_regs.c +++ b/src/jit_regs.c @@ -50,6 +50,7 @@ typedef struct { int last_read; int tot_reads; int tracked; + int overwrite; emit_mode mode; ereg pref_reg; ereg reg; @@ -738,20 +739,25 @@ void hl_regs_flush( jit_ctx *jit ) { int_arr regs_track; int_arr_free(®s_track); + int rarg = 0; + int end_pos = ctx->emit_pos; for(int i=0;ivalue_count + jit->phi_count;i++) { value_info *v = VAL(i); if( v->tracked && v->reg ) { int start = ctx->pos_map[v->start]; - int end = v->last_read < 0 ? start : ctx->pos_map[v->last_read]; - //printf(" @%X %s := %s\n", ctx->pos_map[v->start], jit->mod->code->strings[jit->fun->assigns[(v->tracked - 1) << 1]], val_str(v->reg,v->mode)); - int_arr_add(regs_track, v->tracked - 1); + int end = v->overwrite < 0 ? end_pos : ctx->pos_map[v->last_read]; + int track = (v->tracked - 1) << 1; +// printf(" @%X-%X %s := %s\n", start, end, jit->mod->code->strings[jit->fun->assigns[track]], val_str(v->reg,v->mode)); + int assign = jit->fun->assigns[track+1]; + if( assign < 0 ) assign = rarg++; + int_arr_add(regs_track, assign); int_arr_add(regs_track, start); int_arr_add(regs_track, end); int_arr_add(regs_track, v->reg); } } jit->regs_track = regs_track.values; - jit->regs_track_count = regs_track.cur >> 2; + jit->regs_track_count = regs_track.cur; } void hl_regs_function( jit_ctx *jit ) { @@ -777,6 +783,7 @@ void hl_regs_function( jit_ctx *jit ) { v->pref_reg = UNUSED; v->stack_pos = INVALID; v->last_read = -1; + v->overwrite = -1; if( i < jit->value_count ) { v->id = i; v->start = jit->values_writes[i]; From d356baa4a4126fdfa9d521eef0c50eb50fa2cd85 Mon Sep 17 00:00:00 2001 From: ncannasse Date: Sat, 4 Jul 2026 12:41:22 +0200 Subject: [PATCH 83/83] debug api : remove 32 bit support and add all cpu registers access --- src/std/debug.c | 96 +++++++++++++------------------------------------ 1 file changed, 24 insertions(+), 72 deletions(-) diff --git a/src/std/debug.c b/src/std/debug.c index e2af3584c..2d178b13a 100644 --- a/src/std/debug.c +++ b/src/std/debug.c @@ -280,68 +280,39 @@ HL_API bool hl_debug_resume( int pid, int thread ) { } #ifdef HL_WIN -#define DefineGetReg(type,GetFun) \ - REGDATA *GetFun( type *c, int reg ) { \ - switch( reg ) { \ - case 0: return GET_REG(sp); \ - case 1: return GET_REG(bp); \ - case 2: return GET_REG(ip); \ - case 4: return &c->Dr0; \ - case 5: return &c->Dr1; \ - case 6: return &c->Dr2; \ - case 7: return &c->Dr3; \ - case 8: return &c->Dr6; \ - case 9: return &c->Dr7; \ - case 10: return GET_REG(ax); \ - default: return GET_REG(ax); \ - } \ +DWORD64 *GetContextReg( CONTEXT *c, int reg ) { + switch( reg ) { + case 0: return &c->Rsp; + case 1: return &c->Rbp; + case 2: return &c->Rip; + case 4: return &c->Dr0; + case 5: return &c->Dr1; + case 6: return &c->Dr2; + case 7: return &c->Dr3; + case 8: return &c->Dr6; + case 9: return &c->Dr7; + default: + if( reg & 1 ) + return &c->FltSave.XmmRegisters[(reg-10)>>1].Low; + return &c->Rax + ((reg - 10) >> 1); } - -#define GET_REG(x) &c->E##x -#define REGDATA DWORD - -#ifdef HL_64 -DefineGetReg(WOW64_CONTEXT,GetContextReg32); -# undef GET_REG -# undef REGDATA -# define GET_REG(x) &c->R##x -# define REGDATA DWORD64 -# endif - -DefineGetReg(CONTEXT,GetContextReg); - +} #endif HL_API void *hl_debug_read_register( int pid, int thread, int reg, bool is64 ) { # if defined(HL_WIN) -# ifdef HL_64 - if( !is64 ) { - WOW64_CONTEXT c; - c.ContextFlags = CONTEXT_FULL | CONTEXT_DEBUG_REGISTERS; - if( !Wow64GetThreadContext(OpenTID(thread),&c) ) - return NULL; - if( reg == 3 ) - return (void*)(int_val)c.EFlags; - if( reg == 11 ) - return NULL; // TODO - return (void*)(int_val)*GetContextReg32(&c,reg); - } +# ifndef HL_64 + return NULL; # else - if( is64 ) return NULL; -# endif + if( !is64 ) return NULL; CONTEXT c; c.ContextFlags = CONTEXT_FULL | CONTEXT_DEBUG_REGISTERS; if( !GetThreadContext(OpenTID(thread),&c) ) return NULL; if( reg == 3 ) return (void*)(int_val)c.EFlags; - if( reg == 11 ) -#ifdef HL_64 - return (void*)(int_val)c.FltSave.XmmRegisters[0].Low; -#else - return (void*)*(int_val*)&c.ExtendedRegisters[10*16]; -#endif return (void*)*GetContextReg(&c,reg); +#endif # elif defined(MAC_DEBUG) return mdbg_read_register(pid, thread, get_reg(reg), is64); # elif defined(USE_PTRACE) @@ -361,38 +332,19 @@ HL_API void *hl_debug_read_register( int pid, int thread, int reg, bool is64 ) { HL_API bool hl_debug_write_register( int pid, int thread, int reg, void *value, bool is64 ) { # if defined(HL_WIN) -# ifdef HL_64 - if( !is64 ) { - WOW64_CONTEXT c; - c.ContextFlags = CONTEXT_FULL | CONTEXT_DEBUG_REGISTERS; - if( !Wow64GetThreadContext(OpenTID(thread),&c) ) - return false; - if( reg == 3 ) - c.EFlags = (int)(int_val)value; - else if( reg == 11 ) - return false; // TODO - else - *GetContextReg32(&c,reg) = (DWORD)(int_val)value; - return (bool)Wow64SetThreadContext(OpenTID(thread),&c); - } +# ifndef HL_64 + return NULL; # else - if( is64 ) return false; -# endif CONTEXT c; c.ContextFlags = CONTEXT_FULL | CONTEXT_DEBUG_REGISTERS; if( !GetThreadContext(OpenTID(thread),&c) ) return false; if( reg == 3 ) c.EFlags = (int)(int_val)value; - else if( reg == 11 ) -# ifdef HL_64 - c.FltSave.XmmRegisters[0].Low = (int_val)value; -# else - *(int_val*)&c.ExtendedRegisters[10*16] = (int_val)value; -# endif else - *GetContextReg(&c,reg) = (REGDATA)value; + *GetContextReg(&c,reg) = (DWORD64)value; return (bool)SetThreadContext(OpenTID(thread),&c); +# endif # elif defined(MAC_DEBUG) return mdbg_write_register(pid, thread, get_reg(reg), value, is64); # elif defined(USE_PTRACE)