From 15d5597a9355b5343b86617afc31aec8d6ea184b Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 26 Jul 2022 14:13:11 -0700 Subject: [PATCH] New constant caching insn: opt_getconstant_path Previously YARV bytecode implemented constant caching by having a pair of instructions, opt_getinlinecache and opt_setinlinecache, wrapping a series of getconstant calls (with putobject providing supporting arguments). This commit replaces that pattern with a new instruction, opt_getconstant_path, handling both getting/setting the inline cache and fetching the constant on a cache miss. This is implemented by storing the full constant path as a null-terminated array of IDs inside of the IC structure. idNULL is used to signal an absolute constant reference. $ ./miniruby --dump=insns -e '::Foo::Bar::Baz' == disasm: #@-e:1 (1,0)-(1,13)> (catch: FALSE) 0000 opt_getconstant_path ( 1)[Li] 0002 leave The motivation for this is that we had increasingly found the need to disassemble the instructions between the opt_getinlinecache and opt_setinlinecache in order to determine the constant we are fetching, or otherwise store metadata. This disassembly was done: * In opt_setinlinecache, to register the IC against the constant names it is using for granular invalidation. * In rb_iseq_free, to unregister the IC from the invalidation table. * In YJIT to find the position of a opt_getinlinecache instruction to invalidate it when the cache is populated * In YJIT to register the constant names being used for invalidation. With this change we no longe need disassemly for these (in fact rb_iseq_each is now unused), as the list of constant names being referenced is held in the IC. This should also make it possible to make more optimizations in the future. This may also reduce the size of iseqs, as previously each segment required 32 bytes (on 64-bit platforms) for each constant segment. This implementation only stores one ID per-segment. There should be no significant performance change between this and the previous implementation. Previously opt_getinlinecache was a "leaf" instruction, but it included a jump (almost always to a separate cache line). Now opt_getconstant_path is a non-leaf (it may raise/autoload/call const_missing) but it does not jump. These seem to even out. --- benchmark/vm_const.yml | 6 + common.mk | 2 +- compile.c | 201 ++++++++----- insns.def | 62 ++-- iseq.c | 114 +++---- iseq.h | 1 + test/ruby/test_mjit.rb | 6 +- test/ruby/test_yjit.rb | 8 +- ...erb => _mjit_compile_getconstant_path.erb} | 1 - tool/ruby_vm/views/mjit_compile.inc.erb | 4 +- vm_core.h | 19 +- vm_insnhelper.c | 83 +++--- yjit.h | 4 +- yjit/src/codegen.rs | 29 +- yjit/src/cruby_bindings.inc.rs | 278 +++++++++--------- yjit/src/invariants.rs | 82 ++---- 16 files changed, 476 insertions(+), 424 deletions(-) rename tool/ruby_vm/views/{_mjit_compile_getinlinecache.erb => _mjit_compile_getconstant_path.erb} (94%) diff --git a/benchmark/vm_const.yml b/benchmark/vm_const.yml index 6064d4eed0..8939ca0cd3 100644 --- a/benchmark/vm_const.yml +++ b/benchmark/vm_const.yml @@ -1,7 +1,13 @@ prelude: | Const = 1 + A = B = C = D = E = F = G = H = I = J = K = L = M = N = O = P = Q = R = S = T = U = V = W = X = Y = Z = 1 + def foo + A; B; C; D; E; F; G; H; I; J; K; L; M; N; O; P; Q; R; S; T; U; V; W; X; Y; Z + end benchmark: vm_const: | j = Const k = Const + vm_const_many: | + foo loop_count: 30000000 diff --git a/common.mk b/common.mk index aeb87dfb55..8dc7029d9d 100644 --- a/common.mk +++ b/common.mk @@ -1045,7 +1045,7 @@ $(srcs_vpath)mjit_compile.inc: $(tooldir)/ruby_vm/views/mjit_compile.inc.erb $(i $(tooldir)/ruby_vm/views/_mjit_compile_insn.erb $(tooldir)/ruby_vm/views/_mjit_compile_send.erb \ $(tooldir)/ruby_vm/views/_mjit_compile_ivar.erb \ $(tooldir)/ruby_vm/views/_mjit_compile_insn_body.erb $(tooldir)/ruby_vm/views/_mjit_compile_pc_and_sp.erb \ - $(tooldir)/ruby_vm/views/_mjit_compile_invokebuiltin.erb $(tooldir)/ruby_vm/views/_mjit_compile_getinlinecache.erb + $(tooldir)/ruby_vm/views/_mjit_compile_invokebuiltin.erb $(tooldir)/ruby_vm/views/_mjit_compile_getconstant_path.erb BUILTIN_RB_SRCS = \ $(srcdir)/ast.rb \ diff --git a/compile.c b/compile.c index dc8ed45946..3e9e02cc9a 100644 --- a/compile.c +++ b/compile.c @@ -2251,6 +2251,26 @@ add_adjust_info(struct iseq_insn_info_entry *insns_info, unsigned int *positions return TRUE; } +static ID *array_to_idlist(VALUE arr) { + RUBY_ASSERT(RB_TYPE_P(arr, T_ARRAY)); + long size = RARRAY_LEN(arr); + ID *ids = (ID *)ALLOC_N(ID, size + 1); + for (int i = 0; i < size; i++) { + VALUE sym = RARRAY_AREF(arr, i); + ids[i] = SYM2ID(sym); + } + ids[size] = 0; + return ids; +} + +static VALUE idlist_to_array(const ID *ids) { + VALUE arr = rb_ary_new(); + while (*ids) { + rb_ary_push(arr, ID2SYM(*ids++)); + } + return arr; +} + /** ruby insn object list -> raw instruction sequence */ @@ -2433,6 +2453,21 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) } /* [ TS_IVC | TS_ICVARC | TS_ISE | TS_IC ] */ case TS_IC: /* inline cache: constants */ + { + unsigned int ic_index = ISEQ_COMPILE_DATA(iseq)->ic_index++; + IC ic = &ISEQ_IS_ENTRY_START(body, type)[ic_index].ic_cache; + if (UNLIKELY(ic_index >= body->ic_size)) { + BADINSN_DUMP(anchor, &iobj->link, 0); + COMPILE_ERROR(iseq, iobj->insn_info.line_no, + "iseq_set_sequence: ic_index overflow: index: %d, size: %d", + ic_index, ISEQ_IS_SIZE(body)); + } + + ic->segments = array_to_idlist(operands[j]); + + generated_iseq[code_index + 1 + j] = (VALUE)ic; + } + break; case TS_ISE: /* inline storage entry: `once` insn */ case TS_ICVARC: /* inline cvar cache */ case TS_IVC: /* inline ivar cache */ @@ -2447,11 +2482,6 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) } generated_iseq[code_index + 1 + j] = (VALUE)ic; - if (insn == BIN(opt_getinlinecache) && type == TS_IC) { - // Store the instruction index for opt_getinlinecache on the IC for - // YJIT to invalidate code when opt_setinlinecache runs. - ic->get_insn_idx = (unsigned int)code_index; - } break; } case TS_CALLDATA: @@ -5154,6 +5184,30 @@ compile_massign(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, return COMPILE_OK; } +static VALUE +collect_const_segments(rb_iseq_t *iseq, const NODE *node) +{ + VALUE arr = rb_ary_new(); + for (;;) + { + switch (nd_type(node)) { + case NODE_CONST: + rb_ary_unshift(arr, ID2SYM(node->nd_vid)); + return arr; + case NODE_COLON3: + rb_ary_unshift(arr, ID2SYM(node->nd_mid)); + rb_ary_unshift(arr, ID2SYM(idNULL)); + return arr; + case NODE_COLON2: + rb_ary_unshift(arr, ID2SYM(node->nd_mid)); + node = node->nd_head; + break; + default: + return Qfalse; + } + } +} + static int compile_const_prefix(rb_iseq_t *iseq, const NODE *const node, LINK_ANCHOR *const pref, LINK_ANCHOR *const body) @@ -8891,37 +8945,31 @@ compile_match(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i static int compile_colon2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped) { - const int line = nd_line(node); if (rb_is_const_id(node->nd_mid)) { /* constant */ - LABEL *lend = NEW_LABEL(line); - int ic_index = ISEQ_BODY(iseq)->ic_size++; - - DECL_ANCHOR(pref); - DECL_ANCHOR(body); - - INIT_ANCHOR(pref); - INIT_ANCHOR(body); - CHECK(compile_const_prefix(iseq, node, pref, body)); - if (LIST_INSN_SIZE_ZERO(pref)) { - if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { - ADD_INSN2(ret, node, opt_getinlinecache, lend, INT2FIX(ic_index)); - } - else { + VALUE segments; + if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache && + (segments = collect_const_segments(iseq, node))) { + ISEQ_BODY(iseq)->ic_size++; + ADD_INSN1(ret, node, opt_getconstant_path, segments); + RB_OBJ_WRITTEN(iseq, Qundef, segments); + } else { + /* constant */ + DECL_ANCHOR(pref); + DECL_ANCHOR(body); + + INIT_ANCHOR(pref); + INIT_ANCHOR(body); + CHECK(compile_const_prefix(iseq, node, pref, body)); + if (LIST_INSN_SIZE_ZERO(pref)) { ADD_INSN(ret, node, putnil); + ADD_SEQ(ret, body); } - - ADD_SEQ(ret, body); - - if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { - ADD_INSN1(ret, node, opt_setinlinecache, INT2FIX(ic_index)); - ADD_LABEL(ret, lend); + else { + ADD_SEQ(ret, pref); + ADD_SEQ(ret, body); } } - else { - ADD_SEQ(ret, pref); - ADD_SEQ(ret, body); - } } else { /* function call */ @@ -8938,25 +8986,18 @@ compile_colon2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, static int compile_colon3(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped) { - const int line = nd_line(node); - LABEL *lend = NEW_LABEL(line); - int ic_index = ISEQ_BODY(iseq)->ic_size++; - debugi("colon3#nd_mid", node->nd_mid); /* add cache insn */ if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { - ADD_INSN2(ret, node, opt_getinlinecache, lend, INT2FIX(ic_index)); - ADD_INSN(ret, node, pop); - } - - ADD_INSN1(ret, node, putobject, rb_cObject); - ADD_INSN1(ret, node, putobject, Qtrue); - ADD_INSN1(ret, node, getconstant, ID2SYM(node->nd_mid)); - - if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { - ADD_INSN1(ret, node, opt_setinlinecache, INT2FIX(ic_index)); - ADD_LABEL(ret, lend); + ISEQ_BODY(iseq)->ic_size++; + VALUE segments = rb_ary_new_from_args(2, ID2SYM(idNULL), ID2SYM(node->nd_mid)); + ADD_INSN1(ret, node, opt_getconstant_path, segments); + RB_OBJ_WRITTEN(iseq, Qundef, segments); + } else { + ADD_INSN1(ret, node, putobject, rb_cObject); + ADD_INSN1(ret, node, putobject, Qtrue); + ADD_INSN1(ret, node, getconstant, ID2SYM(node->nd_mid)); } if (popped) { @@ -9457,18 +9498,14 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no case NODE_CONST:{ debugi("nd_vid", node->nd_vid); - if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { - LABEL *lend = NEW_LABEL(line); - int ic_index = body->ic_size++; - - ADD_INSN2(ret, node, opt_getinlinecache, lend, INT2FIX(ic_index)); - ADD_INSN1(ret, node, putobject, Qtrue); - ADD_INSN1(ret, node, getconstant, ID2SYM(node->nd_vid)); - ADD_INSN1(ret, node, opt_setinlinecache, INT2FIX(ic_index)); - ADD_LABEL(ret, lend); - } - else { - ADD_INSN(ret, node, putnil); + if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { + body->ic_size++; + VALUE segments = rb_ary_new_from_args(1, ID2SYM(node->nd_vid)); + ADD_INSN1(ret, node, opt_getconstant_path, segments); + RB_OBJ_WRITTEN(iseq, Qundef, segments); + } + else { + ADD_INSN(ret, node, putnil); ADD_INSN1(ret, node, putobject, Qtrue); ADD_INSN1(ret, node, getconstant, ID2SYM(node->nd_vid)); } @@ -9953,10 +9990,16 @@ insn_data_to_s_detail(INSN *iobj) rb_str_concat(str, opobj_inspect(OPERAND_AT(iobj, j))); break; case TS_IC: /* inline cache */ + rb_str_concat(str, opobj_inspect(OPERAND_AT(iobj, j))); + break; case TS_IVC: /* inline ivar cache */ + rb_str_catf(str, "", FIX2INT(OPERAND_AT(iobj, j))); + break; case TS_ICVARC: /* inline cvar cache */ + rb_str_catf(str, "", FIX2INT(OPERAND_AT(iobj, j))); + break; case TS_ISE: /* inline storage entry */ - rb_str_catf(str, "", FIX2INT(OPERAND_AT(iobj, j))); + rb_str_catf(str, "", FIX2INT(OPERAND_AT(iobj, j))); break; case TS_CALLDATA: /* we store these as call infos at compile time */ { @@ -10352,9 +10395,20 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, } break; case TS_IC: - argv[j] = op; - if (NUM2UINT(op) >= ISEQ_BODY(iseq)->ic_size) { - ISEQ_BODY(iseq)->ic_size = NUM2INT(op) + 1; + { + VALUE segments = rb_ary_new(); + op = rb_to_array_type(op); + + for (int i = 0; i < RARRAY_LEN(op); i++) { + VALUE sym = RARRAY_AREF(op, i); + sym = rb_to_symbol_type(sym); + rb_ary_push(segments, sym); + } + + RB_GC_GUARD(op); + argv[j] = segments; + RB_OBJ_WRITTEN(iseq, Qundef, segments); + ISEQ_BODY(iseq)->ic_size++; } break; case TS_IVC: /* inline ivar cache */ @@ -10548,6 +10602,7 @@ rb_iseq_mark_insn_storage(struct iseq_compile_data_storage *storage) case TS_CDHASH: case TS_ISEQ: case TS_VALUE: + case TS_IC: // constant path array case TS_CALLDATA: // ci is stored. { VALUE op = OPERAND_AT(iobj, j); @@ -11176,6 +11231,12 @@ ibf_dump_code(struct ibf_dump *dump, const rb_iseq_t *iseq) wv = (VALUE)ibf_dump_iseq(dump, (const rb_iseq_t *)op); break; case TS_IC: + { + IC ic = (IC)op; + VALUE arr = idlist_to_array(ic->segments); + wv = ibf_dump_object(dump, arr); + } + break; case TS_ISE: case TS_IVC: case TS_ICVARC: @@ -11220,6 +11281,7 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod struct rb_iseq_constant_body *load_body = ISEQ_BODY(iseq); struct rb_call_data *cd_entries = load_body->call_data; + int ic_index = 0; iseq_bits_t * mark_offset_bits; @@ -11236,7 +11298,6 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod for (code_index=0; code_indexsegments = array_to_idlist(arr); + + code[code_index] = (VALUE)ic; + } + break; case TS_ISE: case TS_ICVARC: case TS_IVC: @@ -11299,12 +11370,6 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod ISE ic = ISEQ_IS_ENTRY_START(load_body, operand_type) + op; code[code_index] = (VALUE)ic; - - if (insn == BIN(opt_getinlinecache) && operand_type == TS_IC) { - // Store the instruction index for opt_getinlinecache on the IC for - // YJIT to invalidate code when opt_setinlinecache runs. - ic->ic_cache.get_insn_idx = insn_index; - } } break; case TS_CALLDATA: diff --git a/insns.def b/insns.def index 06ca31a850..adf350b657 100644 --- a/insns.def +++ b/insns.def @@ -253,6 +253,28 @@ setclassvariable vm_setclassvariable(GET_ISEQ(), GET_CFP(), id, val, ic); } +DEFINE_INSN +opt_getconstant_path +(IC ic) +() +(VALUE val) +// attr bool leaf = false; /* may autoload */ +{ + const ID *segments = ic->segments; + struct iseq_inline_constant_cache_entry *ice = ic->entry; + if (ice && vm_ic_hit_p(ice, GET_EP())) { + val = ice->value; + + VM_ASSERT(val == vm_get_ev_const_chain(ec, segments)); + } else { + ruby_vm_constant_cache_misses++; + val = vm_get_ev_const_chain(ec, segments); + vm_ic_track_const_chain(GET_CFP(), ic, segments); + // INSN_ATTR(width) == 2 + vm_ic_update(GET_ISEQ(), ic, val, GET_EP(), GET_PC() - 2); + } +} + /* Get constant variable id. If klass is Qnil and allow_nil is Qtrue, constants are searched in the current scope. Otherwise, get constant under klass class or module. @@ -1020,46 +1042,6 @@ branchnil /* for optimize */ /**********************************************************/ -/* push inline-cached value and go to dst if it is valid */ -DEFINE_INSN -opt_getinlinecache -(OFFSET dst, IC ic) -() -(VALUE val) -{ - struct iseq_inline_constant_cache_entry *ice = ic->entry; - - // If there isn't an entry, then we're going to walk through the ISEQ - // starting at this instruction until we get to the associated - // opt_setinlinecache and associate this inline cache with every getconstant - // listed in between. We're doing this here instead of when the instructions - // are first compiled because it's possible to turn off inline caches and we - // want this to work in either case. - if (!ice) { - vm_ic_compile(GET_CFP(), ic); - } - - if (ice && vm_ic_hit_p(ice, GET_EP())) { - val = ice->value; - JUMP(dst); - } - else { - ruby_vm_constant_cache_misses++; - val = Qnil; - } -} - -/* set inline cache */ -DEFINE_INSN -opt_setinlinecache -(IC ic) -(VALUE val) -(VALUE val) -// attr bool leaf = false; -{ - vm_ic_update(GET_ISEQ(), ic, val, GET_EP()); -} - /* run iseq only once */ DEFINE_INSN once diff --git a/iseq.c b/iseq.c index 3d40b88a0d..f13d30eb3f 100644 --- a/iseq.c +++ b/iseq.c @@ -102,68 +102,48 @@ compile_data_free(struct iseq_compile_data *compile_data) } } -struct iseq_clear_ic_references_data { - IC ic; -}; - -// This iterator is used to walk through the instructions and clean any -// references to ICs that are contained within this ISEQ out of the VM's -// constant cache table. It passes around a struct that holds the current IC -// we're looking for, which can be NULL (if we haven't hit an opt_getinlinecache -// instruction yet) or set to an IC (if we've hit an opt_getinlinecache and -// haven't yet hit the associated opt_setinlinecache). -static bool -iseq_clear_ic_references_i(VALUE *code, VALUE insn, size_t index, void *data) -{ - struct iseq_clear_ic_references_data *ic_data = (struct iseq_clear_ic_references_data *) data; +static void remove_from_constant_cache(ID id, IC ic) { + rb_vm_t *vm = GET_VM(); + VALUE lookup_result; + st_data_t ic_data = (st_data_t)ic; - switch (insn) { - case BIN(opt_getinlinecache): { - RUBY_ASSERT_ALWAYS(ic_data->ic == NULL); + if (rb_id_table_lookup(vm->constant_cache, id, &lookup_result)) { + st_table *ics = (st_table *)lookup_result; + st_delete(ics, &ic_data, NULL); - ic_data->ic = (IC) code[index + 2]; - return true; - } - case BIN(getconstant): { - if (ic_data->ic != NULL) { - ID id = (ID) code[index + 1]; - rb_vm_t *vm = GET_VM(); - VALUE lookup_result; - - if (rb_id_table_lookup(vm->constant_cache, id, &lookup_result)) { - st_table *ics = (st_table *)lookup_result; - st_data_t ic = (st_data_t)ic_data->ic; - st_delete(ics, &ic, NULL); - - if (ics->num_entries == 0) { - rb_id_table_delete(vm->constant_cache, id); - st_free_table(ics); - } - } + if (ics->num_entries == 0) { + rb_id_table_delete(vm->constant_cache, id); + st_free_table(ics); } - - return true; - } - case BIN(opt_setinlinecache): { - RUBY_ASSERT_ALWAYS(ic_data->ic != NULL); - - ic_data->ic = NULL; - return true; - } - default: - return true; } } // When an ISEQ is being freed, all of its associated ICs are going to go away -// as well. Because of this, we need to walk through the ISEQ, find any -// opt_getinlinecache calls, and clear out the VM's constant cache of associated -// ICs. +// as well. Because of this, we need to iterate over the ICs, and clear them +// from the VM's constant cache. static void iseq_clear_ic_references(const rb_iseq_t *iseq) { - struct iseq_clear_ic_references_data data = { .ic = NULL }; - rb_iseq_each(iseq, 0, iseq_clear_ic_references_i, (void *) &data); + for (unsigned int ic_idx = 0; ic_idx < ISEQ_BODY(iseq)->ic_size; ic_idx++) { + IC ic = &ISEQ_IS_IC_ENTRY(ISEQ_BODY(iseq), ic_idx); + + // Iterate over the IC's constant path's segments and clean any references to + // the ICs out of the VM's constant cache table. + const ID *segments = ic->segments; + + // It's possible that segments is NULL if we overallocated an IC but + // optimizations removed the instruction using it + if (segments == NULL) + continue; + + for (int i = 0; segments[i]; i++) { + ID id = segments[i]; + if (id == idNULL) continue; + remove_from_constant_cache(id, ic); + } + + ruby_xfree((void *)segments); + } } void @@ -593,6 +573,17 @@ rb_iseq_memsize(const rb_iseq_t *iseq) /* body->is_entries */ size += ISEQ_IS_SIZE(body) * sizeof(union iseq_inline_storage_entry); + /* IC entries constant segments */ + for (unsigned int ic_idx = 0; ic_idx < body->ic_size; ic_idx++) { + IC ic = &ISEQ_IS_IC_ENTRY(body, ic_idx); + const ID *ids = ic->segments; + if (!ids) continue; + while (*ids++) { + size += sizeof(ID); + } + size += sizeof(ID); // null terminator + } + /* body->call_data */ size += body->ci_size * sizeof(struct rb_call_data); // TODO: should we count imemo_callinfo? @@ -2175,6 +2166,16 @@ rb_insn_operand_intern(const rb_iseq_t *iseq, } case TS_IC: + { + ret = rb_sprintf("is_entries); + const ID *segments = ((IC)op)->segments; + rb_str_cat2(ret, rb_id2name(*segments++)); + while (*segments) { + rb_str_catf(ret, "::%s", rb_id2name(*segments++)); + } + rb_str_cat2(ret, ">"); + } + break; case TS_IVC: case TS_ICVARC: case TS_ISE: @@ -3011,6 +3012,15 @@ iseq_data_to_ary(const rb_iseq_t *iseq) } break; case TS_IC: + { + VALUE list = rb_ary_new(); + const ID *ids = ((IC)*seq)->segments; + while (*ids) { + rb_ary_push(list, ID2SYM(*ids++)); + } + rb_ary_push(ary, list); + } + break; case TS_IVC: case TS_ICVARC: case TS_ISE: diff --git a/iseq.h b/iseq.h index e7db9b951f..fa328e9a84 100644 --- a/iseq.h +++ b/iseq.h @@ -119,6 +119,7 @@ struct iseq_compile_data { int node_level; int isolated_depth; unsigned int ci_index; + unsigned int ic_index; const rb_compile_option_t *option; struct rb_id_table *ivar_cache_table; const struct rb_builtin_function *builtin_function_table; diff --git a/test/ruby/test_mjit.rb b/test/ruby/test_mjit.rb index 02be88aa32..688b966f43 100644 --- a/test/ruby/test_mjit.rb +++ b/test/ruby/test_mjit.rb @@ -179,7 +179,7 @@ def self.foo end def test_compile_insn_constant - assert_compile_once("#{<<~"begin;"}\n#{<<~"end;"}", result_inspect: '1', insns: %i[getconstant setconstant]) + assert_compile_once("#{<<~"begin;"}\n#{<<~"end;"}", result_inspect: '1', insns: %i[opt_getconstant_path setconstant]) begin; FOO = 1 FOO @@ -490,8 +490,8 @@ def test_compile_insn_objtostring end; end - def test_compile_insn_inlinecache - assert_compile_once('Struct', result_inspect: 'Struct', insns: %i[opt_getinlinecache opt_setinlinecache]) + def test_compile_insn_getconstant_path + assert_compile_once('Struct', result_inspect: 'Struct', insns: %i[opt_getconstant_path]) end def test_compile_insn_once diff --git a/test/ruby/test_yjit.rb b/test/ruby/test_yjit.rb index f8fc4f21ef..82bcf65a9e 100644 --- a/test/ruby/test_yjit.rb +++ b/test/ruby/test_yjit.rb @@ -389,8 +389,8 @@ def test_getspecial_backref assert_compiles("'foo' =~ /(o)./; $2", insns: %i[getspecial], result: nil) end - def test_compile_opt_getinlinecache - assert_compiles(<<~RUBY, insns: %i[opt_getinlinecache], result: 123, call_threshold: 2) + def test_compile_opt_getconstant_path + assert_compiles(<<~RUBY, insns: %i[opt_getconstant_path], result: 123, call_threshold: 2) def get_foo FOO end @@ -402,8 +402,8 @@ def get_foo RUBY end - def test_opt_getinlinecache_slowpath - assert_compiles(<<~RUBY, exits: { opt_getinlinecache: 1 }, result: [42, 42, 1, 1], call_threshold: 2) + def test_opt_getconstant_path_slowpath + assert_compiles(<<~RUBY, exits: { opt_getconstant_path: 1 }, result: [42, 42, 1, 1], call_threshold: 2) class A FOO = 42 class << self diff --git a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb b/tool/ruby_vm/views/_mjit_compile_getconstant_path.erb similarity index 94% rename from tool/ruby_vm/views/_mjit_compile_getinlinecache.erb rename to tool/ruby_vm/views/_mjit_compile_getconstant_path.erb index fa38af4045..c321da9a52 100644 --- a/tool/ruby_vm/views/_mjit_compile_getinlinecache.erb +++ b/tool/ruby_vm/views/_mjit_compile_getconstant_path.erb @@ -17,7 +17,6 @@ % # JIT: Inline everything in IC, and cancel the slow path fprintf(f, " if (vm_inlined_ic_hit_p(0x%"PRIxVALUE", 0x%"PRIxVALUE", (const rb_cref_t *)0x%"PRIxVALUE", reg_cfp->ep)) {", ice->flags, ice->value, (VALUE)ice->ic_cref); fprintf(f, " stack[%d] = 0x%"PRIxVALUE";\n", b->stack_size, ice->value); - fprintf(f, " goto label_%d;\n", pos + insn_len(insn) + (int)dst); fprintf(f, " }"); fprintf(f, " else {"); fprintf(f, " reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size); diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb index 5820f81770..49158b1de6 100644 --- a/tool/ruby_vm/views/mjit_compile.inc.erb +++ b/tool/ruby_vm/views/mjit_compile.inc.erb @@ -65,8 +65,8 @@ switch (insn) { <%= render 'mjit_compile_ivar', locals: { insn: insn } -%> % when 'invokebuiltin', 'opt_invokebuiltin_delegate' <%= render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%> -% when 'opt_getinlinecache' -<%= render 'mjit_compile_getinlinecache', locals: { insn: insn } -%> +% when 'opt_getconstant_path' +<%= render 'mjit_compile_getconstant_path', locals: { insn: insn } -%> % when 'leave', 'opt_invokebuiltin_delegate_leave' % # opt_invokebuiltin_delegate_leave also implements leave insn. We need to handle it here for inlining. % if insn.name == 'opt_invokebuiltin_delegate_leave' diff --git a/vm_core.h b/vm_core.h index c394862ecb..65dda45328 100644 --- a/vm_core.h +++ b/vm_core.h @@ -256,9 +256,19 @@ STATIC_ASSERT(sizeof_iseq_inline_constant_cache_entry, struct iseq_inline_constant_cache { struct iseq_inline_constant_cache_entry *entry; - // For YJIT: the index to the opt_getinlinecache instruction in the same iseq. - // It's set during compile time and constant once set. - unsigned get_insn_idx; + + /** + * A null-terminated list of ids, used to represent a constant's path + * idNULL is used to represent the :: prefix, and 0 is used to donate the end + * of the list. + * + * For example + * FOO {rb_intern("FOO"), 0} + * FOO::BAR {rb_intern("FOO"), rb_intern("BAR"), 0} + * ::FOO {idNULL, rb_intern("FOO"), 0} + * ::FOO::BAR {idNULL, rb_intern("FOO"), rb_intern("BAR"), 0} + */ + const ID *segments; }; struct iseq_inline_iv_cache_entry { @@ -339,6 +349,9 @@ typedef uintptr_t iseq_bits_t; #define ISEQ_IS_SIZE(body) (body->ic_size + body->ivc_size + body->ise_size + body->icvarc_size) +/* [ TS_IVC | TS_ICVARC | TS_ISE | TS_IC ] */ +#define ISEQ_IS_IC_ENTRY(body, idx) (body->is_entries[(idx) + body->ise_size + body->icvarc_size + body->ivc_size].ic_cache); + /* instruction sequence type */ enum rb_iseq_type { ISEQ_TYPE_TOP, diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 2fb1ecb5f7..7f124543d6 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1039,6 +1039,27 @@ vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_ } } +static inline VALUE +vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments) +{ + VALUE val; + int idx = 0; + int allow_nil = TRUE; + val = Qnil; + if (segments[0] == idNULL) { + val = rb_cObject; + idx++; + allow_nil = FALSE; + } + while (segments[idx]) { + ID id = segments[idx++]; + val = vm_get_ev_const(ec, val, id, allow_nil, 0); + allow_nil = FALSE; + } + return val; +} + + static inline VALUE vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise) { @@ -4917,53 +4938,35 @@ vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t num, const VALUE *ptr) #define IMEMO_CONST_CACHE_SHAREABLE IMEMO_FL_USER0 -// This is the iterator used by vm_ic_compile for rb_iseq_each. It is used as a -// callback for each instruction within the ISEQ, and is meant to return a -// boolean indicating whether or not to keep iterating. -// -// This is used to walk through the ISEQ and find all getconstant instructions -// between the starting opt_getinlinecache and the ending opt_setinlinecache and -// associating the inline cache with the constant name components on the VM. -static bool -vm_ic_compile_i(VALUE *code, VALUE insn, size_t index, void *ic) +static void +vm_track_constant_cache(ID id, void *ic) { - if (insn == BIN(opt_setinlinecache)) { - return false; - } + struct rb_id_table *const_cache = GET_VM()->constant_cache; + VALUE lookup_result; + st_table *ics; - if (insn == BIN(getconstant)) { - ID id = code[index + 1]; - struct rb_id_table *const_cache = GET_VM()->constant_cache; - VALUE lookup_result; - st_table *ics; - - if (rb_id_table_lookup(const_cache, id, &lookup_result)) { - ics = (st_table *)lookup_result; - } - else { - ics = st_init_numtable(); - rb_id_table_insert(const_cache, id, (VALUE)ics); - } - - st_insert(ics, (st_data_t) ic, (st_data_t) Qtrue); + if (rb_id_table_lookup(const_cache, id, &lookup_result)) { + ics = (st_table *)lookup_result; + } + else { + ics = st_init_numtable(); + rb_id_table_insert(const_cache, id, (VALUE)ics); } - return true; + st_insert(ics, (st_data_t) ic, (st_data_t) Qtrue); } -// Loop through the instruction sequences starting at the opt_getinlinecache -// call and gather up every getconstant's ID. Associate that with the VM's -// constant cache so that whenever one of the constants changes the inline cache -// will get busted. static void -vm_ic_compile(rb_control_frame_t *cfp, IC ic) +vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments) { - const rb_iseq_t *iseq = cfp->iseq; - RB_VM_LOCK_ENTER(); - { - rb_iseq_each(iseq, cfp->pc - ISEQ_BODY(iseq)->iseq_encoded, vm_ic_compile_i, (void *) ic); + + for (int i = 0; segments[i]; i++) { + ID id = segments[i]; + if (id == idNULL) continue; + vm_track_constant_cache(id, ic); } + RB_VM_LOCK_LEAVE(); } @@ -4995,7 +4998,7 @@ rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep) } static void -vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep) +vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc) { if (ruby_vm_const_missing_count > 0) { ruby_vm_const_missing_count = 0; @@ -5011,7 +5014,9 @@ vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep) #ifndef MJIT_HEADER // MJIT and YJIT can't be on at the same time, so there is no need to // notify YJIT about changes to the IC when running inside MJIT code. - rb_yjit_constant_ic_update(iseq, ic); + RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded); + unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded); + rb_yjit_constant_ic_update(iseq, ic, pos); #endif } diff --git a/yjit.h b/yjit.h index cf420df251..8d483df3cc 100644 --- a/yjit.h +++ b/yjit.h @@ -50,7 +50,7 @@ void rb_yjit_iseq_mark(void *payload); void rb_yjit_iseq_update_references(void *payload); void rb_yjit_iseq_free(void *payload); void rb_yjit_before_ractor_spawn(void); -void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic); +void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx); void rb_yjit_tracing_invalidate_all(void); #else @@ -73,7 +73,7 @@ static inline void rb_yjit_iseq_mark(void *payload) {} static inline void rb_yjit_iseq_update_references(void *payload) {} static inline void rb_yjit_iseq_free(void *payload) {} static inline void rb_yjit_before_ractor_spawn(void) {} -static inline void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic) {} +static inline void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx) {} static inline void rb_yjit_tracing_invalidate_all(void) {} #endif // #if YJIT_BUILD diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index af501158ff..ca910c570d 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -740,9 +740,10 @@ pub fn gen_single_block( .try_into() .unwrap(); - // opt_getinlinecache wants to be in a block all on its own. Cut the block short - // if we run into it. See gen_opt_getinlinecache() for details. - if opcode == YARVINSN_opt_getinlinecache.as_usize() && insn_idx > starting_insn_idx { + // We need opt_getconstant_path to be in a block all on its own. Cut the block short + // if we run into it. This is necessary because we want to invalidate based on the + // instruction's index. + if opcode == YARVINSN_opt_getconstant_path.as_usize() && insn_idx > starting_insn_idx { jump_to_next_insn(&mut jit, &ctx, cb, ocb); break; } @@ -5556,15 +5557,15 @@ fn gen_setclassvariable( KeepCompiling } -fn gen_opt_getinlinecache( +fn gen_opt_getconstant_path( jit: &mut JITState, ctx: &mut Context, cb: &mut CodeBlock, ocb: &mut OutlinedCb, ) -> CodegenStatus { - let jump_offset = jit_get_arg(jit, 0); - let const_cache_as_value = jit_get_arg(jit, 1); + let const_cache_as_value = jit_get_arg(jit, 0); let ic: *const iseq_inline_constant_cache = const_cache_as_value.as_ptr(); + let idlist: *const ID = unsafe { (*ic).segments }; // See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update(). let ice = unsafe { (*ic).entry }; @@ -5606,22 +5607,12 @@ fn gen_opt_getinlinecache( // Invalidate output code on any constant writes associated with // constants referenced within the current block. - assume_stable_constant_names(jit, ocb); + assume_stable_constant_names(jit, ocb, idlist); jit_putobject(jit, ctx, cb, unsafe { (*ice).value }); } - // Jump over the code for filling the cache - let jump_idx = jit_next_insn_idx(jit) + jump_offset.as_u32(); - gen_direct_jump( - jit, - ctx, - BlockId { - iseq: jit.iseq, - idx: jump_idx, - }, - cb, - ); + jump_to_next_insn(jit, ctx, cb, ocb); EndBlock } @@ -5945,7 +5936,7 @@ fn get_gen_fn(opcode: VALUE) -> Option { YARVINSN_opt_size => Some(gen_opt_size), YARVINSN_opt_length => Some(gen_opt_length), YARVINSN_opt_regexpmatch2 => Some(gen_opt_regexpmatch2), - YARVINSN_opt_getinlinecache => Some(gen_opt_getinlinecache), + YARVINSN_opt_getconstant_path => Some(gen_opt_getconstant_path), YARVINSN_invokebuiltin => Some(gen_invokebuiltin), YARVINSN_opt_invokebuiltin_delegate => Some(gen_opt_invokebuiltin_delegate), YARVINSN_opt_invokebuiltin_delegate_leave => Some(gen_opt_invokebuiltin_delegate), diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index fed132588c..21ffd1a5a7 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -560,7 +560,7 @@ pub struct iseq_inline_constant_cache_entry { #[derive(Debug, Copy, Clone)] pub struct iseq_inline_constant_cache { pub entry: *mut iseq_inline_constant_cache_entry, - pub get_insn_idx: ::std::os::raw::c_uint, + pub segments: *const ID, } #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -753,107 +753,107 @@ pub const YARVINSN_getinstancevariable: ruby_vminsn_type = 8; pub const YARVINSN_setinstancevariable: ruby_vminsn_type = 9; pub const YARVINSN_getclassvariable: ruby_vminsn_type = 10; pub const YARVINSN_setclassvariable: ruby_vminsn_type = 11; -pub const YARVINSN_getconstant: ruby_vminsn_type = 12; -pub const YARVINSN_setconstant: ruby_vminsn_type = 13; -pub const YARVINSN_getglobal: ruby_vminsn_type = 14; -pub const YARVINSN_setglobal: ruby_vminsn_type = 15; -pub const YARVINSN_putnil: ruby_vminsn_type = 16; -pub const YARVINSN_putself: ruby_vminsn_type = 17; -pub const YARVINSN_putobject: ruby_vminsn_type = 18; -pub const YARVINSN_putspecialobject: ruby_vminsn_type = 19; -pub const YARVINSN_putstring: ruby_vminsn_type = 20; -pub const YARVINSN_concatstrings: ruby_vminsn_type = 21; -pub const YARVINSN_anytostring: ruby_vminsn_type = 22; -pub const YARVINSN_toregexp: ruby_vminsn_type = 23; -pub const YARVINSN_intern: ruby_vminsn_type = 24; -pub const YARVINSN_newarray: ruby_vminsn_type = 25; -pub const YARVINSN_newarraykwsplat: ruby_vminsn_type = 26; -pub const YARVINSN_duparray: ruby_vminsn_type = 27; -pub const YARVINSN_duphash: ruby_vminsn_type = 28; -pub const YARVINSN_expandarray: ruby_vminsn_type = 29; -pub const YARVINSN_concatarray: ruby_vminsn_type = 30; -pub const YARVINSN_splatarray: ruby_vminsn_type = 31; -pub const YARVINSN_newhash: ruby_vminsn_type = 32; -pub const YARVINSN_newrange: ruby_vminsn_type = 33; -pub const YARVINSN_pop: ruby_vminsn_type = 34; -pub const YARVINSN_dup: ruby_vminsn_type = 35; -pub const YARVINSN_dupn: ruby_vminsn_type = 36; -pub const YARVINSN_swap: ruby_vminsn_type = 37; -pub const YARVINSN_topn: ruby_vminsn_type = 38; -pub const YARVINSN_setn: ruby_vminsn_type = 39; -pub const YARVINSN_adjuststack: ruby_vminsn_type = 40; -pub const YARVINSN_defined: ruby_vminsn_type = 41; -pub const YARVINSN_checkmatch: ruby_vminsn_type = 42; -pub const YARVINSN_checkkeyword: ruby_vminsn_type = 43; -pub const YARVINSN_checktype: ruby_vminsn_type = 44; -pub const YARVINSN_defineclass: ruby_vminsn_type = 45; -pub const YARVINSN_definemethod: ruby_vminsn_type = 46; -pub const YARVINSN_definesmethod: ruby_vminsn_type = 47; -pub const YARVINSN_send: ruby_vminsn_type = 48; -pub const YARVINSN_opt_send_without_block: ruby_vminsn_type = 49; -pub const YARVINSN_objtostring: ruby_vminsn_type = 50; -pub const YARVINSN_opt_str_freeze: ruby_vminsn_type = 51; -pub const YARVINSN_opt_nil_p: ruby_vminsn_type = 52; -pub const YARVINSN_opt_str_uminus: ruby_vminsn_type = 53; -pub const YARVINSN_opt_newarray_max: ruby_vminsn_type = 54; -pub const YARVINSN_opt_newarray_min: ruby_vminsn_type = 55; -pub const YARVINSN_invokesuper: ruby_vminsn_type = 56; -pub const YARVINSN_invokeblock: ruby_vminsn_type = 57; -pub const YARVINSN_leave: ruby_vminsn_type = 58; -pub const YARVINSN_throw: ruby_vminsn_type = 59; -pub const YARVINSN_jump: ruby_vminsn_type = 60; -pub const YARVINSN_branchif: ruby_vminsn_type = 61; -pub const YARVINSN_branchunless: ruby_vminsn_type = 62; -pub const YARVINSN_branchnil: ruby_vminsn_type = 63; -pub const YARVINSN_opt_getinlinecache: ruby_vminsn_type = 64; -pub const YARVINSN_opt_setinlinecache: ruby_vminsn_type = 65; -pub const YARVINSN_once: ruby_vminsn_type = 66; -pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 67; -pub const YARVINSN_opt_plus: ruby_vminsn_type = 68; -pub const YARVINSN_opt_minus: ruby_vminsn_type = 69; -pub const YARVINSN_opt_mult: ruby_vminsn_type = 70; -pub const YARVINSN_opt_div: ruby_vminsn_type = 71; -pub const YARVINSN_opt_mod: ruby_vminsn_type = 72; -pub const YARVINSN_opt_eq: ruby_vminsn_type = 73; -pub const YARVINSN_opt_neq: ruby_vminsn_type = 74; -pub const YARVINSN_opt_lt: ruby_vminsn_type = 75; -pub const YARVINSN_opt_le: ruby_vminsn_type = 76; -pub const YARVINSN_opt_gt: ruby_vminsn_type = 77; -pub const YARVINSN_opt_ge: ruby_vminsn_type = 78; -pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 79; -pub const YARVINSN_opt_and: ruby_vminsn_type = 80; -pub const YARVINSN_opt_or: ruby_vminsn_type = 81; -pub const YARVINSN_opt_aref: ruby_vminsn_type = 82; -pub const YARVINSN_opt_aset: ruby_vminsn_type = 83; -pub const YARVINSN_opt_aset_with: ruby_vminsn_type = 84; -pub const YARVINSN_opt_aref_with: ruby_vminsn_type = 85; -pub const YARVINSN_opt_length: ruby_vminsn_type = 86; -pub const YARVINSN_opt_size: ruby_vminsn_type = 87; -pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 88; -pub const YARVINSN_opt_succ: ruby_vminsn_type = 89; -pub const YARVINSN_opt_not: ruby_vminsn_type = 90; -pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 91; -pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 92; -pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 93; -pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 94; -pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 95; -pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 96; -pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 97; -pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 98; -pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 99; -pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 100; -pub const YARVINSN_trace_nop: ruby_vminsn_type = 101; -pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 102; -pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 103; -pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 104; -pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 105; -pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 106; -pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 107; -pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 108; -pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 109; -pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 110; -pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 111; -pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 112; +pub const YARVINSN_opt_getconstant_path: ruby_vminsn_type = 12; +pub const YARVINSN_getconstant: ruby_vminsn_type = 13; +pub const YARVINSN_setconstant: ruby_vminsn_type = 14; +pub const YARVINSN_getglobal: ruby_vminsn_type = 15; +pub const YARVINSN_setglobal: ruby_vminsn_type = 16; +pub const YARVINSN_putnil: ruby_vminsn_type = 17; +pub const YARVINSN_putself: ruby_vminsn_type = 18; +pub const YARVINSN_putobject: ruby_vminsn_type = 19; +pub const YARVINSN_putspecialobject: ruby_vminsn_type = 20; +pub const YARVINSN_putstring: ruby_vminsn_type = 21; +pub const YARVINSN_concatstrings: ruby_vminsn_type = 22; +pub const YARVINSN_anytostring: ruby_vminsn_type = 23; +pub const YARVINSN_toregexp: ruby_vminsn_type = 24; +pub const YARVINSN_intern: ruby_vminsn_type = 25; +pub const YARVINSN_newarray: ruby_vminsn_type = 26; +pub const YARVINSN_newarraykwsplat: ruby_vminsn_type = 27; +pub const YARVINSN_duparray: ruby_vminsn_type = 28; +pub const YARVINSN_duphash: ruby_vminsn_type = 29; +pub const YARVINSN_expandarray: ruby_vminsn_type = 30; +pub const YARVINSN_concatarray: ruby_vminsn_type = 31; +pub const YARVINSN_splatarray: ruby_vminsn_type = 32; +pub const YARVINSN_newhash: ruby_vminsn_type = 33; +pub const YARVINSN_newrange: ruby_vminsn_type = 34; +pub const YARVINSN_pop: ruby_vminsn_type = 35; +pub const YARVINSN_dup: ruby_vminsn_type = 36; +pub const YARVINSN_dupn: ruby_vminsn_type = 37; +pub const YARVINSN_swap: ruby_vminsn_type = 38; +pub const YARVINSN_topn: ruby_vminsn_type = 39; +pub const YARVINSN_setn: ruby_vminsn_type = 40; +pub const YARVINSN_adjuststack: ruby_vminsn_type = 41; +pub const YARVINSN_defined: ruby_vminsn_type = 42; +pub const YARVINSN_checkmatch: ruby_vminsn_type = 43; +pub const YARVINSN_checkkeyword: ruby_vminsn_type = 44; +pub const YARVINSN_checktype: ruby_vminsn_type = 45; +pub const YARVINSN_defineclass: ruby_vminsn_type = 46; +pub const YARVINSN_definemethod: ruby_vminsn_type = 47; +pub const YARVINSN_definesmethod: ruby_vminsn_type = 48; +pub const YARVINSN_send: ruby_vminsn_type = 49; +pub const YARVINSN_opt_send_without_block: ruby_vminsn_type = 50; +pub const YARVINSN_objtostring: ruby_vminsn_type = 51; +pub const YARVINSN_opt_str_freeze: ruby_vminsn_type = 52; +pub const YARVINSN_opt_nil_p: ruby_vminsn_type = 53; +pub const YARVINSN_opt_str_uminus: ruby_vminsn_type = 54; +pub const YARVINSN_opt_newarray_max: ruby_vminsn_type = 55; +pub const YARVINSN_opt_newarray_min: ruby_vminsn_type = 56; +pub const YARVINSN_invokesuper: ruby_vminsn_type = 57; +pub const YARVINSN_invokeblock: ruby_vminsn_type = 58; +pub const YARVINSN_leave: ruby_vminsn_type = 59; +pub const YARVINSN_throw: ruby_vminsn_type = 60; +pub const YARVINSN_jump: ruby_vminsn_type = 61; +pub const YARVINSN_branchif: ruby_vminsn_type = 62; +pub const YARVINSN_branchunless: ruby_vminsn_type = 63; +pub const YARVINSN_branchnil: ruby_vminsn_type = 64; +pub const YARVINSN_once: ruby_vminsn_type = 65; +pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 66; +pub const YARVINSN_opt_plus: ruby_vminsn_type = 67; +pub const YARVINSN_opt_minus: ruby_vminsn_type = 68; +pub const YARVINSN_opt_mult: ruby_vminsn_type = 69; +pub const YARVINSN_opt_div: ruby_vminsn_type = 70; +pub const YARVINSN_opt_mod: ruby_vminsn_type = 71; +pub const YARVINSN_opt_eq: ruby_vminsn_type = 72; +pub const YARVINSN_opt_neq: ruby_vminsn_type = 73; +pub const YARVINSN_opt_lt: ruby_vminsn_type = 74; +pub const YARVINSN_opt_le: ruby_vminsn_type = 75; +pub const YARVINSN_opt_gt: ruby_vminsn_type = 76; +pub const YARVINSN_opt_ge: ruby_vminsn_type = 77; +pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 78; +pub const YARVINSN_opt_and: ruby_vminsn_type = 79; +pub const YARVINSN_opt_or: ruby_vminsn_type = 80; +pub const YARVINSN_opt_aref: ruby_vminsn_type = 81; +pub const YARVINSN_opt_aset: ruby_vminsn_type = 82; +pub const YARVINSN_opt_aset_with: ruby_vminsn_type = 83; +pub const YARVINSN_opt_aref_with: ruby_vminsn_type = 84; +pub const YARVINSN_opt_length: ruby_vminsn_type = 85; +pub const YARVINSN_opt_size: ruby_vminsn_type = 86; +pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 87; +pub const YARVINSN_opt_succ: ruby_vminsn_type = 88; +pub const YARVINSN_opt_not: ruby_vminsn_type = 89; +pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 90; +pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 91; +pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 92; +pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 93; +pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 94; +pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 95; +pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 96; +pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 97; +pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 98; +pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 99; +pub const YARVINSN_trace_nop: ruby_vminsn_type = 100; +pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 101; +pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 102; +pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 103; +pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 104; +pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 105; +pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 106; +pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 107; +pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 108; +pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 109; +pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 110; +pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 111; +pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 112; pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 113; pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 114; pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 115; @@ -906,44 +906,42 @@ pub const YARVINSN_trace_jump: ruby_vminsn_type = 161; pub const YARVINSN_trace_branchif: ruby_vminsn_type = 162; pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 163; pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 164; -pub const YARVINSN_trace_opt_getinlinecache: ruby_vminsn_type = 165; -pub const YARVINSN_trace_opt_setinlinecache: ruby_vminsn_type = 166; -pub const YARVINSN_trace_once: ruby_vminsn_type = 167; -pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 168; -pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 169; -pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 170; -pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 171; -pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 172; -pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 173; -pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 174; -pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 175; -pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 176; -pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 177; -pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 178; -pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 179; -pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 180; -pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 181; -pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 182; -pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 183; -pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 184; -pub const YARVINSN_trace_opt_aset_with: ruby_vminsn_type = 185; -pub const YARVINSN_trace_opt_aref_with: ruby_vminsn_type = 186; -pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 187; -pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 188; -pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 189; -pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 190; -pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 191; -pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 192; -pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 193; -pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 194; -pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 195; -pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 196; -pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 197; -pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 198; -pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 199; -pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 200; -pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 201; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 202; +pub const YARVINSN_trace_once: ruby_vminsn_type = 165; +pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 166; +pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 167; +pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 168; +pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 169; +pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 170; +pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 171; +pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 172; +pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 173; +pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 174; +pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 175; +pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 176; +pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 177; +pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 178; +pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 179; +pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 180; +pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 181; +pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 182; +pub const YARVINSN_trace_opt_aset_with: ruby_vminsn_type = 183; +pub const YARVINSN_trace_opt_aref_with: ruby_vminsn_type = 184; +pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 185; +pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 186; +pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 187; +pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 188; +pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 189; +pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 190; +pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 191; +pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 192; +pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 193; +pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 194; +pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 195; +pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 196; +pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 197; +pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 198; +pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 199; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 200; pub type ruby_vminsn_type = u32; extern "C" { pub fn rb_vm_insn_addr2opcode(addr: *const ::std::os::raw::c_void) -> ::std::os::raw::c_int; diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs index 6329c70f87..2f99e8dc14 100644 --- a/yjit/src/invariants.rs +++ b/yjit/src/invariants.rs @@ -12,7 +12,6 @@ use crate::yjit::yjit_enabled_p; use std::collections::{HashMap, HashSet}; use std::mem; -use std::os::raw::c_void; // Invariants to track: // assume_bop_not_redefined(jit, INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) @@ -173,57 +172,41 @@ pub fn assume_single_ractor_mode(jit: &mut JITState, ocb: &mut OutlinedCb) -> bo /// subsequent opt_setinlinecache and find all of the name components that are /// associated with this constant (which correspond to the getconstant /// arguments). -pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb) { +pub fn assume_stable_constant_names(jit: &mut JITState, ocb: &mut OutlinedCb, idlist: *const ID) { /// Tracks that a block is assuming that the name component of a constant /// has not changed since the last call to this function. - unsafe extern "C" fn assume_stable_constant_name( - code: *mut VALUE, - insn: VALUE, - index: u64, - data: *mut c_void, - ) -> bool { - if insn.as_u32() == YARVINSN_opt_setinlinecache { - return false; + fn assume_stable_constant_name( + jit: &mut JITState, + id: ID, + ) { + if id == idNULL as u64 { + // Used for :: prefix + return; } - if insn.as_u32() == YARVINSN_getconstant { - let jit = &mut *(data as *mut JITState); + let invariants = Invariants::get_instance(); + invariants + .constant_state_blocks + .entry(id) + .or_default() + .insert(jit.get_block()); + invariants + .block_constant_states + .entry(jit.get_block()) + .or_default() + .insert(id); + } - // The first operand to GETCONSTANT is always the ID associated with - // the constant lookup. We are grabbing this out in order to - // associate this block with the stability of this constant name. - let id = code.add(index.as_usize() + 1).read().as_u64() as ID; - let invariants = Invariants::get_instance(); - invariants - .constant_state_blocks - .entry(id) - .or_default() - .insert(jit.get_block()); - invariants - .block_constant_states - .entry(jit.get_block()) - .or_default() - .insert(id); + for i in 0.. { + match unsafe { *idlist.offset(i) } { + 0 => break, // End of NULL terminated list + id => assume_stable_constant_name(jit, id), } - - true } jit_ensure_block_entry_exit(jit, ocb); - unsafe { - let iseq = jit.get_iseq(); - let encoded = get_iseq_body_iseq_encoded(iseq); - let start_index = jit.get_pc().offset_from(encoded); - - rb_iseq_each( - iseq, - start_index.try_into().unwrap(), - Some(assume_stable_constant_name), - jit as *mut _ as *mut c_void, - ); - }; } /// Called when a basic operator is redefined. Note that all the blocks assuming @@ -450,7 +433,7 @@ pub fn block_assumptions_free(blockref: &BlockRef) { /// Invalidate the block for the matching opt_getinlinecache so it could regenerate code /// using the new value in the constant cache. #[no_mangle] -pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC) { +pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, insn_idx: u32) { // If YJIT isn't enabled, do nothing if !yjit_enabled_p() { return; @@ -464,34 +447,33 @@ pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC) { with_vm_lock(src_loc!(), || { let code = unsafe { get_iseq_body_iseq_encoded(iseq) }; - let get_insn_idx = unsafe { (*ic).get_insn_idx }; // This should come from a running iseq, so direct threading translation // should have been done assert!(unsafe { FL_TEST(iseq.into(), VALUE(ISEQ_TRANSLATED as usize)) } != VALUE(0)); - assert!(get_insn_idx < unsafe { get_iseq_encoded_size(iseq) }); + assert!(insn_idx < unsafe { get_iseq_encoded_size(iseq) }); - // Ensure that the instruction the get_insn_idx is pointing to is in - // fact a opt_getinlinecache instruction. + // Ensure that the instruction the insn_idx is pointing to is in + // fact a opt_getconstant_path instruction. assert_eq!( unsafe { - let opcode_pc = code.add(get_insn_idx.as_usize()); + let opcode_pc = code.add(insn_idx.as_usize()); let translated_opcode: VALUE = opcode_pc.read(); rb_vm_insn_decode(translated_opcode) }, - YARVINSN_opt_getinlinecache.try_into().unwrap() + YARVINSN_opt_getconstant_path.try_into().unwrap() ); // Find the matching opt_getinlinecache and invalidate all the blocks there // RUBY_ASSERT(insn_op_type(BIN(opt_getinlinecache), 1) == TS_IC); - let ic_pc = unsafe { code.add(get_insn_idx.as_usize() + 2) }; + let ic_pc = unsafe { code.add(insn_idx.as_usize() + 1) }; let ic_operand: IC = unsafe { ic_pc.read() }.as_mut_ptr(); if ic == ic_operand { for block in take_version_list(BlockId { iseq, - idx: get_insn_idx, + idx: insn_idx, }) { invalidate_block_version(&block); incr_counter!(invalidate_constant_ic_fill); -- 2.37.1