diff --git a/enumerator.c b/enumerator.c index ab03a273d5e39b..8d02c836e960e0 100644 --- a/enumerator.c +++ b/enumerator.c @@ -280,28 +280,20 @@ enumerator_ptr(VALUE obj) } static void -proc_entry_mark(void *p) +proc_entry_mark_and_move(void *p) { struct proc_entry *ptr = p; - rb_gc_mark_movable(ptr->proc); - rb_gc_mark_movable(ptr->memo); -} - -static void -proc_entry_compact(void *p) -{ - struct proc_entry *ptr = p; - ptr->proc = rb_gc_location(ptr->proc); - ptr->memo = rb_gc_location(ptr->memo); + rb_gc_mark_and_move(&ptr->proc); + rb_gc_mark_and_move(&ptr->memo); } static const rb_data_type_t proc_entry_data_type = { "proc_entry", { - proc_entry_mark, + proc_entry_mark_and_move, RUBY_TYPED_DEFAULT_FREE, NULL, // Nothing allocated externally, so don't need a memsize function - proc_entry_compact, + proc_entry_mark_and_move, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; @@ -1280,26 +1272,19 @@ enumerator_size(VALUE obj) * Yielder */ static void -yielder_mark(void *p) +yielder_mark_and_move(void *p) { struct yielder *ptr = p; - rb_gc_mark_movable(ptr->proc); -} - -static void -yielder_compact(void *p) -{ - struct yielder *ptr = p; - ptr->proc = rb_gc_location(ptr->proc); + rb_gc_mark_and_move(&ptr->proc); } static const rb_data_type_t yielder_data_type = { "yielder", { - yielder_mark, + yielder_mark_and_move, RUBY_TYPED_DEFAULT_FREE, NULL, - yielder_compact, + yielder_mark_and_move, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; @@ -1410,28 +1395,20 @@ yielder_new(void) * Generator */ static void -generator_mark(void *p) -{ - struct generator *ptr = p; - rb_gc_mark_movable(ptr->proc); - rb_gc_mark_movable(ptr->obj); -} - -static void -generator_compact(void *p) +generator_mark_and_move(void *p) { struct generator *ptr = p; - ptr->proc = rb_gc_location(ptr->proc); - ptr->obj = rb_gc_location(ptr->obj); + rb_gc_mark_and_move(&ptr->proc); + rb_gc_mark_and_move(&ptr->obj); } static const rb_data_type_t generator_data_type = { "generator", { - generator_mark, + generator_mark_and_move, RUBY_TYPED_DEFAULT_FREE, NULL, - generator_compact, + generator_mark_and_move, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; @@ -2894,19 +2871,11 @@ stop_result(VALUE self) */ static void -producer_mark(void *p) +producer_mark_and_move(void *p) { struct producer *ptr = p; - rb_gc_mark_movable(ptr->init); - rb_gc_mark_movable(ptr->proc); -} - -static void -producer_compact(void *p) -{ - struct producer *ptr = p; - ptr->init = rb_gc_location(ptr->init); - ptr->proc = rb_gc_location(ptr->proc); + rb_gc_mark_and_move(&ptr->init); + rb_gc_mark_and_move(&ptr->proc); } #define producer_free RUBY_TYPED_DEFAULT_FREE @@ -2920,10 +2889,10 @@ producer_memsize(const void *p) static const rb_data_type_t producer_data_type = { "producer", { - producer_mark, + producer_mark_and_move, producer_free, producer_memsize, - producer_compact, + producer_mark_and_move, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; @@ -3082,17 +3051,10 @@ enumerator_s_produce(int argc, VALUE *argv, VALUE klass) */ static void -enum_chain_mark(void *p) +enum_chain_mark_and_move(void *p) { struct enum_chain *ptr = p; - rb_gc_mark_movable(ptr->enums); -} - -static void -enum_chain_compact(void *p) -{ - struct enum_chain *ptr = p; - ptr->enums = rb_gc_location(ptr->enums); + rb_gc_mark_and_move(&ptr->enums); } #define enum_chain_free RUBY_TYPED_DEFAULT_FREE @@ -3106,10 +3068,10 @@ enum_chain_memsize(const void *p) static const rb_data_type_t enum_chain_data_type = { "chain", { - enum_chain_mark, + enum_chain_mark_and_move, enum_chain_free, enum_chain_memsize, - enum_chain_compact, + enum_chain_mark_and_move, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; @@ -3404,17 +3366,10 @@ enumerator_plus(VALUE obj, VALUE eobj) */ static void -enum_product_mark(void *p) -{ - struct enum_product *ptr = p; - rb_gc_mark_movable(ptr->enums); -} - -static void -enum_product_compact(void *p) +enum_product_mark_and_move(void *p) { struct enum_product *ptr = p; - ptr->enums = rb_gc_location(ptr->enums); + rb_gc_mark_and_move(&ptr->enums); } #define enum_product_free RUBY_TYPED_DEFAULT_FREE @@ -3428,10 +3383,10 @@ enum_product_memsize(const void *p) static const rb_data_type_t enum_product_data_type = { "product", { - enum_product_mark, + enum_product_mark_and_move, enum_product_free, enum_product_memsize, - enum_product_compact, + enum_product_mark_and_move, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; diff --git a/error.c b/error.c index 9a758a7dd711b8..e07c99e6dfde59 100644 --- a/error.c +++ b/error.c @@ -2517,30 +2517,21 @@ typedef struct name_error_message_struct { } name_error_message_t; static void -name_err_mesg_mark(void *p) +name_err_mesg_mark_and_move(void *p) { name_error_message_t *ptr = (name_error_message_t *)p; - rb_gc_mark_movable(ptr->mesg); - rb_gc_mark_movable(ptr->recv); - rb_gc_mark_movable(ptr->name); -} - -static void -name_err_mesg_update(void *p) -{ - name_error_message_t *ptr = (name_error_message_t *)p; - ptr->mesg = rb_gc_location(ptr->mesg); - ptr->recv = rb_gc_location(ptr->recv); - ptr->name = rb_gc_location(ptr->name); + rb_gc_mark_and_move(&ptr->mesg); + rb_gc_mark_and_move(&ptr->recv); + rb_gc_mark_and_move(&ptr->name); } static const rb_data_type_t name_err_mesg_data_type = { "name_err_mesg", { - name_err_mesg_mark, + name_err_mesg_mark_and_move, RUBY_TYPED_DEFAULT_FREE, NULL, // No external memory to report, - name_err_mesg_update, + name_err_mesg_mark_and_move, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; diff --git a/gc.c b/gc.c index 64a22cd1b78fff..90f2b29bfaf3fe 100644 --- a/gc.c +++ b/gc.c @@ -3028,7 +3028,7 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) mark_current_machine_context(ec); MARK_CHECKPOINT("global_symbols"); - rb_sym_global_symbols_mark(); + rb_sym_global_symbols_mark_and_move(); MARK_CHECKPOINT("finish"); @@ -4045,7 +4045,7 @@ rb_gc_update_vm_references(void *objspace) rb_vm_update_references(vm); rb_gc_update_global_tbl(); - rb_sym_global_symbols_update_references(); + rb_sym_global_symbols_mark_and_move(); #if USE_YJIT void rb_yjit_root_update_references(void); // in Rust diff --git a/imemo.c b/imemo.c index 2c721ca9112250..30dae8d583dc00 100644 --- a/imemo.c +++ b/imemo.c @@ -372,6 +372,9 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating) } } else { + RUBY_ASSERT(RB_TYPE_P(cc->klass, T_CLASS) || RB_TYPE_P(cc->klass, T_ICLASS)); + RUBY_ASSERT(IMEMO_TYPE_P((VALUE)cc->cme_, imemo_ment)); + rb_gc_mark_weak((VALUE *)&cc->klass); if ((vm_cc_super_p(cc) || vm_cc_refinement_p(cc))) { rb_gc_mark_movable((VALUE)cc->cme_); diff --git a/internal/symbol.h b/internal/symbol.h index 8571c002896554..b9109b1347ad8c 100644 --- a/internal/symbol.h +++ b/internal/symbol.h @@ -17,8 +17,7 @@ #endif /* symbol.c */ -void rb_sym_global_symbols_mark(void); -void rb_sym_global_symbols_update_references(void); +void rb_sym_global_symbols_mark_and_move(void); VALUE rb_to_symbol_type(VALUE obj); VALUE rb_sym_intern(const char *ptr, long len, rb_encoding *enc); VALUE rb_sym_intern_ascii(const char *ptr, long len); diff --git a/marshal.c b/marshal.c index 7db4bfc6d9122e..f7474ca60edf07 100644 --- a/marshal.c +++ b/marshal.c @@ -2572,19 +2572,19 @@ Init_marshal(void) } static int -marshal_compat_table_mark_i(st_data_t key, st_data_t value, st_data_t _) +marshal_compat_table_mark_and_move_i(st_data_t key, st_data_t value, st_data_t _) { marshal_compat_t *p = (marshal_compat_t *)value; - rb_gc_mark_movable(p->newclass); - rb_gc_mark_movable(p->oldclass); + rb_gc_mark_and_move(&p->newclass); + rb_gc_mark_and_move(&p->oldclass); return ST_CONTINUE; } static void -marshal_compat_table_mark(void *tbl) +marshal_compat_table_mark_and_move(void *tbl) { if (!tbl) return; - st_foreach(tbl, marshal_compat_table_mark_i, 0); + st_foreach(tbl, marshal_compat_table_mark_and_move_i, 0); } static int @@ -2607,29 +2607,13 @@ marshal_compat_table_memsize(const void *data) return st_memsize(data) + sizeof(marshal_compat_t) * st_table_size(data); } -static int -marshal_compat_table_compact_i(st_data_t key, st_data_t value, st_data_t _) -{ - marshal_compat_t *p = (marshal_compat_t *)value; - p->newclass = rb_gc_location(p->newclass); - p->oldclass = rb_gc_location(p->oldclass); - return ST_CONTINUE; -} - -static void -marshal_compat_table_compact(void *tbl) -{ - if (!tbl) return; - st_foreach(tbl, marshal_compat_table_compact_i, 0); -} - static const rb_data_type_t marshal_compat_type = { .wrap_struct_name = "marshal_compat_table", .function = { - .dmark = marshal_compat_table_mark, + .dmark = marshal_compat_table_mark_and_move, .dfree = marshal_compat_table_free, .dsize = marshal_compat_table_memsize, - .dcompact = marshal_compat_table_compact, + .dcompact = marshal_compat_table_mark_and_move, }, .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY, }; diff --git a/set.c b/set.c index f83fb0880ca951..c589fb4523274a 100644 --- a/set.c +++ b/set.c @@ -172,9 +172,7 @@ set_foreach_replace(st_data_t key, st_data_t argp, int error) static int set_replace_ref(st_data_t *key, st_data_t argp, int existing) { - if (rb_gc_location((VALUE)*key) != (VALUE)*key) { - *key = rb_gc_location((VALUE)*key); - } + rb_gc_mark_and_move((VALUE *)key); return ST_CONTINUE; } diff --git a/shape.c b/shape.c index 6e1b49352f922b..d6d05e12d50db9 100644 --- a/shape.c +++ b/shape.c @@ -296,26 +296,13 @@ rb_shape_get_root_shape(void) } static void -shape_tree_mark(void *data) +shape_tree_mark_and_move(void *data) { rb_shape_t *cursor = rb_shape_get_root_shape(); rb_shape_t *end = RSHAPE(rb_shape_tree.next_shape_id - 1); while (cursor <= end) { if (cursor->edges && !SINGLE_CHILD_P(cursor->edges)) { - rb_gc_mark_movable(cursor->edges); - } - cursor++; - } -} - -static void -shape_tree_compact(void *data) -{ - rb_shape_t *cursor = rb_shape_get_root_shape(); - rb_shape_t *end = RSHAPE(rb_shape_tree.next_shape_id - 1); - while (cursor <= end) { - if (cursor->edges && !SINGLE_CHILD_P(cursor->edges)) { - cursor->edges = rb_gc_location(cursor->edges); + rb_gc_mark_and_move(&cursor->edges); } cursor++; } @@ -330,10 +317,10 @@ shape_tree_memsize(const void *data) static const rb_data_type_t shape_tree_type = { .wrap_struct_name = "VM/shape_tree", .function = { - .dmark = shape_tree_mark, + .dmark = shape_tree_mark_and_move, .dfree = NULL, // Nothing to free, done at VM exit in rb_shape_free_all, .dsize = shape_tree_memsize, - .dcompact = shape_tree_compact, + .dcompact = shape_tree_mark_and_move, }, .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, }; diff --git a/symbol.c b/symbol.c index c337cc288ecded..ddb0f1556ba019 100644 --- a/symbol.c +++ b/symbol.c @@ -371,21 +371,12 @@ Init_sym(void) } void -rb_sym_global_symbols_mark(void) +rb_sym_global_symbols_mark_and_move(void) { rb_symbols_t *symbols = &ruby_global_symbols; - rb_gc_mark_movable(symbols->sym_set); - rb_gc_mark_movable(symbols->ids); -} - -void -rb_sym_global_symbols_update_references(void) -{ - rb_symbols_t *symbols = &ruby_global_symbols; - - symbols->sym_set = rb_gc_location(symbols->sym_set); - symbols->ids = rb_gc_location(symbols->ids); + rb_gc_mark_and_move(&symbols->sym_set); + rb_gc_mark_and_move(&symbols->ids); } static int diff --git a/test/ruby/test_zjit.rb b/test/ruby/test_zjit.rb index 8115a601668422..9eca30c7875ac6 100644 --- a/test/ruby/test_zjit.rb +++ b/test/ruby/test_zjit.rb @@ -1043,6 +1043,26 @@ def +(_) = 100 } end + def test_defined_with_defined_values + assert_compiles '["constant", "method", "global-variable"]', %q{ + class Foo; end + def bar; end + $ruby = 1 + + def test = return defined?(Foo), defined?(bar), defined?($ruby) + + test + }, insns: [:defined] + end + + def test_defined_with_undefined_values + assert_compiles '[nil, nil, nil]', %q{ + def test = return defined?(Foo), defined?(bar), defined?($ruby) + + test + }, insns: [:defined] + end + def test_defined_yield assert_compiles "nil", "defined?(yield)" assert_compiles '[nil, nil, "yield"]', %q{ @@ -1432,6 +1452,30 @@ def test(val) = val.nil? }, call_threshold: 2, insns: [:opt_nil_p] end + def test_basic_object_guard_works_with_immediate + assert_compiles 'NilClass', %q{ + class Foo; end + + def test(val) = val.class + + test(Foo.new) + test(Foo.new) + test(nil) + }, call_threshold: 2 + end + + def test_basic_object_guard_works_with_false + assert_compiles 'FalseClass', %q{ + class Foo; end + + def test(val) = val.class + + test(Foo.new) + test(Foo.new) + test(false) + }, call_threshold: 2 + end + private # Assert that every method call in `test_script` can be compiled by ZJIT diff --git a/time.c b/time.c index 4c2b15d90e227d..7159a9309840de 100644 --- a/time.c +++ b/time.c @@ -1888,39 +1888,25 @@ force_make_tm(VALUE time, struct time_object *tobj) } static void -time_mark(void *ptr) +time_mark_and_move(void *ptr) { struct time_object *tobj = ptr; if (!FIXWV_P(tobj->timew)) { - rb_gc_mark_movable(w2v(tobj->timew)); + rb_gc_mark_and_move(&WIDEVAL_GET(tobj->timew)); } - rb_gc_mark_movable(tobj->vtm.year); - rb_gc_mark_movable(tobj->vtm.subsecx); - rb_gc_mark_movable(tobj->vtm.utc_offset); - rb_gc_mark_movable(tobj->vtm.zone); -} - -static void -time_compact(void *ptr) -{ - struct time_object *tobj = ptr; - if (!FIXWV_P(tobj->timew)) { - WIDEVAL_GET(tobj->timew) = WIDEVAL_WRAP(rb_gc_location(w2v(tobj->timew))); - } - - tobj->vtm.year = rb_gc_location(tobj->vtm.year); - tobj->vtm.subsecx = rb_gc_location(tobj->vtm.subsecx); - tobj->vtm.utc_offset = rb_gc_location(tobj->vtm.utc_offset); - tobj->vtm.zone = rb_gc_location(tobj->vtm.zone); + rb_gc_mark_and_move(&tobj->vtm.year); + rb_gc_mark_and_move(&tobj->vtm.subsecx); + rb_gc_mark_and_move(&tobj->vtm.utc_offset); + rb_gc_mark_and_move(&tobj->vtm.zone); } static const rb_data_type_t time_data_type = { .wrap_struct_name = "time", .function = { - .dmark = time_mark, + .dmark = time_mark_and_move, .dfree = RUBY_TYPED_DEFAULT_FREE, .dsize = NULL, - .dcompact = time_compact, + .dcompact = time_mark_and_move, }, .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE, }; diff --git a/vm_method.c b/vm_method.c index 76b1c97d046c2b..c1793c102c4b8a 100644 --- a/vm_method.c +++ b/vm_method.c @@ -30,16 +30,27 @@ mark_cc_entry_i(VALUE ccs_ptr, void *data) VM_ASSERT(vm_ccs_p(ccs)); if (METHOD_ENTRY_INVALIDATED(ccs->cme)) { + /* Before detaching the CCs from this class, we need to invalidate the cc + * since we will no longer be marking the cme on their behalf. + */ + for (int i = 0; i < ccs->len; i++) { + const struct rb_callcache *cc = ccs->entries[i].cc; + if (cc->klass == Qundef) continue; // already invalidated + VM_ASSERT(cc->klass == Qundef || vm_cc_check_cme(cc, ccs->cme)); + VM_ASSERT(!vm_cc_super_p(cc) && !vm_cc_refinement_p(cc)); + vm_cc_invalidate(cc); + } ruby_xfree(ccs); return ID_TABLE_DELETE; } else { rb_gc_mark_movable((VALUE)ccs->cme); - for (int i=0; ilen; i++) { - VM_ASSERT(vm_cc_check_cme(ccs->entries[i].cc, ccs->cme)); + for (int i = 0; i < ccs->len; i++) { + const struct rb_callcache *cc = ccs->entries[i].cc; + VM_ASSERT(cc->klass == Qundef || vm_cc_check_cme(cc, ccs->cme)); - rb_gc_mark_movable((VALUE)ccs->entries[i].cc); + rb_gc_mark_movable((VALUE)cc); } return ID_TABLE_CONTINUE; } diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs index 6c3e95546302ae..9bf11dfc4afdee 100644 --- a/zjit/src/asm/mod.rs +++ b/zjit/src/asm/mod.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; use std::fmt; +use std::ops::Range; use std::rc::Rc; use std::cell::RefCell; use std::mem; @@ -124,7 +125,7 @@ impl CodeBlock { } /// Invoke a callback with write_ptr temporarily adjusted to a given address - pub fn with_write_ptr(&mut self, code_ptr: CodePtr, callback: impl Fn(&mut CodeBlock)) { + pub fn with_write_ptr(&mut self, code_ptr: CodePtr, callback: impl Fn(&mut CodeBlock)) -> Range { // Temporarily update the write_pos. Ignore the dropped_bytes flag at the old address. let old_write_pos = self.write_pos; let old_dropped_bytes = self.dropped_bytes; @@ -134,9 +135,13 @@ impl CodeBlock { // Invoke the callback callback(self); + // Build a code range modified by the callback + let ret = code_ptr..self.get_write_ptr(); + // Restore the original write_pos and dropped_bytes flag. self.dropped_bytes = old_dropped_bytes; self.write_pos = old_write_pos; + ret } /// Get a (possibly dangling) direct pointer into the executable memory block diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 148d01ea862e97..0c7e6883c2759c 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -256,7 +256,7 @@ impl Assembler // Many Arm insns support only 32-bit or 64-bit operands. asm.load with fewer // bits zero-extends the value, so it's safe to recognize it as a 32-bit value. if out_opnd.rm_num_bits() < 32 { - out_opnd.with_num_bits(32).unwrap() + out_opnd.with_num_bits(32) } else { out_opnd } @@ -282,7 +282,7 @@ impl Assembler BitmaskImmediate::new_32b_reg(imm as u32).is_ok()) { Opnd::UImm(imm as u64) } else { - asm.load(opnd).with_num_bits(dest_num_bits).unwrap() + asm.load(opnd).with_num_bits(dest_num_bits) } }, Opnd::UImm(uimm) => { @@ -292,7 +292,7 @@ impl Assembler BitmaskImmediate::new_32b_reg(uimm as u32).is_ok()) { opnd } else { - asm.load(opnd).with_num_bits(dest_num_bits).unwrap() + asm.load(opnd).with_num_bits(dest_num_bits) } }, Opnd::None | Opnd::Value(_) => unreachable!() @@ -360,8 +360,8 @@ impl Assembler match opnd0 { Opnd::Reg(_) | Opnd::VReg { .. } => { match opnd0.rm_num_bits() { - 8 => asm.and(opnd0.with_num_bits(64).unwrap(), Opnd::UImm(0xff)), - 16 => asm.and(opnd0.with_num_bits(64).unwrap(), Opnd::UImm(0xffff)), + 8 => asm.and(opnd0.with_num_bits(64), Opnd::UImm(0xff)), + 16 => asm.and(opnd0.with_num_bits(64), Opnd::UImm(0xffff)), 32 | 64 => opnd0, bits => unreachable!("Invalid number of bits. {}", bits) } @@ -505,7 +505,7 @@ impl Assembler let split_right = split_shifted_immediate(asm, *right); let opnd1 = match split_right { Opnd::VReg { .. } if opnd0.num_bits() != split_right.num_bits() => { - split_right.with_num_bits(opnd0.num_bits().unwrap()).unwrap() + split_right.with_num_bits(opnd0.num_bits().unwrap()) }, _ => split_right }; @@ -1823,7 +1823,7 @@ mod tests { #[test] fn test_emit_test_32b_reg_not_bitmask_imm() { let (mut asm, mut cb) = setup_asm(); - let w0 = Opnd::Reg(X0_REG).with_num_bits(32).unwrap(); + let w0 = Opnd::Reg(X0_REG).with_num_bits(32); asm.test(w0, Opnd::UImm(u32::MAX.into())); // All ones is not encodable with a bitmask immediate, // so this needs one register @@ -1833,7 +1833,7 @@ mod tests { #[test] fn test_emit_test_32b_reg_bitmask_imm() { let (mut asm, mut cb) = setup_asm(); - let w0 = Opnd::Reg(X0_REG).with_num_bits(32).unwrap(); + let w0 = Opnd::Reg(X0_REG).with_num_bits(32); asm.test(w0, Opnd::UImm(0x80000001)); asm.compile_with_num_regs(&mut cb, 0); } diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index b910052dae1b04..1bed45cba5f6aa 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -146,17 +146,29 @@ impl Opnd } } - pub fn with_num_bits(&self, num_bits: u8) -> Option { + /// Return Some(Opnd) with a given num_bits if self has num_bits. + /// None if self doesn't have a num_bits field. + pub fn try_num_bits(&self, num_bits: u8) -> Option { assert!(num_bits == 8 || num_bits == 16 || num_bits == 32 || num_bits == 64); match *self { Opnd::Reg(reg) => Some(Opnd::Reg(reg.with_num_bits(num_bits))), Opnd::Mem(Mem { base, disp, .. }) => Some(Opnd::Mem(Mem { base, disp, num_bits })), Opnd::VReg { idx, .. } => Some(Opnd::VReg { idx, num_bits }), - //Opnd::Stack { idx, stack_size, num_locals, sp_offset, reg_mapping, .. } => Some(Opnd::Stack { idx, num_bits, stack_size, num_locals, sp_offset, reg_mapping }), _ => None, } } + /// Return Opnd with a given num_bits if self has num_bits. + /// Panic otherwise. This should be used only when you know which Opnd self is. + #[track_caller] + pub fn with_num_bits(&self, num_bits: u8) -> Opnd { + if let Some(opnd) = self.try_num_bits(num_bits) { + opnd + } else { + unreachable!("with_num_bits should not be used on: {self:?}"); + } + } + /// Get the size in bits for register/memory operands. pub fn rm_num_bits(&self) -> u8 { self.num_bits().unwrap() @@ -1720,7 +1732,7 @@ impl Assembler while let Some(opnd) = opnd_iter.next() { match *opnd { Opnd::VReg { idx, num_bits } => { - *opnd = Opnd::Reg(reg_mapping[idx].unwrap()).with_num_bits(num_bits).unwrap(); + *opnd = Opnd::Reg(reg_mapping[idx].unwrap()).with_num_bits(num_bits); }, Opnd::Mem(Mem { base: MemBase::VReg(idx), disp, num_bits }) => { let base = MemBase::Reg(reg_mapping[idx].unwrap().reg_no); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index a5439bf2ca1723..3e6bbfa605463e 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -5,7 +5,7 @@ use std::ffi::{c_int, c_void}; use crate::asm::Label; use crate::backend::current::{Reg, ALLOC_REGS}; use crate::invariants::{track_bop_assumption, track_cme_assumption, track_single_ractor_assumption, track_stable_constant_names_assumption}; -use crate::gc::{get_or_create_iseq_payload, append_gc_offsets}; +use crate::gc::{append_gc_offsets, get_or_create_iseq_payload, get_or_create_iseq_payload_ptr}; use crate::state::ZJITState; use crate::stats::{counter_ptr, Counter}; use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; @@ -369,7 +369,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::SideExit { state, reason } => return gen_side_exit(jit, asm, reason, &function.frame_state(*state)), Insn::PutSpecialObject { value_type } => gen_putspecialobject(asm, *value_type), Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state))?, - Insn::Defined { op_type, obj, pushval, v } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v))?, + Insn::Defined { op_type, obj, pushval, v, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), &function.frame_state(*state))?, &Insn::IncrCounter(counter) => return Some(gen_incr_counter(asm, counter)), Insn::ArrayExtend { .. } | Insn::ArrayMax { .. } @@ -438,7 +438,7 @@ fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { ep_opnd } -fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, _obj: VALUE, pushval: VALUE, _tested_value: Opnd) -> Option { +fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, state: &FrameState) -> Option { match op_type as defined_type { DEFINED_YIELD => { // `yield` goes to the block handler stowed in the "local" iseq which is @@ -455,7 +455,17 @@ fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, _obj: VALUE, Some(Qnil.into()) } } - _ => None + _ => { + // Save the PC and SP because the callee may allocate or call #respond_to? + gen_prepare_non_leaf_call(jit, asm, state)?; + + // TODO: Inline the cases for each op_type + // Call vm_defined(ec, reg_cfp, op_type, obj, v) + let def_result = asm_ccall!(asm, rb_vm_defined, EC, CFP, op_type.into(), obj.into(), tested_value); + + asm.cmp(def_result.with_num_bits(8), 0.into()); + Some(asm.csel_ne(pushval.into(), Qnil.into())) + } } } @@ -521,6 +531,7 @@ fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf /// Record a patch point that should be invalidated on a given invariant fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) -> Option<()> { + let payload_ptr = get_or_create_iseq_payload_ptr(jit.iseq); let label = asm.new_label("patch_point").unwrap_label(); let invariant = invariant.clone(); @@ -532,19 +543,19 @@ fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invarian match invariant { Invariant::BOPRedefined { klass, bop } => { let side_exit_ptr = cb.resolve_label(label); - track_bop_assumption(klass, bop, code_ptr, side_exit_ptr); + track_bop_assumption(klass, bop, code_ptr, side_exit_ptr, payload_ptr); } Invariant::MethodRedefined { klass: _, method: _, cme } => { let side_exit_ptr = cb.resolve_label(label); - track_cme_assumption(cme, code_ptr, side_exit_ptr); + track_cme_assumption(cme, code_ptr, side_exit_ptr, payload_ptr); } Invariant::StableConstantNames { idlist } => { let side_exit_ptr = cb.resolve_label(label); - track_stable_constant_names_assumption(idlist, code_ptr, side_exit_ptr); + track_stable_constant_names_assumption(idlist, code_ptr, side_exit_ptr, payload_ptr); } Invariant::SingleRactorMode => { let side_exit_ptr = cb.resolve_label(label); - track_single_ractor_assumption(code_ptr, side_exit_ptr); + track_single_ractor_assumption(code_ptr, side_exit_ptr, payload_ptr); } } }); @@ -1059,7 +1070,8 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard } else if guard_type.is_subtype(types::StaticSymbol) { // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG // Use 8-bit comparison like YJIT does - asm.cmp(val.with_num_bits(8).unwrap(), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + debug_assert!(val.try_num_bits(8).is_some(), "GuardType should not be used for a known constant, but val was: {val:?}"); + asm.cmp(val.try_num_bits(8)?, Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); asm.jne(side_exit(jit, state, GuardType(guard_type))?); } else if guard_type.is_subtype(types::NilClass) { asm.cmp(val, Qnil.into()); @@ -1068,17 +1080,29 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard asm.cmp(val, Qtrue.into()); asm.jne(side_exit(jit, state, GuardType(guard_type))?); } else if guard_type.is_subtype(types::FalseClass) { - assert!(Qfalse.as_i64() == 0); - asm.test(val, val); + asm.cmp(val, Qfalse.into()); asm.jne(side_exit(jit, state, GuardType(guard_type))?); + } else if guard_type.is_immediate() { + // All immediate types' guard should have been handled above + panic!("unexpected immediate guard type: {guard_type}"); } else if let Some(expected_class) = guard_type.runtime_exact_ruby_class() { - asm_comment!(asm, "guard exact class"); + asm_comment!(asm, "guard exact class for non-immediate types"); - // Get the class of the value - let klass = asm.ccall(rb_yarv_class_of as *const u8, vec![val]); + let side_exit = side_exit(jit, state, GuardType(guard_type))?; + + // Check if it's a special constant + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(side_exit.clone()); + + // Check if it's false + asm.cmp(val, Qfalse.into()); + asm.je(side_exit.clone()); + + // Load the class from the object's klass field + let klass = asm.load(Opnd::mem(64, val, RUBY_OFFSET_RBASIC_KLASS)); asm.cmp(klass, Opnd::Value(expected_class)); - asm.jne(side_exit(jit, state, GuardType(guard_type))?); + asm.jne(side_exit); } else { unimplemented!("unsupported type: {guard_type}"); } diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs index 01bcc9fe5d532e..d94d86036baed2 100644 --- a/zjit/src/gc.rs +++ b/zjit/src/gc.rs @@ -1,6 +1,6 @@ // This module is responsible for marking/moving objects on GC. -use std::ffi::c_void; +use std::{ffi::c_void, ops::Range}; use crate::{cruby::*, profile::IseqProfile, state::ZJITState, virtualmem::CodePtr}; /// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. @@ -26,16 +26,16 @@ impl IseqPayload { } } -/// Get the payload object associated with an iseq. Create one if none exists. -pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { +/// Get a pointer to the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload_ptr(iseq: IseqPtr) -> *mut IseqPayload { type VoidPtr = *mut c_void; - let payload_non_null = unsafe { + unsafe { let payload = rb_iseq_get_zjit_payload(iseq); if payload.is_null() { // Allocate a new payload with Box and transfer ownership to the GC. - // We drop the payload with Box::from_raw when the GC frees the iseq and calls us. - // NOTE(alan): Sometimes we read from an iseq without ever writing to it. + // We drop the payload with Box::from_raw when the GC frees the ISEQ and calls us. + // NOTE(alan): Sometimes we read from an ISEQ without ever writing to it. // We allocate in those cases anyways. let iseq_size = get_iseq_encoded_size(iseq); let new_payload = IseqPayload::new(iseq_size); @@ -46,13 +46,23 @@ pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { } else { payload as *mut IseqPayload } - }; + } +} + +/// Get the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { + let payload_non_null = get_or_create_iseq_payload_ptr(iseq); + payload_ptr_as_mut(payload_non_null) +} +/// Convert an IseqPayload pointer to a mutable reference. Only one reference +/// should be kept at a time. +fn payload_ptr_as_mut(payload_ptr: *mut IseqPayload) -> &'static mut IseqPayload { // SAFETY: we should have the VM lock and all other Ruby threads should be asleep. So we have // exclusive mutable access. // Hmm, nothing seems to stop calling this on the same // iseq twice, though, which violates aliasing rules. - unsafe { payload_non_null.as_mut() }.unwrap() + unsafe { payload_ptr.as_mut() }.unwrap() } #[unsafe(no_mangle)] @@ -90,29 +100,12 @@ pub extern "C" fn rb_zjit_iseq_mark(payload: *mut c_void) { } } -/// Append a set of gc_offsets to the iseq's payload -pub fn append_gc_offsets(iseq: IseqPtr, offsets: &Vec) { - let payload = get_or_create_iseq_payload(iseq); - payload.gc_offsets.extend(offsets); - - // Call writebarrier on each newly added value - let cb = ZJITState::get_code_block(); - for &offset in offsets.iter() { - let value_ptr: *const u8 = offset.raw_ptr(cb); - let value_ptr = value_ptr as *const VALUE; - unsafe { - let object = value_ptr.read_unaligned(); - rb_gc_writebarrier(iseq.into(), object); - } - } -} - -/// GC callback for updating GC objects in the per-iseq payload. +/// GC callback for updating GC objects in the per-ISEQ payload. /// This is a mirror of [rb_zjit_iseq_mark]. #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_iseq_update_references(payload: *mut c_void) { let payload = if payload.is_null() { - return; // nothing to mark + return; // nothing to update } else { // SAFETY: The GC takes the VM lock while marking, which // we assert, so we should be synchronized and data race free. @@ -150,3 +143,37 @@ pub extern "C" fn rb_zjit_iseq_update_references(payload: *mut c_void) { } cb.mark_all_executable(); } + +/// Append a set of gc_offsets to the iseq's payload +pub fn append_gc_offsets(iseq: IseqPtr, offsets: &Vec) { + let payload = get_or_create_iseq_payload(iseq); + payload.gc_offsets.extend(offsets); + + // Call writebarrier on each newly added value + let cb = ZJITState::get_code_block(); + for &offset in offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + let value_ptr = value_ptr as *const VALUE; + unsafe { + let object = value_ptr.read_unaligned(); + rb_gc_writebarrier(iseq.into(), object); + } + } +} + +/// Remove GC offsets that overlap with a given removed_range. +/// We do this when invalidation rewrites some code with a jump instruction +/// and GC offsets are corrupted by the rewrite, assuming no on-stack code +/// will step into the instruction with the GC offsets after invalidation. +pub fn remove_gc_offsets(payload_ptr: *mut IseqPayload, removed_range: &Range) { + let payload = payload_ptr_as_mut(payload_ptr); + payload.gc_offsets.retain(|&gc_offset| { + let offset_range = gc_offset..(gc_offset.add_bytes(SIZEOF_VALUE)); + !ranges_overlap(&offset_range, removed_range) + }); +} + +/// Return true if given Range ranges overlap with each other +fn ranges_overlap(left: &Range, right: &Range) -> bool where T: PartialOrd { + left.start < right.end && right.start < left.end +} diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index f4dae3f0ec1352..55041945d4b505 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -472,7 +472,7 @@ pub enum Insn { Test { val: InsnId }, /// Return C `true` if `val` is `Qnil`, else `false`. IsNil { val: InsnId }, - Defined { op_type: usize, obj: VALUE, pushval: VALUE, v: InsnId }, + Defined { op_type: usize, obj: VALUE, pushval: VALUE, v: InsnId, state: InsnId }, GetConstantPath { ic: *const iseq_inline_constant_cache, state: InsnId }, /// Get a global variable named `id` @@ -1173,7 +1173,7 @@ impl Function { &ArrayDup { val, state } => ArrayDup { val: find!(val), state }, &HashDup { val, state } => HashDup { val: find!(val), state }, &CCall { cfun, ref args, name, return_type, elidable } => CCall { cfun, args: find_vec!(args), name, return_type, elidable }, - &Defined { op_type, obj, pushval, v } => Defined { op_type, obj, pushval, v: find!(v) }, + &Defined { op_type, obj, pushval, v, state } => Defined { op_type, obj, pushval, v: find!(v), state: find!(state) }, &DefinedIvar { self_val, pushval, id, state } => DefinedIvar { self_val: find!(self_val), pushval, id, state }, &NewArray { ref elements, state } => NewArray { elements: find_vec!(elements), state: find!(state) }, &NewHash { ref elements, state } => { @@ -2788,7 +2788,8 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let obj = get_arg(pc, 1); let pushval = get_arg(pc, 2); let v = state.stack_pop()?; - state.stack_push(fun.push_insn(block, Insn::Defined { op_type, obj, pushval, v })); + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); + state.stack_push(fun.push_insn(block, Insn::Defined { op_type, obj, pushval, v, state: exit_id })); } YARVINSN_definedivar => { // (ID id, IVC ic, VALUE pushval) @@ -4061,12 +4062,12 @@ mod tests { fn test@:2: bb0(v0:BasicObject): v2:NilClass = Const Value(nil) - v3:BasicObject = Defined constant, v2 - v4:BasicObject = Defined func, v0 - v5:NilClass = Const Value(nil) - v6:BasicObject = Defined global-variable, v5 - v8:ArrayExact = NewArray v3, v4, v6 - Return v8 + v4:BasicObject = Defined constant, v2 + v6:BasicObject = Defined func, v0 + v7:NilClass = Const Value(nil) + v9:BasicObject = Defined global-variable, v7 + v11:ArrayExact = NewArray v4, v6, v9 + Return v11 "#]]); } @@ -5566,7 +5567,7 @@ mod opt_tests { fn test@:5: bb0(v0:BasicObject): PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v6:BasicObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, BasicObject[class_exact*:Object@VALUE(0x1000)] + v6:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] v7:BasicObject = SendWithoutBlockDirect v6, :foo (0x1038) Return v7 "#]]); @@ -5606,7 +5607,7 @@ mod opt_tests { fn test@:6: bb0(v0:BasicObject): PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v6:BasicObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, BasicObject[class_exact*:Object@VALUE(0x1000)] + v6:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] v7:BasicObject = SendWithoutBlockDirect v6, :foo (0x1038) Return v7 "#]]); @@ -5625,7 +5626,7 @@ mod opt_tests { bb0(v0:BasicObject): v2:Fixnum[3] = Const Value(3) PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010) - v7:BasicObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, BasicObject[class_exact*:Object@VALUE(0x1000)] + v7:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] v8:BasicObject = SendWithoutBlockDirect v7, :Integer (0x1038), v2 Return v8 "#]]); @@ -5647,7 +5648,7 @@ mod opt_tests { v2:Fixnum[1] = Const Value(1) v3:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v8:BasicObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, BasicObject[class_exact*:Object@VALUE(0x1000)] + v8:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] v9:BasicObject = SendWithoutBlockDirect v8, :foo (0x1038), v2, v3 Return v9 "#]]); @@ -5670,10 +5671,10 @@ mod opt_tests { fn test@:7: bb0(v0:BasicObject): PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v8:BasicObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, BasicObject[class_exact*:Object@VALUE(0x1000)] + v8:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] v9:BasicObject = SendWithoutBlockDirect v8, :foo (0x1038) PatchPoint MethodRedefined(Object@0x1000, bar@0x1040, cme:0x1048) - v11:BasicObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, BasicObject[class_exact*:Object@VALUE(0x1000)] + v11:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] v12:BasicObject = SendWithoutBlockDirect v11, :bar (0x1038) Return v12 "#]]); @@ -6475,7 +6476,7 @@ mod opt_tests { fn test@:8: bb0(v0:BasicObject, v1:BasicObject): PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) - v7:BasicObject[class_exact:C] = GuardType v1, BasicObject[class_exact:C] + v7:HeapObject[class_exact:C] = GuardType v1, HeapObject[class_exact:C] v8:BasicObject = SendWithoutBlockDirect v7, :foo (0x1038) Return v8 "#]]); @@ -7428,7 +7429,7 @@ mod opt_tests { fn test@:3: bb0(v0:BasicObject): PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v6:BasicObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, BasicObject[class_exact*:Object@VALUE(0x1000)] + v6:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] v7:BasicObject = SendWithoutBlockDirect v6, :foo (0x1038) Return v7 "#]]); @@ -7497,7 +7498,7 @@ mod opt_tests { fn test@:6: bb0(v0:BasicObject, v1:BasicObject): PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) - v7:BasicObject[class_exact:C] = GuardType v1, BasicObject[class_exact:C] + v7:HeapObject[class_exact:C] = GuardType v1, HeapObject[class_exact:C] v8:BasicObject = GetIvar v7, :@foo Return v8 "#]]); @@ -7518,7 +7519,7 @@ mod opt_tests { fn test@:6: bb0(v0:BasicObject, v1:BasicObject): PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) - v7:BasicObject[class_exact:C] = GuardType v1, BasicObject[class_exact:C] + v7:HeapObject[class_exact:C] = GuardType v1, HeapObject[class_exact:C] v8:BasicObject = GetIvar v7, :@foo Return v8 "#]]); diff --git a/zjit/src/hir_type/gen_hir_type.rb b/zjit/src/hir_type/gen_hir_type.rb index 685767898221e6..15aa68a6004c86 100644 --- a/zjit/src/hir_type/gen_hir_type.rb +++ b/zjit/src/hir_type/gen_hir_type.rb @@ -156,6 +156,8 @@ def add_union name, type_names add_union "Subclass", $subclass add_union "BoolExact", [true_exact.name, false_exact.name] add_union "Immediate", [fixnum.name, flonum.name, static_sym.name, nil_exact.name, true_exact.name, false_exact.name, undef_.name] +$bits["HeapObject"] = ["BasicObject & !Immediate"] +$numeric_bits["HeapObject"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Immediate"] # ===== Finished generating the DAG; write Rust code ===== diff --git a/zjit/src/hir_type/hir_type.inc.rs b/zjit/src/hir_type/hir_type.inc.rs index 68039c7f533b82..58508740800da3 100644 --- a/zjit/src/hir_type/hir_type.inc.rs +++ b/zjit/src/hir_type/hir_type.inc.rs @@ -38,6 +38,7 @@ mod bits { pub const HashExact: u64 = 1u64 << 23; pub const HashSubclass: u64 = 1u64 << 24; pub const HeapFloat: u64 = 1u64 << 25; + pub const HeapObject: u64 = BasicObject & !Immediate; pub const Immediate: u64 = FalseClass | Fixnum | Flonum | NilClass | StaticSymbol | TrueClass | Undef; pub const Integer: u64 = Bignum | Fixnum; pub const Module: u64 = Class | ModuleExact | ModuleSubclass; @@ -65,7 +66,7 @@ mod bits { pub const Symbol: u64 = DynamicSymbol | StaticSymbol; pub const TrueClass: u64 = 1u64 << 40; pub const Undef: u64 = 1u64 << 41; - pub const AllBitPatterns: [(&'static str, u64); 65] = [ + pub const AllBitPatterns: [(&'static str, u64); 66] = [ ("Any", Any), ("RubyValue", RubyValue), ("Immediate", Immediate), @@ -75,6 +76,7 @@ mod bits { ("BuiltinExact", BuiltinExact), ("BoolExact", BoolExact), ("TrueClass", TrueClass), + ("HeapObject", HeapObject), ("String", String), ("Subclass", Subclass), ("StringSubclass", StringSubclass), @@ -174,6 +176,7 @@ pub mod types { pub const HashExact: Type = Type::from_bits(bits::HashExact); pub const HashSubclass: Type = Type::from_bits(bits::HashSubclass); pub const HeapFloat: Type = Type::from_bits(bits::HeapFloat); + pub const HeapObject: Type = Type::from_bits(bits::HeapObject); pub const Immediate: Type = Type::from_bits(bits::Immediate); pub const Integer: Type = Type::from_bits(bits::Integer); pub const Module: Type = Type::from_bits(bits::Module); diff --git a/zjit/src/hir_type/mod.rs b/zjit/src/hir_type/mod.rs index 84679c419d9ba9..607ccbde84bfae 100644 --- a/zjit/src/hir_type/mod.rs +++ b/zjit/src/hir_type/mod.rs @@ -248,7 +248,7 @@ impl Type { else if val.class() == unsafe { rb_cString } { types::StringExact } else { // TODO(max): Add more cases for inferring type bits from built-in types - Type { bits: bits::BasicObject, spec: Specialization::TypeExact(val.class()) } + Type { bits: bits::HeapObject, spec: Specialization::TypeExact(val.class()) } } } @@ -497,7 +497,7 @@ impl Type { } } - fn is_immediate(&self) -> bool { + pub fn is_immediate(&self) -> bool { self.is_subtype(types::Immediate) } @@ -583,6 +583,7 @@ mod tests { assert_subtype(Type::fixnum(123), types::Immediate); assert_subtype(types::Fixnum, types::Immediate); assert_not_subtype(types::Bignum, types::Immediate); + assert_not_subtype(types::Integer, types::Immediate); assert_subtype(types::NilClass, types::Immediate); assert_subtype(types::TrueClass, types::Immediate); assert_subtype(types::FalseClass, types::Immediate); @@ -592,6 +593,32 @@ mod tests { assert_not_subtype(types::HeapFloat, types::Immediate); } + #[test] + fn heap_object() { + assert_not_subtype(Type::fixnum(123), types::HeapObject); + assert_not_subtype(types::Fixnum, types::HeapObject); + assert_subtype(types::Bignum, types::HeapObject); + assert_not_subtype(types::Integer, types::HeapObject); + assert_not_subtype(types::NilClass, types::HeapObject); + assert_not_subtype(types::TrueClass, types::HeapObject); + assert_not_subtype(types::FalseClass, types::HeapObject); + assert_not_subtype(types::StaticSymbol, types::HeapObject); + assert_subtype(types::DynamicSymbol, types::HeapObject); + assert_not_subtype(types::Flonum, types::HeapObject); + assert_subtype(types::HeapFloat, types::HeapObject); + assert_not_subtype(types::BasicObject, types::HeapObject); + assert_not_subtype(types::Object, types::HeapObject); + assert_not_subtype(types::Immediate, types::HeapObject); + assert_not_subtype(types::HeapObject, types::Immediate); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("world")); + assert_subtype(left, types::HeapObject); + assert_subtype(right, types::HeapObject); + assert_subtype(left.union(right), types::HeapObject); + }); + } + #[test] fn fixnum_has_ruby_object() { assert_eq!(Type::fixnum(3).ruby_object(), Some(VALUE::fixnum_from_usize(3))); diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs index c8c91dc45b97a1..85bc04fc715253 100644 --- a/zjit/src/invariants.rs +++ b/zjit/src/invariants.rs @@ -1,24 +1,32 @@ use std::{collections::{HashMap, HashSet}, mem}; -use crate::{backend::lir::{asm_comment, Assembler}, cruby::{rb_callable_method_entry_t, ruby_basic_operators, src_loc, with_vm_lock, IseqPtr, RedefinitionFlag, ID}, hir::Invariant, options::debug, state::{zjit_enabled_p, ZJITState}, virtualmem::CodePtr}; +use crate::{backend::lir::{asm_comment, Assembler}, cruby::{rb_callable_method_entry_t, ruby_basic_operators, src_loc, with_vm_lock, IseqPtr, RedefinitionFlag, ID}, gc::IseqPayload, hir::Invariant, options::debug, state::{zjit_enabled_p, ZJITState}, virtualmem::CodePtr}; +use crate::gc::remove_gc_offsets; -macro_rules! compile_jumps { - ($cb:expr, $jumps:expr, $($comment_args:tt)*) => { - for jump in $jumps { - $cb.with_write_ptr(jump.from, |cb| { +macro_rules! compile_patch_points { + ($cb:expr, $patch_points:expr, $($comment_args:tt)*) => { + for patch_point in $patch_points { + let written_range = $cb.with_write_ptr(patch_point.patch_point_ptr, |cb| { let mut asm = Assembler::new(); asm_comment!(asm, $($comment_args)*); - asm.jmp(jump.to.into()); + asm.jmp(patch_point.side_exit_ptr.into()); asm.compile(cb).expect("can write existing code"); }); + // Stop marking GC offsets corrupted by the jump instruction + remove_gc_offsets(patch_point.payload_ptr, &written_range); } }; } +/// When a PatchPoint is invalidated, it generates a jump instruction from `from` to `to`. #[derive(Debug, Eq, Hash, PartialEq)] -struct Jump { - from: CodePtr, - to: CodePtr, +struct PatchPoint { + /// Code pointer to be invalidated + patch_point_ptr: CodePtr, + /// Code pointer to a side exit + side_exit_ptr: CodePtr, + /// Raw pointer to the ISEQ payload + payload_ptr: *mut IseqPayload, } /// Used to track all of the various block references that contain assumptions @@ -32,16 +40,16 @@ pub struct Invariants { no_ep_escape_iseqs: HashSet, /// Map from a class and its associated basic operator to a set of patch points - bop_patch_points: HashMap<(RedefinitionFlag, ruby_basic_operators), HashSet>, + bop_patch_points: HashMap<(RedefinitionFlag, ruby_basic_operators), HashSet>, /// Map from CME to patch points that assume the method hasn't been redefined - cme_patch_points: HashMap<*const rb_callable_method_entry_t, HashSet>, + cme_patch_points: HashMap<*const rb_callable_method_entry_t, HashSet>, /// Map from constant ID to patch points that assume the constant hasn't been redefined - constant_state_patch_points: HashMap>, + constant_state_patch_points: HashMap>, /// Set of patch points that assume that the interpreter is running with only one ractor - single_ractor_patch_points: HashSet, + single_ractor_patch_points: HashSet, } /// Called when a basic operator is redefined. Note that all the blocks assuming @@ -56,13 +64,13 @@ pub extern "C" fn rb_zjit_bop_redefined(klass: RedefinitionFlag, bop: ruby_basic with_vm_lock(src_loc!(), || { let invariants = ZJITState::get_invariants(); - if let Some(jumps) = invariants.bop_patch_points.get(&(klass, bop)) { + if let Some(patch_points) = invariants.bop_patch_points.get(&(klass, bop)) { let cb = ZJITState::get_code_block(); let bop = Invariant::BOPRedefined { klass, bop }; debug!("BOP is redefined: {}", bop); // Invalidate all patch points for this BOP - compile_jumps!(cb, jumps, "BOP is redefined: {}", bop); + compile_patch_points!(cb, patch_points, "BOP is redefined: {}", bop); cb.mark_all_executable(); } @@ -106,12 +114,14 @@ pub fn track_bop_assumption( klass: RedefinitionFlag, bop: ruby_basic_operators, patch_point_ptr: CodePtr, - side_exit_ptr: CodePtr + side_exit_ptr: CodePtr, + payload_ptr: *mut IseqPayload, ) { let invariants = ZJITState::get_invariants(); - invariants.bop_patch_points.entry((klass, bop)).or_default().insert(Jump { - from: patch_point_ptr, - to: side_exit_ptr, + invariants.bop_patch_points.entry((klass, bop)).or_default().insert(PatchPoint { + patch_point_ptr, + side_exit_ptr, + payload_ptr, }); } @@ -119,12 +129,14 @@ pub fn track_bop_assumption( pub fn track_cme_assumption( cme: *const rb_callable_method_entry_t, patch_point_ptr: CodePtr, - side_exit_ptr: CodePtr + side_exit_ptr: CodePtr, + payload_ptr: *mut IseqPayload, ) { let invariants = ZJITState::get_invariants(); - invariants.cme_patch_points.entry(cme).or_default().insert(Jump { - from: patch_point_ptr, - to: side_exit_ptr, + invariants.cme_patch_points.entry(cme).or_default().insert(PatchPoint { + patch_point_ptr, + side_exit_ptr, + payload_ptr, }); } @@ -132,7 +144,8 @@ pub fn track_cme_assumption( pub fn track_stable_constant_names_assumption( idlist: *const ID, patch_point_ptr: CodePtr, - side_exit_ptr: CodePtr + side_exit_ptr: CodePtr, + payload_ptr: *mut IseqPayload, ) { let invariants = ZJITState::get_invariants(); @@ -143,9 +156,10 @@ pub fn track_stable_constant_names_assumption( break; } - invariants.constant_state_patch_points.entry(id).or_default().insert(Jump { - from: patch_point_ptr, - to: side_exit_ptr, + invariants.constant_state_patch_points.entry(id).or_default().insert(PatchPoint { + patch_point_ptr, + side_exit_ptr, + payload_ptr, }); idx += 1; @@ -163,12 +177,12 @@ pub extern "C" fn rb_zjit_cme_invalidate(cme: *const rb_callable_method_entry_t) with_vm_lock(src_loc!(), || { let invariants = ZJITState::get_invariants(); // Get the CMD's jumps and remove the entry from the map as it has been invalidated - if let Some(jumps) = invariants.cme_patch_points.remove(&cme) { + if let Some(patch_points) = invariants.cme_patch_points.remove(&cme) { let cb = ZJITState::get_code_block(); debug!("CME is invalidated: {:?}", cme); // Invalidate all patch points for this CME - compile_jumps!(cb, jumps, "CME is invalidated: {:?}", cme); + compile_patch_points!(cb, patch_points, "CME is invalidated: {:?}", cme); cb.mark_all_executable(); } @@ -185,12 +199,12 @@ pub extern "C" fn rb_zjit_constant_state_changed(id: ID) { with_vm_lock(src_loc!(), || { let invariants = ZJITState::get_invariants(); - if let Some(jumps) = invariants.constant_state_patch_points.get(&id) { + if let Some(patch_points) = invariants.constant_state_patch_points.get(&id) { let cb = ZJITState::get_code_block(); debug!("Constant state changed: {:?}", id); // Invalidate all patch points for this constant ID - compile_jumps!(cb, jumps, "Constant state changed: {:?}", id); + compile_patch_points!(cb, patch_points, "Constant state changed: {:?}", id); cb.mark_all_executable(); } @@ -198,11 +212,12 @@ pub extern "C" fn rb_zjit_constant_state_changed(id: ID) { } /// Track the JIT code that assumes that the interpreter is running with only one ractor -pub fn track_single_ractor_assumption(patch_point_ptr: CodePtr, side_exit_ptr: CodePtr) { +pub fn track_single_ractor_assumption(patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, payload_ptr: *mut IseqPayload) { let invariants = ZJITState::get_invariants(); - invariants.single_ractor_patch_points.insert(Jump { - from: patch_point_ptr, - to: side_exit_ptr, + invariants.single_ractor_patch_points.insert(PatchPoint { + patch_point_ptr, + side_exit_ptr, + payload_ptr, }); } @@ -217,10 +232,10 @@ pub extern "C" fn rb_zjit_before_ractor_spawn() { with_vm_lock(src_loc!(), || { let cb = ZJITState::get_code_block(); - let jumps = mem::take(&mut ZJITState::get_invariants().single_ractor_patch_points); + let patch_points = mem::take(&mut ZJITState::get_invariants().single_ractor_patch_points); // Invalidate all patch points for single ractor mode - compile_jumps!(cb, jumps, "Another ractor spawned, invalidating single ractor mode assumption"); + compile_patch_points!(cb, patch_points, "Another ractor spawned, invalidating single ractor mode assumption"); cb.mark_all_executable(); });