diff --git a/depend b/depend index 7fdf369158c305..b9d91faa2a05a2 100644 --- a/depend +++ b/depend @@ -19732,6 +19732,7 @@ vm_trace.$(OBJEXT): {$(VPATH)}vm_opts.h vm_trace.$(OBJEXT): {$(VPATH)}vm_sync.h vm_trace.$(OBJEXT): {$(VPATH)}vm_trace.c vm_trace.$(OBJEXT): {$(VPATH)}yjit.h +vm_trace.$(OBJEXT): {$(VPATH)}zjit.h weakmap.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h weakmap.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h weakmap.$(OBJEXT): $(CCAN_DIR)/list/list.h diff --git a/ext/-test-/tracepoint/gc_hook.c b/ext/-test-/tracepoint/gc_hook.c index 54c06c54a5b9b7..525be6da632bf7 100644 --- a/ext/-test-/tracepoint/gc_hook.c +++ b/ext/-test-/tracepoint/gc_hook.c @@ -2,6 +2,7 @@ #include "ruby/debug.h" static int invoking; /* TODO: should not be global variable */ +extern VALUE tp_mBug; static VALUE invoke_proc_ensure(VALUE _) @@ -17,9 +18,9 @@ invoke_proc_begin(VALUE proc) } static void -invoke_proc(void *data) +invoke_proc(void *ivar_name) { - VALUE proc = (VALUE)data; + VALUE proc = rb_ivar_get(tp_mBug, rb_intern(ivar_name)); invoking += 1; rb_ensure(invoke_proc_begin, proc, invoke_proc_ensure, 0); } @@ -40,16 +41,16 @@ gc_start_end_i(VALUE tpval, void *data) } static VALUE -set_gc_hook(VALUE module, VALUE proc, rb_event_flag_t event, const char *tp_str, const char *proc_str) +set_gc_hook(VALUE proc, rb_event_flag_t event, const char *tp_str, const char *proc_str) { VALUE tpval; ID tp_key = rb_intern(tp_str); /* disable previous keys */ - if (rb_ivar_defined(module, tp_key) != 0 && - RTEST(tpval = rb_ivar_get(module, tp_key))) { + if (rb_ivar_defined(tp_mBug, tp_key) != 0 && + RTEST(tpval = rb_ivar_get(tp_mBug, tp_key))) { rb_tracepoint_disable(tpval); - rb_ivar_set(module, tp_key, Qnil); + rb_ivar_set(tp_mBug, tp_key, Qnil); } if (RTEST(proc)) { @@ -57,8 +58,9 @@ set_gc_hook(VALUE module, VALUE proc, rb_event_flag_t event, const char *tp_str, rb_raise(rb_eTypeError, "trace_func needs to be Proc"); } - tpval = rb_tracepoint_new(0, event, gc_start_end_i, (void *)proc); - rb_ivar_set(module, tp_key, tpval); + rb_ivar_set(tp_mBug, rb_intern(proc_str), proc); + tpval = rb_tracepoint_new(0, event, gc_start_end_i, (void *)proc_str); + rb_ivar_set(tp_mBug, tp_key, tpval); rb_tracepoint_enable(tpval); } @@ -66,16 +68,16 @@ set_gc_hook(VALUE module, VALUE proc, rb_event_flag_t event, const char *tp_str, } static VALUE -set_after_gc_start(VALUE module, VALUE proc) +set_after_gc_start(VALUE _self, VALUE proc) { - return set_gc_hook(module, proc, RUBY_INTERNAL_EVENT_GC_START, + return set_gc_hook(proc, RUBY_INTERNAL_EVENT_GC_START, "__set_after_gc_start_tpval__", "__set_after_gc_start_proc__"); } static VALUE -start_after_gc_exit(VALUE module, VALUE proc) +start_after_gc_exit(VALUE _self, VALUE proc) { - return set_gc_hook(module, proc, RUBY_INTERNAL_EVENT_GC_EXIT, + return set_gc_hook(proc, RUBY_INTERNAL_EVENT_GC_EXIT, "__set_after_gc_exit_tpval__", "__set_after_gc_exit_proc__"); } diff --git a/ext/-test-/tracepoint/tracepoint.c b/ext/-test-/tracepoint/tracepoint.c index 2826cc038c7546..001d9513b29fb2 100644 --- a/ext/-test-/tracepoint/tracepoint.c +++ b/ext/-test-/tracepoint/tracepoint.c @@ -1,6 +1,8 @@ #include "ruby/ruby.h" #include "ruby/debug.h" +VALUE tp_mBug; + struct tracepoint_track { size_t newobj_count; size_t free_count; @@ -89,8 +91,8 @@ void Init_gc_hook(VALUE); void Init_tracepoint(void) { - VALUE mBug = rb_define_module("Bug"); - Init_gc_hook(mBug); - rb_define_module_function(mBug, "tracepoint_track_objspace_events", tracepoint_track_objspace_events, 0); - rb_define_module_function(mBug, "tracepoint_specify_normal_and_internal_events", tracepoint_specify_normal_and_internal_events, 0); + tp_mBug = rb_define_module("Bug"); // GC root + Init_gc_hook(tp_mBug); + rb_define_module_function(tp_mBug, "tracepoint_track_objspace_events", tracepoint_track_objspace_events, 0); + rb_define_module_function(tp_mBug, "tracepoint_specify_normal_and_internal_events", tracepoint_specify_normal_and_internal_events, 0); } diff --git a/jit.c b/jit.c index 0709c6d8f03561..efecbef35455ca 100644 --- a/jit.c +++ b/jit.c @@ -492,3 +492,51 @@ rb_jit_vm_unlock(unsigned int *recursive_lock_level, const char *file, int line) { rb_vm_lock_leave(recursive_lock_level, file, line); } + +void +rb_iseq_reset_jit_func(const rb_iseq_t *iseq) +{ + RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq)); + iseq->body->jit_entry = NULL; + iseq->body->jit_exception = NULL; + // Enable re-compiling this ISEQ. Event when it's invalidated for TracePoint, + // we'd like to re-compile ISEQs that haven't been converted to trace_* insns. + iseq->body->jit_entry_calls = 0; + iseq->body->jit_exception_calls = 0; +} + +// Callback data for rb_jit_for_each_iseq +struct iseq_callback_data { + rb_iseq_callback callback; + void *data; +}; + +// Heap-walking callback for rb_jit_for_each_iseq +static int +for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data) +{ + const struct iseq_callback_data *callback_data = (struct iseq_callback_data *)data; + VALUE v = (VALUE)vstart; + for (; v != (VALUE)vend; v += stride) { + void *ptr = rb_asan_poisoned_object_p(v); + rb_asan_unpoison_object(v, false); + + if (rb_obj_is_iseq(v)) { + rb_iseq_t *iseq = (rb_iseq_t *)v; + callback_data->callback(iseq, callback_data->data); + } + + if (ptr) { + rb_asan_poison_object(v); + } + } + return 0; +} + +// Walk all ISEQs in the heap and invoke the callback - shared between YJIT and ZJIT +void +rb_jit_for_each_iseq(rb_iseq_callback callback, void *data) +{ + struct iseq_callback_data callback_data = { .callback = callback, .data = data }; + rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data); +} diff --git a/test/-ext-/tracepoint/test_tracepoint.rb b/test/-ext-/tracepoint/test_tracepoint.rb index bf66d8f10514ff..debddd83d043fe 100644 --- a/test/-ext-/tracepoint/test_tracepoint.rb +++ b/test/-ext-/tracepoint/test_tracepoint.rb @@ -83,7 +83,7 @@ def run(hook) end def test_teardown_with_active_GC_end_hook - assert_separately([], 'require("-test-/tracepoint"); Bug.after_gc_exit_hook = proc {}') + assert_separately([], 'require("-test-/tracepoint"); Bug.after_gc_exit_hook = proc {}; GC.start') end end diff --git a/test/.excludes-zjit/ErrorHighlightTest.rb b/test/.excludes-zjit/ErrorHighlightTest.rb deleted file mode 100644 index 2ddee303ddd871..00000000000000 --- a/test/.excludes-zjit/ErrorHighlightTest.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_local_variable_get, 'Test fails with ZJIT') diff --git a/test/.excludes-zjit/TestObjSpace.rb b/test/.excludes-zjit/TestObjSpace.rb deleted file mode 100644 index 28e97f4578e7cf..00000000000000 --- a/test/.excludes-zjit/TestObjSpace.rb +++ /dev/null @@ -1,2 +0,0 @@ -exclude(:test_dump_to_io, 'Test fails with ZJIT') -exclude(:test_dump_to_default, 'Test fails with ZJIT') diff --git a/test/.excludes-zjit/TestParse.rb b/test/.excludes-zjit/TestParse.rb deleted file mode 100644 index e5d3b0944771b2..00000000000000 --- a/test/.excludes-zjit/TestParse.rb +++ /dev/null @@ -1,2 +0,0 @@ -exclude(:test_flip_flop, 'Test fails with ZJIT') -exclude(:test_rescue_in_command_assignment, 'Test fails with ZJIT') diff --git a/test/.excludes-zjit/TestRegexp.rb b/test/.excludes-zjit/TestRegexp.rb deleted file mode 100644 index e344b6d803a905..00000000000000 --- a/test/.excludes-zjit/TestRegexp.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_union, 'Test fails with ZJIT') diff --git a/test/.excludes-zjit/TestTimeout.rb b/test/.excludes-zjit/TestTimeout.rb deleted file mode 100644 index 5e1570fb8f57fd..00000000000000 --- a/test/.excludes-zjit/TestTimeout.rb +++ /dev/null @@ -1,3 +0,0 @@ -exclude(:test_timeout, 'Test hangs with ZJIT') -exclude(:test_nested_timeout, 'Test hangs with ZJIT') -exclude(:test_nested_timeout_error_identity, 'Test hangs with ZJIT') diff --git a/test/io/wait/test_io_wait.rb b/test/io/wait/test_io_wait.rb index 0e011a473f3485..445363714c0912 100644 --- a/test/io/wait/test_io_wait.rb +++ b/test/io/wait/test_io_wait.rb @@ -114,7 +114,7 @@ def test_wait_readable_eof ret = nil assert_nothing_raised(Timeout::Error) do q.push(true) - t = EnvUtil.apply_timeout_scale(0.1) + t = EnvUtil.apply_timeout_scale(1) Timeout.timeout(t) { ret = @r.wait_readable } end assert_equal @r, ret diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 7aba333e92377f..d88b4f07f6de19 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -747,7 +747,7 @@ def test_interrupt_in_finalizer Signal.trap(:INT, 'DEFAULT') pid = $$ Thread.start do - 10.times { + 1000.times { sleep 0.1 Process.kill("INT", pid) rescue break } diff --git a/test/ruby/test_settracefunc.rb b/test/ruby/test_settracefunc.rb index fac6dd818533d0..ccf24521694484 100644 --- a/test/ruby/test_settracefunc.rb +++ b/test/ruby/test_settracefunc.rb @@ -2724,7 +2724,7 @@ def obj.example end def test_disable_local_tracepoint_in_trace - assert_normal_exit <<-EOS + assert_normal_exit(<<-EOS, timeout: 60) def foo trace = TracePoint.new(:b_return){|tp| tp.disable diff --git a/test/ruby/test_zjit.rb b/test/ruby/test_zjit.rb index e79e80fb44ce9b..0353a48eecab72 100644 --- a/test/ruby/test_zjit.rb +++ b/test/ruby/test_zjit.rb @@ -2202,6 +2202,44 @@ def test } end + def test_global_tracepoint + assert_compiles 'true', %q{ + def foo = 1 + + foo + foo + + called = false + + tp = TracePoint.new(:return) { |event| + if event.method_id == :foo + called = true + end + } + tp.enable do + foo + end + called + } + end + + def test_local_tracepoint + assert_compiles 'true', %q{ + def foo = 1 + + foo + foo + + called = false + + tp = TracePoint.new(:return) { |_| called = true } + tp.enable(target: method(:foo)) do + foo + end + called + } + end + private # Assert that every method call in `test_script` can be compiled by ZJIT diff --git a/vm_trace.c b/vm_trace.c index cb4feff147168e..1069d36dd08762 100644 --- a/vm_trace.c +++ b/vm_trace.c @@ -35,6 +35,7 @@ #include "vm_core.h" #include "ruby/ractor.h" #include "yjit.h" +#include "zjit.h" #include "builtin.h" @@ -135,6 +136,7 @@ update_global_event_hook(rb_event_flag_t prev_events, rb_event_flag_t new_events // Do this after event flags updates so other ractors see updated vm events // when they wake up. rb_yjit_tracing_invalidate_all(); + rb_zjit_tracing_invalidate_all(); } } @@ -1285,6 +1287,7 @@ rb_tracepoint_enable_for_target(VALUE tpval, VALUE target, VALUE target_line) } rb_yjit_tracing_invalidate_all(); + rb_zjit_tracing_invalidate_all(); ruby_vm_event_local_num++; diff --git a/yjit.c b/yjit.c index bca0df96fdf33f..b38f860ed5e627 100644 --- a/yjit.c +++ b/yjit.c @@ -413,18 +413,6 @@ rb_iseq_set_yjit_payload(const rb_iseq_t *iseq, void *payload) iseq->body->yjit_payload = payload; } -void -rb_iseq_reset_jit_func(const rb_iseq_t *iseq) -{ - RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq)); - iseq->body->jit_entry = NULL; - iseq->body->jit_exception = NULL; - // Enable re-compiling this ISEQ. Event when it's invalidated for TracePoint, - // we'd like to re-compile ISEQs that haven't been converted to trace_* insns. - iseq->body->jit_entry_calls = 0; - iseq->body->jit_exception_calls = 0; -} - rb_proc_t * rb_yjit_get_proc_ptr(VALUE procv) { @@ -643,41 +631,6 @@ rb_yjit_constcache_shareable(const struct iseq_inline_constant_cache_entry *ice) return (ice->flags & IMEMO_CONST_CACHE_SHAREABLE) != 0; } -// Used for passing a callback and other data over rb_objspace_each_objects -struct iseq_callback_data { - rb_iseq_callback callback; - void *data; -}; - -// Heap-walking callback for rb_yjit_for_each_iseq(). -static int -for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data) -{ - const struct iseq_callback_data *callback_data = (struct iseq_callback_data *)data; - VALUE v = (VALUE)vstart; - for (; v != (VALUE)vend; v += stride) { - void *ptr = rb_asan_poisoned_object_p(v); - rb_asan_unpoison_object(v, false); - - if (rb_obj_is_iseq(v)) { - rb_iseq_t *iseq = (rb_iseq_t *)v; - callback_data->callback(iseq, callback_data->data); - } - - asan_poison_object_if(ptr, v); - } - return 0; -} - -// Iterate through the whole GC heap and invoke a callback for each iseq. -// Used for global code invalidation. -void -rb_yjit_for_each_iseq(rb_iseq_callback callback, void *data) -{ - struct iseq_callback_data callback_data = { .callback = callback, .data = data }; - rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data); -} - // For running write barriers from Rust. Required when we add a new edge in the // object graph from `old` to `young`. void diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index d30fb6c7796448..2fc85431e0439d 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -333,7 +333,6 @@ fn main() { .allowlist_function("rb_yjit_constcache_shareable") .allowlist_function("rb_iseq_reset_jit_func") .allowlist_function("rb_yjit_dump_iseq_loc") - .allowlist_function("rb_yjit_for_each_iseq") .allowlist_function("rb_yjit_obj_written") .allowlist_function("rb_yjit_str_simple_append") .allowlist_function("rb_RSTRING_PTR") @@ -355,6 +354,7 @@ fn main() { .allowlist_function("rb_jit_multi_ractor_p") .allowlist_function("rb_jit_vm_lock_then_barrier") .allowlist_function("rb_jit_vm_unlock") + .allowlist_function("rb_jit_for_each_iseq") .allowlist_type("robject_offsets") // from vm_sync.h diff --git a/yjit/src/core.rs b/yjit/src/core.rs index d42726bcc77691..f8c80c0c860130 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -1818,7 +1818,7 @@ pub fn for_each_iseq(mut callback: F) { callback(iseq); } let mut data: &mut dyn FnMut(IseqPtr) = &mut callback; - unsafe { rb_yjit_for_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) }; + unsafe { rb_jit_for_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) }; } /// Iterate over all on-stack ISEQs diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 4d52b675a0de1f..429330168b0f29 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1185,7 +1185,6 @@ extern "C" { pub fn rb_full_cfunc_return(ec: *mut rb_execution_context_t, return_value: VALUE); pub fn rb_iseq_get_yjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void; pub fn rb_iseq_set_yjit_payload(iseq: *const rb_iseq_t, payload: *mut ::std::os::raw::c_void); - pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); pub fn rb_yjit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t; pub fn rb_get_symbol_id(namep: VALUE) -> ID; pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE; @@ -1219,7 +1218,6 @@ extern "C" { pub fn rb_RSTRUCT_SET(st: VALUE, k: ::std::os::raw::c_int, v: VALUE); pub fn rb_ENCODING_GET(obj: VALUE) -> ::std::os::raw::c_int; pub fn rb_yjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool; - pub fn rb_yjit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); pub fn rb_yjit_obj_written( old: VALUE, young: VALUE, @@ -1328,4 +1326,6 @@ extern "C" { file: *const ::std::os::raw::c_char, line: ::std::os::raw::c_int, ); + pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); } diff --git a/zjit.h b/zjit.h index adf47046f83b37..45e91fa43c77ca 100644 --- a/zjit.h +++ b/zjit.h @@ -23,6 +23,7 @@ void rb_zjit_constant_state_changed(ID id); void rb_zjit_iseq_mark(void *payload); void rb_zjit_iseq_update_references(void *payload); void rb_zjit_before_ractor_spawn(void); +void rb_zjit_tracing_invalidate_all(void); #else #define rb_zjit_enabled_p false static inline void rb_zjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception) {} @@ -33,6 +34,7 @@ static inline void rb_zjit_cme_invalidate(const rb_callable_method_entry_t *cme) static inline void rb_zjit_invalidate_ep_is_bp(const rb_iseq_t *iseq) {} static inline void rb_zjit_constant_state_changed(ID id) {} static inline void rb_zjit_before_ractor_spawn(void) {} +static inline void rb_zjit_tracing_invalidate_all(void) {} #endif // #if USE_ZJIT #endif // #ifndef ZJIT_H diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index e57d0ae0154c48..c608d482e2c987 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -349,7 +349,6 @@ fn main() { .allowlist_function("rb_full_cfunc_return") .allowlist_function("rb_assert_(iseq|cme)_handle") .allowlist_function("rb_IMEMO_TYPE_P") - .allowlist_function("rb_iseq_reset_jit_func") .allowlist_function("rb_RSTRING_PTR") .allowlist_function("rb_RSTRING_LEN") .allowlist_function("rb_ENCODING_GET") @@ -368,6 +367,8 @@ fn main() { .allowlist_function("rb_jit_multi_ractor_p") .allowlist_function("rb_jit_vm_lock_then_barrier") .allowlist_function("rb_jit_vm_unlock") + .allowlist_function("rb_jit_for_each_iseq") + .allowlist_function("rb_iseq_reset_jit_func") .allowlist_type("robject_offsets") // from vm_sync.h diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs index 5e582ce28203ad..08d3571c2d6b6a 100644 --- a/zjit/src/asm/mod.rs +++ b/zjit/src/asm/mod.rs @@ -330,7 +330,7 @@ pub fn imm_num_bits(imm: i64) -> u8 return 32; } - return 64; + 64 } /// Compute the number of bits needed to encode an unsigned value @@ -347,7 +347,7 @@ pub fn uimm_num_bits(uimm: u64) -> u8 return 32; } - return 64; + 64 } #[cfg(test)] diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 584914c83392cb..ef5a4aadd6bc22 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -140,17 +140,17 @@ fn emit_load_value(cb: &mut CodeBlock, rd: A64Opnd, value: u64) -> usize { // If the value fits into a single movz // instruction, then we'll use that. movz(cb, rd, A64Opnd::new_uimm(current), 0); - return 1; + 1 } else if u16::try_from(!value).is_ok() { // For small negative values, use a single movn movn(cb, rd, A64Opnd::new_uimm(!value), 0); - return 1; + 1 } else if BitmaskImmediate::try_from(current).is_ok() { // Otherwise, if the immediate can be encoded // with the special bitmask immediate encoding, // we'll use that. mov(cb, rd, A64Opnd::new_uimm(current)); - return 1; + 1 } else { // Finally we'll fall back to encoding the value // using movz for the first 16 bits and movk for @@ -176,7 +176,7 @@ fn emit_load_value(cb: &mut CodeBlock, rd: A64Opnd, value: u64) -> usize { movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 48); num_insns += 1; } - return num_insns; + num_insns } } @@ -1107,8 +1107,8 @@ impl Assembler // be stored is first and the address is second. However in // our IR we have the address first and the register second. match dest_num_bits { - 64 | 32 => stur(cb, src.into(), dest.into()), - 16 => sturh(cb, src.into(), dest.into()), + 64 | 32 => stur(cb, src, dest), + 16 => sturh(cb, src, dest), num_bits => panic!("unexpected dest num_bits: {} (src: {:#?}, dest: {:#?})", num_bits, src, dest), } }, diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 068e3f69dc76b6..03cd5253bc2729 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -1568,7 +1568,7 @@ impl Assembler let side_exit_label = if let Some(label) = label { Target::Label(label) } else { - self.new_label("side_exit".into()) + self.new_label("side_exit") }; self.write_label(side_exit_label.clone()); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 66ad5ba3a9d0cb..0d461cb0aa15dd 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -104,48 +104,28 @@ pub extern "C" fn rb_zjit_iseq_gen_entry_point(iseq: IseqPtr, _ec: EcPtr) -> *co /// Compile an entry point for a given ISEQ fn gen_iseq_entry_point(cb: &mut CodeBlock, iseq: IseqPtr) -> Result { // Compile ISEQ into High-level IR - let function = match compile_iseq(iseq) { - Ok(function) => function, - Err(err) => { - incr_counter!(failed_iseq_count); - return Err(err); - } - }; + let function = compile_iseq(iseq).inspect_err(|_| { + incr_counter!(failed_iseq_count); + })?; // Compile the High-level IR - let start_ptr = match gen_iseq(cb, iseq, Some(&function)) { - Ok(start_ptr) => start_ptr, - Err(err) => { - debug!("Failed to compile iseq: gen_iseq failed: {}", iseq_get_location(iseq, 0)); - return Err(err); - } - }; + let start_ptr = gen_iseq(cb, iseq, Some(&function)).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq, 0)); + })?; // Compile an entry point to the JIT code - let entry_ptr = match gen_entry(cb, iseq, &function, start_ptr) { - Ok(entry_ptr) => entry_ptr, - Err(err) => { - debug!("Failed to compile iseq: gen_entry failed: {}", iseq_get_location(iseq, 0)); - return Err(err); - } - }; - - // Return a JIT code address - Ok(entry_ptr) + gen_entry(cb, iseq, &function, start_ptr).inspect_err(|err| { + debug!("{err:?}: gen_entry failed: {}", iseq_get_location(iseq, 0)); + }) } /// Stub a branch for a JIT-to-JIT call fn gen_iseq_call(cb: &mut CodeBlock, caller_iseq: IseqPtr, iseq_call: &Rc>) -> Result<(), CompileError> { // Compile a function stub - let stub_ptr = match gen_function_stub(cb, iseq_call.clone()) { - Ok(stub_ptr) => stub_ptr, - Err(err) => { - // Failed to compile the stub. Bail out of compiling the caller ISEQ. - debug!("Failed to compile iseq: could not compile stub: {} -> {}", - iseq_get_location(caller_iseq, 0), iseq_get_location(iseq_call.borrow().iseq, 0)); - return Err(err); - } - }; + let stub_ptr = gen_function_stub(cb, iseq_call.clone()).inspect_err(|err| { + debug!("{err:?}: gen_function_stub failed: {} -> {}", + iseq_get_location(caller_iseq, 0), iseq_get_location(iseq_call.borrow().iseq, 0)); + })?; // Update the JIT-to-JIT call to call the stub let stub_addr = stub_ptr.raw_ptr(cb); @@ -505,7 +485,7 @@ fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); let pushval = asm.load(pushval.into()); asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); - asm.csel_e(Qnil.into(), pushval.into()) + asm.csel_e(Qnil.into(), pushval) } else { Qnil.into() } @@ -581,7 +561,7 @@ fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) { let payload_ptr = get_or_create_iseq_payload_ptr(jit.iseq); let label = asm.new_label("patch_point").unwrap_label(); - let invariant = invariant.clone(); + let invariant = *invariant; // Compile a side exit. Fill nop instructions if the last patch point is too close. asm.patch_point(build_side_exit(jit, state, PatchPoint(invariant), Some(label))); @@ -943,7 +923,7 @@ fn gen_send( gen_incr_counter(asm, Counter::dynamic_send_count); // Save PC and SP - gen_save_pc(asm, state); + gen_prepare_call_with_gc(asm, state); gen_save_sp(asm, state.stack().len()); // Spill locals and stack @@ -979,7 +959,7 @@ fn gen_send_without_block( asm_comment!(asm, "spill frame state"); // Save PC and SP - gen_save_pc(asm, state); + gen_prepare_call_with_gc(asm, state); gen_save_sp(asm, state.stack().len()); // Spill locals and stack @@ -1012,7 +992,7 @@ fn gen_send_without_block_direct( state: &FrameState, ) -> lir::Opnd { // Save cfp->pc and cfp->sp for the caller frame - gen_save_pc(asm, state); + gen_prepare_call_with_gc(asm, state); gen_save_sp(asm, state.stack().len() - args.len() - 1); // -1 for receiver gen_spill_locals(jit, asm, state); @@ -1363,9 +1343,13 @@ fn gen_incr_counter(asm: &mut Assembler, counter: Counter) { } } -/// Save the incremented PC on the CFP. -/// This is necessary when callees can raise or allocate. -fn gen_save_pc(asm: &mut Assembler, state: &FrameState) { +/// Save the current PC on the CFP as a preparation for calling a C function +/// that may allocate objects and trigger GC. Use gen_prepare_non_leaf_call() +/// if it may raise exceptions or call arbitrary methods. +/// +/// Unlike YJIT, we don't need to save the stack slots to protect them from GC +/// because the backend spills all live registers onto the C stack on CCall. +fn gen_prepare_call_with_gc(asm: &mut Assembler, state: &FrameState) { let opcode: usize = state.get_opcode().try_into().unwrap(); let next_pc: *const VALUE = unsafe { state.pc.offset(insn_len(opcode) as isize) }; @@ -1409,7 +1393,7 @@ fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { // TODO: Lazily materialize caller frames when needed // Save PC for backtraces and allocation tracing - gen_save_pc(asm, state); + gen_prepare_call_with_gc(asm, state); // Save SP and spill the virtual stack in case it raises an exception // and the interpreter uses the stack for handling the exception @@ -1420,15 +1404,6 @@ fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameS gen_spill_locals(jit, asm, state); } -/// Prepare for calling a C function that may allocate objects and trigger GC. -/// Use gen_prepare_non_leaf_call() if it may also call an arbitrary method. -fn gen_prepare_call_with_gc(asm: &mut Assembler, state: &FrameState) { - // Save PC for allocation tracing - gen_save_pc(asm, state); - // Unlike YJIT, we don't need to save the stack to protect them from GC - // because the backend spills all live registers onto the C stack on asm.ccall(). -} - /// Frame metadata written by gen_push_frame() struct ControlFrame { recv: Opnd, @@ -1547,14 +1522,13 @@ fn build_side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReaso locals.push(jit.get_opnd(insn_id)); } - let target = Target::SideExit { + Target::SideExit { pc: state.pc, stack, locals, reason, label, - }; - target + } } /// Return true if a given ISEQ is known to escape EP to the heap on entry. @@ -1652,13 +1626,10 @@ c_callable! { // Otherwise, attempt to compile the ISEQ. We have to mark_all_executable() beyond this point. let code_ptr = with_time_stat(compile_time_ns, || function_stub_hit_body(cb, &iseq_call)); - let code_ptr = match code_ptr { - Ok(code_ptr) => code_ptr, - Err(compile_error) => { - prepare_for_exit(iseq, cfp, sp, &compile_error); - ZJITState::get_exit_trampoline_with_counter() - } - }; + let code_ptr = code_ptr.unwrap_or_else(|compile_error| { + prepare_for_exit(iseq, cfp, sp, &compile_error); + ZJITState::get_exit_trampoline_with_counter() + }); cb.mark_all_executable(); code_ptr.raw_ptr(cb) }) @@ -1668,13 +1639,9 @@ c_callable! { /// Compile an ISEQ for a function stub fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &Rc>) -> Result { // Compile the stubbed ISEQ - let code_ptr = match gen_iseq(cb, iseq_call.borrow().iseq, None) { - Ok(code_ptr) => code_ptr, - Err(err) => { - debug!("Failed to compile iseq: gen_iseq failed: {}", iseq_get_location(iseq_call.borrow().iseq, 0)); - return Err(err); - } - }; + let code_ptr = gen_iseq(cb, iseq_call.borrow().iseq, None).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq_call.borrow().iseq, 0)); + })?; // Update the stub to call the code pointer let code_addr = code_ptr.raw_ptr(cb); @@ -1694,7 +1661,7 @@ fn gen_function_stub(cb: &mut CodeBlock, iseq_call: Rc>) -> Re // Call function_stub_hit using the shared trampoline. See `gen_function_stub_hit_trampoline`. // Use load_into instead of mov, which is split on arm64, to avoid clobbering ALLOC_REGS. - asm.load_into(SCRATCH_OPND, Opnd::const_ptr(Rc::into_raw(iseq_call).into())); + asm.load_into(SCRATCH_OPND, Opnd::const_ptr(Rc::into_raw(iseq_call))); asm.jmp(ZJITState::get_function_stub_hit_trampoline().into()); asm.compile(cb).map(|(code_ptr, gc_offsets)| { diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 899ed4d892895f..b82edc6633fcfe 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -88,7 +88,7 @@ #![allow(unused_imports)] use std::convert::From; -use std::ffi::{CString, CStr}; +use std::ffi::{c_void, CString, CStr}; use std::fmt::{Debug, Formatter}; use std::os::raw::{c_char, c_int, c_uint}; use std::panic::{catch_unwind, UnwindSafe}; @@ -293,6 +293,17 @@ pub fn iseq_opcode_at_idx(iseq: IseqPtr, insn_idx: u32) -> u32 { unsafe { rb_iseq_opcode_at_pc(iseq, pc) as u32 } } +/// Iterate over all existing ISEQs +pub fn for_each_iseq(mut callback: F) { + unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) { + // SAFETY: points to the local below + let callback: &mut &mut dyn FnMut(IseqPtr) -> bool = unsafe { std::mem::transmute(&mut *data) }; + callback(iseq); + } + let mut data: &mut dyn FnMut(IseqPtr) = &mut callback; + unsafe { rb_jit_for_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) }; +} + /// Return a poison value to be set above the stack top to verify leafness. #[cfg(not(test))] pub fn vm_stack_canary() -> u64 { diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 88b90976975bc8..95727566881511 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1029,4 +1029,6 @@ unsafe extern "C" { file: *const ::std::os::raw::c_char, line: ::std::os::raw::c_int, ); + pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); } diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index afd158211dfd3f..8a5dd2e3989515 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -1012,7 +1012,7 @@ impl + PartialEq> UnionFind { /// Private. Return the internal representation of the forwarding pointer for a given element. fn at(&self, idx: T) -> Option { - self.forwarded.get(idx.into()).map(|x| *x).flatten() + self.forwarded.get(idx.into()).copied().flatten() } /// Private. Set the internal representation of the forwarding pointer for the given element diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs index 14fea76d1b499c..9935336bc0fc76 100644 --- a/zjit/src/invariants.rs +++ b/zjit/src/invariants.rs @@ -269,3 +269,25 @@ pub extern "C" fn rb_zjit_before_ractor_spawn() { cb.mark_all_executable(); }); } + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_tracing_invalidate_all() { + use crate::gc::{get_or_create_iseq_payload, IseqStatus}; + use crate::cruby::{for_each_iseq, rb_iseq_reset_jit_func}; + + if !zjit_enabled_p() { + return; + } + + // Stop other ractors since we are going to patch machine code. + with_vm_lock(src_loc!(), || { + debug!("Invalidating all ZJIT compiled code due to TracePoint"); + + for_each_iseq(|iseq| { + let payload = get_or_create_iseq_payload(iseq); + + payload.status = IseqStatus::NotCompiled; + unsafe { rb_iseq_reset_jit_func(iseq) }; + }); + }); +} diff --git a/zjit/src/options.rs b/zjit/src/options.rs index 00fb9c50a57127..f0a7f0bc7b60fe 100644 --- a/zjit/src/options.rs +++ b/zjit/src/options.rs @@ -253,8 +253,8 @@ fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { ("perf", "") => options.perf = true, - ("allowed-iseqs", _) if opt_val != "" => options.allowed_iseqs = Some(parse_jit_list(opt_val)), - ("log-compiled-iseqs", _) if opt_val != "" => { + ("allowed-iseqs", _) if !opt_val.is_empty() => options.allowed_iseqs = Some(parse_jit_list(opt_val)), + ("log-compiled-iseqs", _) if !opt_val.is_empty() => { // Truncate the file if it exists std::fs::OpenOptions::new() .create(true) @@ -336,7 +336,7 @@ pub extern "C" fn rb_zjit_option_enabled_p(_ec: EcPtr, _self: VALUE) -> VALUE { #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_stats_enabled_p(_ec: EcPtr, _self: VALUE) -> VALUE { // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. - if unsafe { OPTIONS.as_ref() }.map_or(false, |opts| opts.stats) { + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats) { Qtrue } else { Qfalse