diff --git a/hash.c b/hash.c index 6f3ffb78ba2055..7256063d72e079 100644 --- a/hash.c +++ b/hash.c @@ -71,10 +71,6 @@ #define HASH_DEBUG 0 #endif -#if HASH_DEBUG -#include "internal/gc.h" -#endif - #define SET_DEFAULT(hash, ifnone) ( \ FL_UNSET_RAW(hash, RHASH_PROC_DEFAULT), \ RHASH_SET_IFNONE(hash, ifnone)) diff --git a/inits.c b/inits.c index 29087a6306481b..b4e58ea25a1cec 100644 --- a/inits.c +++ b/inits.c @@ -90,7 +90,6 @@ rb_call_builtin_inits(void) #define BUILTIN(n) CALL(builtin_##n) BUILTIN(kernel); BUILTIN(yjit); - // BUILTIN(yjit_hook) is called after rb_yjit_init() BUILTIN(gc); BUILTIN(ractor); BUILTIN(numeric); @@ -109,6 +108,7 @@ rb_call_builtin_inits(void) BUILTIN(nilclass); BUILTIN(marshal); BUILTIN(zjit); + BUILTIN(yjit_hook); Init_builtin_prelude(); } #undef CALL diff --git a/lib/tsort.gemspec b/lib/tsort.gemspec index 8970cbe8261770..4e0ef0507df008 100644 --- a/lib/tsort.gemspec +++ b/lib/tsort.gemspec @@ -19,6 +19,7 @@ Gem::Specification.new do |spec| spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage + spec.metadata["changelog_uri"] = "#{spec.homepage}/releases" dir, gemspec = File.split(__FILE__) excludes = %W[ diff --git a/ruby.c b/ruby.c index 0d09e7ce61e1ab..cc67b0b25db8f8 100644 --- a/ruby.c +++ b/ruby.c @@ -1833,12 +1833,6 @@ ruby_opt_init(ruby_cmdline_options_t *opt) } #endif -#if USE_YJIT - // Call yjit_hook.rb after rb_yjit_init() to use `RubyVM::YJIT.enabled?` - void Init_builtin_yjit_hook(); - Init_builtin_yjit_hook(); -#endif - rb_namespace_init_done(); ruby_init_prelude(); ruby_set_script_name(opt->script_name); diff --git a/test/ruby/test_zjit.rb b/test/ruby/test_zjit.rb index 0dcdb8e4cb8275..b78d53e682233c 100644 --- a/test/ruby/test_zjit.rb +++ b/test/ruby/test_zjit.rb @@ -819,6 +819,13 @@ def a(n1,n2,n3,n4,n5,n6,n7,n8) = self } end + def test_spilled_param_new_arary + assert_compiles '[:ok]', %q{ + def a(n1,n2,n3,n4,n5,n6,n7,n8) = [n8] + a(0,0,0,0,0,0,0, :ok) + } + end + def test_opt_aref_with assert_compiles ':ok', %q{ def aref_with(hash) = hash["key"] @@ -882,6 +889,48 @@ def test = X end end + def test_constant_invalidation + assert_compiles '123', <<~RUBY, call_threshold: 2, insns: [:opt_getconstant_path] + class C; end + def test = C + test + test + + C = 123 + test + RUBY + end + + def test_constant_path_invalidation + assert_compiles '["Foo::C", "Foo::C", "Bar::C"]', <<~RUBY, call_threshold: 2, insns: [:opt_getconstant_path] + module A + module B; end + end + + module Foo + C = "Foo::C" + end + + module Bar + C = "Bar::C" + end + + A::B = Foo + + def test = A::B::C + + result = [] + + result << test + result << test + + A::B = Bar + + result << test + result + RUBY + end + def test_dupn assert_compiles '[[1], [1, 1], :rhs, [nil, :rhs]]', <<~RUBY, insns: [:dupn] def test(array) = (array[1, 2] ||= :rhs) diff --git a/vm_method.c b/vm_method.c index 84d0ed2f9e4c07..fa81d56c74119d 100644 --- a/vm_method.c +++ b/vm_method.c @@ -148,6 +148,7 @@ rb_clear_constant_cache_for_id(ID id) } rb_yjit_constant_state_changed(id); + rb_zjit_constant_state_changed(id); } static void diff --git a/yjit.rb b/yjit.rb index e8ba3cdd28752a..e4fafa729eea75 100644 --- a/yjit.rb +++ b/yjit.rb @@ -64,7 +64,6 @@ def self.enable(stats: false, log: false, mem_size: nil, call_threshold: nil) end at_exit { print_and_dump_stats } if stats - call_yjit_hooks Primitive.rb_yjit_enable(stats, stats != :quiet, log, log != :quiet, mem_size, call_threshold) end diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index 7c8ce7bce5c69d..61b6f233269269 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -105,6 +105,9 @@ fn main() { .allowlist_var("SHAPE_ID_NUM_BITS") .allowlist_var("SHAPE_ID_HAS_IVAR_MASK") + // From ruby/internal/eval.h + .allowlist_function("rb_funcall") + // From ruby/internal/intern/object.h .allowlist_function("rb_obj_is_kind_of") .allowlist_function("rb_obj_frozen_p") @@ -269,6 +272,7 @@ fn main() { .allowlist_function("rb_float_new") // From vm_core.h + .allowlist_var("rb_cRubyVM") .allowlist_var("rb_mRubyVMFrozenCore") .allowlist_var("VM_BLOCK_HANDLER_NONE") .allowlist_type("vm_frame_env_flags") @@ -383,6 +387,7 @@ fn main() { .allowlist_function("rb_ivar_defined") .allowlist_function("rb_ivar_get") .allowlist_function("rb_mod_name") + .allowlist_function("rb_const_get") // From internal/vm.h .allowlist_var("rb_vm_insns_count") diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index 725a29fa70cf23..f7a08b3b18b537 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -601,9 +601,15 @@ pub fn rust_str_to_ruby(str: &str) -> VALUE { /// Produce a Ruby symbol from a Rust string slice pub fn rust_str_to_sym(str: &str) -> VALUE { + let id = rust_str_to_id(str); + unsafe { rb_id2sym(id) } +} + +/// Produce an ID from a Rust string slice +pub fn rust_str_to_id(str: &str) -> ID { let c_str = CString::new(str).unwrap(); let c_ptr: *const c_char = c_str.as_ptr(); - unsafe { rb_id2sym(rb_intern(c_ptr)) } + unsafe { rb_intern(c_ptr) } } /// Produce an owned Rust String from a C char pointer diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 666f5c4f28d614..8268b889199170 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1019,6 +1019,7 @@ extern "C" { pub fn rb_gc_location(obj: VALUE) -> VALUE; pub fn rb_gc_writebarrier(old: VALUE, young: VALUE); pub fn rb_class_get_superclass(klass: VALUE) -> VALUE; + pub fn rb_funcall(recv: VALUE, mid: ID, n: ::std::os::raw::c_int, ...) -> VALUE; pub static mut rb_mKernel: VALUE; pub static mut rb_cBasicObject: VALUE; pub static mut rb_cArray: VALUE; @@ -1080,6 +1081,7 @@ extern "C" { pub fn rb_ivar_get(obj: VALUE, name: ID) -> VALUE; pub fn rb_ivar_defined(obj: VALUE, name: ID) -> VALUE; pub fn rb_attr_get(obj: VALUE, name: ID) -> VALUE; + pub fn rb_const_get(space: VALUE, name: ID) -> VALUE; pub fn rb_obj_info_dump(obj: VALUE); pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE; pub fn rb_obj_equal(obj1: VALUE, obj2: VALUE) -> VALUE; @@ -1102,6 +1104,7 @@ extern "C" { klass: VALUE, id: ID, ) -> *const rb_callable_method_entry_t; + pub static mut rb_cRubyVM: VALUE; pub static mut rb_mRubyVMFrozenCore: VALUE; pub static mut rb_block_param_proxy: VALUE; pub fn rb_vm_ep_local_ep(ep: *const VALUE) -> *const VALUE; diff --git a/yjit/src/yjit.rs b/yjit/src/yjit.rs index 4b2b5214f4da13..8df1163d64b725 100644 --- a/yjit/src/yjit.rs +++ b/yjit/src/yjit.rs @@ -54,6 +54,12 @@ fn yjit_init() { // TODO: need to make sure that command-line options have been // initialized by CRuby + // Call YJIT hooks before enabling YJIT to avoid compiling the hooks themselves + unsafe { + let yjit = rb_const_get(rb_cRubyVM, rust_str_to_id("YJIT")); + rb_funcall(yjit, rust_str_to_id("call_yjit_hooks"), 0); + } + // Catch panics to avoid UB for unwinding into C frames. // See https://doc.rust-lang.org/nomicon/exception-safety.html let result = std::panic::catch_unwind(|| { diff --git a/yjit_hook.rb b/yjit_hook.rb index 8f0f38aaf1f535..610a7be3303e2c 100644 --- a/yjit_hook.rb +++ b/yjit_hook.rb @@ -1,8 +1,3 @@ -# If YJIT is enabled, load the YJIT-only version of builtin methods -if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled? - RubyVM::YJIT.send(:call_yjit_hooks) -end - # Remove the helper defined in kernel.rb class Module undef :with_yjit diff --git a/zjit.h b/zjit.h index 724ae4abd05b15..e354d4b122d0cf 100644 --- a/zjit.h +++ b/zjit.h @@ -14,6 +14,7 @@ void rb_zjit_profile_enable(const rb_iseq_t *iseq); void rb_zjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop); void rb_zjit_cme_invalidate(const rb_callable_method_entry_t *cme); void rb_zjit_invalidate_ep_is_bp(const rb_iseq_t *iseq); +void rb_zjit_constant_state_changed(ID id); void rb_zjit_iseq_mark(void *payload); void rb_zjit_iseq_update_references(void *payload); #else @@ -24,6 +25,7 @@ static inline void rb_zjit_profile_enable(const rb_iseq_t *iseq) {} static inline void rb_zjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop) {} static inline void rb_zjit_cme_invalidate(const rb_callable_method_entry_t *cme) {} static inline void rb_zjit_invalidate_ep_is_bp(const rb_iseq_t *iseq) {} +static inline void rb_zjit_constant_state_changed(ID id) {} #endif // #if USE_YJIT #endif // #ifndef ZJIT_H diff --git a/zjit/src/asm/arm64/opnd.rs b/zjit/src/asm/arm64/opnd.rs index 28422b747652d8..a77958f7e6eeec 100644 --- a/zjit/src/asm/arm64/opnd.rs +++ b/zjit/src/asm/arm64/opnd.rs @@ -119,6 +119,9 @@ pub const X20_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 20 }; pub const X21_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 21 }; pub const X22_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 22 }; +// frame pointer (base pointer) +pub const X29_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 29 }; + // link register pub const X30_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 30 }; diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 88ccad8e091ed1..42dc31c90fd5cc 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -29,6 +29,7 @@ pub const C_ARG_OPNDS: [Opnd; 6] = [ pub const C_RET_REG: Reg = X0_REG; pub const C_RET_OPND: Opnd = Opnd::Reg(X0_REG); pub const NATIVE_STACK_PTR: Opnd = Opnd::Reg(XZR_REG); +pub const NATIVE_BASE_PTR: Opnd = Opnd::Reg(X29_REG); // These constants define the way we work with Arm64's stack pointer. The stack // pointer always needs to be aligned to a 16-byte boundary. @@ -911,18 +912,54 @@ impl Assembler cb.write_byte(0); } }, - Insn::FrameSetup => { + &Insn::FrameSetup { preserved, mut slot_count } => { + const { assert!(SIZEOF_VALUE == 8, "alignment logic relies on SIZEOF_VALUE == 8"); } + // Preserve X29 and set up frame record stp_pre(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, -16)); - - // X29 (frame_pointer) = SP mov(cb, X29, C_SP_REG); - }, - Insn::FrameTeardown => { + + for regs in preserved.chunks(2) { + // For the body, store pairs and move SP + if let [reg0, reg1] = regs { + stp_pre(cb, reg1.into(), reg0.into(), A64Opnd::new_mem(128, C_SP_REG, -16)); + } else if let [reg] = regs { + // For overhang, store but don't move SP. Combine movement with + // movement for slots below. + stur(cb, reg.into(), A64Opnd::new_mem(64, C_SP_REG, -8)); + slot_count += 1; + } else { + unreachable!("chunks(2)"); + } + } + // Align slot_count + if slot_count % 2 == 1 { + slot_count += 1 + } + if slot_count > 0 { + let slot_offset = (slot_count * SIZEOF_VALUE) as u64; + // Bail when asked to reserve too many slots in one instruction. + ShiftedImmediate::try_from(slot_offset).ok()?; + sub(cb, C_SP_REG, C_SP_REG, A64Opnd::new_uimm(slot_offset)); + } + } + Insn::FrameTeardown { preserved } => { + // Restore preserved registers below frame pointer. + let mut base_offset = 0; + for regs in preserved.chunks(2) { + if let [reg0, reg1] = regs { + base_offset -= 16; + ldp(cb, reg1.into(), reg0.into(), A64Opnd::new_mem(128, X29, base_offset)); + } else if let [reg] = regs { + ldur(cb, reg.into(), A64Opnd::new_mem(64, X29, base_offset - 8)); + } else { + unreachable!("chunks(2)"); + } + } + // SP = X29 (frame pointer) mov(cb, C_SP_REG, X29); - ldp_post(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, 16)); - }, + } Insn::Add { left, right, out } => { // Usually, we issue ADDS, so you could branch on overflow, but ADDS with // out=31 refers to out=XZR, which discards the sum. So, instead of ADDS @@ -1482,11 +1519,73 @@ mod tests { fn test_emit_frame() { let (mut asm, mut cb) = setup_asm(); - asm.frame_setup(); - asm.frame_teardown(); + asm.frame_setup(&[], 0); + asm.frame_teardown(&[]); asm.compile_with_num_regs(&mut cb, 0); } + #[test] + fn frame_setup_and_teardown() { + const THREE_REGS: &'static [Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG)]; + // Test 3 preserved regs (odd), odd slot_count + { + let (mut asm, mut cb) = setup_asm(); + asm.frame_setup(THREE_REGS, 3); + asm.frame_teardown(THREE_REGS); + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm!(cb, "fd7bbfa9fd030091f44fbfa9f5831ff8ff8300d1b44f7fa9b5835ef8bf030091fd7bc1a8", " + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + "); + } + + // Test 3 preserved regs (odd), even slot_count + { + let (mut asm, mut cb) = setup_asm(); + asm.frame_setup(THREE_REGS, 4); + asm.frame_teardown(THREE_REGS); + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm!(cb, "fd7bbfa9fd030091f44fbfa9f5831ff8ffc300d1b44f7fa9b5835ef8bf030091fd7bc1a8", " + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x30 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + "); + } + + // Test 4 preserved regs (even), odd slot_count + { + static FOUR_REGS: &'static [Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG), Opnd::Reg(X22_REG)]; + let (mut asm, mut cb) = setup_asm(); + asm.frame_setup(FOUR_REGS, 3); + asm.frame_teardown(FOUR_REGS); + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm!(cb, "fd7bbfa9fd030091f44fbfa9f657bfa9ff8300d1b44f7fa9b6577ea9bf030091fd7bc1a8", " + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stp x22, x21, [sp, #-0x10]! + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldp x22, x21, [x29, #-0x20] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + "); + } + } + #[test] fn test_emit_je_fits_into_bcond() { let (mut asm, mut cb) = setup_asm(); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 7bac210bee6689..36e783bd4e658a 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -12,10 +12,12 @@ use crate::asm::{CodeBlock, Label}; pub use crate::backend::current::{ Reg, EC, CFP, SP, - NATIVE_STACK_PTR, + NATIVE_STACK_PTR, NATIVE_BASE_PTR, C_ARG_OPNDS, C_RET_REG, C_RET_OPND, }; +pub static JIT_PRESERVED_REGS: &'static [Opnd] = &[CFP, SP, EC]; + // Memory operand base #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum MemBase @@ -291,8 +293,6 @@ pub enum Target context: Option, /// We use this to enrich asm comments. reason: SideExitReason, - /// The number of bytes we need to adjust the C stack pointer by. - c_stack_bytes: usize, /// Some if the side exit should write this label. We use it for patch points. label: Option