diff --git a/insns.def b/insns.def index c35869eb09fe66..c9de6129120fdc 100644 --- a/insns.def +++ b/insns.def @@ -1575,6 +1575,7 @@ opt_empty_p (CALL_DATA cd) (VALUE recv) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_empty_p(recv); @@ -1603,6 +1604,7 @@ opt_not (CALL_DATA cd) (VALUE recv) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_not(GET_ISEQ(), cd, recv); diff --git a/zjit.rb b/zjit.rb index cf0896e107203e..86d6643df21dd0 100644 --- a/zjit.rb +++ b/zjit.rb @@ -56,9 +56,11 @@ def stats_string :total_insn_count, :vm_insn_count, :zjit_insn_count, + :zjit_dynamic_dispatch, :ratio_in_zjit, ], buf:, stats:) print_counters_with_prefix(prefix: 'exit_', prompt: 'side exit reasons', buf:, stats:, limit: 20) + print_counters_with_prefix(prefix: 'specific_exit_', prompt: 'specific side exit reasons', buf:, stats:, limit: 20) buf end diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 7e317d49913e1a..0639176fd63f8b 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -1604,6 +1604,18 @@ impl Assembler Opnd::mem(64, SCRATCH_OPND, 0) }; self.incr_counter(counter_opnd, 1.into()); + + asm_comment!(self, "increment a specific exit counter"); + let counter = crate::stats::side_exit_reason_counter(reason); + self.load_into(SCRATCH_OPND, Opnd::const_ptr(crate::stats::counter_ptr(counter))); + let counter_opnd = if cfg!(target_arch = "aarch64") { // See arm64_split() + // Using C_CRET_OPND since arm64_emit uses both SCRATCH0 and SCRATCH1 for IncrCounter. + self.lea_into(C_RET_OPND, Opnd::mem(64, SCRATCH_OPND, 0)); + C_RET_OPND + } else { // x86_emit expects Opnd::Mem + Opnd::mem(64, SCRATCH_OPND, 0) + }; + self.incr_counter(counter_opnd, 1.into()); } asm_comment!(self, "exit to the interpreter"); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 2b71596a171272..fd804e5a426cd5 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -858,6 +858,10 @@ fn gen_send_without_block( cd: *const rb_call_data, state: &FrameState, ) -> lir::Opnd { + if get_option!(stats) { + gen_incr_counter(asm, Counter::zjit_dynamic_dispatch); + } + // Note that it's incorrect to use this frame state to side exit because // the state might not be on the boundary of an interpreter instruction. // For example, `opt_str_uminus` pushes to the stack and then sends. diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index c804ecce863ada..d10e3cc8a0eab7 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -696,7 +696,9 @@ pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 229; pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 230; pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 231; pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 232; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 234; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 235; pub type ruby_vminsn_type = u32; pub type rb_iseq_callback = ::std::option::Option< unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs index c9ebcebc86d81b..36965c2c00dc1f 100644 --- a/zjit/src/cruby_methods.rs +++ b/zjit/src/cruby_methods.rs @@ -171,8 +171,12 @@ pub fn init() -> Annotations { annotate!(rb_cModule, "===", types::BoolExact, no_gc, leaf); annotate!(rb_cArray, "length", types::Fixnum, no_gc, leaf, elidable); annotate!(rb_cArray, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cArray, "empty?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cHash, "empty?", types::BoolExact, no_gc, leaf, elidable); annotate!(rb_cNilClass, "nil?", types::TrueClass, no_gc, leaf, elidable); annotate!(rb_mKernel, "nil?", types::FalseClass, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "==", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!", types::BoolExact, no_gc, leaf, elidable); annotate_builtin!(rb_mKernel, "Float", types::Float); annotate_builtin!(rb_mKernel, "Integer", types::Integer); diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index d269baf8848762..039ba6de5b010a 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -2661,10 +2661,16 @@ fn insn_idx_at_offset(idx: u32, offset: i64) -> u32 { ((idx as isize) + (offset as isize)) as u32 } -fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec { +struct BytecodeInfo { + jump_targets: Vec, + has_send: bool, +} + +fn compute_bytecode_info(iseq: *const rb_iseq_t) -> BytecodeInfo { let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; let mut insn_idx = 0; let mut jump_targets = HashSet::new(); + let mut has_send = false; while insn_idx < iseq_size { // Get the current pc and opcode let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; @@ -2688,12 +2694,13 @@ fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec { jump_targets.insert(insn_idx); } } + YARVINSN_send => has_send = true, _ => {} } } let mut result = jump_targets.into_iter().collect::>(); result.sort(); - result + BytecodeInfo { jump_targets: result, has_send } } #[derive(Debug, PartialEq)] @@ -2800,7 +2807,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let mut profiles = ProfileOracle::new(payload); let mut fun = Function::new(iseq); // Compute a map of PC->Block by finding jump targets - let jump_targets = compute_jump_targets(iseq); + let BytecodeInfo { jump_targets, has_send } = compute_bytecode_info(iseq); let mut insn_idx_to_block = HashMap::new(); for insn_idx in jump_targets { if insn_idx == 0 { @@ -3120,7 +3127,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { } YARVINSN_getlocal_WC_0 => { let ep_offset = get_arg(pc, 0).as_u32(); - if iseq_type == ISEQ_TYPE_EVAL { + if iseq_type == ISEQ_TYPE_EVAL || has_send { // On eval, the locals are always on the heap, so read the local using EP. state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0 })); } else { @@ -3138,7 +3145,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let ep_offset = get_arg(pc, 0).as_u32(); let val = state.stack_pop()?; state.setlocal(ep_offset, val); - if iseq_type == ISEQ_TYPE_EVAL { + if iseq_type == ISEQ_TYPE_EVAL || has_send { // On eval, the locals are always on the heap, so write the local using EP. fun.push_insn(block, Insn::SetLocal { val, ep_offset, level: 0 }); } @@ -4674,9 +4681,10 @@ mod tests { assert_snapshot!(hir_string("test"), @r" fn test@:3: bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = Send v1, 0x1000, :each + v3:BasicObject = GetLocal l0, EP@3 + v5:BasicObject = Send v3, 0x1000, :each CheckInterrupts - Return v4 + Return v5 "); } @@ -4742,11 +4750,12 @@ mod tests { eval(" def test(a) = foo(&a) "); - assert_snapshot!(hir_string("test"), @r#" - fn test@:2: - bb0(v0:BasicObject, v1:BasicObject): - SideExit UnknownCallType - "#); + assert_snapshot!(hir_string("test"), @r" + fn test@:2: + bb0(v0:BasicObject, v1:BasicObject): + v3:BasicObject = GetLocal l0, EP@3 + SideExit UnknownCallType + "); } #[test] @@ -7195,6 +7204,30 @@ mod opt_tests { "); } + #[test] + fn reload_local_across_send() { + eval(" + def foo(&block) = 1 + def test + a = 1 + foo {|| } + a + end + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:4: + bb0(v0:BasicObject): + v3:Fixnum[1] = Const Value(1) + SetLocal l0, EP@3, v3 + v6:BasicObject = Send v0, 0x1000, :foo + v7:BasicObject = GetLocal l0, EP@3 + CheckInterrupts + Return v7 + "); + } + #[test] fn dont_specialize_call_to_iseq_with_rest() { eval(" @@ -8117,6 +8150,79 @@ mod opt_tests { "); } + #[test] + fn test_specialize_basicobject_not_to_ccall() { + eval(" + def test(a) = !a + + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:2: + bb0(v0:BasicObject, v1:BasicObject): + PatchPoint MethodRedefined(Array@0x1000, !@0x1008, cme:0x1010) + v9:ArrayExact = GuardType v1, ArrayExact + v10:BoolExact = CCall !@0x1038, v9 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_array_empty_p_to_ccall() { + eval(" + def test(a) = a.empty? + + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:2: + bb0(v0:BasicObject, v1:BasicObject): + PatchPoint MethodRedefined(Array@0x1000, empty?@0x1008, cme:0x1010) + v9:ArrayExact = GuardType v1, ArrayExact + v10:BoolExact = CCall empty?@0x1038, v9 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_hash_empty_p_to_ccall() { + eval(" + def test(a) = a.empty? + + test({}) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:2: + bb0(v0:BasicObject, v1:BasicObject): + PatchPoint MethodRedefined(Hash@0x1000, empty?@0x1008, cme:0x1010) + v9:HashExact = GuardType v1, HashExact + v10:BoolExact = CCall empty?@0x1038, v9 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_basic_object_eq_to_ccall() { + eval(" + class C; end + def test(a, b) = a == b + + test(C.new, C.new) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): + PatchPoint MethodRedefined(C@0x1000, ==@0x1008, cme:0x1010) + v10:HeapObject[class_exact:C] = GuardType v1, HeapObject[class_exact:C] + v11:BoolExact = CCall ==@0x1038, v10, v2 + CheckInterrupts + Return v11 + "); + } + #[test] fn test_guard_fixnum_and_fixnum() { eval(" diff --git a/zjit/src/hir_type/mod.rs b/zjit/src/hir_type/mod.rs index c18b2735bee834..d6d43984251e57 100644 --- a/zjit/src/hir_type/mod.rs +++ b/zjit/src/hir_type/mod.rs @@ -246,6 +246,8 @@ impl Type { else if val.is_true() { types::TrueClass } else if val.is_false() { types::FalseClass } else if val.class() == unsafe { rb_cString } { types::StringExact } + else if val.class() == unsafe { rb_cArray } { types::ArrayExact } + else if val.class() == unsafe { rb_cHash } { types::HashExact } else { // TODO(max): Add more cases for inferring type bits from built-in types Type { bits: bits::HeapObject, spec: Specialization::TypeExact(val.class()) } diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index 771d90cb0ec426..b311f0ba94edf8 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -66,6 +66,8 @@ fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) { YARVINSN_opt_ge => profile_operands(profiler, profile, 2), YARVINSN_opt_and => profile_operands(profiler, profile, 2), YARVINSN_opt_or => profile_operands(profiler, profile, 2), + YARVINSN_opt_empty_p => profile_operands(profiler, profile, 1), + YARVINSN_opt_not => profile_operands(profiler, profile, 1), YARVINSN_opt_send_without_block => { let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); let argc = unsafe { vm_ci_argc((*cd).ci) }; diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index b754404a66e002..10692d83f57428 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -72,6 +72,9 @@ make_counters! { // The number of times YARV instructions are executed on JIT code zjit_insn_count, + // The number of times we do a dynamic dispatch from JIT code + zjit_dynamic_dispatch, + // failed_: Compilation failure reasons failed_iseq_stack_too_large, failed_hir_compile, @@ -81,6 +84,23 @@ make_counters! { // exit_: Side exit reasons (ExitCounters shares the same prefix) exit_compilation_failure, + + // specific_exit_: Side exits counted by type, not by PC + specific_exit_unknown_newarray_send, + specific_exit_unknown_call_type, + specific_exit_unknown_opcode, + specific_exit_unhandled_instruction, + specific_exit_fixnum_add_overflow, + specific_exit_fixnum_sub_overflow, + specific_exit_fixnum_mult_overflow, + specific_exit_guard_type_failure, + specific_exit_guard_bit_equals_failure, + specific_exit_patchpoint, + specific_exit_callee_side_exit, + specific_exit_obj_to_string_fallback, + specific_exit_unknown_special_variable, + specific_exit_unhandled_defined_type, + specific_exit_interrupt, } /// Increase a counter by a specified amount @@ -107,6 +127,27 @@ pub fn exit_counter_ptr(pc: *const VALUE) -> *mut u64 { unsafe { exit_counters.get_unchecked_mut(opcode as usize) } } +pub fn side_exit_reason_counter(reason: crate::hir::SideExitReason) -> Counter { + use crate::hir::SideExitReason; + match reason { + SideExitReason::UnknownNewarraySend(_) => Counter::specific_exit_unknown_newarray_send, + SideExitReason::UnknownCallType => Counter::specific_exit_unknown_call_type, + SideExitReason::UnknownOpcode(_) => Counter::specific_exit_unknown_opcode, + SideExitReason::UnhandledInstruction(_) => Counter::specific_exit_unhandled_instruction, + SideExitReason::FixnumAddOverflow => Counter::specific_exit_fixnum_add_overflow, + SideExitReason::FixnumSubOverflow => Counter::specific_exit_fixnum_sub_overflow, + SideExitReason::FixnumMultOverflow => Counter::specific_exit_fixnum_mult_overflow, + SideExitReason::GuardType(_) => Counter::specific_exit_guard_type_failure, + SideExitReason::GuardBitEquals(_) => Counter::specific_exit_guard_bit_equals_failure, + SideExitReason::PatchPoint(_) => Counter::specific_exit_patchpoint, + SideExitReason::CalleeSideExit => Counter::specific_exit_callee_side_exit, + SideExitReason::ObjToStringFallback => Counter::specific_exit_obj_to_string_fallback, + SideExitReason::UnknownSpecialVariable(_) => Counter::specific_exit_unknown_special_variable, + SideExitReason::UnhandledDefinedType(_) => Counter::specific_exit_unhandled_defined_type, + SideExitReason::Interrupt => Counter::specific_exit_interrupt, + } +} + /// Return a Hash object that contains ZJIT statistics #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> VALUE {