[pull] master from ruby:master

insns.def

@@ -1575,6 +1575,7 @@ opt_empty_p
 (CALL_DATA cd)
 (VALUE recv)
 (VALUE val)
+// attr bool zjit_profile = true;
 {
     val = vm_opt_empty_p(recv);
 
@@ -1603,6 +1604,7 @@ opt_not
 (CALL_DATA cd)
 (VALUE recv)
 (VALUE val)
+// attr bool zjit_profile = true;
 {
     val = vm_opt_not(GET_ISEQ(), cd, recv);
 

zjit.rb

@@ -56,9 +56,11 @@ def stats_string
       :total_insn_count,
       :vm_insn_count,
       :zjit_insn_count,
+      :zjit_dynamic_dispatch,
       :ratio_in_zjit,
     ], buf:, stats:)
     print_counters_with_prefix(prefix: 'exit_', prompt: 'side exit reasons', buf:, stats:, limit: 20)
+    print_counters_with_prefix(prefix: 'specific_exit_', prompt: 'specific side exit reasons', buf:, stats:, limit: 20)
 
     buf
   end

zjit/src/backend/lir.rs

@@ -1604,6 +1604,18 @@ impl Assembler
                         Opnd::mem(64, SCRATCH_OPND, 0)
                     };
                     self.incr_counter(counter_opnd, 1.into());
+
+                    asm_comment!(self, "increment a specific exit counter");
+                    let counter = crate::stats::side_exit_reason_counter(reason);
+                    self.load_into(SCRATCH_OPND, Opnd::const_ptr(crate::stats::counter_ptr(counter)));
+                    let counter_opnd = if cfg!(target_arch = "aarch64") { // See arm64_split()
+                        // Using C_CRET_OPND since arm64_emit uses both SCRATCH0 and SCRATCH1 for IncrCounter.
+                        self.lea_into(C_RET_OPND, Opnd::mem(64, SCRATCH_OPND, 0));
+                        C_RET_OPND
+                    } else { // x86_emit expects Opnd::Mem
+                        Opnd::mem(64, SCRATCH_OPND, 0)
+                    };
+                    self.incr_counter(counter_opnd, 1.into());
                 }
 
                 asm_comment!(self, "exit to the interpreter");

zjit/src/codegen.rs

@@ -858,6 +858,10 @@ fn gen_send_without_block(
     cd: *const rb_call_data,
     state: &FrameState,
 ) -> lir::Opnd {
+    if get_option!(stats) {
+        gen_incr_counter(asm, Counter::zjit_dynamic_dispatch);
+    }
+
     // Note that it's incorrect to use this frame state to side exit because
     // the state might not be on the boundary of an interpreter instruction.
     // For example, `opt_str_uminus` pushes to the stack and then sends.

zjit/src/cruby_methods.rs

@@ -171,8 +171,12 @@ pub fn init() -> Annotations {
     annotate!(rb_cModule, "===", types::BoolExact, no_gc, leaf);
     annotate!(rb_cArray, "length", types::Fixnum, no_gc, leaf, elidable);
     annotate!(rb_cArray, "size", types::Fixnum, no_gc, leaf, elidable);
+    annotate!(rb_cArray, "empty?", types::BoolExact, no_gc, leaf, elidable);
+    annotate!(rb_cHash, "empty?", types::BoolExact, no_gc, leaf, elidable);
     annotate!(rb_cNilClass, "nil?", types::TrueClass, no_gc, leaf, elidable);
     annotate!(rb_mKernel, "nil?", types::FalseClass, no_gc, leaf, elidable);
+    annotate!(rb_cBasicObject, "==", types::BoolExact, no_gc, leaf, elidable);
+    annotate!(rb_cBasicObject, "!", types::BoolExact, no_gc, leaf, elidable);
 
     annotate_builtin!(rb_mKernel, "Float", types::Float);
     annotate_builtin!(rb_mKernel, "Integer", types::Integer);

zjit/src/hir.rs

@@ -2661,10 +2661,16 @@ fn insn_idx_at_offset(idx: u32, offset: i64) -> u32 {
     ((idx as isize) + (offset as isize)) as u32
 }
 
-fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec<u32> {
+struct BytecodeInfo {
+    jump_targets: Vec<u32>,
+    has_send: bool,
+}
+
+fn compute_bytecode_info(iseq: *const rb_iseq_t) -> BytecodeInfo {
     let iseq_size = unsafe { get_iseq_encoded_size(iseq) };
     let mut insn_idx = 0;
     let mut jump_targets = HashSet::new();
+    let mut has_send = false;
     while insn_idx < iseq_size {
         // Get the current pc and opcode
         let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) };
@@ -2688,12 +2694,13 @@ fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec<u32> {
                     jump_targets.insert(insn_idx);
                 }
             }
+            YARVINSN_send => has_send = true,
             _ => {}
         }
     }
     let mut result = jump_targets.into_iter().collect::<Vec<_>>();
     result.sort();
-    result
+    BytecodeInfo { jump_targets: result, has_send }
 }
 
 #[derive(Debug, PartialEq)]
@@ -2800,7 +2807,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
     let mut profiles = ProfileOracle::new(payload);
     let mut fun = Function::new(iseq);
     // Compute a map of PC->Block by finding jump targets
-    let jump_targets = compute_jump_targets(iseq);
+    let BytecodeInfo { jump_targets, has_send } = compute_bytecode_info(iseq);
     let mut insn_idx_to_block = HashMap::new();
     for insn_idx in jump_targets {
         if insn_idx == 0 {
@@ -3120,7 +3127,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
                 }
                 YARVINSN_getlocal_WC_0 => {
                     let ep_offset = get_arg(pc, 0).as_u32();
-                    if iseq_type == ISEQ_TYPE_EVAL {
+                    if iseq_type == ISEQ_TYPE_EVAL || has_send {
                         // On eval, the locals are always on the heap, so read the local using EP.
                         state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0 }));
                     } else {
@@ -3138,7 +3145,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
                     let ep_offset = get_arg(pc, 0).as_u32();
                     let val = state.stack_pop()?;
                     state.setlocal(ep_offset, val);
-                    if iseq_type == ISEQ_TYPE_EVAL {
+                    if iseq_type == ISEQ_TYPE_EVAL || has_send {
                         // On eval, the locals are always on the heap, so write the local using EP.
                         fun.push_insn(block, Insn::SetLocal { val, ep_offset, level: 0 });
                     }
@@ -4674,9 +4681,10 @@ mod tests {
         assert_snapshot!(hir_string("test"), @r"
         fn test@<compiled>:3:
         bb0(v0:BasicObject, v1:BasicObject):
-          v4:BasicObject = Send v1, 0x1000, :each
+          v3:BasicObject = GetLocal l0, EP@3
+          v5:BasicObject = Send v3, 0x1000, :each
           CheckInterrupts
-          Return v4
+          Return v5
         ");
     }
 
@@ -4742,11 +4750,12 @@ mod tests {
         eval("
             def test(a) = foo(&a)
         ");
-        assert_snapshot!(hir_string("test"), @r#"
-            fn test@<compiled>:2:
-            bb0(v0:BasicObject, v1:BasicObject):
-              SideExit UnknownCallType
-        "#);
+        assert_snapshot!(hir_string("test"), @r"
+        fn test@<compiled>:2:
+        bb0(v0:BasicObject, v1:BasicObject):
+          v3:BasicObject = GetLocal l0, EP@3
+          SideExit UnknownCallType
+        ");
     }
 
     #[test]
@@ -7195,6 +7204,30 @@ mod opt_tests {
         ");
     }
 
+    #[test]
+    fn reload_local_across_send() {
+        eval("
+            def foo(&block) = 1
+            def test
+              a = 1
+              foo {|| }
+              a
+            end
+            test
+            test
+        ");
+        assert_snapshot!(hir_string("test"), @r"
+        fn test@<compiled>:4:
+        bb0(v0:BasicObject):
+          v3:Fixnum[1] = Const Value(1)
+          SetLocal l0, EP@3, v3
+          v6:BasicObject = Send v0, 0x1000, :foo
+          v7:BasicObject = GetLocal l0, EP@3
+          CheckInterrupts
+          Return v7
+        ");
+    }
+
     #[test]
     fn dont_specialize_call_to_iseq_with_rest() {
         eval("
@@ -8117,6 +8150,79 @@ mod opt_tests {
         ");
     }
 
+    #[test]
+    fn test_specialize_basicobject_not_to_ccall() {
+        eval("
+            def test(a) = !a
+
+            test([])
+        ");
+        assert_snapshot!(hir_string("test"), @r"
+        fn test@<compiled>:2:
+        bb0(v0:BasicObject, v1:BasicObject):
+          PatchPoint MethodRedefined(Array@0x1000, !@0x1008, cme:0x1010)
+          v9:ArrayExact = GuardType v1, ArrayExact
+          v10:BoolExact = CCall !@0x1038, v9
+          CheckInterrupts
+          Return v10
+        ");
+    }
+
+    #[test]
+    fn test_specialize_array_empty_p_to_ccall() {
+        eval("
+            def test(a) = a.empty?
+
+            test([])
+        ");
+        assert_snapshot!(hir_string("test"), @r"
+        fn test@<compiled>:2:
+        bb0(v0:BasicObject, v1:BasicObject):
+          PatchPoint MethodRedefined(Array@0x1000, empty?@0x1008, cme:0x1010)
+          v9:ArrayExact = GuardType v1, ArrayExact
+          v10:BoolExact = CCall empty?@0x1038, v9
+          CheckInterrupts
+          Return v10
+        ");
+    }
+
+    #[test]
+    fn test_specialize_hash_empty_p_to_ccall() {
+        eval("
+            def test(a) = a.empty?
+
+            test({})
+        ");
+        assert_snapshot!(hir_string("test"), @r"
+        fn test@<compiled>:2:
+        bb0(v0:BasicObject, v1:BasicObject):
+          PatchPoint MethodRedefined(Hash@0x1000, empty?@0x1008, cme:0x1010)
+          v9:HashExact = GuardType v1, HashExact
+          v10:BoolExact = CCall empty?@0x1038, v9
+          CheckInterrupts
+          Return v10
+        ");
+    }
+
+    #[test]
+    fn test_specialize_basic_object_eq_to_ccall() {
+        eval("
+            class C; end
+            def test(a, b) = a == b
+
+            test(C.new, C.new)
+        ");
+        assert_snapshot!(hir_string("test"), @r"
+        fn test@<compiled>:3:
+        bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject):
+          PatchPoint MethodRedefined(C@0x1000, ==@0x1008, cme:0x1010)
+          v10:HeapObject[class_exact:C] = GuardType v1, HeapObject[class_exact:C]
+          v11:BoolExact = CCall ==@0x1038, v10, v2
+          CheckInterrupts
+          Return v11
+        ");
+    }
+
     #[test]
     fn test_guard_fixnum_and_fixnum() {
         eval("

zjit/src/hir_type/mod.rs

@@ -246,6 +246,8 @@ impl Type {
         else if val.is_true() { types::TrueClass }
         else if val.is_false() { types::FalseClass }
         else if val.class() == unsafe { rb_cString } { types::StringExact }
+        else if val.class() == unsafe { rb_cArray } { types::ArrayExact }
+        else if val.class() == unsafe { rb_cHash } { types::HashExact }
         else {
             // TODO(max): Add more cases for inferring type bits from built-in types
             Type { bits: bits::HeapObject, spec: Specialization::TypeExact(val.class()) }

zjit/src/profile.rs

@@ -66,6 +66,8 @@ fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) {
         YARVINSN_opt_ge    => profile_operands(profiler, profile, 2),
         YARVINSN_opt_and   => profile_operands(profiler, profile, 2),
         YARVINSN_opt_or    => profile_operands(profiler, profile, 2),
+        YARVINSN_opt_empty_p => profile_operands(profiler, profile, 1),
+        YARVINSN_opt_not   => profile_operands(profiler, profile, 1),
         YARVINSN_opt_send_without_block => {
             let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr();
             let argc = unsafe { vm_ci_argc((*cd).ci) };

zjit/src/stats.rs

@@ -72,6 +72,9 @@ make_counters! {
     // The number of times YARV instructions are executed on JIT code
     zjit_insn_count,
 
+    // The number of times we do a dynamic dispatch from JIT code
+    zjit_dynamic_dispatch,
+
     // failed_: Compilation failure reasons
     failed_iseq_stack_too_large,
     failed_hir_compile,
@@ -81,6 +84,23 @@ make_counters! {
 
     // exit_: Side exit reasons (ExitCounters shares the same prefix)
     exit_compilation_failure,
+
+    // specific_exit_: Side exits counted by type, not by PC
+    specific_exit_unknown_newarray_send,
+    specific_exit_unknown_call_type,
+    specific_exit_unknown_opcode,
+    specific_exit_unhandled_instruction,
+    specific_exit_fixnum_add_overflow,
+    specific_exit_fixnum_sub_overflow,
+    specific_exit_fixnum_mult_overflow,
+    specific_exit_guard_type_failure,
+    specific_exit_guard_bit_equals_failure,
+    specific_exit_patchpoint,
+    specific_exit_callee_side_exit,
+    specific_exit_obj_to_string_fallback,
+    specific_exit_unknown_special_variable,
+    specific_exit_unhandled_defined_type,
+    specific_exit_interrupt,
 }
 
 /// Increase a counter by a specified amount
@@ -107,6 +127,27 @@ pub fn exit_counter_ptr(pc: *const VALUE) -> *mut u64 {
     unsafe { exit_counters.get_unchecked_mut(opcode as usize) }
 }
 
+pub fn side_exit_reason_counter(reason: crate::hir::SideExitReason) -> Counter {
+    use crate::hir::SideExitReason;
+    match reason {
+        SideExitReason::UnknownNewarraySend(_) => Counter::specific_exit_unknown_newarray_send,
+        SideExitReason::UnknownCallType => Counter::specific_exit_unknown_call_type,
+        SideExitReason::UnknownOpcode(_) => Counter::specific_exit_unknown_opcode,
+        SideExitReason::UnhandledInstruction(_) => Counter::specific_exit_unhandled_instruction,
+        SideExitReason::FixnumAddOverflow => Counter::specific_exit_fixnum_add_overflow,
+        SideExitReason::FixnumSubOverflow => Counter::specific_exit_fixnum_sub_overflow,
+        SideExitReason::FixnumMultOverflow => Counter::specific_exit_fixnum_mult_overflow,
+        SideExitReason::GuardType(_) => Counter::specific_exit_guard_type_failure,
+        SideExitReason::GuardBitEquals(_) => Counter::specific_exit_guard_bit_equals_failure,
+        SideExitReason::PatchPoint(_) => Counter::specific_exit_patchpoint,
+        SideExitReason::CalleeSideExit => Counter::specific_exit_callee_side_exit,
+        SideExitReason::ObjToStringFallback => Counter::specific_exit_obj_to_string_fallback,
+        SideExitReason::UnknownSpecialVariable(_) => Counter::specific_exit_unknown_special_variable,
+        SideExitReason::UnhandledDefinedType(_) => Counter::specific_exit_unhandled_defined_type,
+        SideExitReason::Interrupt => Counter::specific_exit_interrupt,
+    }
+}
+
 /// Return a Hash object that contains ZJIT statistics
 #[unsafe(no_mangle)]
 pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> VALUE {