[pull] master from ruby:master

insns.def

@@ -1137,6 +1137,7 @@ invokeblock
 // attr bool handles_sp = true;
 // attr rb_snum_t sp_inc = sp_inc_of_invokeblock(cd->ci);
 // attr rb_snum_t comptime_sp_inc = sp_inc_of_invokeblock(ci);
+// attr bool zjit_profile = true;
 {
     VALUE bh = VM_BLOCK_HANDLER_NONE;
     val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);

vm_insnhelper.c

@@ -5497,6 +5497,12 @@ vm_invoke_proc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
     return vm_invoke_block(ec, reg_cfp, calling, ci, is_lambda, block_handler);
 }
 
+enum rb_block_handler_type
+rb_vm_block_handler_type(VALUE block_handler)
+{
+    return vm_block_handler_type(block_handler);
+}
+
 static inline VALUE
 vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
                 struct rb_calling_info *calling, const struct rb_callinfo *ci,
@@ -6059,6 +6065,12 @@ vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqv
     }
 }
 
+VALUE
+rb_vm_get_block_handler(rb_control_frame_t *reg_cfp)
+{
+    return VM_CF_BLOCK_HANDLER(reg_cfp);
+}
+
 static VALUE
 vm_invokeblock_i(struct rb_execution_context_struct *ec,
                  struct rb_control_frame_struct *reg_cfp,

zjit.c

@@ -301,6 +301,10 @@ rb_zjit_class_has_default_allocator(VALUE klass)
     return alloc == rb_class_allocate_instance;
 }
 
+
+VALUE rb_vm_get_block_handler(rb_control_frame_t *reg_cfp);
+enum rb_block_handler_type rb_vm_block_handler_type(VALUE block_handler);
+
 // Primitives used by zjit.rb. Don't put other functions below, which wouldn't use them.
 VALUE rb_zjit_assert_compiles(rb_execution_context_t *ec, VALUE self);
 VALUE rb_zjit_stats(rb_execution_context_t *ec, VALUE self, VALUE target_key);

zjit.rb

@@ -164,6 +164,7 @@ def stats_string
     print_counters_with_prefix(prefix: 'unspecialized_send_without_block_def_type_', prompt: 'not optimized method types for send_without_block', buf:, stats:, limit: 20)
     print_counters_with_prefix(prefix: 'not_optimized_yarv_insn_', prompt: 'not optimized instructions', buf:, stats:, limit: 20)
     print_counters_with_prefix(prefix: 'send_fallback_', prompt: 'send fallback reasons', buf:, stats:, limit: 20)
+    print_counters_with_prefix(prefix: 'invokeblock_handler_', prompt: 'invokeblock handler', buf:, stats:, limit: 10)
 
     # Show most popular unsupported call features. Because each call can
     # use multiple complex features, a decrease in this number does not

zjit/bindgen/src/main.rs

@@ -399,6 +399,8 @@ fn main() {
         .allowlist_function("rb_yarv_str_eql_internal")
         .allowlist_function("rb_str_neq_internal")
         .allowlist_function("rb_yarv_ary_entry_internal")
+        .allowlist_function("rb_vm_get_block_handler")
+        .allowlist_function("rb_vm_block_handler_type")
         .allowlist_function("rb_FL_TEST")
         .allowlist_function("rb_FL_TEST_RAW")
         .allowlist_function("rb_RB_TYPE_P")

zjit/src/distribution.rs

@@ -114,10 +114,18 @@ impl<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> Distributi
         self.kind == DistributionKind::Monomorphic
     }
 
+    pub fn is_polymorphic(&self) -> bool {
+        self.kind == DistributionKind::Polymorphic
+    }
+
     pub fn is_skewed_polymorphic(&self) -> bool {
         self.kind == DistributionKind::SkewedPolymorphic
     }
 
+    pub fn is_megamorphic(&self) -> bool {
+        self.kind == DistributionKind::Megamorphic
+    }
+
     pub fn is_skewed_megamorphic(&self) -> bool {
         self.kind == DistributionKind::SkewedMegamorphic
     }

zjit/src/hir.rs

@@ -2286,7 +2286,7 @@ impl Function {
                         };
                         let ci = unsafe { get_call_data_ci(cd) }; // info about the call site
 
-                        // If the call site info indicates that the `Function` has `VM_CALL_ARGS_SPLAT` set, then
+                        // If the call site info indicates that the `Function` has overly complex arguments, then
                         // do not optimize into a `SendWithoutBlockDirect`.
                         let flags = unsafe { rb_vm_ci_flag(ci) };
                         if unspecializable_call_type(flags) {
@@ -4265,13 +4265,13 @@ fn num_locals(iseq: *const rb_iseq_t) -> usize {
 
 /// If we can't handle the type of send (yet), bail out.
 fn unhandled_call_type(flags: u32) -> Result<(), CallType> {
-    if (flags & VM_CALL_KWARG) != 0 { return Err(CallType::Kwarg); }
     if (flags & VM_CALL_TAILCALL) != 0 { return Err(CallType::Tailcall); }
     Ok(())
 }
 
-/// If a given call uses splatting or block arguments, then we won't specialize.
+/// If a given call uses overly complex arguments, then we won't specialize.
 fn unspecializable_call_type(flags: u32) -> bool {
+    ((flags & VM_CALL_KWARG) != 0) ||
     ((flags & VM_CALL_ARGS_SPLAT) != 0) ||
     ((flags & VM_CALL_ARGS_BLOCKARG) != 0)
 }
@@ -4407,6 +4407,34 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
             // profiled cfp->self.
             if opcode == YARVINSN_getinstancevariable || opcode == YARVINSN_trace_getinstancevariable {
                 profiles.profile_self(&exit_state, self_param);
+            } else if opcode == YARVINSN_invokeblock || opcode == YARVINSN_trace_invokeblock {
+                if get_option!(stats) {
+                    let iseq_insn_idx = exit_state.insn_idx;
+                    if let Some(operand_types) = profiles.payload.profile.get_operand_types(iseq_insn_idx) {
+                        if let [self_type_distribution] = &operand_types[..] {
+                            let summary = TypeDistributionSummary::new(&self_type_distribution);
+                            if summary.is_monomorphic() {
+                                let obj = summary.bucket(0).class();
+                                let bh_type = unsafe { rb_vm_block_handler_type(obj) };
+                                if bh_type == block_handler_type_iseq {
+                                    fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_iseq));
+                                } else if bh_type == block_handler_type_ifunc {
+                                    fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_ifunc));
+                                } else {
+                                    fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_other));
+                                }
+                            } else if summary.is_skewed_polymorphic() || summary.is_polymorphic() {
+                                fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_polymorphic));
+                            } else if summary.is_skewed_megamorphic() || summary.is_megamorphic() {
+                                fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_megamorphic));
+                            } else {
+                                fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_no_profiles));
+                            }
+                        } else {
+                            fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_no_profiles));
+                        }
+                    }
+                }
             } else {
                 profiles.profile_stack(&exit_state);
             }

zjit/src/hir/opt_tests.rs

@@ -2659,7 +2659,10 @@ mod hir_opt_tests {
           Jump bb2(v4)
         bb2(v6:BasicObject):
           v10:Fixnum[1] = Const Value(1)
-          SideExit UnhandledCallType(Kwarg)
+          IncrCounter complex_arg_pass_caller_kwarg
+          v12:BasicObject = SendWithoutBlock v6, :foo, v10
+          CheckInterrupts
+          Return v12
         ");
     }
 
@@ -2682,7 +2685,10 @@ mod hir_opt_tests {
           Jump bb2(v4)
         bb2(v6:BasicObject):
           v10:Fixnum[1] = Const Value(1)
-          SideExit UnhandledCallType(Kwarg)
+          IncrCounter complex_arg_pass_caller_kwarg
+          v12:BasicObject = SendWithoutBlock v6, :foo, v10
+          CheckInterrupts
+          Return v12
         ");
     }
 

zjit/src/hir/tests.rs

@@ -1572,7 +1572,9 @@ pub mod hir_build_tests {
           Jump bb2(v5, v6)
         bb2(v8:BasicObject, v9:BasicObject):
           v13:Fixnum[1] = Const Value(1)
-          SideExit UnhandledCallType(Kwarg)
+          v15:BasicObject = SendWithoutBlock v8, :foo, v13
+          CheckInterrupts
+          Return v15
         ");
     }
 

zjit/src/profile.rs

@@ -43,6 +43,10 @@ impl Profiler {
     fn peek_at_self(&self) -> VALUE {
         unsafe { rb_get_cfp_self(self.cfp) }
     }
+
+    fn peek_at_block_handler(&self) -> VALUE {
+        unsafe { rb_vm_get_block_handler(self.cfp) }
+    }
 }
 
 /// API called from zjit_* instruction. opcode is the bare (non-zjit_*) instruction.
@@ -83,6 +87,7 @@ fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) {
         YARVINSN_opt_length    => profile_operands(profiler, profile, 1),
         YARVINSN_opt_size      => profile_operands(profiler, profile, 1),
         YARVINSN_opt_succ      => profile_operands(profiler, profile, 1),
+        YARVINSN_invokeblock   => profile_block_handler(profiler, profile),
         YARVINSN_opt_send_without_block | YARVINSN_send => {
             let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr();
             let argc = unsafe { vm_ci_argc((*cd).ci) };
@@ -135,6 +140,17 @@ fn profile_self(profiler: &mut Profiler, profile: &mut IseqProfile) {
     types[0].observe(ty);
 }
 
+fn profile_block_handler(profiler: &mut Profiler, profile: &mut IseqProfile) {
+    let types = &mut profile.opnd_types[profiler.insn_idx];
+    if types.is_empty() {
+        types.resize(1, TypeDistribution::new());
+    }
+    let obj = profiler.peek_at_block_handler();
+    let ty = ProfiledType::object(obj);
+    unsafe { rb_gc_writebarrier(profiler.iseq.into(), ty.class()) };
+    types[0].observe(ty);
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Flags(u32);
 
@@ -147,6 +163,8 @@ impl Flags {
     const IS_T_OBJECT: u32 = 1 << 2;
     /// Object is a struct with embedded fields
     const IS_STRUCT_EMBEDDED: u32 = 1 << 3;
+    /// Set if the ProfiledType is used for profiling specific objects, not just classes/shapes
+    const IS_OBJECT_PROFILING: u32 = 1 << 4;
 
     pub fn none() -> Self { Self(Self::NONE) }
 
@@ -155,6 +173,7 @@ impl Flags {
     pub fn is_embedded(self) -> bool { (self.0 & Self::IS_EMBEDDED) != 0 }
     pub fn is_t_object(self) -> bool { (self.0 & Self::IS_T_OBJECT) != 0 }
     pub fn is_struct_embedded(self) -> bool { (self.0 & Self::IS_STRUCT_EMBEDDED) != 0 }
+    pub fn is_object_profiling(self) -> bool { (self.0 & Self::IS_OBJECT_PROFILING) != 0 }
 }
 
 /// opt_send_without_block/opt_plus/... should store:
@@ -182,6 +201,14 @@ impl Default for ProfiledType {
 }
 
 impl ProfiledType {
+    /// Profile the object itself
+    fn object(obj: VALUE) -> Self {
+        let mut flags = Flags::none();
+        flags.0 |= Flags::IS_OBJECT_PROFILING;
+        Self { class: obj, shape: INVALID_SHAPE_ID, flags }
+    }
+
+    /// Profile the class and shape of the given object
     fn new(obj: VALUE) -> Self {
         if obj == Qfalse {
             return Self { class: unsafe { rb_cFalseClass },
@@ -251,6 +278,9 @@ impl ProfiledType {
     }
 
     pub fn is_string(&self) -> bool {
+        if self.flags.is_object_profiling() {
+            panic!("should not call is_string on object-profiled ProfiledType");
+        }
         // Fast paths for immediates and exact-class
         if self.flags.is_immediate() {
             return false;

zjit/src/stats.rs

@@ -287,6 +287,13 @@ make_counters! {
     // The number of times we ran a dynamic check
     guard_type_count,
     guard_shape_count,
+
+    invokeblock_handler_monomorphic_iseq,
+    invokeblock_handler_monomorphic_ifunc,
+    invokeblock_handler_monomorphic_other,
+    invokeblock_handler_polymorphic,
+    invokeblock_handler_megamorphic,
+    invokeblock_handler_no_profiles,
 }
 
 /// Increase a counter by a specified amount