[pull] master from ruby:master

common.mk

@@ -1133,7 +1133,7 @@ $(srcs_vpath)insns_info.inc: $(tooldir)/ruby_vm/views/insns_info.inc.erb $(inc_c
   $(tooldir)/ruby_vm/views/_insn_type_chars.erb $(tooldir)/ruby_vm/views/_insn_name_info.erb \
   $(tooldir)/ruby_vm/views/_insn_len_info.erb $(tooldir)/ruby_vm/views/_insn_operand_info.erb \
   $(tooldir)/ruby_vm/views/_attributes.erb $(tooldir)/ruby_vm/views/_comptime_insn_stack_increase.erb \
-  $(tooldir)/ruby_vm/views/_zjit_helpers.erb
+  $(tooldir)/ruby_vm/views/_zjit_helpers.erb $(tooldir)/ruby_vm/views/_insn_leaf_info.erb
 $(srcs_vpath)vmtc.inc: $(tooldir)/ruby_vm/views/vmtc.inc.erb $(inc_common_headers)
 $(srcs_vpath)vm.inc: $(tooldir)/ruby_vm/views/vm.inc.erb $(inc_common_headers) \
   $(tooldir)/ruby_vm/views/_insn_entry.erb $(tooldir)/ruby_vm/views/_trace_instruction.erb \

insns.def

@@ -1206,7 +1206,7 @@ jump
 ()
 ()
 /* Same discussion as leave. */
-// attr bool leaf = leafness_of_check_ints; /* has rb_threadptr_execute_interrupts() */
+// attr bool leaf = false; /* has rb_threadptr_execute_interrupts() */
 {
     RUBY_VM_CHECK_INTS(ec);
     JUMP(dst);
@@ -1219,7 +1219,7 @@ branchif
 (VALUE val)
 ()
 /* Same discussion as jump. */
-// attr bool leaf = leafness_of_check_ints; /* has rb_threadptr_execute_interrupts() */
+// attr bool leaf = false; /* has rb_threadptr_execute_interrupts() */
 {
     if (RTEST(val)) {
         RUBY_VM_CHECK_INTS(ec);
@@ -1234,7 +1234,7 @@ branchunless
 (VALUE val)
 ()
 /* Same discussion as jump. */
-// attr bool leaf = leafness_of_check_ints; /* has rb_threadptr_execute_interrupts() */
+// attr bool leaf = false; /* has rb_threadptr_execute_interrupts() */
 {
     if (!RTEST(val)) {
         RUBY_VM_CHECK_INTS(ec);
@@ -1249,7 +1249,7 @@ branchnil
 (VALUE val)
 ()
 /* Same discussion as jump. */
-// attr bool leaf = leafness_of_check_ints; /* has rb_threadptr_execute_interrupts() */
+// attr bool leaf = false; /* has rb_threadptr_execute_interrupts() */
 {
     if (NIL_P(val)) {
         RUBY_VM_CHECK_INTS(ec);

template/Makefile.in

@@ -679,6 +679,7 @@ $(INSNS): $(srcdir)/insns.def vm_opts.h \
 	  $(tooldir)/ruby_vm/views/_comptime_insn_stack_increase.erb \
 	  $(tooldir)/ruby_vm/views/_copyright.erb \
 	  $(tooldir)/ruby_vm/views/_insn_entry.erb \
+	  $(tooldir)/ruby_vm/views/_insn_leaf_info.erb \
 	  $(tooldir)/ruby_vm/views/_insn_len_info.erb \
 	  $(tooldir)/ruby_vm/views/_insn_name_info.erb \
 	  $(tooldir)/ruby_vm/views/_insn_operand_info.erb \

test/ruby/test_zjit.rb

@@ -196,6 +196,50 @@ def test
     }, insns: [:setglobal]
   end
 
+  def test_getlocal_after_eval
+    assert_compiles '2', %q{
+      def test
+        a = 1
+        eval('a = 2')
+        a
+      end
+      test
+    }
+  end
+
+  def test_getlocal_after_instance_eval
+    assert_compiles '2', %q{
+      def test
+        a = 1
+        instance_eval('a = 2')
+        a
+      end
+      test
+    }
+  end
+
+  def test_getlocal_after_module_eval
+    assert_compiles '2', %q{
+      def test
+        a = 1
+        Kernel.module_eval('a = 2')
+        a
+      end
+      test
+    }
+  end
+
+  def test_getlocal_after_class_eval
+    assert_compiles '2', %q{
+      def test
+        a = 1
+        Kernel.class_eval('a = 2')
+        a
+      end
+      test
+    }
+  end
+
   def test_setlocal
     assert_compiles '3', %q{
       def test(n)
@@ -1453,8 +1497,7 @@ def entry = jit_frame1      # 3
   end
 
   def test_bop_invalidation
-    omit 'Invalidation on BOP redefinition is not implemented yet'
-    assert_compiles '', %q{
+    assert_compiles '100', %q{
       def test
         eval(<<~RUBY)
           class Integer

tool/ruby_vm/models/bare_instructions.rb

@@ -107,14 +107,6 @@ def handles_sp?
     /\b(false|0)\b/ !~ @attrs.fetch('handles_sp').expr.expr
   end
 
-  def always_leaf?
-    @attrs.fetch('leaf').expr.expr == 'true;'
-  end
-
-  def leaf_without_check_ints?
-    @attrs.fetch('leaf').expr.expr == 'leafness_of_check_ints;'
-  end
-
   def handle_canary stmt
     # Stack canary is basically a good thing that we want to add, however:
     #

tool/ruby_vm/views/_insn_leaf_info.erb

@@ -0,0 +1,18 @@
+MAYBE_UNUSED(static bool insn_leaf(int insn, const VALUE *opes));
+static bool
+insn_leaf(int insn, const VALUE *opes)
+{
+    switch (insn) {
+% RubyVM::Instructions.each do |insn|
+%   next if insn.is_a?(RubyVM::TraceInstructions) || insn.is_a?(RubyVM::ZJITInstructions)
+      case <%= insn.bin %>:
+        return attr_leaf_<%= insn.name %>(<%=
+          insn.operands.map.with_index do |ope, i|
+            "(#{ope[:type]})opes[#{i}]"
+          end.join(', ')
+        %>);
+% end
+      default:
+        return false;
+    }
+}

tool/ruby_vm/views/_leaf_helpers.erb

@@ -10,10 +10,6 @@
 
 #include "iseq.h"
 
-// This is used to tell JIT that this insn would be leaf if CHECK_INTS didn't exist.
-// It should be used only when RUBY_VM_CHECK_INTS is directly written in insns.def.
-static bool leafness_of_check_ints = false;
-
 static bool
 leafness_of_defined(rb_num_t op_type)
 {
@@ -25,7 +21,7 @@ leafness_of_defined(rb_num_t op_type)
       case DEFINED_YIELD:
       case DEFINED_REF:
       case DEFINED_ZSUPER:
-        return false;
+        return true;
       case DEFINED_CONST:
       case DEFINED_CONST_FROM:
         /* has rb_autoload_load(); */

tool/ruby_vm/views/insns_info.inc.erb

@@ -21,5 +21,6 @@
 <%= render 'sp_inc_helpers' %>
 <%= render 'zjit_helpers' %>
 <%= render 'attributes' %>
+<%= render 'insn_leaf_info' %>
 <%= render 'comptime_insn_stack_increase' %>
 #endif

vm.c

@@ -1061,7 +1061,7 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co
     // Invalidate JIT code that assumes cfp->ep == vm_base_ptr(cfp).
     if (env->iseq) {
         rb_yjit_invalidate_ep_is_bp(env->iseq);
-        rb_zjit_invalidate_ep_is_bp(env->iseq);
+        rb_zjit_invalidate_no_ep_escape(env->iseq);
     }
 
     return (VALUE)env;

vm_eval.c

@@ -1985,6 +1985,10 @@ eval_string_with_cref(VALUE self, VALUE src, rb_cref_t *cref, VALUE file, int li
     block.as.captured.code.iseq = cfp->iseq;
     block.type = block_type_iseq;
 
+    // EP is not escaped to the heap here, but captured and reused by another frame.
+    // ZJIT's locals are incompatible with it unlike YJIT's, so invalidate the ISEQ for ZJIT.
+    rb_zjit_invalidate_no_ep_escape(cfp->iseq);
+
     iseq = eval_make_iseq(src, file, line, &block);
     if (!iseq) {
         rb_exc_raise(ec->errinfo);

zjit.c

@@ -333,6 +333,12 @@ rb_zjit_defined_ivar(VALUE obj, ID id, VALUE pushval)
     return result ? pushval : Qnil;
 }
 
+bool
+rb_zjit_insn_leaf(int insn, const VALUE *opes)
+{
+    return insn_leaf(insn, opes);
+}
+
 // Primitives used by zjit.rb. Don't put other functions below, which wouldn't use them.
 VALUE rb_zjit_assert_compiles(rb_execution_context_t *ec, VALUE self);
 VALUE rb_zjit_stats(rb_execution_context_t *ec, VALUE self, VALUE target_key);

zjit.h

@@ -18,7 +18,7 @@ void rb_zjit_profile_insn(uint32_t insn, rb_execution_context_t *ec);
 void rb_zjit_profile_enable(const rb_iseq_t *iseq);
 void rb_zjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop);
 void rb_zjit_cme_invalidate(const rb_callable_method_entry_t *cme);
-void rb_zjit_invalidate_ep_is_bp(const rb_iseq_t *iseq);
+void rb_zjit_invalidate_no_ep_escape(const rb_iseq_t *iseq);
 void rb_zjit_constant_state_changed(ID id);
 void rb_zjit_iseq_mark(void *payload);
 void rb_zjit_iseq_update_references(void *payload);
@@ -31,7 +31,7 @@ static inline void rb_zjit_profile_insn(uint32_t insn, rb_execution_context_t *e
 static inline void rb_zjit_profile_enable(const rb_iseq_t *iseq) {}
 static inline void rb_zjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop) {}
 static inline void rb_zjit_cme_invalidate(const rb_callable_method_entry_t *cme) {}
-static inline void rb_zjit_invalidate_ep_is_bp(const rb_iseq_t *iseq) {}
+static inline void rb_zjit_invalidate_no_ep_escape(const rb_iseq_t *iseq) {}
 static inline void rb_zjit_constant_state_changed(ID id) {}
 static inline void rb_zjit_before_ractor_spawn(void) {}
 static inline void rb_zjit_tracing_invalidate_all(void) {}

zjit/bindgen/src/main.rs

@@ -358,6 +358,7 @@ fn main() {
         .allowlist_function("rb_zjit_print_exception")
         .allowlist_function("rb_zjit_singleton_class_p")
         .allowlist_function("rb_zjit_defined_ivar")
+        .allowlist_function("rb_zjit_insn_leaf")
         .allowlist_type("robject_offsets")
         .allowlist_type("rstring_offsets")
         .allowlist_var("RB_INVALID_SHAPE_ID")

zjit/src/codegen.rs

@@ -9,7 +9,7 @@ use std::slice;
 
 use crate::asm::Label;
 use crate::backend::current::{Reg, ALLOC_REGS};
-use crate::invariants::{track_bop_assumption, track_cme_assumption, track_single_ractor_assumption, track_stable_constant_names_assumption, track_no_trace_point_assumption};
+use crate::invariants::{track_bop_assumption, track_cme_assumption, track_no_ep_escape_assumption, track_no_trace_point_assumption, track_single_ractor_assumption, track_stable_constant_names_assumption};
 use crate::gc::{append_gc_offsets, get_or_create_iseq_payload, get_or_create_iseq_payload_ptr, IseqPayload, IseqStatus};
 use crate::state::ZJITState;
 use crate::stats::{exit_counter_for_compile_error, incr_counter, incr_counter_by, CompileError};
@@ -581,25 +581,24 @@ fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invarian
 
     // Remember the current address as a patch point
     asm.pos_marker(move |code_ptr, cb| {
+        let side_exit_ptr = cb.resolve_label(label);
         match invariant {
             Invariant::BOPRedefined { klass, bop } => {
-                let side_exit_ptr = cb.resolve_label(label);
                 track_bop_assumption(klass, bop, code_ptr, side_exit_ptr, payload_ptr);
             }
             Invariant::MethodRedefined { klass: _, method: _, cme } => {
-                let side_exit_ptr = cb.resolve_label(label);
                 track_cme_assumption(cme, code_ptr, side_exit_ptr, payload_ptr);
             }
             Invariant::StableConstantNames { idlist } => {
-                let side_exit_ptr = cb.resolve_label(label);
                 track_stable_constant_names_assumption(idlist, code_ptr, side_exit_ptr, payload_ptr);
             }
             Invariant::NoTracePoint => {
-                let side_exit_ptr = cb.resolve_label(label);
                 track_no_trace_point_assumption(code_ptr, side_exit_ptr, payload_ptr);
             }
+            Invariant::NoEPEscape(iseq) => {
+                track_no_ep_escape_assumption(iseq, code_ptr, side_exit_ptr, payload_ptr);
+            }
             Invariant::SingleRactorMode => {
-                let side_exit_ptr = cb.resolve_label(label);
                 track_single_ractor_assumption(code_ptr, side_exit_ptr, payload_ptr);
             }
         }
@@ -871,7 +870,7 @@ fn gen_entry_param(asm: &mut Assembler, iseq: IseqPtr, local_idx: usize) -> lir:
     // If the ISEQ does not escape EP, we can optimize the local variable access using the SP register.
     if !iseq_entry_escapes_ep(iseq) {
         // Create a reference to the local variable using the SP register. We assume EP == BP.
-        // TODO: Implement the invalidation in rb_zjit_invalidate_ep_is_bp()
+        // TODO: Implement the invalidation in rb_zjit_invalidate_no_ep_escape()
         let offs = -(SIZEOF_VALUE_I32 * (ep_offset + 1));
         Opnd::mem(64, SP, offs)
     } else {
@@ -966,14 +965,10 @@ fn gen_send(
     unsafe extern "C" {
         fn rb_vm_send(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE;
     }
-    let ret = asm.ccall(
+    asm.ccall(
         rb_vm_send as *const u8,
         vec![EC, CFP, (cd as usize).into(), VALUE(blockiseq as usize).into()],
-    );
-    // TODO: Add a PatchPoint here that can side-exit the function if the callee messed with
-    // the frame's locals
-
-    ret
+    )
 }
 
 /// Compile a dynamic dispatch without block
@@ -1002,14 +997,10 @@ fn gen_send_without_block(
     unsafe extern "C" {
         fn rb_vm_opt_send_without_block(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE;
     }
-    let ret = asm.ccall(
+    asm.ccall(
         rb_vm_opt_send_without_block as *const u8,
         vec![EC, CFP, (cd as usize).into()],
-    );
-    // TODO(max): Add a PatchPoint here that can side-exit the function if the callee messed with
-    // the frame's locals
-
-    ret
+    )
 }
 
 /// Compile a direct jump to an ISEQ call without block
@@ -1058,8 +1049,6 @@ fn gen_send_without_block_direct(
     let iseq_call = IseqCall::new(iseq);
     let dummy_ptr = cb.get_write_ptr().raw_ptr(cb);
     jit.iseq_calls.push(iseq_call.clone());
-    // TODO(max): Add a PatchPoint here that can side-exit the function if the callee messed with
-    // the frame's locals
     let ret = asm.ccall_with_iseq_call(dummy_ptr, c_args, &iseq_call);
 
     // If a callee side-exits, i.e. returns Qundef, propagate the return value to the caller.
@@ -1099,14 +1088,10 @@ fn gen_invokesuper(
     unsafe extern "C" {
         fn rb_vm_invokesuper(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE;
     }
-    let ret = asm.ccall(
+    asm.ccall(
         rb_vm_invokesuper as *const u8,
         vec![EC, CFP, (cd as usize).into(), VALUE(blockiseq as usize).into()],
-    );
-    // TODO: Add a PatchPoint here that can side-exit the function if the callee messed with
-    // the frame's locals
-
-    ret
+    )
 }
 
 /// Compile a string resurrection