[pull] master from ruby:master

doc/contributing/building_ruby.md

@@ -307,9 +307,6 @@ RUBY_TEST_TIMEOUT_SCALE=5 SYNTAX_SUGGEST_TIMEOUT=600 make check
 
 Please note, however, the following caveats!
 
-* ASAN will not work properly on any currently released version of Ruby; the
-  necessary support is currently only present on Ruby's master branch (and the
-  whole test suite passes only as of commit [Revision 9d0a5148]).
 * Due to [Bug #20243], Clang generates code for threadlocal variables which
   doesn't work with M:N threading. Thus, it's necessary to disable M:N
   threading support at build time for now (with the `-DUSE_MN_THREADS=0`

gc/mmtk/src/api.rs

@@ -107,10 +107,7 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder {
     if let Some(threads) = mmtk_builder_default_parse_threads() {
         if !builder.options.threads.set(threads) {
             // MMTk will validate it and reject 0.
-            eprintln!(
-                "[FATAL] Failed to set the number of MMTk threads to {}",
-                threads
-            );
+            eprintln!("[FATAL] Failed to set the number of MMTk threads to {threads}");
             std::process::exit(1);
         }
     }
@@ -120,10 +117,7 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder {
     let heap_max = mmtk_builder_default_parse_heap_max();
 
     if heap_min >= heap_max {
-        eprintln!(
-            "[FATAL] MMTK_HEAP_MIN({}) >= MMTK_HEAP_MAX({})",
-            heap_min, heap_max
-        );
+        eprintln!("[FATAL] MMTK_HEAP_MIN({heap_min}) >= MMTK_HEAP_MAX({heap_max})");
         std::process::exit(1);
     }
 

gc/mmtk/src/binding.rs

@@ -119,7 +119,7 @@ impl RubyBinding {
     }
 
     pub fn register_wb_unprotected_object(&self, object: ObjectReference) {
-        debug!("Registering WB-unprotected object: {}", object);
+        debug!("Registering WB-unprotected object: {object}");
         let mut objects = self.wb_unprotected_objects.lock().unwrap();
         objects.insert(object);
     }

gc/mmtk/src/collection.rs

@@ -41,10 +41,7 @@ impl Collection<Ruby> for VMCollection {
                 .name("MMTk Worker Thread".to_string())
                 .spawn(move || {
                     let ordinal = worker.ordinal;
-                    debug!(
-                        "Hello! This is MMTk Worker Thread running! ordinal: {}",
-                        ordinal
-                    );
+                    debug!("Hello! This is MMTk Worker Thread running! ordinal: {ordinal}");
                     crate::register_gc_thread(thread::current().id());
                     let ptr_worker = &mut *worker as *mut GCWorker<Ruby>;
                     let gc_thread_tls =
@@ -55,10 +52,7 @@ impl Collection<Ruby> for VMCollection {
                         GCThreadTLS::to_vwt(gc_thread_tls),
                         worker,
                     );
-                    debug!(
-                        "An MMTk Worker Thread is quitting. Good bye! ordinal: {}",
-                        ordinal
-                    );
+                    debug!("An MMTk Worker Thread is quitting. Good bye! ordinal: {ordinal}");
                     crate::unregister_gc_thread(thread::current().id());
                 })
                 .unwrap(),

gc/mmtk/src/scanning.rs

@@ -47,11 +47,7 @@ impl Scanning<Ruby> for VMScanning {
             );
             let forwarded_target = object_tracer.trace_object(target_object);
             if forwarded_target != target_object {
-                trace!(
-                    "  Forwarded target {} -> {}",
-                    target_object,
-                    forwarded_target
-                );
+                trace!("  Forwarded target {target_object} -> {forwarded_target}");
             }
             forwarded_target
         };
@@ -251,15 +247,11 @@ impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for ScanWbUnprotectedRoots<F> {
         VMScanning::collect_object_roots_in("wb_unprot_roots", gc_tls, &mut self.factory, || {
             for object in self.objects.iter().copied() {
                 if object.is_reachable() {
-                    debug!(
-                        "[wb_unprot_roots] Visiting WB-unprotected object (parent): {}",
-                        object
-                    );
+                    debug!("[wb_unprot_roots] Visiting WB-unprotected object (parent): {object}");
                     (upcalls().scan_object_ruby_style)(object);
                 } else {
                     debug!(
-                        "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {}",
-                        object
+                        "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {object}"
                     );
                 }
             }

gc/mmtk/src/weak_proc.rs

@@ -103,7 +103,7 @@ impl GCWork<Ruby> for ProcessObjFreeCandidates {
 
         let n_cands = obj_free_candidates.len();
 
-        debug!("Total: {} candidates", n_cands);
+        debug!("Total: {n_cands} candidates");
 
         // Process obj_free
         let mut new_candidates = Vec::new();
@@ -112,11 +112,7 @@ impl GCWork<Ruby> for ProcessObjFreeCandidates {
             if object.is_reachable() {
                 // Forward and add back to the candidate list.
                 let new_object = object.forward();
-                trace!(
-                    "Forwarding obj_free candidate: {} -> {}",
-                    object,
-                    new_object
-                );
+                trace!("Forwarding obj_free candidate: {object} -> {new_object}");
                 new_candidates.push(new_object);
             } else {
                 (upcalls().call_obj_free)(object);
@@ -158,11 +154,10 @@ trait GlobalTableProcessingWork {
         let forward_object = |_worker, object: ObjectReference, _pin| {
             debug_assert!(
                 mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(),
-                "{} is not an MMTk object",
-                object
+                "{object} is not an MMTk object"
             );
             let result = object.forward();
-            trace!("Forwarding reference: {} -> {}", object, result);
+            trace!("Forwarding reference: {object} -> {result}");
             result
         };
 
@@ -216,14 +211,10 @@ impl GCWork<Ruby> for UpdateWbUnprotectedObjectsList {
             if object.is_reachable() {
                 // Forward and add back to the candidate list.
                 let new_object = object.forward();
-                trace!(
-                    "Forwarding WB-unprotected object: {} -> {}",
-                    object,
-                    new_object
-                );
+                trace!("Forwarding WB-unprotected object: {object} -> {new_object}");
                 objects.insert(new_object);
             } else {
-                trace!("Removing WB-unprotected object from list: {}", object);
+                trace!("Removing WB-unprotected object from list: {object}");
             }
         }
 

tool/zjit_bisect.rb

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+require 'logger'
+require 'open3'
+require 'tempfile'
+require 'timeout'
+
+RUBY = ARGV[0] || raise("Usage: ruby jit_bisect.rb <path_to_ruby> <options>")
+OPTIONS = ARGV[1] || raise("Usage: ruby jit_bisect.rb <path_to_ruby> <options>")
+TIMEOUT_SEC = 5
+LOGGER = Logger.new($stdout)
+
+# From https://github.com/tekknolagi/omegastar
+# MIT License
+# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms
+# Attempt to reduce the `items` argument as much as possible, returning the
+# shorter version. `fixed` will always be used as part of the items when
+# running `command`.
+# `command` should return True if the command succeeded (the failure did not
+# reproduce) and False if the command failed (the failure reproduced).
+def bisect_impl(command, fixed, items, indent="")
+  LOGGER.info("#{indent}step fixed[#{fixed.length}] and items[#{items.length}]")
+  while items.length > 1
+    LOGGER.info("#{indent}#{fixed.length + items.length} candidates")
+    # Return two halves of the given list. For odd-length lists, the second
+    # half will be larger.
+    half = items.length / 2
+    left = items[0...half]
+    right = items[half..]
+    if !command.call(fixed + left)
+      items = left
+      next
+    end
+    if !command.call(fixed + right)
+      items = right
+      next
+    end
+    # We need something from both halves to trigger the failure. Try
+    # holding each half fixed and bisecting the other half to reduce the
+    # candidates.
+    new_right = bisect_impl(command, fixed + left, right, indent + "< ")
+    new_left = bisect_impl(command, fixed + new_right, left, indent + "> ")
+    return new_left + new_right
+  end
+  items
+end
+
+# From https://github.com/tekknolagi/omegastar
+# MIT License
+# Copyright (c) 2024 Maxwell Bernstein and Meta Platforms
+def run_bisect(command, items)
+  LOGGER.info("Verifying items")
+  if command.call(items)
+    raise StandardError.new("Command succeeded with full items")
+  end
+  if !command.call([])
+    raise StandardError.new("Command failed with empty items")
+  end
+  bisect_impl(command, [], items)
+end
+
+def run_with_jit_list(ruby, options, jit_list)
+  # Make a new temporary file containing the JIT list
+  Tempfile.create("jit_list") do |temp_file|
+    temp_file.write(jit_list.join("\n"))
+    temp_file.flush
+    temp_file.close
+    # Run the JIT with the temporary file
+    Open3.capture3("#{ruby} --zjit-allowed-iseqs=#{temp_file.path} #{options}")
+  end
+end
+
+# Try running with no JIT list to get a stable baseline
+_, stderr, status = run_with_jit_list(RUBY, OPTIONS, [])
+if !status.success?
+  raise "Command failed with empty JIT list: #{stderr}"
+end
+# Collect the JIT list from the failing Ruby process
+jit_list = nil
+Tempfile.create "jit_list" do |temp_file|
+  Open3.capture3("#{RUBY} --zjit-log-compiled-iseqs=#{temp_file.path} #{OPTIONS}")
+  jit_list = File.readlines(temp_file.path).map(&:strip).reject(&:empty?)
+end
+LOGGER.info("Starting with JIT list of #{jit_list.length} items.")
+# Now narrow it down
+command = lambda do |items|
+  status = Timeout.timeout(TIMEOUT_SEC) do
+    _, _, status = run_with_jit_list(RUBY, OPTIONS, items)
+    status
+  end
+  status.success?
+end
+result = run_bisect(command, jit_list)
+File.open("jitlist.txt", "w") do |file|
+  file.puts(result)
+end
+puts "Reduced JIT list (available in jitlist.txt):"
+puts result

zjit/src/codegen.rs

@@ -121,39 +121,46 @@ fn gen_iseq_entry_point_body(cb: &mut CodeBlock, iseq: IseqPtr) -> *const u8 {
     };
 
     // Compile the High-level IR
-    let (start_ptr, mut branch_iseqs) = match gen_function(cb, iseq, &function) {
-        Some((start_ptr, gc_offsets, jit)) => {
-            // Remember the block address to reuse it later
-            let payload = get_or_create_iseq_payload(iseq);
-            payload.start_ptr = Some(start_ptr);
-            append_gc_offsets(iseq, &gc_offsets);
-
-            // Compile an entry point to the JIT code
-            (gen_entry(cb, iseq, &function, start_ptr), jit.branch_iseqs)
-        },
-        None => (None, vec![]),
+    let Some((start_ptr, gc_offsets, jit)) = gen_function(cb, iseq, &function) else {
+        debug!("Failed to compile iseq: gen_function failed: {}", iseq_get_location(iseq, 0));
+        return std::ptr::null();
+    };
+
+    // Compile an entry point to the JIT code
+    let Some(entry_ptr) = gen_entry(cb, iseq, &function, start_ptr) else {
+        debug!("Failed to compile iseq: gen_entry failed: {}", iseq_get_location(iseq, 0));
+        return std::ptr::null();
     };
 
+    let mut branch_iseqs = jit.branch_iseqs;
+
     // Recursively compile callee ISEQs
+    let caller_iseq = iseq;
     while let Some((branch, iseq)) = branch_iseqs.pop() {
         // Disable profiling. This will be the last use of the profiling information for the ISEQ.
         unsafe { rb_zjit_profile_disable(iseq); }
 
         // Compile the ISEQ
-        if let Some((callee_ptr, callee_branch_iseqs)) = gen_iseq(cb, iseq) {
-            let callee_addr = callee_ptr.raw_ptr(cb);
-            branch.regenerate(cb, |asm| {
-                asm.ccall(callee_addr, vec![]);
-            });
-            branch_iseqs.extend(callee_branch_iseqs);
-        } else {
+        let Some((callee_ptr, callee_branch_iseqs)) = gen_iseq(cb, iseq) else {
             // Failed to compile the callee. Bail out of compiling this graph of ISEQs.
+            debug!("Failed to compile iseq: could not compile callee: {} -> {}",
+                   iseq_get_location(caller_iseq, 0), iseq_get_location(iseq, 0));
             return std::ptr::null();
-        }
+        };
+        let callee_addr = callee_ptr.raw_ptr(cb);
+        branch.regenerate(cb, |asm| {
+            asm.ccall(callee_addr, vec![]);
+        });
+        branch_iseqs.extend(callee_branch_iseqs);
     }
 
-    // Return a JIT code address or a null pointer
-    start_ptr.map(|start_ptr| start_ptr.raw_ptr(cb)).unwrap_or(std::ptr::null())
+    // Remember the block address to reuse it later
+    let payload = get_or_create_iseq_payload(iseq);
+    payload.start_ptr = Some(start_ptr);
+    append_gc_offsets(iseq, &gc_offsets);
+
+    // Return a JIT code address
+    entry_ptr.raw_ptr(cb)
 }
 
 /// Write an entry to the perf map in /tmp
@@ -284,6 +291,10 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, function: &Function) -> Optio
             let iseq_name = iseq_get_location(iseq, 0);
             register_with_perf(iseq_name, start_usize, code_size);
         }
+        if ZJITState::should_log_compiled_iseqs() {
+            let iseq_name = iseq_get_location(iseq, 0);
+            ZJITState::log_compile(iseq_name);
+        }
     }
     result
 }
@@ -1173,7 +1184,8 @@ fn compile_iseq(iseq: IseqPtr) -> Option<Function> {
     let mut function = match iseq_to_hir(iseq) {
         Ok(function) => function,
         Err(err) => {
-            debug!("ZJIT: iseq_to_hir: {err:?}");
+            let name = crate::cruby::iseq_get_location(iseq, 0);
+            debug!("ZJIT: iseq_to_hir: {err:?}: {name}");
             return None;
         }
     };

zjit/src/hir.rs

@@ -2480,6 +2480,7 @@ pub enum ParseError {
     UnknownParameterType(ParameterType),
     MalformedIseq(u32), // insn_idx into iseq_encoded
     Validation(ValidationError),
+    NotAllowed,
 }
 
 /// Return the number of locals in the current ISEQ (includes parameters)
@@ -2545,6 +2546,9 @@ fn filter_unknown_parameter_type(iseq: *const rb_iseq_t) -> Result<(), ParseErro
 
 /// Compile ISEQ into High-level IR
 pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> {
+    if !ZJITState::can_compile_iseq(iseq) {
+        return Err(ParseError::NotAllowed);
+    }
     filter_unknown_parameter_type(iseq)?;
     let payload = get_or_create_iseq_payload(iseq);
     let mut profiles = ProfileOracle::new(payload);