diff --git a/.github/workflows/cygwin.yml b/.github/workflows/cygwin.yml index 95242138274f30..879a9d586608ae 100644 --- a/.github/workflows/cygwin.yml +++ b/.github/workflows/cygwin.yml @@ -45,7 +45,7 @@ jobs: - name: Setup Cygwin uses: cygwin/cygwin-install-action@master with: - packages: ruby gcc-core make autoconf libtool libssl-devel libyaml-devel libffi-devel zlib-devel + packages: ruby gcc-core make autoconf libtool libssl-devel libyaml-devel libffi-devel zlib-devel rubygems - name: configure run: | @@ -53,12 +53,11 @@ jobs: ./configure --disable-install-doc shell: C:\cygwin\bin\bash.EXE --noprofile --norc -e -o igncr -o pipefail {0} - # This fails with: tool/outdate-bundled-gems.rb:3:in 'Kernel#require': cannot load such file -- rubygems (LoadError) - #- name: Extract bundled gems - # run: | - # make ruby -j5 - # make extract-gems - # shell: C:\cygwin\bin\bash.EXE --noprofile --norc -e -o igncr -o pipefail {0} + - name: Extract bundled gems + run: | + make ruby -j5 + make extract-gems + shell: C:\cygwin\bin\bash.EXE --noprofile --norc -e -o igncr -o pipefail {0} - name: make all timeout-minutes: 30 diff --git a/test/-ext-/thread/test_lock_native_thread.rb b/test/-ext-/thread/test_lock_native_thread.rb index 8a5ba78838d6bf..b4044b2b935074 100644 --- a/test/-ext-/thread/test_lock_native_thread.rb +++ b/test/-ext-/thread/test_lock_native_thread.rb @@ -15,6 +15,8 @@ class TestThreadLockNativeThread < Test::Unit::TestCase def test_lock_native_thread + omit "LSAN reports memory leak because NT is not freed for MN thread" if Test::Sanitizers.lsan_enabled? + assert_separately([{'RUBY_MN_THREADS' => '1'}], <<-RUBY) require '-test-/thread/lock_native_thread' @@ -28,6 +30,8 @@ def test_lock_native_thread end def test_lock_native_thread_tls + omit "LSAN reports memory leak because NT is not freed for MN thread" if Test::Sanitizers.lsan_enabled? + assert_separately([{'RUBY_MN_THREADS' => '1'}], <<-RUBY) require '-test-/thread/lock_native_thread' tn = 10 diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 0a19035dc1584f..7d72acfe14056f 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -380,7 +380,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::SendWithoutBlock { cd, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), // Give up SendWithoutBlockDirect for 6+ args since asm.ccall() doesn't support it. Insn::SendWithoutBlockDirect { cd, state, args, .. } if args.len() + 1 > C_ARG_OPNDS.len() => // +1 for self - gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), SendFallbackReason::SendWithoutBlockDirectTooManyArgs), + gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), SendFallbackReason::TooManyArgsForLir), Insn::SendWithoutBlockDirect { cme, iseq, recv, args, state, .. } => gen_send_without_block_direct(cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state)), &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), @@ -1264,10 +1264,21 @@ fn gen_send_without_block_direct( // Set up arguments let mut c_args = vec![recv]; - c_args.extend(args); + c_args.extend(&args); + + let num_optionals_passed = if unsafe { get_iseq_flags_has_opt(iseq) } { + // See vm_call_iseq_setup_normal_opt_start in vm_inshelper.c + let lead_num = unsafe { get_iseq_body_param_lead_num(iseq) } as u32; + let opt_num = unsafe { get_iseq_body_param_opt_num(iseq) } as u32; + assert!(args.len() as u32 <= lead_num + opt_num); + let num_optionals_passed = args.len() as u32 - lead_num; + num_optionals_passed + } else { + 0 + }; // Make a method call. The target address will be rewritten once compiled. - let iseq_call = IseqCall::new(iseq); + let iseq_call = IseqCall::new(iseq, num_optionals_passed); let dummy_ptr = cb.get_write_ptr().raw_ptr(cb); jit.iseq_calls.push(iseq_call.clone()); let ret = asm.ccall_with_iseq_call(dummy_ptr, c_args, &iseq_call); @@ -2129,7 +2140,8 @@ c_callable! { // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; let iseq = iseq_call.iseq.get(); - let pc = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; // TODO: handle opt_pc once supported + let entry_insn_idxs = crate::hir::jit_entry_insns(iseq); + let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idxs[iseq_call.jit_entry_idx.to_usize()]) }; unsafe { rb_set_cfp_pc(cfp, pc) }; // JIT-to-JIT calls don't set SP or fill nils to uninitialized (non-argument) locals. @@ -2189,13 +2201,8 @@ fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); })?; - // We currently don't support JIT-to-JIT calls for ISEQs with optional arguments. - // So we only need to use jit_entry_ptrs[0] for now. TODO(Shopify/ruby#817): Support optional arguments. - let Some(&jit_entry_ptr) = jit_entry_ptrs.first() else { - return Err(CompileError::JitToJitOptional) - }; - // Update the stub to call the code pointer + let jit_entry_ptr = jit_entry_ptrs[iseq_call.jit_entry_idx.to_usize()]; let code_addr = jit_entry_ptr.raw_ptr(cb); let iseq = iseq_call.iseq.get(); iseq_call.regenerate(cb, |asm| { @@ -2407,7 +2414,7 @@ impl Assembler { /// Store info about a JIT entry point pub struct JITEntry { - /// Index that corresponds to jit_entry_insns() + /// Index that corresponds to [crate::hir::jit_entry_insns] jit_entry_idx: usize, /// Position where the entry point starts start_addr: Cell>, @@ -2430,6 +2437,9 @@ pub struct IseqCall { /// Callee ISEQ that start_addr jumps to pub iseq: Cell, + /// Index that corresponds to [crate::hir::jit_entry_insns] + jit_entry_idx: u32, + /// Position where the call instruction starts start_addr: Cell>, @@ -2441,11 +2451,12 @@ pub type IseqCallRef = Rc; impl IseqCall { /// Allocate a new IseqCall - fn new(iseq: IseqPtr) -> IseqCallRef { + fn new(iseq: IseqPtr, jit_entry_idx: u32) -> IseqCallRef { let iseq_call = IseqCall { iseq: Cell::new(iseq), start_addr: Cell::new(None), end_addr: Cell::new(None), + jit_entry_idx, }; Rc::new(iseq_call) } diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs index 7fba755a6facda..8dc53302835d23 100644 --- a/zjit/src/cruby_methods.rs +++ b/zjit/src/cruby_methods.rs @@ -226,6 +226,18 @@ pub fn init() -> Annotations { annotate!(rb_cBasicObject, "initialize", inline_basic_object_initialize); annotate!(rb_cInteger, "succ", inline_integer_succ); annotate!(rb_cInteger, "^", inline_integer_xor); + annotate!(rb_cInteger, "==", inline_integer_eq); + annotate!(rb_cInteger, "+", inline_integer_plus); + annotate!(rb_cInteger, "-", inline_integer_minus); + annotate!(rb_cInteger, "*", inline_integer_mult); + annotate!(rb_cInteger, "/", inline_integer_div); + annotate!(rb_cInteger, "%", inline_integer_mod); + annotate!(rb_cInteger, "&", inline_integer_and); + annotate!(rb_cInteger, "|", inline_integer_or); + annotate!(rb_cInteger, ">", inline_integer_gt); + annotate!(rb_cInteger, ">=", inline_integer_ge); + annotate!(rb_cInteger, "<", inline_integer_lt); + annotate!(rb_cInteger, "<=", inline_integer_le); annotate!(rb_cString, "to_s", inline_string_to_s, types::StringExact); let thread_singleton = unsafe { rb_singleton_class(rb_cThread) }; annotate!(thread_singleton, "current", types::BasicObject, no_gc, leaf); @@ -420,6 +432,84 @@ fn inline_integer_xor(fun: &mut hir::Function, block: hir::BlockId, recv: hir::I None } +fn try_inline_fixnum_op(fun: &mut hir::Function, block: hir::BlockId, f: &dyn Fn(hir::InsnId, hir::InsnId) -> hir::Insn, bop: u32, left: hir::InsnId, right: hir::InsnId, state: hir::InsnId) -> Option { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, INTEGER_REDEFINED_OP_FLAG) } { + // If the basic operation is already redefined, we cannot optimize it. + return None; + } + if fun.likely_a(left, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + if bop == BOP_NEQ { + // For opt_neq, the interpreter checks that both neq and eq are unchanged. + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop: BOP_EQ }, state }); + } + // Rely on the MethodRedefined PatchPoint for other bops. + let left = fun.coerce_to(block, left, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + return Some(fun.push_insn(block, f(left, right))); + } + None +} + +fn inline_integer_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumEq { left, right }, BOP_EQ, recv, other, state) +} + +fn inline_integer_plus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAdd { left, right, state }, BOP_PLUS, recv, other, state) +} + +fn inline_integer_minus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumSub { left, right, state }, BOP_MINUS, recv, other, state) +} + +fn inline_integer_mult(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMult { left, right, state }, BOP_MULT, recv, other, state) +} + +fn inline_integer_div(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumDiv { left, right, state }, BOP_DIV, recv, other, state) +} + +fn inline_integer_mod(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMod { left, right, state }, BOP_MOD, recv, other, state) +} + +fn inline_integer_and(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAnd { left, right, }, BOP_AND, recv, other, state) +} + +fn inline_integer_or(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumOr { left, right, }, BOP_OR, recv, other, state) +} + +fn inline_integer_gt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGt { left, right }, BOP_GT, recv, other, state) +} + +fn inline_integer_ge(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGe { left, right }, BOP_GE, recv, other, state) +} + +fn inline_integer_lt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLt { left, right }, BOP_LT, recv, other, state) +} + +fn inline_integer_le(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLe { left, right }, BOP_LE, recv, other, state) +} + fn inline_basic_object_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option { let &[other] = args else { return None; }; let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); @@ -429,6 +519,10 @@ fn inline_basic_object_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hi fn inline_basic_object_neq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option { let &[other] = args else { return None; }; + let result = try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumNeq { left, right }, BOP_NEQ, recv, other, state); + if result.is_some() { + return result; + } let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { return None; diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 9b4495c6e6aeec..4df3ddbb26f162 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -9,7 +9,7 @@ use crate::{ cast::IntoUsize, codegen::local_idx_to_ep_offset, cruby::*, payload::{get_or_create_iseq_payload, IseqPayload}, options::{debug, get_option, DumpHIR}, state::ZJITState }; use std::{ - cell::RefCell, collections::{HashMap, HashSet, VecDeque}, ffi::{c_void, c_uint, CStr}, fmt::Display, mem::{align_of, size_of}, ptr, slice::Iter + cell::RefCell, collections::{HashMap, HashSet, VecDeque}, ffi::{c_void, c_uint, c_int, CStr}, fmt::Display, mem::{align_of, size_of}, ptr, slice::Iter }; use crate::hir_type::{Type, types}; use crate::bitset::BitSet; @@ -593,15 +593,21 @@ pub enum SendFallbackReason { SendWithoutBlockCfuncArrayVariadic, SendWithoutBlockNotOptimizedMethodType(MethodType), SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType), - SendWithoutBlockDirectTooManyArgs, + SendWithoutBlockBopRedefined, + SendWithoutBlockOperandsNotFixnum, SendPolymorphic, SendMegamorphic, SendNoProfiles, + SendCfuncVariadic, + SendCfuncArrayVariadic, SendNotOptimizedMethodType(MethodType), CCallWithFrameTooManyArgs, ObjToStringNotString, + TooManyArgsForLir, /// The Proc object for a BMETHOD is not defined by an ISEQ. (See `enum rb_block_type`.) BmethodNonIseqProc, + /// Caller supplies too few or too many arguments than what the callee's parameters expects. + ArgcParamMismatch, /// The call has at least one feature on the caller or callee side that the optimizer does not /// support. ComplexArgPass, @@ -1453,7 +1459,7 @@ pub enum ValidationError { MiscValidationError(InsnId, String), } -fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq_t) -> bool { +fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq_t, send_insn: InsnId, args: &[InsnId]) -> bool { let mut can_send = true; let mut count_failure = |counter| { can_send = false; @@ -1462,12 +1468,30 @@ fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq use Counter::*; if unsafe { rb_get_iseq_flags_has_rest(iseq) } { count_failure(complex_arg_pass_param_rest) } - if unsafe { rb_get_iseq_flags_has_opt(iseq) } { count_failure(complex_arg_pass_param_opt) } + if unsafe { rb_get_iseq_flags_has_post(iseq) } { count_failure(complex_arg_pass_param_post) } if unsafe { rb_get_iseq_flags_has_kw(iseq) } { count_failure(complex_arg_pass_param_kw) } if unsafe { rb_get_iseq_flags_has_kwrest(iseq) } { count_failure(complex_arg_pass_param_kwrest) } if unsafe { rb_get_iseq_flags_has_block(iseq) } { count_failure(complex_arg_pass_param_block) } if unsafe { rb_get_iseq_flags_forwardable(iseq) } { count_failure(complex_arg_pass_param_forwardable) } + if !can_send { + function.set_dynamic_send_reason(send_insn, ComplexArgPass); + return false; + } + + // Check argument count against callee's parameters. Note that correctness for this calculation + // relies on rejecting features above. + let lead_num = unsafe { get_iseq_body_param_lead_num(iseq) }; + let opt_num = unsafe { get_iseq_body_param_opt_num(iseq) }; + can_send = c_int::try_from(args.len()) + .as_ref() + .map(|argc| (lead_num..=lead_num + opt_num).contains(argc)) + .unwrap_or(false); + if !can_send { + function.set_dynamic_send_reason(send_insn, ArgcParamMismatch); + return false + } + can_send } @@ -1481,15 +1505,14 @@ pub struct Function { /// of entry block params after infer_types() fills Empty to all insn_types. param_types: Vec, - // TODO: get method name and source location from the ISEQ - insns: Vec, union_find: std::cell::RefCell>, insn_types: Vec, blocks: Vec, /// Entry block for the interpreter entry_block: BlockId, - /// Entry block for JIT-to-JIT calls + /// Entry block for JIT-to-JIT calls. Length will be `opt_num+1`, for callers + /// fulfilling `(0..=opt_num)` optional parameters. jit_entry_blocks: Vec, profiles: Option, } @@ -2048,9 +2071,8 @@ impl Function { for jit_entry_block in self.jit_entry_blocks.iter() { let entry_params = self.blocks[jit_entry_block.0].params.iter(); let param_types = self.param_types.iter(); - assert_eq!( - entry_params.len(), - param_types.len(), + assert!( + param_types.len() >= entry_params.len(), "param types should be initialized before type inference", ); for (param, param_type) in std::iter::zip(entry_params, param_types) { @@ -2245,31 +2267,10 @@ impl Function { if 0 != ci_flags & VM_CALL_FORWARDING { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_forwarding)); } } - fn try_rewrite_fixnum_op(&mut self, block: BlockId, orig_insn_id: InsnId, f: &dyn Fn(InsnId, InsnId) -> Insn, bop: u32, left: InsnId, right: InsnId, state: InsnId) { - if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, INTEGER_REDEFINED_OP_FLAG) } { - // If the basic operation is already redefined, we cannot optimize it. - self.push_insn_id(block, orig_insn_id); - return; - } - if self.likely_a(left, types::Fixnum, state) && self.likely_a(right, types::Fixnum, state) { - if bop == BOP_NEQ { - // For opt_neq, the interpreter checks that both neq and eq are unchanged. - self.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop: BOP_EQ }, state }); - } - self.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop }, state }); - let left = self.coerce_to(block, left, types::Fixnum, state); - let right = self.coerce_to(block, right, types::Fixnum, state); - let result = self.push_insn(block, f(left, right)); - self.make_equal_to(orig_insn_id, result); - self.insn_types[result.0] = self.infer_type(result); - } else { - self.push_insn_id(block, orig_insn_id); - } - } - fn rewrite_if_frozen(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, klass: u32, bop: u32, state: InsnId) { if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { // If the basic operation is already redefined, we cannot optimize it. + self.set_dynamic_send_reason(orig_insn_id, SendWithoutBlockBopRedefined); self.push_insn_id(block, orig_insn_id); return; } @@ -2313,32 +2314,6 @@ impl Function { assert!(self.blocks[block.0].insns.is_empty()); for insn_id in old_insns { match self.find(insn_id) { - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(plus) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumAdd { left, right, state }, BOP_PLUS, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minus) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumSub { left, right, state }, BOP_MINUS, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(mult) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumMult { left, right, state }, BOP_MULT, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(div) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumDiv { left, right, state }, BOP_DIV, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(modulo) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumMod { left, right, state }, BOP_MOD, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(eq) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumEq { left, right }, BOP_EQ, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(neq) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumNeq { left, right }, BOP_NEQ, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(lt) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumLt { left, right }, BOP_LT, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(le) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumLe { left, right }, BOP_LE, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(gt) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumGt { left, right }, BOP_GT, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(ge) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumGe { left, right }, BOP_GE, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(and) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumAnd { left, right }, BOP_AND, recv, args[0], state), - Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(or) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumOr { left, right }, BOP_OR, recv, args[0], state), Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(freeze) && args.is_empty() => self.try_rewrite_freeze(block, insn_id, recv, state), Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => @@ -2403,8 +2378,7 @@ impl Function { // Only specialize positional-positional calls // TODO(max): Handle other kinds of parameter passing let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; - if !can_direct_send(self, block, iseq) { - self.set_dynamic_send_reason(insn_id, ComplexArgPass); + if !can_direct_send(self, block, iseq, insn_id, args.as_slice()) { self.push_insn_id(block, insn_id); continue; } self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); @@ -2429,8 +2403,7 @@ impl Function { let capture = unsafe { proc_block.as_.captured.as_ref() }; let iseq = unsafe { *capture.code.iseq.as_ref() }; - if !can_direct_send(self, block, iseq) { - self.set_dynamic_send_reason(insn_id, ComplexArgPass); + if !can_direct_send(self, block, iseq, insn_id, args.as_slice()) { self.push_insn_id(block, insn_id); continue; } // Can't pass a block to a block for now @@ -2844,10 +2817,12 @@ impl Function { // Do method lookup let cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; if cme.is_null() { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedMethodType(MethodType::Null)); return Err(()); } // Filter for C methods + // TODO(max): Handle VM_METHOD_TYPE_ALIAS let def_type = unsafe { get_cme_def_type(cme) }; if def_type != VM_METHOD_TYPE_CFUNC { return Err(()); @@ -2911,10 +2886,12 @@ impl Function { // Variadic method -1 => { // func(int argc, VALUE *argv, VALUE recv) + fun.set_dynamic_send_reason(send_insn_id, SendCfuncVariadic); Err(()) } -2 => { // (self, args_ruby_array) + fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic); Err(()) } _ => unreachable!("unknown cfunc kind: argc={argc}") @@ -2949,6 +2926,7 @@ impl Function { // Do method lookup let mut cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; if cme.is_null() { + fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::Null)); return Err(()); } @@ -3010,6 +2988,10 @@ impl Function { fun.blocks[block.0].insns.extend(insns); fun.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } fun.remove_block(tmp_block); return Ok(()); } @@ -3083,6 +3065,10 @@ impl Function { fun.blocks[block.0].insns.extend(insns); fun.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } fun.remove_block(tmp_block); return Ok(()); } @@ -4310,7 +4296,8 @@ fn insn_idx_at_offset(idx: u32, offset: i64) -> u32 { } /// List of insn_idx that starts a JIT entry block -fn jit_entry_insns(iseq: IseqPtr) -> Vec { +pub fn jit_entry_insns(iseq: IseqPtr) -> Vec { + // TODO(alan): Make an iterator type for this instead of copying all of the opt_table each call let opt_num = unsafe { get_iseq_body_param_opt_num(iseq) }; if opt_num > 0 { let mut result = vec![]; @@ -4320,10 +4307,6 @@ fn jit_entry_insns(iseq: IseqPtr) -> Vec { let insn_idx = unsafe { opt_table.offset(opt_idx).read().as_u32() }; result.push(insn_idx); } - - // Deduplicate entries with HashSet since opt_table may have duplicated entries, e.g. proc { |a=a| a } - result.sort(); - result.dedup(); result } else { vec![0] @@ -4389,7 +4372,6 @@ pub enum ParseError { StackUnderflow(FrameState), MalformedIseq(u32), // insn_idx into iseq_encoded Validation(ValidationError), - FailedOptionalArguments, NotAllowed, } @@ -4470,28 +4452,45 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { // Compute a map of PC->Block by finding jump targets let jit_entry_insns = jit_entry_insns(iseq); let BytecodeInfo { jump_targets, has_blockiseq } = compute_bytecode_info(iseq, &jit_entry_insns); + + // Make all empty basic blocks. The ordering of the BBs matters as it is taken as a schedule + // in the backend without a scheduling pass. TODO: Higher quality scheduling during lowering. let mut insn_idx_to_block = HashMap::new(); + // Make blocks for optionals first, and put them right next to their JIT entrypoint + for insn_idx in jit_entry_insns.iter().copied() { + let jit_entry_block = fun.new_block(insn_idx); + fun.jit_entry_blocks.push(jit_entry_block); + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); + } + // Make blocks for the rest of the jump targets for insn_idx in jump_targets { - // Prepend a JIT entry block if it's a jit_entry_insn. - // compile_entry_block() assumes that a JIT entry block jumps to the next block. - if jit_entry_insns.contains(&insn_idx) { - let jit_entry_block = fun.new_block(insn_idx); - fun.jit_entry_blocks.push(jit_entry_block); - } - insn_idx_to_block.insert(insn_idx, fun.new_block(insn_idx)); + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); + } + // Done, drop `mut`. + let insn_idx_to_block = insn_idx_to_block; + + // Compile an entry_block for the interpreter + compile_entry_block(&mut fun, jit_entry_insns.as_slice(), &insn_idx_to_block); + + // Compile all JIT-to-JIT entry blocks + for (jit_entry_idx, insn_idx) in jit_entry_insns.iter().enumerate() { + let target_block = insn_idx_to_block.get(insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + compile_jit_entry_block(&mut fun, jit_entry_idx, target_block); } // Check if the EP is escaped for the ISEQ from the beginning. We give up // optimizing locals in that case because they're shared with other frames. let ep_escaped = iseq_escapes_ep(iseq); - // Compile an entry_block for the interpreter - compile_entry_block(&mut fun, &jit_entry_insns); - - // Iteratively fill out basic blocks using a queue + // Iteratively fill out basic blocks using a queue. // TODO(max): Basic block arguments at edges let mut queue = VecDeque::new(); - queue.push_back((FrameState::new(iseq), insn_idx_to_block[&0], /*insn_idx=*/0, /*local_inval=*/false)); + for &insn_idx in jit_entry_insns.iter() { + queue.push_back((FrameState::new(iseq), insn_idx_to_block[&insn_idx], /*insn_idx=*/insn_idx, /*local_inval=*/false)); + } // Keep compiling blocks until the queue becomes empty let mut visited = HashSet::new(); @@ -4501,16 +4500,11 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { if visited.contains(&block) { continue; } visited.insert(block); - // Compile a JIT entry to the block if it's a jit_entry_insn - if let Some(block_idx) = jit_entry_insns.iter().position(|&idx| idx == insn_idx) { - compile_jit_entry_block(&mut fun, block_idx, block); - } - // Load basic block params first let self_param = fun.push_insn(block, Insn::Param); let mut state = { let mut result = FrameState::new(iseq); - let local_size = if insn_idx == 0 { num_locals(iseq) } else { incoming_state.locals.len() }; + let local_size = if jit_entry_insns.contains(&insn_idx) { num_locals(iseq) } else { incoming_state.locals.len() }; for _ in 0..local_size { result.locals.push(fun.push_insn(block, Insn::Param)); } @@ -5362,14 +5356,6 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { } } - // Bail out if there's a JIT entry block whose target block was not compiled - // due to an unsupported instruction in the middle of the ISEQ. - for jit_entry_block in fun.jit_entry_blocks.iter() { - if fun.blocks[jit_entry_block.0].insns.is_empty() { - return Err(ParseError::FailedOptionalArguments); - } - } - fun.set_param_types(); fun.infer_types(); @@ -5389,44 +5375,46 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { } /// Compile an entry_block for the interpreter -fn compile_entry_block(fun: &mut Function, jit_entry_insns: &[u32]) { +fn compile_entry_block(fun: &mut Function, jit_entry_insns: &[u32], insn_idx_to_block: &HashMap) { let entry_block = fun.entry_block; - fun.push_insn(entry_block, Insn::EntryPoint { jit_entry_idx: None }); - - // Prepare entry_state with basic block params - let (self_param, entry_state) = compile_entry_state(fun, entry_block); - - // Jump to target blocks + let (self_param, entry_state) = compile_entry_state(fun); let mut pc: Option = None; - let &last_entry_insn = jit_entry_insns.last().unwrap(); - for (jit_entry_block_idx, &jit_entry_insn) in jit_entry_insns.iter().enumerate() { - let jit_entry_block = fun.jit_entry_blocks[jit_entry_block_idx]; - let target_block = BlockId(jit_entry_block.0 + 1); // jit_entry_block precedes the jump target block - - if jit_entry_insn == last_entry_insn { - // If it's the last possible entry, jump to the target_block without checking PC - fun.push_insn(entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); - } else { - // Otherwise, jump to the target_block only if PC matches. - let pc = pc.unwrap_or_else(|| { - let insn_id = fun.push_insn(entry_block, Insn::LoadPC); - pc = Some(insn_id); - insn_id - }); - let expected_pc = fun.push_insn(entry_block, Insn::Const { - val: Const::CPtr(unsafe { rb_iseq_pc_at_idx(fun.iseq, jit_entry_insn) } as *const u8), - }); - let test_id = fun.push_insn(entry_block, Insn::IsBitEqual { left: pc, right: expected_pc }); - fun.push_insn(entry_block, Insn::IfTrue { - val: test_id, - target: BranchEdge { target: target_block, args: entry_state.as_args(self_param) }, - }); + let &all_opts_passed_insn_idx = jit_entry_insns.last().unwrap(); + + // Check-and-jump for each missing optional PC + for &jit_entry_insn in jit_entry_insns.iter() { + if jit_entry_insn == all_opts_passed_insn_idx { + continue; } + let target_block = insn_idx_to_block.get(&jit_entry_insn) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + // Load PC once at the start of the block, shared among all cases + let pc = *pc.get_or_insert_with(|| fun.push_insn(entry_block, Insn::LoadPC)); + let expected_pc = fun.push_insn(entry_block, Insn::Const { + val: Const::CPtr(unsafe { rb_iseq_pc_at_idx(fun.iseq, jit_entry_insn) } as *const u8), + }); + let test_id = fun.push_insn(entry_block, Insn::IsBitEqual { left: pc, right: expected_pc }); + fun.push_insn(entry_block, Insn::IfTrue { + val: test_id, + target: BranchEdge { target: target_block, args: entry_state.as_args(self_param) }, + }); } + + // Terminate the block with a jump to the block with all optionals passed + let target_block = insn_idx_to_block.get(&all_opts_passed_insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + fun.push_insn(entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); } /// Compile initial locals for an entry_block for the interpreter -fn compile_entry_state(fun: &mut Function, entry_block: BlockId) -> (InsnId, FrameState) { +fn compile_entry_state(fun: &mut Function) -> (InsnId, FrameState) { + let entry_block = fun.entry_block; + fun.push_insn(entry_block, Insn::EntryPoint { jit_entry_idx: None }); + let iseq = fun.iseq; let param_size = unsafe { get_iseq_body_param_size(iseq) }.to_usize(); let rest_param_idx = iseq_rest_param_idx(iseq); @@ -5452,21 +5440,27 @@ fn compile_jit_entry_block(fun: &mut Function, jit_entry_idx: usize, target_bloc fun.push_insn(jit_entry_block, Insn::EntryPoint { jit_entry_idx: Some(jit_entry_idx) }); // Prepare entry_state with basic block params - let (self_param, entry_state) = compile_jit_entry_state(fun, jit_entry_block); + let (self_param, entry_state) = compile_jit_entry_state(fun, jit_entry_block, jit_entry_idx); // Jump to target_block fun.push_insn(jit_entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); } /// Compile params and initial locals for a jit_entry_block -fn compile_jit_entry_state(fun: &mut Function, jit_entry_block: BlockId) -> (InsnId, FrameState) { +fn compile_jit_entry_state(fun: &mut Function, jit_entry_block: BlockId, jit_entry_idx: usize) -> (InsnId, FrameState) { let iseq = fun.iseq; let param_size = unsafe { get_iseq_body_param_size(iseq) }.to_usize(); + let opt_num: usize = unsafe { get_iseq_body_param_opt_num(iseq) }.try_into().expect("iseq param opt_num >= 0"); + let lead_num: usize = unsafe { get_iseq_body_param_lead_num(iseq) }.try_into().expect("iseq param lead_num >= 0"); + let passed_opt_num = jit_entry_idx; let self_param = fun.push_insn(jit_entry_block, Insn::Param); let mut entry_state = FrameState::new(iseq); for local_idx in 0..num_locals(iseq) { - if local_idx < param_size { + if (lead_num + passed_opt_num..lead_num + opt_num).contains(&local_idx) { + // Omitted optionals are locals, so they start as nils before their code run + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } else if local_idx < param_size { entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Param)); } else { entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); @@ -5889,13 +5883,15 @@ mod graphviz_tests { bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject)  PatchPoint NoTracePoint  PatchPoint NoTracePoint  - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, 29)  - v25:Fixnum = GuardType v11, Fixnum  - v26:Fixnum = GuardType v12, Fixnum  - v27:Fixnum = FixnumOr v25, v26  + PatchPoint NoTracePoint  + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010)  + v26:Fixnum = GuardType v11, Fixnum  + v27:Fixnum = GuardType v12, Fixnum  + v28:Fixnum = FixnumOr v26, v27  + IncrCounter inline_cfunc_optimized_send_count  PatchPoint NoTracePoint  CheckInterrupts  - Return v27  + Return v28  >]; } "#); diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 55123f47aa9416..2c56120bf621be 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -109,13 +109,15 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[1] = Const Value(1) v12:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v29:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v33:Fixnum[3] = Const Value(3) + IncrCounter inline_cfunc_optimized_send_count v17:Fixnum[3] = Const Value(3) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v30:Fixnum[6] = Const Value(6) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v34:Fixnum[6] = Const Value(6) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v30 + Return v34 "); } @@ -138,13 +140,15 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[5] = Const Value(5) v12:Fixnum[3] = Const Value(3) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v29:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v33:Fixnum[2] = Const Value(2) + IncrCounter inline_cfunc_optimized_send_count v17:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v30:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v34:Fixnum[1] = Const Value(1) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v30 + Return v34 "); } @@ -167,10 +171,11 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[0] = Const Value(0) v12:Fixnum[1073741825] = Const Value(1073741825) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v22:Fixnum[-1073741825] = Const Value(-1073741825) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v24:Fixnum[-1073741825] = Const Value(-1073741825) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v22 + Return v24 "); } @@ -193,10 +198,11 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[6] = Const Value(6) v12:Fixnum[7] = Const Value(7) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v22:Fixnum[42] = Const Value(42) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v24:Fixnum[42] = Const Value(42) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v22 + Return v24 "); } @@ -220,17 +226,20 @@ mod hir_opt_tests { Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): v13:Fixnum[0] = Const Value(0) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v32:Fixnum = GuardType v9, Fixnum - v39:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v33:Fixnum = GuardType v9, Fixnum + v45:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count v20:Fixnum[0] = Const Value(0) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v35:Fixnum = GuardType v9, Fixnum - v40:Fixnum[0] = Const Value(0) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v41:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v38:Fixnum = GuardType v9, Fixnum + v46:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count + PatchPoint MethodRedefined(Integer@0x1000, +@0x1038, cme:0x1040) + v47:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v41 + Return v47 "); } @@ -257,8 +266,9 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[1] = Const Value(1) v12:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v38:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v40:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v22:Fixnum[3] = Const Value(3) CheckInterrupts @@ -289,13 +299,15 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[1] = Const Value(1) v12:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LE) - v51:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v55:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v21:Fixnum[2] = Const Value(2) v23:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LE) - v53:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v57:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v33:Fixnum[3] = Const Value(3) CheckInterrupts @@ -326,8 +338,9 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[2] = Const Value(2) v12:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GT) - v38:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) + v40:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v22:Fixnum[3] = Const Value(3) CheckInterrupts @@ -358,13 +371,15 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[2] = Const Value(2) v12:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GE) - v51:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v55:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v21:Fixnum[2] = Const Value(2) v23:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GE) - v53:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v57:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v33:Fixnum[3] = Const Value(3) CheckInterrupts @@ -395,8 +410,9 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[1] = Const Value(1) v12:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v38:FalseClass = Const Value(false) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v40:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v31:Fixnum[4] = Const Value(4) CheckInterrupts @@ -427,8 +443,9 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[2] = Const Value(2) v12:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v38:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v40:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v22:Fixnum[3] = Const Value(3) CheckInterrupts @@ -459,9 +476,10 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[1] = Const Value(1) v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_NEQ) - v39:TrueClass = Const Value(true) + v41:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v22:Fixnum[3] = Const Value(3) CheckInterrupts @@ -492,9 +510,10 @@ mod hir_opt_tests { bb2(v6:BasicObject): v10:Fixnum[2] = Const Value(2) v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_NEQ) - v39:FalseClass = Const Value(false) + v41:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts v31:Fixnum[4] = Const Value(4) CheckInterrupts @@ -568,11 +587,12 @@ mod hir_opt_tests { Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): v14:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v23:Fixnum = GuardType v9, Fixnum - v24:Fixnum = FixnumAdd v23, v14 + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v24, v14 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v24 + Return v25 "); } @@ -919,6 +939,87 @@ mod hir_opt_tests { "); } + #[test] + fn test_optimize_send_direct_no_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + PatchPoint NoSingletonClass(Object@0x1000) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_direct_one_optional_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + PatchPoint NoSingletonClass(Object@0x1000) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_direct_all_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3, 4 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[4] = Const Value(4) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + PatchPoint NoSingletonClass(Object@0x1000) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + CheckInterrupts + Return v23 + "); + } + #[test] fn test_optimize_variadic_ccall() { eval(" @@ -971,9 +1072,12 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - v19:BasicObject = SendWithoutBlock v11, :+, v12 + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + IncrCounter inline_iseq_optimized_send_count + v28:Fixnum[100] = Const Value(100) CheckInterrupts - Return v19 + Return v28 "); } @@ -995,12 +1099,13 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v25:Fixnum = GuardType v11, Fixnum - v26:Fixnum = GuardType v12, Fixnum - v27:Fixnum = FixnumAdd v25, v26 + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumAdd v26, v27 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v27 + Return v28 "); } @@ -1022,11 +1127,12 @@ mod hir_opt_tests { Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): v14:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v23:Fixnum = GuardType v9, Fixnum - v24:Fixnum = FixnumAdd v23, v14 + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v24, v14 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v24 + Return v25 "); } @@ -1048,11 +1154,12 @@ mod hir_opt_tests { Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): v13:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v23:Fixnum = GuardType v9, Fixnum - v24:Fixnum = FixnumAdd v13, v23 + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v13, v24 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v24 + Return v25 "); } @@ -1074,12 +1181,13 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v25:Fixnum = GuardType v11, Fixnum - v26:Fixnum = GuardType v12, Fixnum - v27:BoolExact = FixnumLt v25, v26 + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:BoolExact = FixnumLt v26, v27 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v27 + Return v28 "); } @@ -1101,11 +1209,12 @@ mod hir_opt_tests { Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): v14:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v23:Fixnum = GuardType v9, Fixnum - v24:BoolExact = FixnumLt v23, v14 + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BoolExact = FixnumLt v24, v14 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v24 + Return v25 "); } @@ -1127,11 +1236,12 @@ mod hir_opt_tests { Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): v13:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v23:Fixnum = GuardType v9, Fixnum - v24:BoolExact = FixnumLt v13, v23 + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BoolExact = FixnumLt v13, v24 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v24 + Return v25 "); } @@ -1674,9 +1784,10 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1704,9 +1815,10 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1734,9 +1846,10 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1764,10 +1877,11 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_DIV) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum - v31:Fixnum = FixnumDiv v29, v30 + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + v32:Fixnum = FixnumDiv v30, v31 + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1795,10 +1909,11 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MOD) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum - v31:Fixnum = FixnumMod v29, v30 + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + v32:Fixnum = FixnumMod v30, v31 + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1826,9 +1941,10 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1856,9 +1972,10 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LE) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1886,9 +2003,10 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GT) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum + PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1916,9 +2034,10 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GE) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1946,9 +2065,10 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v29:Fixnum = GuardType v11, Fixnum - v30:Fixnum = GuardType v12, Fixnum + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -1976,10 +2096,11 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_NEQ) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) v30:Fixnum = GuardType v11, Fixnum - v31:Fixnum = GuardType v12, Fixnum + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v32:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count v23:Fixnum[5] = Const Value(5) CheckInterrupts Return v23 @@ -2549,10 +2670,12 @@ mod hir_opt_tests { Jump bb2(v4) bb2(v6:BasicObject): v11:Fixnum[1] = Const Value(1) - IncrCounter complex_arg_pass_param_opt - v13:BasicObject = SendWithoutBlock v6, :foo, v11 + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + PatchPoint NoSingletonClass(Object@0x1000) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v11 CheckInterrupts - Return v13 + Return v21 "); } @@ -2640,6 +2763,31 @@ mod hir_opt_tests { "); } + #[test] + fn dont_specialize_call_to_post_param_iseq() { + eval(" + def foo(opt=80, post) = post + def test = foo(10) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[10] = Const Value(10) + IncrCounter complex_arg_pass_param_post + v13:BasicObject = SendWithoutBlock v6, :foo, v11 + CheckInterrupts + Return v13 + "); + } + #[test] fn dont_specialize_call_to_iseq_with_kw() { eval(" @@ -2692,6 +2840,56 @@ mod hir_opt_tests { "); } + #[test] + fn dont_specialize_call_to_iseq_with_param_kw() { + eval(" + def foo(int: 1) = int + 1 + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter complex_arg_pass_param_kw + v11:BasicObject = SendWithoutBlock v6, :foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_param_kwrest() { + eval(" + def foo(**kwargs) = kwargs.keys + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter complex_arg_pass_param_kwrest + v11:BasicObject = SendWithoutBlock v6, :foo + CheckInterrupts + Return v11 + "); + } + #[test] fn dont_replace_get_constant_path_with_empty_ic() { eval(" @@ -2946,7 +3144,6 @@ mod hir_opt_tests { v13:NilClass = Const Value(nil) PatchPoint MethodRedefined(Hash@0x1008, new@0x1010, cme:0x1018) v46:HashExact = ObjectAllocClass Hash:VALUE(0x1008) - IncrCounter complex_arg_pass_param_opt IncrCounter complex_arg_pass_param_kw IncrCounter complex_arg_pass_param_block v20:BasicObject = SendWithoutBlock v46, :initialize @@ -4566,12 +4763,13 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, 28) - v25:Fixnum = GuardType v11, Fixnum - v26:Fixnum = GuardType v12, Fixnum - v27:Fixnum = FixnumAnd v25, v26 + PatchPoint MethodRedefined(Integer@0x1000, &@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumAnd v26, v27 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v27 + Return v28 "); } @@ -4594,12 +4792,13 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v6, v7, v8) bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, 29) - v25:Fixnum = GuardType v11, Fixnum - v26:Fixnum = GuardType v12, Fixnum - v27:Fixnum = FixnumOr v25, v26 + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumOr v26, v27 + IncrCounter inline_cfunc_optimized_send_count CheckInterrupts - Return v27 + Return v28 "); } @@ -6141,6 +6340,55 @@ mod hir_opt_tests { "); } + #[test] + fn test_dont_optimize_when_passing_too_many_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo(3, 3, 3) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[3] = Const Value(3) + v14:Fixnum[3] = Const Value(3) + v16:Fixnum[3] = Const Value(3) + v18:BasicObject = SendWithoutBlock v10, :foo, v12, v14, v16 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_few_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:BasicObject = SendWithoutBlock v10, :foo + CheckInterrupts + Return v12 + "); + } + #[test] fn test_dont_inline_integer_succ_with_args() { eval(" diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 4caf0f838fac43..abf2f9497c2875 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -133,15 +133,16 @@ pub mod hir_build_tests { v5:CBool = IsBitEqual v3, v4 IfTrue v5, bb2(v1, v2) Jump bb4(v1, v2) - bb1(v9:BasicObject, v10:BasicObject): + bb1(v9:BasicObject): EntryPoint JIT(0) + v10:NilClass = Const Value(nil) Jump bb2(v9, v10) - bb2(v12:BasicObject, v13:BasicObject): - v16:Fixnum[1] = Const Value(1) - Jump bb4(v12, v16) - bb3(v20:BasicObject, v21:BasicObject): + bb2(v16:BasicObject, v17:BasicObject): + v20:Fixnum[1] = Const Value(1) + Jump bb4(v16, v20) + bb3(v13:BasicObject, v14:BasicObject): EntryPoint JIT(1) - Jump bb4(v20, v21) + Jump bb4(v13, v14) bb4(v23:BasicObject, v24:BasicObject): v28:Fixnum[123] = Const Value(123) CheckInterrupts @@ -806,17 +807,18 @@ pub mod hir_build_tests { v6:CBool = IsBitEqual v4, v5 IfTrue v6, bb2(v1, v2, v3) Jump bb4(v1, v2, v3) - bb1(v10:BasicObject, v11:BasicObject): + bb1(v10:BasicObject): EntryPoint JIT(0) + v11:NilClass = Const Value(nil) v12:NilClass = Const Value(nil) Jump bb2(v10, v11, v12) - bb2(v14:BasicObject, v15:BasicObject, v16:NilClass): - v20:Fixnum[1] = Const Value(1) - Jump bb4(v14, v20, v20) - bb3(v26:BasicObject, v27:BasicObject): + bb2(v19:BasicObject, v20:BasicObject, v21:NilClass): + v25:Fixnum[1] = Const Value(1) + Jump bb4(v19, v25, v25) + bb3(v15:BasicObject, v16:BasicObject): EntryPoint JIT(1) - v28:NilClass = Const Value(nil) - Jump bb4(v26, v27, v28) + v17:NilClass = Const Value(nil) + Jump bb4(v15, v16, v17) bb4(v30:BasicObject, v31:BasicObject, v32:NilClass|Fixnum): v38:ArrayExact = NewArray v31, v32 CheckInterrupts @@ -831,7 +833,34 @@ pub mod hir_build_tests { TracePoint.new(:line) {}.enable test "); - assert_compile_fails("test", ParseError::FailedOptionalArguments); + assert_snapshot!(hir_string("test"), @r" + fn test@:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal l0, SP@5 + v3:NilClass = Const Value(nil) + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v6:CBool = IsBitEqual v4, v5 + IfTrue v6, bb2(v1, v2, v3) + Jump bb4(v1, v2, v3) + bb1(v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v10, v11, v12) + bb2(v19:BasicObject, v20:BasicObject, v21:NilClass): + SideExit UnhandledYARVInsn(trace_putobject_INT2FIX_1_) + bb3(v15:BasicObject, v16:BasicObject): + EntryPoint JIT(1) + v17:NilClass = Const Value(nil) + Jump bb4(v15, v16, v17) + bb4(v26:BasicObject, v27:BasicObject, v28:NilClass): + v34:ArrayExact = NewArray v27, v28 + CheckInterrupts + Return v34 + "); } #[test] @@ -839,7 +868,30 @@ pub mod hir_build_tests { eval(" def test(a = (def foo = nil)) = a "); - assert_compile_fails("test", ParseError::FailedOptionalArguments); + assert_snapshot!(hir_string("test"), @r" + fn test@:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal l0, SP@4 + v3:CPtr = LoadPC + v4:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v5:CBool = IsBitEqual v3, v4 + IfTrue v5, bb2(v1, v2) + Jump bb4(v1, v2) + bb1(v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v9, v10) + bb2(v16:BasicObject, v17:BasicObject): + SideExit UnhandledYARVInsn(definemethod) + bb3(v13:BasicObject, v14:BasicObject): + EntryPoint JIT(1) + Jump bb4(v13, v14) + bb4(v22:BasicObject, v23:BasicObject): + CheckInterrupts + Return v23 + "); } #[test] @@ -854,12 +906,16 @@ pub mod hir_build_tests { v1:BasicObject = LoadSelf v2:BasicObject = GetLocal l0, SP@4 Jump bb2(v1, v2) - bb1(v5:BasicObject, v6:BasicObject): + bb1(v5:BasicObject): EntryPoint JIT(0) + v6:NilClass = Const Value(nil) Jump bb2(v5, v6) - bb2(v8:BasicObject, v9:BasicObject): + bb3(v9:BasicObject, v10:BasicObject): + EntryPoint JIT(1) + Jump bb2(v9, v10) + bb2(v12:BasicObject, v13:BasicObject): CheckInterrupts - Return v9 + Return v13 "); } diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 5103a549ba8e93..b0ca28d258506a 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -178,17 +178,22 @@ make_counters! { send_fallback_send_without_block_cfunc_array_variadic, send_fallback_send_without_block_not_optimized_method_type, send_fallback_send_without_block_not_optimized_method_type_optimized, - send_fallback_send_without_block_direct_too_many_args, + send_fallback_too_many_args_for_lir, + send_fallback_send_without_block_bop_redefined, + send_fallback_send_without_block_operands_not_fixnum, send_fallback_send_polymorphic, send_fallback_send_megamorphic, send_fallback_send_no_profiles, send_fallback_send_not_optimized_method_type, send_fallback_ccall_with_frame_too_many_args, + send_fallback_argc_param_mismatch, // The call has at least one feature on the caller or callee side // that the optimizer does not support. send_fallback_one_or_more_complex_arg_pass, send_fallback_bmethod_non_iseq_proc, send_fallback_obj_to_string_not_string, + send_fallback_send_cfunc_variadic, + send_fallback_send_cfunc_array_variadic, send_fallback_uncategorized, } @@ -210,7 +215,6 @@ make_counters! { compile_error_register_spill_on_alloc, compile_error_parse_stack_underflow, compile_error_parse_malformed_iseq, - compile_error_parse_failed_optional_arguments, compile_error_parse_not_allowed, compile_error_validation_block_has_no_terminator, compile_error_validation_terminator_not_at_end, @@ -274,7 +278,7 @@ make_counters! { // Unsupported parameter features complex_arg_pass_param_rest, - complex_arg_pass_param_opt, + complex_arg_pass_param_post, complex_arg_pass_param_kw, complex_arg_pass_param_kwrest, complex_arg_pass_param_block, @@ -354,7 +358,6 @@ pub enum CompileError { ExceptionHandler, OutOfMemory, ParseError(ParseError), - JitToJitOptional, } /// Return a raw pointer to the exit counter for a given CompileError @@ -367,11 +370,9 @@ pub fn exit_counter_for_compile_error(compile_error: &CompileError) -> Counter { IseqStackTooLarge => compile_error_iseq_stack_too_large, ExceptionHandler => compile_error_exception_handler, OutOfMemory => compile_error_out_of_memory, - JitToJitOptional => compile_error_jit_to_jit_optional, ParseError(parse_error) => match parse_error { StackUnderflow(_) => compile_error_parse_stack_underflow, MalformedIseq(_) => compile_error_parse_malformed_iseq, - FailedOptionalArguments => compile_error_parse_failed_optional_arguments, NotAllowed => compile_error_parse_not_allowed, Validation(validation) => match validation { BlockHasNoTerminator(_) => compile_error_validation_block_has_no_terminator, @@ -472,11 +473,16 @@ pub fn send_fallback_counter(reason: crate::hir::SendFallbackReason) -> Counter SendWithoutBlockNotOptimizedMethodType(_) => send_fallback_send_without_block_not_optimized_method_type, SendWithoutBlockNotOptimizedMethodTypeOptimized(_) => send_fallback_send_without_block_not_optimized_method_type_optimized, - SendWithoutBlockDirectTooManyArgs => send_fallback_send_without_block_direct_too_many_args, + TooManyArgsForLir => send_fallback_too_many_args_for_lir, + SendWithoutBlockBopRedefined => send_fallback_send_without_block_bop_redefined, + SendWithoutBlockOperandsNotFixnum => send_fallback_send_without_block_operands_not_fixnum, SendPolymorphic => send_fallback_send_polymorphic, SendMegamorphic => send_fallback_send_megamorphic, SendNoProfiles => send_fallback_send_no_profiles, + SendCfuncVariadic => send_fallback_send_cfunc_variadic, + SendCfuncArrayVariadic => send_fallback_send_cfunc_array_variadic, ComplexArgPass => send_fallback_one_or_more_complex_arg_pass, + ArgcParamMismatch => send_fallback_argc_param_mismatch, BmethodNonIseqProc => send_fallback_bmethod_non_iseq_proc, SendNotOptimizedMethodType(_) => send_fallback_send_not_optimized_method_type, CCallWithFrameTooManyArgs => send_fallback_ccall_with_frame_too_many_args,