diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs index 86176c0ec9bae5..9b792f5f376325 100644 --- a/zjit/src/asm/mod.rs +++ b/zjit/src/asm/mod.rs @@ -16,7 +16,7 @@ pub mod x86_64; pub mod arm64; /// Index to a label created by cb.new_label() -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] pub struct Label(pub usize); /// The object that knows how to encode the branch instruction. diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 428d4bff779084..79c07d42ada67c 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -2,6 +2,7 @@ use std::mem::take; use crate::asm::{CodeBlock, Label}; use crate::asm::arm64::*; +use crate::codegen::split_patch_point; use crate::cruby::*; use crate::backend::lir::*; use crate::options::asm_dump; @@ -826,6 +827,14 @@ impl Assembler { *opnds = vec![]; asm.push_insn(insn); } + // For compile_exits, support splitting simple return values here + Insn::CRet(opnd) => { + match opnd { + Opnd::Reg(C_RET_REG) => {}, + _ => asm.load_into(C_RET_OPND, *opnd), + } + asm.cret(C_RET_OPND); + } Insn::Lea { opnd, out } => { *opnd = split_only_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); let mem_out = split_memory_write(out, SCRATCH0_OPND); @@ -894,6 +903,9 @@ impl Assembler { } } } + &mut Insn::PatchPoint { ref target, invariant, payload } => { + split_patch_point(asm, target, invariant, payload); + } _ => { asm.push_insn(insn); } @@ -1514,7 +1526,7 @@ impl Assembler { Insn::Jonz(opnd, target) => { emit_cmp_zero_jump(cb, opnd.into(), false, target.clone()); }, - Insn::PatchPoint(_) | + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in arm64_scratch_split"), Insn::PadPatchPoint => { // If patch points are too close to each other or the end of the block, fill nop instructions if let Some(last_patch_pos) = last_patch_pos { @@ -1694,7 +1706,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, pc: 0 as _, stack: vec![], locals: vec![], label: None }; + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } }; asm.push_insn(Insn::Joz(val64, side_exit)); asm.parallel_mov(vec![(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)), (C_ARG_OPNDS[1], Opnd::mem(64, SP, -8))]); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 69b030608be776..cb8382a43c940c 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -6,9 +6,10 @@ use std::rc::Rc; use std::sync::{Arc, Mutex}; use crate::codegen::local_size_and_idx_to_ep_offset; use crate::cruby::{Qundef, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; -use crate::hir::SideExitReason; +use crate::hir::{Invariant, SideExitReason}; use crate::options::{TraceExits, debug, get_option}; use crate::cruby::VALUE; +use crate::payload::IseqPayload; use crate::stats::{exit_counter_ptr, exit_counter_ptr_for_opcode, side_exit_counter, CompileError}; use crate::virtualmem::CodePtr; use crate::asm::{CodeBlock, Label}; @@ -25,7 +26,7 @@ pub use crate::backend::current::{ pub static JIT_PRESERVED_REGS: &[Opnd] = &[CFP, SP, EC]; // Memory operand base -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum MemBase { /// Register: Every Opnd::Mem should have MemBase::Reg as of emit. @@ -37,7 +38,7 @@ pub enum MemBase } // Memory location -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct Mem { // Base register number or instruction index @@ -87,7 +88,7 @@ impl fmt::Debug for Mem { } /// Operand to an IR instruction -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] pub enum Opnd { None, // For insns with no output @@ -298,6 +299,14 @@ impl From for Opnd { } } +/// Context for a side exit. If `SideExit` matches, it reuses the same code. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct SideExit { + pub pc: Opnd, + pub stack: Vec, + pub locals: Vec, +} + /// Branch target (something that we can jump to) /// for branch instructions #[derive(Clone, Debug)] @@ -309,13 +318,10 @@ pub enum Target Label(Label), /// Side exit to the interpreter SideExit { - pc: *const VALUE, - stack: Vec, - locals: Vec, - /// We use this to enrich asm comments. + /// Context used for compiling the side exit + exit: SideExit, + /// We use this to increment exit counters reason: SideExitReason, - /// Some if the side exit should write this label. We use it for patch points. - label: Option