From 27a749b4bfbd1500b72044189afd2da8d65d7843 Mon Sep 17 00:00:00 2001 From: Demilade Sonuga Date: Sat, 23 Aug 2025 20:41:50 +0100 Subject: [PATCH 1/3] Get rid of mismatched_lifetime_syntaxes warning --- src/fastalloc/vregset.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fastalloc/vregset.rs b/src/fastalloc/vregset.rs index 77287dcc..1d08bf5d 100644 --- a/src/fastalloc/vregset.rs +++ b/src/fastalloc/vregset.rs @@ -58,7 +58,7 @@ impl VRegSet { self.items[self.head.index()].next == self.head } - pub fn iter(&self) -> VRegSetIter { + pub fn iter(&self) -> VRegSetIter<'_> { VRegSetIter { curr_item: self.items[self.head.index()].next, head: self.head, From 9e55e261496f6dda6a7bdebf09a42bf6013f0abd Mon Sep 17 00:00:00 2001 From: Demilade Sonuga Date: Sat, 23 Aug 2025 21:37:09 +0100 Subject: [PATCH 2/3] Add support for an arbitrary number of operands with any constraints to fastalloc Prior to this commit in fastalloc, Any constraints were allocated just like Reg constraints. Now, rather than treating Reg constraints as equivalent to Any constraints, operands with Reg constraints get their set of available registers separate from operands that have Any constraints. This comes with the drawback of having to reserve registers for the Reg operands and evict vregs in them before processing use and def operands. Any defs that are never used are also allocated directly into spillslots because there is no point allocating them into registers. --- doc/FASTALLOC.md | 59 ++++++--- src/fastalloc/iter.rs | 4 + src/fastalloc/mod.rs | 299 +++++++++++++++++++++++++++++++++--------- 3 files changed, 282 insertions(+), 80 deletions(-) diff --git a/doc/FASTALLOC.md b/doc/FASTALLOC.md index 79dac025..612543b1 100644 --- a/doc/FASTALLOC.md +++ b/doc/FASTALLOC.md @@ -42,17 +42,22 @@ During allocation, it's necessary to determine which VReg is in a PReg to generate the right move(s) for eviction. `vreg_in_preg` is a vector that stores this information. -## Available PRegs For Use In Instruction (`available_pregs`) +## Available PRegs For Use In Instruction (`available_pregs_for_regs`, `available_pregs_for_any`) -This is a 2-tuple of `PRegSet`s, a bitset of physical registers, one for +These are a 2-tuples of `PRegSet`s, a bitset of physical registers, one for the instruction's early phase and one for the late phase. They are used to determine which registers are available for use in the early/late phases of an instruction. -Prior to the beginning of any instruction's allocation, this set is reset -to include all allocatable physical registers, some of which may already +Prior to the beginning of any instruction's allocation, `available_pregs_for_regs` +is reset to include all allocatable physical registers, some of which may already contain a VReg. +The two sets have the same function, except that `available_pregs_for_regs` is +used to determine which registers are available for operands with a register-only +constraint while `available_pregs_for_any` is used to determine which registers +are available for operands with no constraints. + ## VReg Liverange Location Info (`vreg_to_live_inst_range`) This is a vector of 3-tuples containing the beginning and the end @@ -71,11 +76,11 @@ in four phases: selection, assignment, eviction, and edit insertion. ## Allocation Phase: Selection -In this phase, a PReg is selected from `available_pregs` for the -operand based on the operand constraints. Depending on the operand's -position the selected PReg is removed from either the early or late -phase or both, indicating that the PReg is no longer available for -allocation by other operands in that phase. +In this phase, a PReg is selected from `available_pregs_for_regs` or +`available_pregs_for_any` for the operand based on the operand constraints. +Depending on the operand's position, the selected PReg is removed from either +the early or late phase or both, indicating that the PReg is no longer available +for allocation by other operands in that phase. ## Allocation Phase: Assignment @@ -88,8 +93,9 @@ In this phase, the previous VReg in the allocation assigned to an operand is evicted, if any. During eviction, a dedicated spillslot is allocated for the evicted -VReg and an edit is inserted after the instruction to move from the -slot to the allocation it's expected to be in after the instruction. +VReg, if it doesn't have a spillslot yet, and an edit is inserted +after the instruction to move from the slot to the allocation +it's expected to be in after the instruction. ## Allocation Phase: Edit Insertion @@ -125,11 +131,11 @@ virtual registers will be in their dedicated spillslots. # Instruction Allocation To allocate a single instruction, the first step is to reset the -`available_pregs` sets to all allocatable PRegs. +`available_pregs_for_regs` sets to all allocatable PRegs. Next, the selection phase is carried out for all operands with fixed register constraints: the registers they are constrained to use are -marked as unavailable in the `available_pregs` set, depending on the +marked as unavailable in the `available_pregs_for_regs` set, depending on the phase that they are valid in. If the operand is an early use or late def operand, then the register will be marked as unavailable in the early set or late set, respectively. Otherwise, the PReg is marked @@ -137,20 +143,29 @@ as unavailable in both the early and late sets, because a PReg assigned to an early def or late use operand cannot be reused by another operand in the same instruction. -After selection for fixed register operands, the eviction phase is +Next, all clobbers are removed from the early and late `available_pregs_for_regs` +sets to avoid allocating a clobber to a def. + +Next, registers are reserved for register-only operands and marked as +unavailable in `available_pregs_for_regs`. +Then `available_pregs_for_any` for the instruction is derived from +`available_pregs_for_regs` by marking all other registers not reserved as +available. This is to avoid a situation where operands with no +constraints take up all available registers, leaving none for operands +with register-only constraints. + +After selection for register-only operands, the eviction phase is carried out for fixed register operands. Any VReg in their selected registers, indicated by `vreg_in_preg`, is evicted: a dedicated spillslot is allocated for the VReg (if it doesn't have one already), an edit is inserted to move from the slot to the PReg, which is where the VReg expected to be after the instruction, and its current allocation in `vreg_allocs` is set to the spillslot. - -Next, all clobbers are removed from the early and late `available_pregs` -sets to avoid allocating a clobber to a def. +The same is then done for clobbers, then register-only operands. Next, the selection, assignment, eviction, and edit insertion phases are carried out for all def operands. When each def operand's allocation is -complete, the def operands is immediately freed, marking the end of the +complete, the def operand is immediately freed, marking the end of the VReg's liverange. It is removed from the `live_vregs` set, its allocation in `vreg_allocs` is set to none, and if it was in a PReg, that PReg's entry in `vreg_in_preg` is set to none. The selection and eviction phases @@ -187,7 +202,9 @@ allocation and no edit insertion needs to be done either. On the other hand, if the VReg's current allocation is not within constraints, the selection and eviction phases are carried out for non-fixed operands. First, a set of PRegs that can be drawn from is -created from `available_pregs`. For early uses and late defs, +created from `available_pregs_for_regs` or `available_pregs_for_any`, +depending on whether the operand has a register-only constraint +or no constraint. For early uses and late defs, this draw-from set is the early set or late set, respectively. For late uses and early defs, the draw-from set is an intersection of the available early and late sets (because a PReg used for a late @@ -196,8 +213,8 @@ likewise, a PReg used for an early def can't be reassigned to another operand in the late phase). The LRU for the VReg's regclass is then traversed from the end to find the least recently used PReg in the draw-from set. Once a PReg is found, -it is marked as the most recently used in the LRU, unavailable in the -`available_pregs` sets, and whatever VReg was in it before is evicted. +it is marked as the most recently used in the LRU, unavailable in both +available pregs sets, and whatever VReg was in it before is evicted. The assignment phase is carried out next. The final allocation for the operand is set to the selected register. diff --git a/src/fastalloc/iter.rs b/src/fastalloc/iter.rs index d1437559..86d124ce 100644 --- a/src/fastalloc/iter.rs +++ b/src/fastalloc/iter.rs @@ -33,6 +33,10 @@ impl<'a> Operands<'a> { pub fn fixed(&self) -> impl Iterator + 'a { self.matches(|op| matches!(op.constraint(), OperandConstraint::FixedReg(_))) } + + pub fn any_reg(&self) -> impl Iterator + 'a { + self.matches(|op| matches!(op.constraint(), OperandConstraint::Reg)) + } } impl<'a> core::ops::Index for Operands<'a> { diff --git a/src/fastalloc/mod.rs b/src/fastalloc/mod.rs index e0bc9343..d7258e2c 100644 --- a/src/fastalloc/mod.rs +++ b/src/fastalloc/mod.rs @@ -10,7 +10,7 @@ use alloc::{vec, vec::Vec}; use core::convert::TryInto; use core::fmt; use core::iter::FromIterator; -use core::ops::{Index, IndexMut}; +use core::ops::{BitAnd, BitOr, Index, IndexMut, Not}; mod iter; mod lru; @@ -183,6 +183,41 @@ struct PartedByOperandPos { items: [T; 2], } +impl Copy for PartedByOperandPos {} + +impl + Copy> BitAnd for PartedByOperandPos { + type Output = Self; + fn bitand(self, other: Self) -> Self { + Self { + items: [ + self.items[0] & other.items[0], + self.items[1] & other.items[1], + ], + } + } +} + +impl + Copy> BitOr for PartedByOperandPos { + type Output = Self; + fn bitor(self, other: Self) -> Self { + Self { + items: [ + self.items[0] | other.items[0], + self.items[1] | other.items[1], + ], + } + } +} + +impl Not for PartedByOperandPos { + type Output = Self; + fn not(self) -> Self { + Self { + items: [self.items[0].invert(), self.items[1].invert()], + } + } +} + impl Index for PartedByOperandPos { type Output = T; fn index(&self, index: OperandPos) -> &Self::Output { @@ -222,10 +257,17 @@ pub struct Env<'a, F: Function> { /// that uses the `i`th operand in the current instruction as its input. reused_input_to_reuse_op: Vec, /// The set of registers that can be used for allocation in the - /// early and late phases of an instruction. + /// early and late phases of an instruction for operands with a + /// Reg constraint. /// Allocatable registers that contain no vregs, registers that can be /// evicted can be in the set, and fixed stack slots are in this set. - available_pregs: PartedByOperandPos, + available_pregs_for_regs: PartedByOperandPos, + /// This is `available_pregs_for_regs` but for operands + /// with an Any constraint. This split is made here to allow for + /// reservation of registers for Reg-only operands, so as to avoid + /// a scenario where available registers are allocated to Any operands + /// without leaving any for Reg-only operands. + available_pregs_for_any: PartedByOperandPos, init_available_pregs: PRegSet, allocatable_regs: PRegSet, stack: Stack<'a, F>, @@ -278,7 +320,7 @@ impl<'a, F: Function> Env<'a, F> { env.scratch_by_class[2], ], }; - trace!("{:?}", env); + trace!("{:#?}", env); let (allocs, max_operand_len) = Allocs::new(func); let fixed_stack_slots = PRegSet::from_iter(env.fixed_stack_slots.iter().cloned()); Self { @@ -308,7 +350,10 @@ impl<'a, F: Function> Env<'a, F> { }, reused_input_to_reuse_op: vec![usize::MAX; max_operand_len as usize], init_available_pregs, - available_pregs: PartedByOperandPos { + available_pregs_for_regs: PartedByOperandPos { + items: [init_available_pregs, init_available_pregs], + }, + available_pregs_for_any: PartedByOperandPos { items: [init_available_pregs, init_available_pregs], }, allocs, @@ -320,7 +365,7 @@ impl<'a, F: Function> Env<'a, F> { fn reset_available_pregs_and_scratch_regs(&mut self) { trace!("Resetting the available pregs"); - self.available_pregs = PartedByOperandPos { + self.available_pregs_for_regs = PartedByOperandPos { items: [self.init_available_pregs, self.init_available_pregs], }; self.edits.scratch_regs = self.edits.dedicated_scratch_regs.clone(); @@ -332,15 +377,16 @@ impl<'a, F: Function> Env<'a, F> { class: RegClass, pos: InstPosition, ) -> Result<(), RegAllocError> { - let avail_regs = - self.available_pregs[OperandPos::Late] & self.available_pregs[OperandPos::Early]; + let avail_regs = self.available_pregs_for_any[OperandPos::Late] + & self.available_pregs_for_any[OperandPos::Early]; + trace!("Checking {avail_regs} for scratch register for {class:?}"); if let Some(preg) = self.lrus[class].last(avail_regs) { if self.vreg_in_preg[preg.index()] != VReg::invalid() { self.evict_vreg_in_preg(inst, preg, pos)?; } self.edits.scratch_regs[class] = Some(preg); - self.available_pregs[OperandPos::Early].remove(preg); - self.available_pregs[OperandPos::Late].remove(preg); + self.available_pregs_for_any[OperandPos::Early].remove(preg); + self.available_pregs_for_any[OperandPos::Late].remove(preg); Ok(()) } else { Err(RegAllocError::TooManyLiveRegs) @@ -365,33 +411,41 @@ impl<'a, F: Function> Env<'a, F> { Ok(()) } - fn reserve_reg_for_fixed_operand( - &mut self, + fn get_spillslot(&mut self, vreg: VReg) -> SpillSlot { + if self.vreg_spillslots[vreg.vreg()].is_invalid() { + self.vreg_spillslots[vreg.vreg()] = self.stack.allocstack(vreg.class()); + } + self.vreg_spillslots[vreg.vreg()] + } + + fn reserve_reg_for_operand( + &self, op: Operand, op_idx: usize, preg: PReg, + available_pregs: &mut PartedByOperandPos, ) -> Result<(), RegAllocError> { - trace!("Reserving register {preg} for fixed operand {op}"); - let early_avail_pregs = self.available_pregs[OperandPos::Early]; - let late_avail_pregs = self.available_pregs[OperandPos::Late]; + trace!("Reserving register {preg} for operand {op}"); + let early_avail_pregs = available_pregs[OperandPos::Early]; + let late_avail_pregs = available_pregs[OperandPos::Late]; match (op.pos(), op.kind()) { (OperandPos::Early, OperandKind::Use) => { if op.as_fixed_nonallocatable().is_none() && !early_avail_pregs.contains(preg) { return Err(RegAllocError::TooManyLiveRegs); } - self.available_pregs[OperandPos::Early].remove(preg); + available_pregs[OperandPos::Early].remove(preg); if self.reused_input_to_reuse_op[op_idx] != usize::MAX { if op.as_fixed_nonallocatable().is_none() && !late_avail_pregs.contains(preg) { return Err(RegAllocError::TooManyLiveRegs); } - self.available_pregs[OperandPos::Late].remove(preg); + available_pregs[OperandPos::Late].remove(preg); } } (OperandPos::Late, OperandKind::Def) => { if op.as_fixed_nonallocatable().is_none() && !late_avail_pregs.contains(preg) { return Err(RegAllocError::TooManyLiveRegs); } - self.available_pregs[OperandPos::Late].remove(preg); + available_pregs[OperandPos::Late].remove(preg); } _ => { if op.as_fixed_nonallocatable().is_none() @@ -399,8 +453,8 @@ impl<'a, F: Function> Env<'a, F> { { return Err(RegAllocError::TooManyLiveRegs); } - self.available_pregs[OperandPos::Early].remove(preg); - self.available_pregs[OperandPos::Late].remove(preg); + available_pregs[OperandPos::Early].remove(preg); + available_pregs[OperandPos::Late].remove(preg); } } Ok(()) @@ -411,7 +465,7 @@ impl<'a, F: Function> Env<'a, F> { match op.constraint() { OperandConstraint::Any => { if let Some(preg) = alloc.as_reg() { - if !self.available_pregs[op.pos()].contains(preg) { + if !self.available_pregs_for_any[op.pos()].contains(preg) { // If a register isn't in the available pregs list, then // there are two cases: either it's reserved for a // fixed register constraint or a vreg allocated in the instruction @@ -437,7 +491,7 @@ impl<'a, F: Function> Env<'a, F> { return false; } if let Some(preg) = alloc.as_reg() { - if !self.available_pregs[op.pos()].contains(preg) { + if !self.available_pregs_for_regs[op.pos()].contains(preg) { trace!("The vreg in {preg}: {}", self.vreg_in_preg[preg.index()]); self.vreg_in_preg[preg.index()] == op.vreg() } else { @@ -505,26 +559,20 @@ impl<'a, F: Function> Env<'a, F> { ); } - /// Allocates a physical register for the operand `op`. - fn alloc_reg_for_operand( - &mut self, - inst: Inst, + fn select_suitable_reg_in_lru( + &self, op: Operand, - ) -> Result { - trace!("available regs: {}", self.available_pregs); - trace!("Int LRU: {:?}", self.lrus[RegClass::Int]); - trace!("Float LRU: {:?}", self.lrus[RegClass::Float]); - trace!("Vector LRU: {:?}", self.lrus[RegClass::Vector]); - trace!(""); + available_pregs: &PartedByOperandPos, + ) -> Result { let draw_from = match (op.pos(), op.kind()) { (OperandPos::Late, OperandKind::Use) | (OperandPos::Early, OperandKind::Def) | (OperandPos::Late, OperandKind::Def) if matches!(op.constraint(), OperandConstraint::Reuse(_)) => { - self.available_pregs[OperandPos::Late] & self.available_pregs[OperandPos::Early] + available_pregs[OperandPos::Late] & available_pregs[OperandPos::Early] } - _ => self.available_pregs[op.pos()], + _ => available_pregs[op.pos()], }; if draw_from.is_empty(op.class()) { trace!("No registers available for {op}"); @@ -537,23 +585,39 @@ impl<'a, F: Function> Env<'a, F> { ); return Err(RegAllocError::TooManyLiveRegs); }; + Ok(preg) + } + + /// Allocates a physical register for the operand `op`. + fn alloc_reg_for_operand( + &mut self, + inst: Inst, + op: Operand, + available_pregs: &mut PartedByOperandPos, + ) -> Result { + trace!("available regs: {}", available_pregs); + trace!("Int LRU: {:?}", self.lrus[RegClass::Int]); + trace!("Float LRU: {:?}", self.lrus[RegClass::Float]); + trace!("Vector LRU: {:?}", self.lrus[RegClass::Vector]); + trace!(""); + let preg = self.select_suitable_reg_in_lru(op, available_pregs)?; if self.vreg_in_preg[preg.index()] != VReg::invalid() { self.evict_vreg_in_preg(inst, preg, InstPosition::After)?; } trace!("The allocated register for vreg {}: {}", op.vreg(), preg); self.lrus[op.class()].poke(preg); - self.available_pregs[op.pos()].remove(preg); + available_pregs[op.pos()].remove(preg); match (op.pos(), op.kind()) { (OperandPos::Late, OperandKind::Use) => { - self.available_pregs[OperandPos::Early].remove(preg); + available_pregs[OperandPos::Early].remove(preg); } (OperandPos::Early, OperandKind::Def) => { - self.available_pregs[OperandPos::Late].remove(preg); + available_pregs[OperandPos::Late].remove(preg); } (OperandPos::Late, OperandKind::Def) if matches!(op.constraint(), OperandConstraint::Reuse(_)) => { - self.available_pregs[OperandPos::Early].remove(preg); + available_pregs[OperandPos::Early].remove(preg); } _ => (), }; @@ -569,8 +633,31 @@ impl<'a, F: Function> Env<'a, F> { op_idx: usize, ) -> Result { let new_alloc = match op.constraint() { - OperandConstraint::Any => self.alloc_reg_for_operand(inst, op)?, - OperandConstraint::Reg => self.alloc_reg_for_operand(inst, op)?, + OperandConstraint::Any => { + if op.kind() == OperandKind::Def + && self.vreg_allocs[op.vreg().vreg()] == Allocation::none() + { + // This def is never used, so it can just be put in its spillslot. + Allocation::stack(self.get_spillslot(op.vreg())) + } else { + let mut available_pregs = self.available_pregs_for_any; + let result = self.alloc_reg_for_operand(inst, op, &mut available_pregs); + self.available_pregs_for_any = available_pregs; + match result { + Ok(alloc) => alloc, + Err(RegAllocError::TooManyLiveRegs) => { + Allocation::stack(self.get_spillslot(op.vreg())) + } + Err(err) => return Err(err), + } + } + } + OperandConstraint::Reg => { + let mut available_pregs = self.available_pregs_for_regs; + let alloc = self.alloc_reg_for_operand(inst, op, &mut available_pregs)?; + self.available_pregs_for_regs = available_pregs; + alloc + } OperandConstraint::FixedReg(preg) => { trace!("The fixed preg: {} for operand {}", preg, op); @@ -610,7 +697,10 @@ impl<'a, F: Function> Env<'a, F> { return Ok(()); } if !self.allocd_within_constraint(op) { - trace!("{op} isn't allocated within constraints."); + trace!( + "{op} isn't allocated within constraints (the alloc: {}).", + self.vreg_allocs[op.vreg().vreg()] + ); let curr_alloc = self.vreg_allocs[op.vreg().vreg()]; let new_alloc = self.alloc_operand(inst, op, op_idx)?; if curr_alloc.is_none() { @@ -664,13 +754,16 @@ impl<'a, F: Function> Env<'a, F> { if self.allocatable_regs.contains(preg) { self.lrus[preg.class()].poke(preg); } - self.available_pregs[op.pos()].remove(preg); + self.available_pregs_for_regs[op.pos()].remove(preg); + self.available_pregs_for_any[op.pos()].remove(preg); match (op.pos(), op.kind()) { (OperandPos::Late, OperandKind::Use) => { - self.available_pregs[OperandPos::Early].remove(preg); + self.available_pregs_for_regs[OperandPos::Early].remove(preg); + self.available_pregs_for_any[OperandPos::Early].remove(preg); } (OperandPos::Early, OperandKind::Def) => { - self.available_pregs[OperandPos::Late].remove(preg); + self.available_pregs_for_regs[OperandPos::Late].remove(preg); + self.available_pregs_for_any[OperandPos::Late].remove(preg); } _ => (), }; @@ -683,12 +776,20 @@ impl<'a, F: Function> Env<'a, F> { ); } trace!( - "Late available regs: {}", - self.available_pregs[OperandPos::Late] + "Late available regs for Reg-only: {}", + self.available_pregs_for_regs[OperandPos::Late] ); trace!( - "Early available regs: {}", - self.available_pregs[OperandPos::Early] + "Early available regs for Reg-only: {}", + self.available_pregs_for_regs[OperandPos::Early] + ); + trace!( + "Late available regs for Any: {}", + self.available_pregs_for_any[OperandPos::Late] + ); + trace!( + "Early available regs for Any: {}", + self.available_pregs_for_any[OperandPos::Early] ); Ok(()) } @@ -707,8 +808,10 @@ impl<'a, F: Function> Env<'a, F> { // To avoid this scenario, clobbers should be removed from both late // and early reg sets. let all_but_clobbers = clobbers.invert(); - self.available_pregs[OperandPos::Late].intersect_from(all_but_clobbers); - self.available_pregs[OperandPos::Early].intersect_from(all_but_clobbers); + self.available_pregs_for_regs[OperandPos::Late].intersect_from(all_but_clobbers); + self.available_pregs_for_regs[OperandPos::Early].intersect_from(all_but_clobbers); + // No need to do the same for `available_pregs_for_any` because it is derived from + // `available_pregs_for_regs` after this operation. } /// If instruction `inst` is a branch in `block`, @@ -769,8 +872,8 @@ impl<'a, F: Function> Env<'a, F> { let resolved_vec = vec_parallel_moves.resolve(); let mut scratch_regs = self.edits.scratch_regs.clone(); let mut num_spillslots = self.stack.num_spillslots; - let mut avail_regs = - self.available_pregs[OperandPos::Early] & self.available_pregs[OperandPos::Late]; + let mut avail_regs = self.available_pregs_for_any[OperandPos::Early] + & self.available_pregs_for_any[OperandPos::Late]; trace!("Resolving parallel moves"); for (resolved, class) in [ @@ -842,11 +945,71 @@ impl<'a, F: Function> Env<'a, F> { let OperandConstraint::FixedReg(preg) = op.constraint() else { unreachable!(); }; - self.reserve_reg_for_fixed_operand(op, op_idx, preg)?; + let mut available_pregs = self.available_pregs_for_regs; + self.reserve_reg_for_operand(op, op_idx, preg, &mut available_pregs)?; + self.available_pregs_for_regs = available_pregs; + if self.allocatable_regs.contains(preg) { self.lrus[preg.class()].poke(preg); } } + + self.remove_clobbers_from_available_pregs(clobbers); + + // Need to reserve enough registers for operands with Reg-only constraints + // This reservation is done here, before any evictions, so that `available_pregs_for_any` + // can be used for scratch registers when generating moves. + self.available_pregs_for_any = self.available_pregs_for_regs; + for (op_idx, op) in operands.any_reg() { + let mut available_pregs = self.available_pregs_for_any; + // Always allocate for a late use, because it will reserve only registers + // that are available in both the early and late phases. + // Consider a scenario: + // use v0 (reg), use v1 (reg), late use v2 (reg) + // If the available registers reserved are: + // early: [p0] + // late: [p0, p1, p2] + // Then this should be enough for all vregs: just allocated p0 to v2, + // p1 and p2 to v0 and v1. + // But if v0 and v1 are allocated first and p0 is allocated to, say, v0 then + // p2 will be left for v2. But it will not be suitable for v2 because it is + // not available in the early phase. + // To avoid this scenario, only registers available in both phases are reserved. + let alloc_for_op = Operand::new( + op.vreg(), + op.constraint(), + OperandKind::Use, + OperandPos::Late, + ); + trace!("Making reservation for {op} as {alloc_for_op}"); + let preg = self.select_suitable_reg_in_lru(alloc_for_op, &available_pregs)?; + self.reserve_reg_for_operand(alloc_for_op, op_idx, preg, &mut available_pregs)?; + self.available_pregs_for_any = available_pregs; + if self.allocatable_regs.contains(preg) { + self.lrus[op.class()].poke(preg); + } + } + // Remove the registers available for Any constraints from the + // Reg registers + self.available_pregs_for_regs = + self.available_pregs_for_regs & !self.available_pregs_for_any; + trace!( + "Late available regs for Reg-only: {}", + self.available_pregs_for_regs[OperandPos::Late] + ); + trace!( + "Early available regs for Reg-only: {}", + self.available_pregs_for_regs[OperandPos::Early] + ); + trace!( + "Late available regs for Any: {}", + self.available_pregs_for_any[OperandPos::Late] + ); + trace!( + "Early available regs for Any: {}", + self.available_pregs_for_any[OperandPos::Early] + ); + for (_, op) in operands.fixed() { let OperandConstraint::FixedReg(preg) = op.constraint() else { unreachable!(); @@ -864,7 +1027,6 @@ impl<'a, F: Function> Env<'a, F> { self.vreg_in_preg[preg.index()] = VReg::invalid(); } } - self.remove_clobbers_from_available_pregs(clobbers); for preg in clobbers { if self.vreg_in_preg[preg.index()] != VReg::invalid() { trace!( @@ -875,6 +1037,26 @@ impl<'a, F: Function> Env<'a, F> { self.vreg_in_preg[preg.index()] = VReg::invalid(); } } + for op in operands.0.iter() { + if op.constraint() == OperandConstraint::Reg + || matches!(op.constraint(), OperandConstraint::FixedReg(_)) + { + continue; + } + if let Some(preg) = self.vreg_allocs[op.vreg().vreg()].as_reg() { + if self.available_pregs_for_regs[OperandPos::Early].contains(preg) + || self.available_pregs_for_regs[OperandPos::Late].contains(preg) + { + trace!( + "Evicting {} from Reg-only {preg}", + self.vreg_in_preg[preg.index()] + ); + self.evict_vreg_in_preg(inst, preg, InstPosition::After)?; + self.vreg_in_preg[preg.index()] = VReg::invalid(); + } + } + } + for (op_idx, op) in operands.def_ops() { trace!("Allocating def operands {op}"); if let OperandConstraint::Reuse(reused_idx) = op.constraint() { @@ -915,6 +1097,9 @@ impl<'a, F: Function> Env<'a, F> { self.process_operand_allocation(inst, op, op_idx)?; } } + // Put registers unused for Reg-only back into `available_pregs_for_any` + self.available_pregs_for_any = self.available_pregs_for_any | self.available_pregs_for_regs; + for (op_idx, op) in operands.use_ops() { if op.as_fixed_nonallocatable().is_some() { continue; @@ -965,12 +1150,8 @@ impl<'a, F: Function> Env<'a, F> { block, self.func.block_params(block) ); - trace!( - "Available pregs: {}", - self.available_pregs[OperandPos::Early] - ); self.reset_available_pregs_and_scratch_regs(); - let avail_regs_for_scratch = self.available_pregs[OperandPos::Early]; + let avail_regs_for_scratch = self.available_pregs_for_regs[OperandPos::Early]; let first_inst = self.func.block_insns(block).first(); // We need to check for the registers that are still live. // These registers are either livein or block params From 239181b903f6196b77d5c8aabac414618e942129 Mon Sep 17 00:00:00 2001 From: Demilade Sonuga Date: Sun, 24 Aug 2025 18:46:09 +0100 Subject: [PATCH 3/3] Add support for stack-only constraints in fastalloc --- src/fastalloc/mod.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/fastalloc/mod.rs b/src/fastalloc/mod.rs index d7258e2c..81654e63 100644 --- a/src/fastalloc/mod.rs +++ b/src/fastalloc/mod.rs @@ -508,9 +508,7 @@ impl<'a, F: Function> Env<'a, F> { unreachable!() } - OperandConstraint::Stack => { - panic!("Stack constraints not supported in fastalloc"); - } + OperandConstraint::Stack => self.edits.is_stack(alloc), } } @@ -668,9 +666,7 @@ impl<'a, F: Function> Env<'a, F> { unreachable!(); } - OperandConstraint::Stack => { - panic!("Stack operand constraints not supported in fastalloc"); - } + OperandConstraint::Stack => Allocation::stack(self.get_spillslot(op.vreg())), }; self.allocs[(inst.index(), op_idx)] = new_alloc; Ok(new_alloc)