diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs index f26b326f93ff..f7bd0ed3df71 100644 --- a/cranelift/codegen/meta/src/gen_inst.rs +++ b/cranelift/codegen/meta/src/gen_inst.rs @@ -336,7 +336,7 @@ fn gen_instruction_data_impl(formats: &[Rc], fmt: &mut Format fmtln!(fmt, "::core::hash::Hash::hash(&{len}, state);"); fmt.add_block(&format!("for &block in {blocks}"), |fmt| { fmtln!(fmt, "::core::hash::Hash::hash(&block.block(pool), state);"); - fmt.add_block("for &arg in block.args_slice(pool)", |fmt| { + fmt.add_block("for arg in block.args(pool)", |fmt| { fmtln!(fmt, "::core::hash::Hash::hash(&arg, state);"); }); }); @@ -964,6 +964,7 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo let mut tmpl_types = Vec::new(); let mut into_args = Vec::new(); let mut block_args = Vec::new(); + let mut lifetime_param = None; for op in &inst.operands_in { if op.kind.is_block() { args.push(format!("{}_label: {}", op.name, "ir::Block")); @@ -972,7 +973,14 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo op.name, "Destination basic block" )); - args.push(format!("{}_args: {}", op.name, "&[Value]")); + let lifetime = *lifetime_param.get_or_insert_with(|| { + tmpl_types.insert(0, "'a".to_string()); + "'a" + }); + args.push(format!( + "{}_args: impl IntoIterator", + op.name, lifetime, + )); args_doc.push(format!("- {}_args: {}", op.name, "Block arguments")); block_args.push(op); diff --git a/cranelift/codegen/meta/src/shared/entities.rs b/cranelift/codegen/meta/src/shared/entities.rs index 3df9ecaa171f..cc3cc9c399b5 100644 --- a/cranelift/codegen/meta/src/shared/entities.rs +++ b/cranelift/codegen/meta/src/shared/entities.rs @@ -43,6 +43,9 @@ pub(crate) struct EntityRefs { /// A reference to a jump table declared in the function preamble. pub(crate) jump_table: OperandKind, + /// A reference to an exception table declared in the function preamble. + pub(crate) exception_table: OperandKind, + /// A variable-sized list of value operands. Use for Block and function call arguments. pub(crate) varargs: OperandKind, } @@ -84,6 +87,8 @@ impl EntityRefs { jump_table: new("table", "ir::JumpTable", "A jump table."), + exception_table: new("exception", "ir::ExceptionTable", "An exception table."), + varargs: OperandKind::new( "", "&[Value]", diff --git a/cranelift/codegen/meta/src/shared/formats.rs b/cranelift/codegen/meta/src/shared/formats.rs index 86d54338d63d..4ff60bef627c 100644 --- a/cranelift/codegen/meta/src/shared/formats.rs +++ b/cranelift/codegen/meta/src/shared/formats.rs @@ -12,6 +12,8 @@ pub(crate) struct Formats { pub(crate) brif: Rc, pub(crate) call: Rc, pub(crate) call_indirect: Rc, + pub(crate) try_call: Rc, + pub(crate) try_call_indirect: Rc, pub(crate) cond_trap: Rc, pub(crate) float_compare: Rc, pub(crate) func_addr: Rc, @@ -133,6 +135,18 @@ impl Formats { .varargs() .build(), + try_call: Builder::new("TryCall") + .imm(&entities.func_ref) + .varargs() + .imm(&&entities.exception_table) + .build(), + + try_call_indirect: Builder::new("TryCallIndirect") + .value() + .varargs() + .imm(&&entities.exception_table) + .build(), + func_addr: Builder::new("FuncAddr").imm(&entities.func_ref).build(), atomic_rmw: Builder::new("AtomicRmw") diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs index 81cf8d5b59ee..6a86e3e9bede 100644 --- a/cranelift/codegen/meta/src/shared/instructions.rs +++ b/cranelift/codegen/meta/src/shared/instructions.rs @@ -307,6 +307,75 @@ fn define_control_flow( .with_doc("function to call, declared by `function`")]) .operands_out(vec![Operand::new("addr", iAddr)]), ); + + ig.push( + Inst::new( + "try_call", + r#" + Call a function, catching the specified exceptions. + + Call the function pointed to by `callee` with the given arguments. On + normal return, branch to the first target, with function returns + available as `retN` block arguments. On exceptional return, + look up the thrown exception tag in the provided exception table; + if the tag matches one of the targets, branch to the matching + target with the exception payloads available as `exnN` block arguments. + If no tag matches, then propagate the exception up the stack. + + It is the Cranelift embedder's responsibility to define the meaning + of tags: they are accepted by this instruction and passed through + to unwind metadata tables in Cranelift's output. Actual unwinding is + outside the purview of the core Cranelift compiler. + + Payload values on exception are passed in fixed register(s) that are + defined by the platform and ABI. See the documentation on `CallConv` + for details. + "#, + &formats.try_call, + ) + .operands_in(vec![ + Operand::new("callee", &entities.func_ref) + .with_doc("function to call, declared by `function`"), + Operand::new("args", &entities.varargs).with_doc("call arguments"), + Operand::new("ET", &entities.exception_table).with_doc("exception table"), + ]) + .call() + .branches(), + ); + + ig.push( + Inst::new( + "try_call_indirect", + r#" + Call a function, catching the specified exceptions. + + Call the function pointed to by `callee` with the given arguments. On + normal return, branch to the first target, with function returns + available as `retN` block arguments. On exceptional return, + look up the thrown exception tag in the provided exception table; + if the tag matches one of the targets, branch to the matching + target with the exception payloads available as `exnN` block arguments. + If no tag matches, then propagate the exception up the stack. + + It is the Cranelift embedder's responsibility to define the meaning + of tags: they are accepted by this instruction and passed through + to unwind metadata tables in Cranelift's output. Actual unwinding is + outside the purview of the core Cranelift compiler. + + Payload values on exception are passed in fixed register(s) that are + defined by the platform and ABI. See the documentation on `CallConv` + for details. + "#, + &formats.try_call_indirect, + ) + .operands_in(vec![ + Operand::new("callee", iAddr).with_doc("address of function to call"), + Operand::new("args", &entities.varargs).with_doc("call arguments"), + Operand::new("ET", &entities.exception_table).with_doc("exception table"), + ]) + .call() + .branches(), + ); } #[inline(never)] diff --git a/cranelift/codegen/src/dominator_tree.rs b/cranelift/codegen/src/dominator_tree.rs index 92f7c143a48b..4441a4909d4e 100644 --- a/cranelift/codegen/src/dominator_tree.rs +++ b/cranelift/codegen/src/dominator_tree.rs @@ -644,7 +644,7 @@ mod tests { cur.insert_block(block1); let v1 = cur.ins().iconst(I32, 1); let v2 = cur.ins().iadd(v0, v1); - cur.ins().jump(block0, &[v2]); + cur.ins().jump(block0, &[v2.into()]); cur.insert_block(block2); cur.ins().return_(&[v0]); diff --git a/cranelift/codegen/src/dominator_tree/simple.rs b/cranelift/codegen/src/dominator_tree/simple.rs index d7117cbc073b..853158bc80b1 100644 --- a/cranelift/codegen/src/dominator_tree/simple.rs +++ b/cranelift/codegen/src/dominator_tree/simple.rs @@ -365,7 +365,7 @@ mod tests { cur.insert_block(block1); let v1 = cur.ins().iconst(I32, 1); let v2 = cur.ins().iadd(v0, v1); - cur.ins().jump(block0, &[v2]); + cur.ins().jump(block0, &[v2.into()]); cur.insert_block(block2); cur.ins().return_(&[v0]); diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs index 7a345264cc4c..7be7af565780 100644 --- a/cranelift/codegen/src/inst_predicates.rs +++ b/cranelift/codegen/src/inst_predicates.rs @@ -200,6 +200,16 @@ pub(crate) fn visit_block_succs( } } + ir::InstructionData::TryCall { exception, .. } + | ir::InstructionData::TryCallIndirect { exception, .. } => { + let pool = &f.dfg.value_lists; + let exdata = &f.stencil.dfg.exception_tables[*exception]; + + for dest in exdata.all_branches() { + visit(inst, dest.block(pool), false); + } + } + inst => debug_assert!(!inst.opcode().is_branch()), } } diff --git a/cranelift/codegen/src/ir/builder.rs b/cranelift/codegen/src/ir/builder.rs index 6032f07f5212..24d9beaee660 100644 --- a/cranelift/codegen/src/ir/builder.rs +++ b/cranelift/codegen/src/ir/builder.rs @@ -6,8 +6,8 @@ use crate::ir; use crate::ir::instructions::InstructionFormat; use crate::ir::types; +use crate::ir::{BlockArg, Inst, Opcode, Type, Value}; use crate::ir::{DataFlowGraph, InstructionData}; -use crate::ir::{Inst, Opcode, Type, Value}; /// Base trait for instruction builders. /// diff --git a/cranelift/codegen/src/ir/dfg.rs b/cranelift/codegen/src/ir/dfg.rs index 593d4cdb4a33..7ee5938768d1 100644 --- a/cranelift/codegen/src/ir/dfg.rs +++ b/cranelift/codegen/src/ir/dfg.rs @@ -8,9 +8,9 @@ use crate::ir::instructions::{CallInfo, InstructionData}; use crate::ir::pcc::Fact; use crate::ir::user_stack_maps::{UserStackMapEntry, UserStackMapEntryVec}; use crate::ir::{ - types, Block, BlockCall, ConstantData, ConstantPool, DynamicType, ExtFuncData, FuncRef, - Immediate, Inst, JumpTables, RelSourceLoc, SigRef, Signature, Type, Value, - ValueLabelAssignments, ValueList, ValueListPool, + types, Block, BlockArg, BlockCall, ConstantData, ConstantPool, DynamicType, ExceptionTables, + ExtFuncData, FuncRef, Immediate, Inst, JumpTables, RelSourceLoc, SigRef, Signature, Type, + Value, ValueLabelAssignments, ValueList, ValueListPool, }; use crate::packed_option::ReservedValue; use crate::write::write_operands; @@ -158,6 +158,9 @@ pub struct DataFlowGraph { /// Jump tables used in this function. pub jump_tables: JumpTables, + + /// Exception tables used in this function. + pub exception_tables: ExceptionTables, } impl DataFlowGraph { @@ -178,6 +181,7 @@ impl DataFlowGraph { constants: ConstantPool::new(), immediates: PrimaryMap::new(), jump_tables: JumpTables::new(), + exception_tables: ExceptionTables::new(), } } @@ -226,8 +230,12 @@ impl DataFlowGraph { } /// Make a BlockCall, bundling together the block and its arguments. - pub fn block_call(&mut self, block: Block, args: &[Value]) -> BlockCall { - BlockCall::new(block, args, &mut self.value_lists) + pub fn block_call<'a>( + &mut self, + block: Block, + args: impl IntoIterator, + ) -> BlockCall { + BlockCall::new(block, args.into_iter().copied(), &mut self.value_lists) } /// Get the total number of values. @@ -437,13 +445,18 @@ impl DataFlowGraph { // Rewrite InstructionData in `self.insts`. for inst in self.insts.0.values_mut() { - inst.map_values(&mut self.value_lists, &mut self.jump_tables, |arg| { - if let ValueData::Alias { original, .. } = self.values[arg].into() { - original - } else { - arg - } - }); + inst.map_values( + &mut self.value_lists, + &mut self.jump_tables, + &mut self.exception_tables, + |arg| { + if let ValueData::Alias { original, .. } = self.values[arg].into() { + original + } else { + arg + } + }, + ); } // - `results` and block-params in `blocks` are not aliases, by @@ -843,15 +856,16 @@ impl DataFlowGraph { &'dfg self, inst: Inst, ) -> impl DoubleEndedIterator + 'dfg { - self.inst_args(inst) - .iter() - .chain( - self.insts[inst] - .branch_destination(&self.jump_tables) - .into_iter() - .flat_map(|branch| branch.args_slice(&self.value_lists).iter()), - ) - .copied() + self.inst_args(inst).iter().copied().chain( + self.insts[inst] + .branch_destination(&self.jump_tables, &self.exception_tables) + .into_iter() + .flat_map(|branch| { + branch + .args(&self.value_lists) + .filter_map(|arg| arg.as_value()) + }), + ) } /// Map a function over the values of the instruction. @@ -859,7 +873,12 @@ impl DataFlowGraph { where F: FnMut(Value) -> Value, { - self.insts[inst].map_values(&mut self.value_lists, &mut self.jump_tables, body); + self.insts[inst].map_values( + &mut self.value_lists, + &mut self.jump_tables, + &mut self.exception_tables, + body, + ); } /// Overwrite the instruction's value references with values from the iterator. @@ -869,9 +888,12 @@ impl DataFlowGraph { where I: Iterator, { - self.insts[inst].map_values(&mut self.value_lists, &mut self.jump_tables, |_| { - values.next().unwrap() - }); + self.insts[inst].map_values( + &mut self.value_lists, + &mut self.jump_tables, + &mut self.exception_tables, + |_| values.next().unwrap(), + ); } /// Get all value arguments on `inst` as a slice. @@ -1078,18 +1100,22 @@ impl DataFlowGraph { /// Get the call signature of a direct or indirect call instruction. /// Returns `None` if `inst` is not a call instruction. pub fn call_signature(&self, inst: Inst) -> Option { - match self.insts[inst].analyze_call(&self.value_lists) { + match self.insts[inst].analyze_call(&self.value_lists, &self.exception_tables) { CallInfo::NotACall => None, CallInfo::Direct(f, _) => Some(self.ext_funcs[f].signature), + CallInfo::DirectWithSig(_, s, _) => Some(s), CallInfo::Indirect(s, _) => Some(s), } } - /// Like `call_signature` but returns none for tail call instructions. - fn non_tail_call_signature(&self, inst: Inst) -> Option { + /// Like `call_signature` but returns none for tail call + /// instructions and try-call (exception-handling invoke) + /// instructions. + fn non_tail_call_or_try_call_signature(&self, inst: Inst) -> Option { let sig = self.call_signature(inst)?; match self.insts[inst].opcode() { ir::Opcode::ReturnCall | ir::Opcode::ReturnCallIndirect => None, + ir::Opcode::TryCall | ir::Opcode::TryCallIndirect => None, _ => Some(sig), } } @@ -1097,7 +1123,7 @@ impl DataFlowGraph { // Only for use by the verifier. Everyone else should just use // `dfg.inst_results(inst).len()`. pub(crate) fn num_expected_results_for_verifier(&self, inst: Inst) -> usize { - match self.non_tail_call_signature(inst) { + match self.non_tail_call_or_try_call_signature(inst) { Some(sig) => self.signatures[sig].returns.len(), None => { let constraints = self.insts[inst].opcode().constraints(); @@ -1112,7 +1138,7 @@ impl DataFlowGraph { inst: Inst, ctrl_typevar: Type, ) -> impl iter::ExactSizeIterator + 'a { - return match self.non_tail_call_signature(inst) { + return match self.non_tail_call_or_try_call_signature(inst) { Some(sig) => InstResultTypes::Signature(self, sig, 0), None => { let constraints = self.insts[inst].opcode().constraints(); diff --git a/cranelift/codegen/src/ir/entities.rs b/cranelift/codegen/src/ir/entities.rs index 005007471b35..4712bbad2425 100644 --- a/cranelift/codegen/src/ir/entities.rs +++ b/cranelift/codegen/src/ir/entities.rs @@ -384,6 +384,55 @@ impl SigRef { } } +/// An opaque exception tag. +/// +/// Exception tags are used to denote the identity of an exception for +/// matching by catch-handlers in exception tables. +/// +/// The index space is arbitrary and is given meaning only by the +/// embedder of Cranelift. Cranelift will carry through these tags +/// from exception tables to the handler metadata produced as output +/// (for use by the embedder's unwinder). +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ExceptionTag(u32); +entity_impl!(ExceptionTag, "tag"); + +impl ExceptionTag { + /// Create a new exception tag from its arbitrary index. + /// + /// This method is for use by the parser. + pub fn with_number(n: u32) -> Option { + if n < u32::MAX { + Some(Self(n)) + } else { + None + } + } +} + +/// An opaque reference to an exception table. +/// +/// `ExceptionTable`s are used for describing exception catch handlers on +/// `try_call` and `try_call_indirect` instructions. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ExceptionTable(u32); +entity_impl!(ExceptionTable, "extable"); + +impl ExceptionTable { + /// Create a new exception table reference from its number. + /// + /// This method is for use by the parser. + pub fn with_number(n: u32) -> Option { + if n < u32::MAX { + Some(Self(n)) + } else { + None + } + } +} + /// An opaque reference to any of the entities defined in this module that can appear in CLIF IR. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] @@ -414,6 +463,8 @@ pub enum AnyEntity { FuncRef(FuncRef), /// A function call signature. SigRef(SigRef), + /// An exception table. + ExceptionTable(ExceptionTable), /// A function's stack limit StackLimit, } @@ -434,6 +485,7 @@ impl fmt::Display for AnyEntity { Self::Constant(r) => r.fmt(f), Self::FuncRef(r) => r.fmt(f), Self::SigRef(r) => r.fmt(f), + Self::ExceptionTable(r) => r.fmt(f), Self::StackLimit => write!(f, "stack_limit"), } } @@ -517,6 +569,12 @@ impl From for AnyEntity { } } +impl From for AnyEntity { + fn from(r: ExceptionTable) -> Self { + Self::ExceptionTable(r) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/cranelift/codegen/src/ir/exception_table.rs b/cranelift/codegen/src/ir/exception_table.rs new file mode 100644 index 000000000000..a5a23c5fde5c --- /dev/null +++ b/cranelift/codegen/src/ir/exception_table.rs @@ -0,0 +1,193 @@ +//! Exception tables: catch handlers on `try_call` instructions. +//! +//! An exception table describes where execution flows after returning +//! from `try_call`. It contains both the "normal" destination -- the +//! block to branch to when the function returns without throwing an +//! exception -- and any "catch" destinations associated with +//! particular exception tags. Each target indicates the arguments to +//! pass to the block that receives control. +//! +//! Like other side-tables (e.g., jump tables), each exception table +//! must be used by only one instruction. Sharing is not permitted +//! because it can complicate transforms (how does one change the +//! table used by only one instruction if others also use it?). +//! +//! In order to allow the `try_call` instruction itself to remain +//! small, the exception table also contains the signature ID of the +//! called function. + +use crate::ir::entities::{ExceptionTag, SigRef}; +use crate::ir::instructions::ValueListPool; +use crate::ir::BlockCall; +use alloc::vec::Vec; +use core::fmt::{self, Display, Formatter}; +use cranelift_entity::packed_option::PackedOption; +#[cfg(feature = "enable-serde")] +use serde_derive::{Deserialize, Serialize}; + +/// Contents of an exception table. +/// +/// The "no exception" target for is stored as the last element of the +/// underlying vector. It can be accessed through the `normal_return` +/// and `normal_return_mut` functions. Exceptional catch clauses may +/// be iterated using the `catches` and `catches_mut` functions. All +/// targets may be iterated over using the `all_targets` and +/// `all_targets_mut` functions. +#[derive(Debug, Clone, PartialEq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ExceptionTableData { + /// All BlockCalls packed together. This is necessary because the + /// rest of the compiler expects to be able to grab a slice of + /// branch targets for any branch instruction. The last BlockCall + /// is the normal-return destination, and the rest correspond to + /// the tags in `tags` below. Thus, we have the invariant that + /// `targets.len() == tags.len() + 1`. + targets: Vec, + + /// Tags corresponding to targets other than the first one. + /// + /// A tag value of `None` indicates a catch-all handler. The + /// catch-all handler matches only if no other handler matches, + /// regardless of the order in this vector. + /// + /// `tags[i]` corresponds to `targets[i]`. Note that there will be + /// one more `targets` element than `tags` because the last + /// element in `targets` is the normal-return path. + tags: Vec>, + + /// The signature of the function whose invocation is associated + /// with this handler table. + sig: SigRef, +} + +impl ExceptionTableData { + /// Create new exception-table data. + /// + /// This data represents the destinations upon return from + /// `try_call` or `try_call_indirect` instruction. There are two + /// possibilities: "normal return" (no exception thrown), or an + /// exceptional return corresponding to one of the listed + /// exception tags. + /// + /// The given tags are passed through to the metadata provided + /// alongside the provided function body, and Cranelift itself + /// does not implement an unwinder; thus, the meaning of the tags + /// is ultimately up to the embedder of Cranelift. The tags are + /// wrapped in `Option` to allow encoding a "catch-all" handler. + /// + /// The BlockCalls must have signatures that match the targeted + /// blocks, as usual. These calls are allowed to use + /// `BlockArg::TryCallRet` in the normal-return case, with types + /// corresponding to the signature's return values, and + /// `BlockArg::TryCallExn` in the exceptional-return cases, with + /// types corresponding to native machine words and an arity + /// corresponding to the number of payload values that the calling + /// convention and platform support. (See [`isa::CallConv`] for + /// more details.) + pub fn new( + sig: SigRef, + normal_return: BlockCall, + tags_and_targets: impl IntoIterator, BlockCall)>, + ) -> Self { + let mut targets = vec![]; + let mut tags = vec![]; + for (tag, target) in tags_and_targets { + tags.push(tag.into()); + targets.push(target); + } + targets.push(normal_return); + + ExceptionTableData { targets, tags, sig } + } + + /// Return a value that can display the contents of this exception + /// table. + pub fn display<'a>(&'a self, pool: &'a ValueListPool) -> DisplayExceptionTable<'a> { + DisplayExceptionTable { table: self, pool } + } + + /// Get the default target for the non-exceptional return case. + pub fn normal_return(&self) -> &BlockCall { + self.targets.last().unwrap() + } + + /// Get the default target for the non-exceptional return case. + pub fn normal_return_mut(&mut self) -> &mut BlockCall { + self.targets.last_mut().unwrap() + } + + /// Get the targets for exceptional return cases, together with + /// their tags. + pub fn catches(&self) -> impl Iterator, &BlockCall)> + '_ { + self.tags + .iter() + .map(|tag| tag.expand()) + // Skips the last entry of `targets` (the normal return) + // because `tags` is one element shorter. + .zip(self.targets.iter()) + } + + /// Get the targets for exceptional return cases, together with + /// their tags. + pub fn catches_mut( + &mut self, + ) -> impl Iterator, &mut BlockCall)> + '_ { + self.tags + .iter() + .map(|tag| tag.expand()) + // Skips the last entry of `targets` (the normal return) + // because `tags` is one element shorter. + .zip(self.targets.iter_mut()) + } + + /// Get all branch targets. + pub fn all_branches(&self) -> &[BlockCall] { + &self.targets[..] + } + + /// Get all branch targets. + pub fn all_branches_mut(&mut self) -> &mut [BlockCall] { + &mut self.targets[..] + } + + /// Get the signature of the function called with this exception + /// table. + pub fn signature(&self) -> SigRef { + self.sig + } +} + +/// A wrapper for the context required to display a +/// [ExceptionTableData]. +pub struct DisplayExceptionTable<'a> { + table: &'a ExceptionTableData, + pool: &'a ValueListPool, +} + +impl<'a> Display for DisplayExceptionTable<'a> { + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + write!( + fmt, + "{}, {}, [", + self.table.sig, + self.table.normal_return().display(self.pool) + )?; + let mut first = true; + for (tag, block_call) in self.table.catches() { + if first { + write!(fmt, " ")?; + first = false; + } else { + write!(fmt, ", ")?; + } + if let Some(tag) = tag { + write!(fmt, "{}: {}", tag, block_call.display(self.pool))?; + } else { + write!(fmt, "default: {}", block_call.display(self.pool))?; + } + } + let space = if first { "" } else { " " }; + write!(fmt, "{space}]")?; + Ok(()) + } +} diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index ebf7f08cb85c..ec3e5bb5de34 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -282,7 +282,9 @@ impl FunctionStencil { /// Rewrite the branch destination to `new_dest` if the destination matches `old_dest`. /// Does nothing if called with a non-jump or non-branch instruction. pub fn rewrite_branch_destination(&mut self, inst: Inst, old_dest: Block, new_dest: Block) { - for dest in self.dfg.insts[inst].branch_destination_mut(&mut self.dfg.jump_tables) { + for dest in self.dfg.insts[inst] + .branch_destination_mut(&mut self.dfg.jump_tables, &mut self.dfg.exception_tables) + { if dest.block(&self.dfg.value_lists) == old_dest { dest.set_block(new_dest, &mut self.dfg.value_lists) } @@ -312,7 +314,7 @@ impl FunctionStencil { pub fn block_successors(&self, block: Block) -> impl DoubleEndedIterator + '_ { self.layout.last_inst(block).into_iter().flat_map(|inst| { self.dfg.insts[inst] - .branch_destination(&self.dfg.jump_tables) + .branch_destination(&self.dfg.jump_tables, &self.dfg.exception_tables) .iter() .map(|block| block.block(&self.dfg.value_lists)) }) diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs index 8aa8d7cb0e5c..66f093c504c6 100644 --- a/cranelift/codegen/src/ir/instructions.rs +++ b/cranelift/codegen/src/ir/instructions.rs @@ -21,7 +21,8 @@ use crate::ir::{ self, condcodes::{FloatCC, IntCC}, trapcode::TrapCode, - types, Block, FuncRef, MemFlags, SigRef, StackSlot, Type, Value, + types, Block, ExceptionTable, ExceptionTables, FuncRef, MemFlags, SigRef, StackSlot, Type, + Value, }; /// Some instructions use an external list of argument values because there is not enough space in @@ -34,6 +35,8 @@ pub type ValueListPool = entity::ListPool; /// A pair of a Block and its arguments, stored in a single EntityList internally. /// +/// Block arguments are semantically a `BlockArg`. +/// /// NOTE: We don't expose either value_to_block or block_to_value outside of this module because /// this operation is not generally safe. However, as the two share the same underlying layout, /// they can be stored in the same value pool. @@ -69,10 +72,14 @@ impl BlockCall { } /// Construct a BlockCall with the given block and arguments. - pub fn new(block: Block, args: &[Value], pool: &mut ValueListPool) -> Self { + pub fn new( + block: Block, + args: impl Iterator, + pool: &mut ValueListPool, + ) -> Self { let mut values = ValueList::default(); values.push(Self::block_to_value(block), pool); - values.extend(args.iter().copied(), pool); + values.extend(args.map(|arg| arg.encode_as_value()), pool); Self { values } } @@ -88,18 +95,36 @@ impl BlockCall { } /// Append an argument to the block args. - pub fn append_argument(&mut self, arg: Value, pool: &mut ValueListPool) { - self.values.push(arg, pool); + pub fn append_argument(&mut self, arg: impl Into, pool: &mut ValueListPool) { + self.values.push(arg.into().encode_as_value(), pool); } - /// Return a slice for the arguments of this block. - pub fn args_slice<'a>(&self, pool: &'a ValueListPool) -> &'a [Value] { - &self.values.as_slice(pool)[1..] + /// Return the length of the argument list. + pub fn len(&self, pool: &ValueListPool) -> usize { + self.values.len(pool) - 1 } - /// Return a slice for the arguments of this block. - pub fn args_slice_mut<'a>(&'a mut self, pool: &'a mut ValueListPool) -> &'a mut [Value] { - &mut self.values.as_mut_slice(pool)[1..] + /// Return an iterator over the arguments of this block. + pub fn args<'a>( + &self, + pool: &'a ValueListPool, + ) -> impl ExactSizeIterator + DoubleEndedIterator + use<'a> + { + self.values.as_slice(pool)[1..] + .iter() + .map(|value| BlockArg::decode_from_value(*value)) + } + + /// Traverse the arguments with a closure that can mutate them. + pub fn update_args BlockArg>( + &mut self, + pool: &mut ValueListPool, + mut f: F, + ) { + for raw in self.values.as_mut_slice(pool)[1..].iter_mut() { + let new = f(BlockArg::decode_from_value(*raw)); + *raw = new.encode_as_value(); + } } /// Remove the argument at ix from the argument list. @@ -113,11 +138,17 @@ impl BlockCall { } /// Appends multiple elements to the arguments. - pub fn extend(&mut self, elements: I, pool: &mut ValueListPool) + pub fn extend(&mut self, elements: I, pool: &mut ValueListPool) where - I: IntoIterator, + I: IntoIterator, + T: Into, { - self.values.extend(elements, pool) + self.values.extend( + elements + .into_iter() + .map(|elem| elem.into().encode_as_value()), + pool, + ) } /// Return a value that can display this block call. @@ -144,10 +175,9 @@ pub struct DisplayBlockCall<'a> { impl<'a> Display for DisplayBlockCall<'a> { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "{}", self.block.block(&self.pool))?; - let args = self.block.args_slice(&self.pool); - if !args.is_empty() { + if self.block.len(self.pool) > 0 { write!(f, "(")?; - for (ix, arg) in args.iter().enumerate() { + for (ix, arg) in self.block.args(self.pool).enumerate() { if ix > 0 { write!(f, ", ")?; } @@ -159,6 +189,94 @@ impl<'a> Display for DisplayBlockCall<'a> { } } +/// A `BlockArg` is a sum type of `Value`, `TryCallRet`, and +/// `TryCallExn`. The latter two are values that are generated "on the +/// edge" out of a `try_call` instruction into a successor block. We +/// use special arguments rather than special values for these because +/// they are not definable as SSA values at a certain program point -- +/// only when the `BlockCall` is executed. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum BlockArg { + /// An ordinary value, usable at the branch instruction using this + /// `BlockArg`, whose value is passed as an argument. + Value(Value), + + /// A return value of a `try_call`'s called function. Signatures + /// allow multiple return values, so this carries an index. This + /// may be used only on the normal (non-exceptional) `BlockCall` + /// out of a `try_call` or `try_call_indirect` instruction. + TryCallRet(u32), + + /// An exception payload value of a `try_call`. Some ABIs may + /// allow multiple payload values, so this carries an index. Its + /// type is defined by the ABI of the called function. This may be + /// used only on an exceptional `BlockCall` out of a `try_call` or + /// `try_call_indirect` instruction. + TryCallExn(u32), +} + +impl BlockArg { + /// Encode this block argument as a `Value` for storage in the + /// value pool. Internal to `BlockCall`, must not be used + /// elsewhere to avoid exposing the raw bit encoding. + fn encode_as_value(&self) -> Value { + let (tag, payload) = match *self { + BlockArg::Value(v) => (0, v.as_bits()), + BlockArg::TryCallRet(i) => (1, i), + BlockArg::TryCallExn(i) => (2, i), + }; + assert!(payload < (1 << 30)); + let raw = (tag << 30) | payload; + Value::from_bits(raw) + } + + /// Decode a raw `Value` encoding of this block argument. + fn decode_from_value(v: Value) -> Self { + let raw = v.as_u32(); + let tag = raw >> 30; + let payload = raw & ((1 << 30) - 1); + match tag { + 0 => BlockArg::Value(Value::from_bits(payload)), + 1 => BlockArg::TryCallRet(payload), + 2 => BlockArg::TryCallExn(payload), + _ => unreachable!(), + } + } + + /// Return this argument as a `Value`, if it is one, or `None` + /// otherwise. + pub fn as_value(&self) -> Option { + match *self { + BlockArg::Value(v) => Some(v), + _ => None, + } + } + + /// Update the contained value, if any. + pub fn map_value Value>(&self, mut f: F) -> Self { + match *self { + BlockArg::Value(v) => BlockArg::Value(f(v)), + other => other, + } + } +} + +impl Display for BlockArg { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + BlockArg::Value(v) => write!(f, "{v}"), + BlockArg::TryCallRet(i) => write!(f, "ret{i}"), + BlockArg::TryCallExn(i) => write!(f, "exn{i}"), + } + } +} + +impl From for BlockArg { + fn from(value: Value) -> BlockArg { + BlockArg::Value(value) + } +} + // Include code generated by `cranelift-codegen/meta/src/gen_inst.rs`. This file contains: // // - The `pub enum InstructionFormat` enum with all the instruction formats. @@ -305,11 +423,18 @@ impl InstructionData { /// Get the destinations of this instruction, if it's a branch. /// /// `br_table` returns the empty slice. - pub fn branch_destination<'a>(&'a self, jump_tables: &'a ir::JumpTables) -> &'a [BlockCall] { + pub fn branch_destination<'a>( + &'a self, + jump_tables: &'a ir::JumpTables, + exception_tables: &'a ir::ExceptionTables, + ) -> &'a [BlockCall] { match self { Self::Jump { destination, .. } => std::slice::from_ref(destination), Self::Brif { blocks, .. } => blocks.as_slice(), Self::BranchTable { table, .. } => jump_tables.get(*table).unwrap().all_branches(), + Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => { + exception_tables.get(*exception).unwrap().all_branches() + } _ => { debug_assert!(!self.opcode().is_branch()); &[] @@ -323,6 +448,7 @@ impl InstructionData { pub fn branch_destination_mut<'a>( &'a mut self, jump_tables: &'a mut ir::JumpTables, + exception_tables: &'a mut ir::ExceptionTables, ) -> &'a mut [BlockCall] { match self { Self::Jump { destination, .. } => std::slice::from_mut(destination), @@ -330,6 +456,12 @@ impl InstructionData { Self::BranchTable { table, .. } => { jump_tables.get_mut(*table).unwrap().all_branches_mut() } + Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => { + exception_tables + .get_mut(*exception) + .unwrap() + .all_branches_mut() + } _ => { debug_assert!(!self.opcode().is_branch()); &mut [] @@ -343,16 +475,15 @@ impl InstructionData { &mut self, pool: &mut ValueListPool, jump_tables: &mut ir::JumpTables, + exception_tables: &mut ir::ExceptionTables, mut f: impl FnMut(Value) -> Value, ) { for arg in self.arguments_mut(pool) { *arg = f(*arg); } - for block in self.branch_destination_mut(jump_tables) { - for arg in block.args_slice_mut(pool) { - *arg = f(*arg); - } + for block in self.branch_destination_mut(jump_tables, exception_tables) { + block.update_args(pool, |arg| arg.map_value(|val| f(val))); } } @@ -437,7 +568,11 @@ impl InstructionData { /// Return information about a call instruction. /// /// Any instruction that can call another function reveals its call signature here. - pub fn analyze_call<'a>(&'a self, pool: &'a ValueListPool) -> CallInfo<'a> { + pub fn analyze_call<'a>( + &'a self, + pool: &'a ValueListPool, + exception_tables: &ExceptionTables, + ) -> CallInfo<'a> { match *self { Self::Call { func_ref, ref args, .. @@ -445,6 +580,23 @@ impl InstructionData { Self::CallIndirect { sig_ref, ref args, .. } => CallInfo::Indirect(sig_ref, &args.as_slice(pool)[1..]), + Self::TryCall { + func_ref, + ref args, + exception, + .. + } => { + let exdata = &exception_tables[exception]; + CallInfo::DirectWithSig(func_ref, exdata.signature(), args.as_slice(pool)) + } + Self::TryCallIndirect { + exception, + ref args, + .. + } => { + let exdata = &exception_tables[exception]; + CallInfo::Indirect(exdata.signature(), &args.as_slice(pool)[1..]) + } Self::Ternary { opcode: Opcode::StackSwitch, .. @@ -495,6 +647,16 @@ impl InstructionData { _ => {} } } + + /// Get the exception table, if any, associated with this instruction. + pub fn exception_table(&self) -> Option { + match self { + Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => { + Some(*exception) + } + _ => None, + } + } } /// Information about call instructions. @@ -508,6 +670,11 @@ pub enum CallInfo<'a> { /// This is an indirect call with the specified signature. See `DataFlowGraph.signatures`. Indirect(SigRef, &'a [Value]), + + /// This is a direct call to an external function declared in the + /// preamble, but the signature is also known by other means: + /// e.g., from an exception table entry. + DirectWithSig(FuncRef, SigRef, &'a [Value]), } /// Value type constraints for a given opcode. diff --git a/cranelift/codegen/src/ir/jumptable.rs b/cranelift/codegen/src/ir/jumptable.rs index 8e1e15c7d621..bca05c21b09f 100644 --- a/cranelift/codegen/src/ir/jumptable.rs +++ b/cranelift/codegen/src/ir/jumptable.rs @@ -113,13 +113,14 @@ mod tests { use super::JumpTableData; use crate::entity::EntityRef; use crate::ir::instructions::ValueListPool; - use crate::ir::{Block, BlockCall, Value}; + use crate::ir::{Block, BlockArg, BlockCall, Value}; + use alloc::vec::Vec; use std::string::ToString; #[test] fn empty() { let mut pool = ValueListPool::default(); - let def = BlockCall::new(Block::new(0), &[], &mut pool); + let def = BlockCall::new(Block::new(0), core::iter::empty(), &mut pool); let jt = JumpTableData::new(def, &[]); @@ -145,10 +146,10 @@ mod tests { let e1 = Block::new(1); let e2 = Block::new(2); - let def = BlockCall::new(e0, &[], &mut pool); - let b1 = BlockCall::new(e1, &[v0], &mut pool); - let b2 = BlockCall::new(e2, &[], &mut pool); - let b3 = BlockCall::new(e1, &[v1], &mut pool); + let def = BlockCall::new(e0, core::iter::empty(), &mut pool); + let b1 = BlockCall::new(e1, core::iter::once(v0.into()), &mut pool); + let b2 = BlockCall::new(e2, core::iter::empty(), &mut pool); + let b3 = BlockCall::new(e1, core::iter::once(v1.into()), &mut pool); let jt = JumpTableData::new(def, &[b1, b2, b3]); @@ -161,8 +162,14 @@ mod tests { assert_eq!(jt.all_branches(), [def, b1, b2, b3]); assert_eq!(jt.as_slice(), [b1, b2, b3]); - assert_eq!(jt.as_slice()[0].args_slice(&pool), [v0]); - assert_eq!(jt.as_slice()[1].args_slice(&pool), []); - assert_eq!(jt.as_slice()[2].args_slice(&pool), [v1]); + assert_eq!( + jt.as_slice()[0].args(&pool).collect::>(), + [BlockArg::Value(v0)] + ); + assert_eq!(jt.as_slice()[1].args(&pool).collect::>(), []); + assert_eq!( + jt.as_slice()[2].args(&pool).collect::>(), + [BlockArg::Value(v1)] + ); } } diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index e6f082d70a8d..ee052a3c92af 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -7,6 +7,7 @@ pub mod constant; pub mod dfg; pub mod dynamic_type; pub mod entities; +mod exception_table; mod extfunc; mod extname; pub mod function; @@ -38,9 +39,11 @@ pub use crate::ir::constant::{ConstantData, ConstantPool}; pub use crate::ir::dfg::{BlockData, DataFlowGraph, ValueDef}; pub use crate::ir::dynamic_type::{dynamic_to_fixed, DynamicTypeData, DynamicTypes}; pub use crate::ir::entities::{ - Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Immediate, Inst, - JumpTable, MemoryType, SigRef, StackSlot, UserExternalNameRef, Value, + Block, Constant, DynamicStackSlot, DynamicType, ExceptionTable, ExceptionTag, FuncRef, + GlobalValue, Immediate, Inst, JumpTable, MemoryType, SigRef, StackSlot, UserExternalNameRef, + Value, }; +pub use crate::ir::exception_table::ExceptionTableData; pub use crate::ir::extfunc::{ AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature, }; @@ -48,7 +51,7 @@ pub use crate::ir::extname::{ExternalName, UserExternalName, UserFuncName}; pub use crate::ir::function::Function; pub use crate::ir::globalvalue::GlobalValueData; pub use crate::ir::instructions::{ - BlockCall, InstructionData, Opcode, ValueList, ValueListPool, VariableArgs, + BlockArg, BlockCall, InstructionData, Opcode, ValueList, ValueListPool, VariableArgs, }; pub use crate::ir::jumptable::JumpTableData; pub use crate::ir::known_symbol::KnownSymbol; @@ -72,6 +75,9 @@ use crate::entity::{entity_impl, PrimaryMap, SecondaryMap}; /// Map of jump tables. pub type JumpTables = PrimaryMap; +/// Map of exception tables. +pub type ExceptionTables = PrimaryMap; + /// Source locations for instructions. pub(crate) type SourceLocs = SecondaryMap; diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 1c5eb7b96b70..178e44cd0fc9 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -1090,10 +1090,11 @@ impl ABIMachineSpec for AArch64MachineDeps { } ], defs: smallvec![], - clobbers: Self::get_regs_clobbered_by_call(call_conv), + clobbers: Self::get_regs_clobbered_by_call(call_conv, false), caller_conv: call_conv, callee_conv: call_conv, callee_pop_size: 0, + try_call_info: None, }), }); insts @@ -1123,9 +1124,10 @@ impl ABIMachineSpec for AArch64MachineDeps { } } - fn get_regs_clobbered_by_call(call_conv: isa::CallConv) -> PRegSet { + fn get_regs_clobbered_by_call(call_conv: isa::CallConv, is_exception: bool) -> PRegSet { match call_conv { isa::CallConv::Winch => WINCH_CLOBBERS, + _ if is_exception => ALL_CLOBBERS, _ => DEFAULT_AAPCS_CLOBBERS, } } @@ -1200,6 +1202,14 @@ impl ABIMachineSpec for AArch64MachineDeps { // retval. regs::writable_xreg(9) } + + fn exception_payload_regs(call_conv: isa::CallConv) -> &'static [Reg] { + const PAYLOAD_REGS: &'static [Reg] = &[regs::xreg(0), regs::xreg(1)]; + match call_conv { + isa::CallConv::SystemV | isa::CallConv::Tail => PAYLOAD_REGS, + _ => &[], + } + } } impl AArch64MachineDeps { @@ -1519,8 +1529,77 @@ const fn winch_clobbers() -> PRegSet { .with(vreg_preg(31)) } +const fn all_clobbers() -> PRegSet { + PRegSet::empty() + // integer registers: x0 to x28 inclusive. (x29 is FP, x30 is + // LR, x31 is SP/ZR.) + .with(xreg_preg(0)) + .with(xreg_preg(1)) + .with(xreg_preg(2)) + .with(xreg_preg(3)) + .with(xreg_preg(4)) + .with(xreg_preg(5)) + .with(xreg_preg(6)) + .with(xreg_preg(7)) + .with(xreg_preg(8)) + .with(xreg_preg(9)) + .with(xreg_preg(10)) + .with(xreg_preg(11)) + .with(xreg_preg(12)) + .with(xreg_preg(13)) + .with(xreg_preg(14)) + .with(xreg_preg(15)) + .with(xreg_preg(16)) + .with(xreg_preg(17)) + .with(xreg_preg(18)) + .with(xreg_preg(19)) + .with(xreg_preg(20)) + .with(xreg_preg(21)) + .with(xreg_preg(22)) + .with(xreg_preg(23)) + .with(xreg_preg(24)) + .with(xreg_preg(25)) + .with(xreg_preg(26)) + .with(xreg_preg(27)) + .with(xreg_preg(28)) + // vector registers: v0 to v31 inclusive. + .with(vreg_preg(0)) + .with(vreg_preg(1)) + .with(vreg_preg(2)) + .with(vreg_preg(3)) + .with(vreg_preg(4)) + .with(vreg_preg(5)) + .with(vreg_preg(6)) + .with(vreg_preg(7)) + .with(vreg_preg(8)) + .with(vreg_preg(9)) + .with(vreg_preg(10)) + .with(vreg_preg(11)) + .with(vreg_preg(12)) + .with(vreg_preg(13)) + .with(vreg_preg(14)) + .with(vreg_preg(15)) + .with(vreg_preg(16)) + .with(vreg_preg(17)) + .with(vreg_preg(18)) + .with(vreg_preg(19)) + .with(vreg_preg(20)) + .with(vreg_preg(21)) + .with(vreg_preg(22)) + .with(vreg_preg(23)) + .with(vreg_preg(24)) + .with(vreg_preg(25)) + .with(vreg_preg(26)) + .with(vreg_preg(27)) + .with(vreg_preg(28)) + .with(vreg_preg(29)) + .with(vreg_preg(30)) + .with(vreg_preg(31)) +} + const DEFAULT_AAPCS_CLOBBERS: PRegSet = default_aapcs_clobbers(); const WINCH_CLOBBERS: PRegSet = winch_clobbers(); +const ALL_CLOBBERS: PRegSet = all_clobbers(); fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { fn preg(r: Reg) -> PReg { diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 7b82b1ce791c..b81124aab612 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -4414,6 +4414,12 @@ (decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) (extern constructor gen_call_indirect gen_call_indirect) +(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call gen_try_call) + +(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call_indirect gen_try_call_indirect) + ;; Helpers for pinned register manipulation. (decl write_pinned_reg (Reg) SideEffectNoResult) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 0d6e11e32f80..c9f84066da8d 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -2944,6 +2944,14 @@ impl MachInstEmit for Inst { } sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + if info.callee_pop_size > 0 { let callee_pop_size = i32::try_from(info.callee_pop_size).expect("callee popped more than 2GB"); @@ -2959,6 +2967,15 @@ impl MachInstEmit for Inst { |needed_space| Some(Inst::EmitIsland { needed_space }), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = Inst::Jump { + dest: BranchTarget::Label(try_call.continuation), + }; + jmp.emit(sink, emit_info, state); + } + // We produce an island above if needed, so disable // the worst-case-size check in this case. start_off = sink.cur_offset(); @@ -2974,6 +2991,14 @@ impl MachInstEmit for Inst { } sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + if info.callee_pop_size > 0 { let callee_pop_size = i32::try_from(info.callee_pop_size).expect("callee popped more than 2GB"); @@ -2989,6 +3014,15 @@ impl MachInstEmit for Inst { |needed_space| Some(Inst::EmitIsland { needed_space }), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = Inst::Jump { + dest: BranchTarget::Label(try_call.continuation), + }; + jmp.emit(sink, emit_info, state); + } + // We produce an island above if needed, so disable // the worst-case-size check in this case. start_off = sink.cur_offset(); diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 84dae78933f5..f8abf42d0cc8 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -932,7 +932,7 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { Inst::MachOTlsGetAddr { rd, .. } => { collector.reg_fixed_def(rd, regs::xreg(0)); let mut clobbers = - AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::AppleAarch64); + AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::AppleAarch64, false); clobbers.remove(regs::xreg_preg(0)); collector.reg_clobbers(clobbers); } @@ -979,6 +979,8 @@ impl MachInst for Inst { fn is_included_in_clobbers(&self) -> bool { let (caller, callee) = match self { Inst::Args { .. } => return false, + Inst::Call { info } if info.try_call_info.is_some() => return true, + Inst::CallInd { info } if info.try_call_info.is_some() => return true, Inst::Call { info } => (info.caller_conv, info.callee_conv), Inst::CallInd { info } => (info.caller_conv, info.callee_conv), _ => return true, @@ -995,8 +997,8 @@ impl MachInst for Inst { // // See the note in [crate::isa::aarch64::abi::is_caller_save_reg] for // more information on this ABI-implementation hack. - let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller); - let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee); + let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller, false); + let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee, false); let mut all_clobbers = caller_clobbers; all_clobbers.union_from(callee_clobbers); @@ -1021,11 +1023,13 @@ impl MachInst for Inst { match self { &Inst::Rets { .. } => MachTerminator::Ret, &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall, - &Inst::Jump { .. } => MachTerminator::Uncond, - &Inst::CondBr { .. } => MachTerminator::Cond, - &Inst::TestBitAndBranch { .. } => MachTerminator::Cond, - &Inst::IndirectBr { .. } => MachTerminator::Indirect, - &Inst::JTSequence { .. } => MachTerminator::Indirect, + &Inst::Jump { .. } => MachTerminator::Branch, + &Inst::CondBr { .. } => MachTerminator::Branch, + &Inst::TestBitAndBranch { .. } => MachTerminator::Branch, + &Inst::IndirectBr { .. } => MachTerminator::Branch, + &Inst::JTSequence { .. } => MachTerminator::Branch, + &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, + &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, _ => MachTerminator::None, } } @@ -1200,6 +1204,16 @@ fn mem_finalize_for_show(mem: &AMode, access_ty: Type, state: &EmitState) -> (St (mem_str, mem) } +fn pretty_print_try_call(info: &TryCallInfo) -> String { + let dests = info + .exception_dests + .iter() + .map(|(tag, label)| format!("{tag:?}: {label:?}")) + .collect::>() + .join(", "); + format!("; b {:?}; catch [{dests}]", info.continuation) +} + impl Inst { fn print_with_state(&self, state: &mut EmitState) -> String { fn op_name(alu_op: ALUOp) -> &'static str { @@ -2565,10 +2579,22 @@ impl Inst { format!("{op} {rd}, {rn}") } } - &Inst::Call { .. } => format!("bl 0"), + &Inst::Call { ref info } => { + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("bl 0{try_call}") + } &Inst::CallInd { ref info } => { let rn = pretty_print_reg(info.dest); - format!("blr {rn}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("blr {rn}{try_call}") } &Inst::ReturnCall { ref info } => { let mut s = format!( diff --git a/cranelift/codegen/src/isa/aarch64/inst/regs.rs b/cranelift/codegen/src/isa/aarch64/inst/regs.rs index fd1abb7fffb4..f5a6eb526daf 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/regs.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/regs.rs @@ -20,8 +20,8 @@ pub const PINNED_REG: u8 = 21; /// Get a reference to an X-register (integer register). Do not use /// this for xsp / xzr; we have two special registers for those. -pub fn xreg(num: u8) -> Reg { - Reg::from(xreg_preg(num)) +pub const fn xreg(num: u8) -> Reg { + Reg::from_real_reg(xreg_preg(num)) } /// Get the given X-register as a PReg. diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 3c02482118ab..65e0d20e6d2c 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -2511,6 +2511,13 @@ (rule (lower (call_indirect sig_ref val inputs)) (gen_call_indirect sig_ref val inputs)) +(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets) + (gen_try_call sig_ref extname dist et inputs targets)) + +(rule (lower_branch (try_call_indirect val inputs et) targets) + (if-let (exception_sig sig_ref) et) + (gen_try_call_indirect sig_ref val et inputs targets)) + ;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; N.B.: the Ret itself is generated by the ABI. diff --git a/cranelift/codegen/src/isa/call_conv.rs b/cranelift/codegen/src/isa/call_conv.rs index cb21d67be3c5..cb869c7ca4f9 100644 --- a/cranelift/codegen/src/isa/call_conv.rs +++ b/cranelift/codegen/src/isa/call_conv.rs @@ -1,3 +1,5 @@ +use crate::ir::types; +use crate::ir::Type; use crate::settings::{self, LibcallCallConv}; use core::fmt; use core::str; @@ -14,7 +16,16 @@ pub enum CallConv { Fast, /// Smallest caller code size, not ABI-stable. Cold, - /// Supports tail calls, not ABI-stable. + /// Supports tail calls, not ABI-stable except for exception + /// payload registers. + /// + /// On exception resume, a caller to a `tail`-convention function + /// assumes that the exception payload values are in the following + /// registers (per platform): + /// - x86-64: rax, rdx + /// - aarch64: x0, x1 + /// - riscv64: a0, a1 + /// - pulley{32,64}: x0, x1 // // Currently, this is basically sys-v except that callees pop stack // arguments, rather than callers. Expected to change even more in the @@ -69,6 +80,26 @@ impl CallConv { _ => false, } } + + /// Does this calling convention support exceptions? + pub fn supports_exceptions(&self) -> bool { + match self { + CallConv::Tail => true, + _ => false, + } + } + + /// What types do the exception payload value(s) have? + pub fn exception_payload_types(&self, pointer_ty: Type) -> &[Type] { + match self { + CallConv::Tail => match pointer_ty { + types::I32 => &[types::I32, types::I32], + types::I64 => &[types::I64, types::I64], + _ => unreachable!(), + }, + _ => &[], + } + } } impl fmt::Display for CallConv { diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs index 0f4a4efbe296..ab09eeec4692 100644 --- a/cranelift/codegen/src/isa/pulley_shared/abi.rs +++ b/cranelift/codegen/src/isa/pulley_shared/abi.rs @@ -532,8 +532,15 @@ where MACHINE_ENV.get_or_init(create_reg_environment) } - fn get_regs_clobbered_by_call(_call_conv_of_callee: isa::CallConv) -> PRegSet { - DEFAULT_CLOBBERS + fn get_regs_clobbered_by_call( + _call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet { + if is_exception { + ALL_CLOBBERS + } else { + DEFAULT_CLOBBERS + } } fn compute_frame_layout( @@ -600,6 +607,14 @@ where // retval. Writable::from_reg(regs::x_reg(15)) } + + fn exception_payload_regs(_call_conv: isa::CallConv) -> &'static [Reg] { + const PAYLOAD_REGS: &'static [Reg] = &[ + Reg::from_real_reg(regs::px_reg(0)), + Reg::from_real_reg(regs::px_reg(1)), + ]; + PAYLOAD_REGS + } } /// Different styles of management of fp/lr and clobbered registers. @@ -934,6 +949,104 @@ const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() .with(pv_reg(30)) .with(pv_reg(31)); +const ALL_CLOBBERS: PRegSet = PRegSet::empty() + .with(px_reg(0)) + .with(px_reg(1)) + .with(px_reg(2)) + .with(px_reg(3)) + .with(px_reg(4)) + .with(px_reg(5)) + .with(px_reg(6)) + .with(px_reg(7)) + .with(px_reg(8)) + .with(px_reg(9)) + .with(px_reg(10)) + .with(px_reg(11)) + .with(px_reg(12)) + .with(px_reg(13)) + .with(px_reg(14)) + .with(px_reg(15)) + .with(px_reg(16)) + .with(px_reg(17)) + .with(px_reg(18)) + .with(px_reg(19)) + .with(px_reg(20)) + .with(px_reg(21)) + .with(px_reg(22)) + .with(px_reg(23)) + .with(px_reg(24)) + .with(px_reg(25)) + .with(px_reg(26)) + .with(px_reg(27)) + .with(px_reg(28)) + .with(px_reg(29)) + .with(px_reg(30)) + .with(px_reg(31)) + .with(pf_reg(0)) + .with(pf_reg(1)) + .with(pf_reg(2)) + .with(pf_reg(3)) + .with(pf_reg(4)) + .with(pf_reg(5)) + .with(pf_reg(6)) + .with(pf_reg(7)) + .with(pf_reg(8)) + .with(pf_reg(9)) + .with(pf_reg(10)) + .with(pf_reg(11)) + .with(pf_reg(12)) + .with(pf_reg(13)) + .with(pf_reg(14)) + .with(pf_reg(15)) + .with(pf_reg(16)) + .with(pf_reg(17)) + .with(pf_reg(18)) + .with(pf_reg(19)) + .with(pf_reg(20)) + .with(pf_reg(21)) + .with(pf_reg(22)) + .with(pf_reg(23)) + .with(pf_reg(24)) + .with(pf_reg(25)) + .with(pf_reg(26)) + .with(pf_reg(27)) + .with(pf_reg(28)) + .with(pf_reg(29)) + .with(pf_reg(30)) + .with(pf_reg(31)) + .with(pv_reg(0)) + .with(pv_reg(1)) + .with(pv_reg(2)) + .with(pv_reg(3)) + .with(pv_reg(4)) + .with(pv_reg(5)) + .with(pv_reg(6)) + .with(pv_reg(7)) + .with(pv_reg(8)) + .with(pv_reg(9)) + .with(pv_reg(10)) + .with(pv_reg(11)) + .with(pv_reg(12)) + .with(pv_reg(13)) + .with(pv_reg(14)) + .with(pv_reg(15)) + .with(pv_reg(16)) + .with(pv_reg(17)) + .with(pv_reg(18)) + .with(pv_reg(19)) + .with(pv_reg(20)) + .with(pv_reg(21)) + .with(pv_reg(22)) + .with(pv_reg(23)) + .with(pv_reg(24)) + .with(pv_reg(25)) + .with(pv_reg(26)) + .with(pv_reg(27)) + .with(pv_reg(28)) + .with(pv_reg(29)) + .with(pv_reg(30)) + .with(pv_reg(31)); + fn create_reg_environment() -> MachineEnv { // Prefer caller-saved registers over callee-saved registers, because that // way we don't need to emit code to save and restore them if we don't diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle index bdca3c8d978f..2387d7c0ea35 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst.isle +++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle @@ -714,6 +714,12 @@ (decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) (extern constructor gen_call_indirect gen_call_indirect) +(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call gen_try_call) + +(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call_indirect gen_try_call_indirect) + ;;;; Helpers for Sign extension ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Sign extend a `Value` to at least 32-bit diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index 1a5095e6fe21..d0b741467758 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -184,6 +184,14 @@ fn pulley_emit

( } sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + let adjust = -i32::try_from(info.callee_pop_size).unwrap(); for i in PulleyMachineDeps::

::gen_sp_reg_adjust(adjust) { >::from(i).emit(sink, emit_info, state); @@ -196,6 +204,15 @@ fn pulley_emit

( |space_needed| Some(>::from(Inst::EmitIsland { space_needed })), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = InstAndKind::

::from(Inst::Jump { + label: try_call.continuation, + }); + jmp.emit(sink, emit_info, state); + } + // We produce an island above if needed, so disable // the worst-case-size check in this case. *start_offset = sink.cur_offset(); @@ -211,6 +228,14 @@ fn pulley_emit

( sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + let adjust = -i32::try_from(info.callee_pop_size).unwrap(); for i in PulleyMachineDeps::

::gen_sp_reg_adjust(adjust) { >::from(i).emit(sink, emit_info, state); @@ -223,6 +248,15 @@ fn pulley_emit

( |space_needed| Some(>::from(Inst::EmitIsland { space_needed })), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = InstAndKind::

::from(Inst::Jump { + label: try_call.continuation, + }); + jmp.emit(sink, emit_info, state); + } + // We produce an island above if needed, so disable // the worst-case-size check in this case. *start_offset = sink.cur_offset(); @@ -541,10 +575,12 @@ fn pulley_emit

( } Inst::EmitIsland { space_needed } => { - let label = sink.get_label(); - >::from(Inst::Jump { label }).emit(sink, emit_info, state); - sink.emit_island(space_needed + 8, &mut state.ctrl_plane); - sink.bind_label(label, &mut state.ctrl_plane); + if sink.island_needed(*space_needed) { + let label = sink.get_label(); + >::from(Inst::Jump { label }).emit(sink, emit_info, state); + sink.emit_island(space_needed + 8, &mut state.ctrl_plane); + sink.bind_label(label, &mut state.ctrl_plane); + } } } } diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs index 1956b2367867..49d3180c5687 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs @@ -10,6 +10,7 @@ use crate::isa::FunctionAlignment; use crate::{machinst::*, trace}; use crate::{settings, CodegenError, CodegenResult}; use alloc::string::{String, ToString}; +use alloc::vec::Vec; use regalloc2::RegClass; use smallvec::SmallVec; @@ -446,15 +447,17 @@ where } fn is_term(&self) -> MachTerminator { - match self.inst { + match &self.inst { Inst::Raw { raw: RawInst::Ret { .. }, } | Inst::Rets { .. } => MachTerminator::Ret, - Inst::Jump { .. } => MachTerminator::Uncond, - Inst::BrIf { .. } => MachTerminator::Cond, - Inst::BrTable { .. } => MachTerminator::Indirect, - Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => MachTerminator::Indirect, + Inst::Jump { .. } => MachTerminator::Branch, + Inst::BrIf { .. } => MachTerminator::Branch, + Inst::BrTable { .. } => MachTerminator::Branch, + Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => MachTerminator::RetCall, + Inst::Call { info } if info.try_call_info.is_some() => MachTerminator::Branch, + Inst::IndirectCall { info } if info.try_call_info.is_some() => MachTerminator::Branch, _ => MachTerminator::None, } } @@ -584,6 +587,16 @@ pub fn reg_name(reg: Reg) -> String { } } +fn pretty_print_try_call(info: &TryCallInfo) -> String { + let dests = info + .exception_dests + .iter() + .map(|(tag, label)| format!("{tag:?}: {label:?}")) + .collect::>() + .join(", "); + format!("; jump {:?}; catch [{dests}]", info.continuation) +} + impl Inst { fn print_with_state

(&self, _state: &mut EmitState

) -> String where @@ -636,12 +649,22 @@ impl Inst { } Inst::Call { info } => { - format!("call {info:?}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("call {info:?}{try_call}") } Inst::IndirectCall { info } => { let callee = format_reg(*info.dest); - format!("indirect_call {callee}, {info:?}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("indirect_call {callee}, {info:?}{try_call}") } Inst::ReturnCall { info } => { diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle index 8371f9c60b91..7e4a421c10bb 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower.isle +++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle @@ -146,6 +146,13 @@ (rule (lower (call_indirect sig_ref val inputs)) (gen_call_indirect sig_ref val inputs)) +(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets) + (gen_try_call sig_ref extname dist et inputs targets)) + +(rule (lower_branch (try_call_indirect val inputs et) targets) + (if-let (exception_sig sig_ref) et) + (gen_try_call_indirect sig_ref val et inputs targets)) + ;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (return_call (func_ref_data sig_ref extname dist) args)) diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs index 5364b78dd0d2..5df160884887 100644 --- a/cranelift/codegen/src/isa/riscv64/abi.rs +++ b/cranelift/codegen/src/isa/riscv64/abi.rs @@ -610,10 +610,11 @@ impl ABIMachineSpec for Riscv64MachineDeps { } ], defs: smallvec![], - clobbers: Self::get_regs_clobbered_by_call(call_conv), + clobbers: Self::get_regs_clobbered_by_call(call_conv, false), caller_conv: call_conv, callee_conv: call_conv, callee_pop_size: 0, + try_call_info: None, }), }); insts @@ -637,8 +638,15 @@ impl ABIMachineSpec for Riscv64MachineDeps { MACHINE_ENV.get_or_init(create_reg_environment) } - fn get_regs_clobbered_by_call(_call_conv_of_callee: isa::CallConv) -> PRegSet { - DEFAULT_CLOBBERS + fn get_regs_clobbered_by_call( + _call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet { + if is_exception { + ALL_CLOBBERS + } else { + DEFAULT_CLOBBERS + } } fn compute_frame_layout( @@ -727,6 +735,14 @@ impl ABIMachineSpec for Riscv64MachineDeps { // retval. Writable::from_reg(regs::x_reg(12)) } + + fn exception_payload_regs(call_conv: isa::CallConv) -> &'static [Reg] { + const PAYLOAD_REGS: &'static [Reg] = &[regs::a0(), regs::a1()]; + match call_conv { + isa::CallConv::SystemV | isa::CallConv::Tail => PAYLOAD_REGS, + _ => &[], + } + } } impl Riscv64ABICallSite { @@ -902,6 +918,104 @@ const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() .with(pv_reg(30)) .with(pv_reg(31)); +const ALL_CLOBBERS: PRegSet = PRegSet::empty() + // Specials: x0 is the zero register; x1 is the return address; x2 is SP. + .with(px_reg(3)) + .with(px_reg(4)) + .with(px_reg(5)) + .with(px_reg(6)) + .with(px_reg(7)) + .with(px_reg(8)) + .with(px_reg(9)) + .with(px_reg(10)) + .with(px_reg(11)) + .with(px_reg(12)) + .with(px_reg(13)) + .with(px_reg(14)) + .with(px_reg(15)) + .with(px_reg(16)) + .with(px_reg(17)) + .with(px_reg(18)) + .with(px_reg(19)) + .with(px_reg(20)) + .with(px_reg(21)) + .with(px_reg(22)) + .with(px_reg(23)) + .with(px_reg(24)) + .with(px_reg(25)) + .with(px_reg(26)) + .with(px_reg(27)) + .with(px_reg(28)) + .with(px_reg(29)) + .with(px_reg(30)) + .with(px_reg(31)) + // F Regs + .with(pf_reg(0)) + .with(pf_reg(1)) + .with(pf_reg(2)) + .with(pf_reg(3)) + .with(pf_reg(4)) + .with(pf_reg(5)) + .with(pf_reg(6)) + .with(pf_reg(7)) + .with(pf_reg(8)) + .with(pf_reg(9)) + .with(pf_reg(10)) + .with(pf_reg(11)) + .with(pf_reg(12)) + .with(pf_reg(13)) + .with(pf_reg(14)) + .with(pf_reg(15)) + .with(pf_reg(16)) + .with(pf_reg(17)) + .with(pf_reg(18)) + .with(pf_reg(19)) + .with(pf_reg(20)) + .with(pf_reg(21)) + .with(pf_reg(22)) + .with(pf_reg(23)) + .with(pf_reg(24)) + .with(pf_reg(25)) + .with(pf_reg(26)) + .with(pf_reg(27)) + .with(pf_reg(28)) + .with(pf_reg(29)) + .with(pf_reg(30)) + .with(pf_reg(31)) + // V Regs + .with(pv_reg(0)) + .with(pv_reg(1)) + .with(pv_reg(2)) + .with(pv_reg(3)) + .with(pv_reg(4)) + .with(pv_reg(5)) + .with(pv_reg(6)) + .with(pv_reg(7)) + .with(pv_reg(8)) + .with(pv_reg(9)) + .with(pv_reg(10)) + .with(pv_reg(11)) + .with(pv_reg(12)) + .with(pv_reg(13)) + .with(pv_reg(14)) + .with(pv_reg(15)) + .with(pv_reg(16)) + .with(pv_reg(17)) + .with(pv_reg(18)) + .with(pv_reg(19)) + .with(pv_reg(20)) + .with(pv_reg(21)) + .with(pv_reg(22)) + .with(pv_reg(23)) + .with(pv_reg(24)) + .with(pv_reg(25)) + .with(pv_reg(26)) + .with(pv_reg(27)) + .with(pv_reg(28)) + .with(pv_reg(29)) + .with(pv_reg(30)) + .with(pv_reg(31)); + fn create_reg_environment() -> MachineEnv { // Some C Extension instructions can only use a subset of the registers. // x8 - x15, f8 - f15, v8 - v15 so we should prefer to use those since diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle index 7d6b5c96adac..e9004fdcd2d1 100644 --- a/cranelift/codegen/src/isa/riscv64/inst.isle +++ b/cranelift/codegen/src/isa/riscv64/inst.isle @@ -2913,6 +2913,13 @@ (rule (lower_branch (br_table index _) targets) (lower_br_table index targets)) +(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets) + (gen_try_call sig_ref extname dist et inputs targets)) + +(rule (lower_branch (try_call_indirect val inputs et) targets) + (if-let (exception_sig sig_ref) et) + (gen_try_call_indirect sig_ref val et inputs targets)) + (decl load_ra () Reg) (extern constructor load_ra load_ra) @@ -3017,6 +3024,12 @@ (decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) (extern constructor gen_call_indirect gen_call_indirect) +(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call gen_try_call) + +(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call_indirect gen_try_call_indirect) + ;;; this is trying to imitate aarch64 `madd` instruction. (decl madd (XReg XReg XReg) XReg) (rule diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs index 36e72db5c2e8..a2883f0fc711 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs @@ -1128,6 +1128,14 @@ impl Inst { sink.push_user_stack_map(state, offset, s); } + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap(); if callee_pop_size > 0 { for inst in Riscv64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) { @@ -1142,6 +1150,15 @@ impl Inst { |needed_space| Some(Inst::EmitIsland { needed_space }), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = Inst::Jal { + label: try_call.continuation, + }; + jmp.emit(sink, emit_info, state); + } + *start_off = sink.cur_offset(); } &Inst::CallInd { ref info } => { @@ -1159,6 +1176,14 @@ impl Inst { sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap(); if callee_pop_size > 0 { for inst in Riscv64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) { @@ -1173,6 +1198,15 @@ impl Inst { |needed_space| Some(Inst::EmitIsland { needed_space }), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = Inst::Jal { + label: try_call.continuation, + }; + jmp.emit(sink, emit_info, state); + } + *start_off = sink.cur_offset(); } @@ -2598,10 +2632,12 @@ impl Inst { } Inst::EmitIsland { needed_space } => { - let jump_around_label = sink.get_label(); - Inst::gen_jump(jump_around_label).emit(sink, emit_info, state); - sink.emit_island(needed_space + 4, &mut state.ctrl_plane); - sink.bind_label(jump_around_label, &mut state.ctrl_plane); + if sink.island_needed(*needed_space) { + let jump_around_label = sink.get_label(); + Inst::gen_jump(jump_around_label).emit(sink, emit_info, state); + sink.emit_island(needed_space + 4, &mut state.ctrl_plane); + sink.bind_label(jump_around_label, &mut state.ctrl_plane); + } } } } diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs index d75e0c1c827c..05e38bbf7a91 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs @@ -390,7 +390,8 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { Inst::ElfTlsGetAddr { rd, .. } => { // x10 is a0 which is both the first argument and the first return value. collector.reg_fixed_def(rd, a0()); - let mut clobbers = Riscv64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV); + let mut clobbers = + Riscv64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV, false); clobbers.remove(px_reg(10)); collector.reg_clobbers(clobbers); } @@ -751,12 +752,14 @@ impl MachInst for Inst { fn is_term(&self) -> MachTerminator { match self { - &Inst::Jal { .. } => MachTerminator::Uncond, - &Inst::CondBr { .. } => MachTerminator::Cond, - &Inst::Jalr { .. } => MachTerminator::Uncond, + &Inst::Jal { .. } => MachTerminator::Branch, + &Inst::CondBr { .. } => MachTerminator::Branch, + &Inst::Jalr { .. } => MachTerminator::Branch, &Inst::Rets { .. } => MachTerminator::Ret, - &Inst::BrTable { .. } => MachTerminator::Indirect, + &Inst::BrTable { .. } => MachTerminator::Branch, &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall, + &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, + &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, _ => MachTerminator::None, } } @@ -874,6 +877,16 @@ pub fn reg_name(reg: Reg) -> String { } } +fn pretty_print_try_call(info: &TryCallInfo) -> String { + let dests = info + .exception_dests + .iter() + .map(|(tag, label)| format!("{tag:?}: {label:?}")) + .collect::>() + .join(", "); + format!("; j {:?}; catch [{dests}]", info.continuation) +} + impl Inst { fn print_with_state(&self, _state: &mut EmitState) -> String { let format_reg = |reg: Reg| -> String { reg_name(reg) }; @@ -1305,10 +1318,22 @@ impl Inst { format!("slli {rd},{rn},{shift_bits}; {op} {rd},{rd},{shift_bits}") }; } - &MInst::Call { ref info } => format!("call {}", info.dest.display(None)), + &MInst::Call { ref info } => { + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("call {}{try_call}", info.dest.display(None)) + } &MInst::CallInd { ref info } => { let rd = format_reg(info.dest); - format!("callind {rd}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("callind {rd}{try_call}") } &MInst::ReturnCall { ref info } => { let mut s = format!( diff --git a/cranelift/codegen/src/isa/riscv64/inst/regs.rs b/cranelift/codegen/src/isa/riscv64/inst/regs.rs index ffdc484a00c7..1231567f8703 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/regs.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/regs.rs @@ -10,14 +10,14 @@ use regalloc2::{PReg, RegClass, VReg}; // first argument of function call #[inline] -pub fn a0() -> Reg { +pub const fn a0() -> Reg { x_reg(10) } // second argument of function call #[inline] #[allow(dead_code)] -pub fn a1() -> Reg { +pub const fn a1() -> Reg { x_reg(11) } @@ -135,10 +135,10 @@ pub fn writable_spilltmp_reg2() -> Writable { } #[inline] -pub fn x_reg(enc: usize) -> Reg { +pub const fn x_reg(enc: usize) -> Reg { let p_reg = PReg::new(enc, RegClass::Int); let v_reg = VReg::new(p_reg.index(), p_reg.class()); - Reg::from(v_reg) + Reg::from_virtual_reg(v_reg) } pub const fn px_reg(enc: usize) -> PReg { PReg::new(enc, RegClass::Int) diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs index 34999f1d79c1..7dd2c53c7c66 100644 --- a/cranelift/codegen/src/isa/s390x/abi.rs +++ b/cranelift/codegen/src/isa/s390x/abi.rs @@ -902,7 +902,11 @@ impl ABIMachineSpec for S390xMachineDeps { } } - fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet { + fn get_regs_clobbered_by_call( + call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet { + assert!(!is_exception); match call_conv_of_callee { isa::CallConv::Tail => TAIL_CLOBBERS, _ => SYSV_CLOBBERS, diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index a8fc78cd86bc..c4d71dea30e5 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -914,7 +914,8 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor false, - &Inst::Call { ref info, .. } => info.caller_conv != info.callee_conv, + &Inst::Call { ref info, .. } => { + info.caller_conv != info.callee_conv || info.try_call_info.is_some() + } &Inst::ElfTlsGetOffset { .. } => false, _ => true, } @@ -1069,10 +1072,10 @@ impl MachInst for Inst { match self { &Inst::Rets { .. } => MachTerminator::Ret, &Inst::ReturnCall { .. } => MachTerminator::RetCall, - &Inst::Jump { .. } => MachTerminator::Uncond, - &Inst::CondBr { .. } => MachTerminator::Cond, - &Inst::IndirectBr { .. } => MachTerminator::Indirect, - &Inst::JTSequence { .. } => MachTerminator::Indirect, + &Inst::Jump { .. } => MachTerminator::Branch, + &Inst::CondBr { .. } => MachTerminator::Branch, + &Inst::IndirectBr { .. } => MachTerminator::Branch, + &Inst::JTSequence { .. } => MachTerminator::Branch, _ => MachTerminator::None, } } diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index 4226c8f9832d..842d514af2d4 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -278,7 +278,10 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { let sig_data = &self.lower_ctx.sigs()[abi]; // Get clobbers: all caller-saves. These may include return value // regs, which we will remove from the clobber set later. - let clobbers = S390xMachineDeps::get_regs_clobbered_by_call(sig_data.call_conv()); + let clobbers = S390xMachineDeps::get_regs_clobbered_by_call( + sig_data.call_conv(), + /* is_exception = */ false, + ); let callee_pop_size = if sig_data.call_conv() == CallConv::Tail { sig_data.sized_stack_arg_space() as u32 } else { @@ -292,6 +295,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { callee_pop_size, caller_conv: self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()), callee_conv: self.lower_ctx.sigs()[abi].call_conv(), + try_call_info: None, }); (info, outputs) } diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index c1bbff697931..c6ed91cc4f86 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -875,10 +875,11 @@ impl ABIMachineSpec for X64ABIMachineSpec { }, ], defs: smallvec![], - clobbers: Self::get_regs_clobbered_by_call(call_conv), + clobbers: Self::get_regs_clobbered_by_call(call_conv, false), callee_pop_size, callee_conv: call_conv, caller_conv: call_conv, + try_call_info: None, }))); insts } @@ -906,10 +907,14 @@ impl ABIMachineSpec for X64ABIMachineSpec { } } - fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet { + fn get_regs_clobbered_by_call( + call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet { match call_conv_of_callee { CallConv::Winch => ALL_CLOBBERS, CallConv::WindowsFastcall => WINDOWS_CLOBBERS, + _ if is_exception => ALL_CLOBBERS, _ => SYSV_CLOBBERS, } } @@ -981,6 +986,14 @@ impl ABIMachineSpec for X64ABIMachineSpec { // supported calling conventions. Writable::from_reg(regs::r11()) } + + fn exception_payload_regs(call_conv: isa::CallConv) -> &'static [Reg] { + const PAYLOAD_REGS: &'static [Reg] = &[regs::rax(), regs::rdx()]; + match call_conv { + isa::CallConv::SystemV | isa::CallConv::Tail => PAYLOAD_REGS, + _ => &[], + } + } } impl X64CallSite { diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index c8338adae6d3..5e3d21024168 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -2477,6 +2477,12 @@ (decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) (extern constructor gen_call_indirect gen_call_indirect) +(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call gen_try_call) + +(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call_indirect gen_try_call_indirect) + ;;;; Helpers for emitting stack switches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl x64_stack_switch_basic (Gpr Gpr Gpr) Gpr) diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 01f8e281e065..b086c215c92d 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1613,6 +1613,14 @@ pub(crate) fn emit( sink.put4(0); sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = call_info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + // Reclaim the outgoing argument area that was released by the callee, to ensure that // StackAMode values are always computed from a consistent SP. if call_info.callee_pop_size > 0 { @@ -1631,6 +1639,15 @@ pub(crate) fn emit( |inst| inst.emit(sink, info, state), |_space_needed| None, ); + + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = call_info.try_call_info.as_ref() { + let jmp = Inst::JmpKnown { + dst: try_call.continuation, + }; + jmp.emit(sink, info, state); + } } Inst::ReturnCallKnown { info: call_info } => { @@ -1702,6 +1719,14 @@ pub(crate) fn emit( sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = call_info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + // Reclaim the outgoing argument area that was released by the callee, to ensure that // StackAMode values are always computed from a consistent SP. if call_info.callee_pop_size > 0 { @@ -1720,6 +1745,13 @@ pub(crate) fn emit( |inst| inst.emit(sink, info, state), |_space_needed| None, ); + + if let Some(try_call) = call_info.try_call_info.as_ref() { + let jmp = Inst::JmpKnown { + dst: try_call.continuation, + }; + jmp.emit(sink, info, state); + } } Inst::Args { .. } => {} diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 79740b9d5165..d95b88fe3752 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -5123,7 +5123,7 @@ fn test_x64_emit() { for (insn, expected_encoding, expected_printing) in insns { // Check the printed text is as expected. let actual_printing = insn.pretty_print_inst(&mut Default::default()); - assert_eq!(expected_printing, actual_printing); + assert_eq!(expected_printing, actual_printing.trim()); let mut buffer = MachBuffer::new(); insn.emit(&mut buffer, &emit_info, &mut Default::default()); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 5ee15133c24b..6d6d3ac3efba 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -11,6 +11,7 @@ use crate::isa::{CallConv, FunctionAlignment}; use crate::{machinst::*, trace}; use crate::{settings, CodegenError, CodegenResult}; use alloc::boxed::Box; +use alloc::vec::Vec; use smallvec::{smallvec, SmallVec}; use std::fmt::{self, Write}; use std::string::{String, ToString}; @@ -1645,13 +1646,23 @@ impl PrettyPrint for Inst { Inst::CallKnown { info } => { let op = ljustify("call".to_string()); - format!("{op} {:?}", info.dest) + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("{op} {:?}{try_call}", info.dest) } Inst::CallUnknown { info } => { let dest = info.dest.pretty_print(8); let op = ljustify("call".to_string()); - format!("{op} *{dest}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("{op} *{dest}{try_call}") } Inst::ReturnCallKnown { info } => { @@ -1972,6 +1983,16 @@ impl PrettyPrint for Inst { } } +fn pretty_print_try_call(info: &TryCallInfo) -> String { + let dests = info + .exception_dests + .iter() + .map(|(tag, label)| format!("{tag:?}: {label:?}")) + .collect::>() + .join(", "); + format!("; jmp {:?}; catch [{dests}]", info.continuation) +} + impl fmt::Debug for Inst { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { write!(fmt, "{}", self.pretty_print_inst(&mut Default::default())) @@ -2468,7 +2489,7 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { // TODO(https://github.com/bytecodealliance/regalloc2/issues/145): // This shouldn't be a fixed register constraint. r10 is caller-saved, so this // should be safe to use. - collector.reg_fixed_use(reg, regs::r10()) + collector.reg_fixed_use(reg, regs::r10()); } _ => dest.get_operands(collector), } @@ -2696,7 +2717,8 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { // pseudoinstruction (and relocation that it emits) is specific to // ELF systems; other x86-64 targets with other conventions (i.e., // Windows) use different TLS strategies. - let mut clobbers = X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV); + let mut clobbers = + X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV, false); clobbers.remove(regs::gpr_preg(regs::ENC_RAX)); collector.reg_clobbers(clobbers); } @@ -2795,10 +2817,14 @@ impl MachInst for Inst { &Self::ReturnCallKnown { .. } | &Self::ReturnCallUnknown { .. } => { MachTerminator::RetCall } - &Self::JmpKnown { .. } => MachTerminator::Uncond, - &Self::JmpCond { .. } => MachTerminator::Cond, - &Self::JmpCondOr { .. } => MachTerminator::Cond, - &Self::JmpTableSeq { .. } => MachTerminator::Indirect, + &Self::JmpKnown { .. } => MachTerminator::Branch, + &Self::JmpCond { .. } => MachTerminator::Branch, + &Self::JmpCondOr { .. } => MachTerminator::Branch, + &Self::JmpTableSeq { .. } => MachTerminator::Branch, + &Self::CallKnown { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, + &Self::CallUnknown { ref info } if info.try_call_info.is_some() => { + MachTerminator::Branch + } // All other cases are boring. _ => MachTerminator::None, } diff --git a/cranelift/codegen/src/isa/x64/inst/regs.rs b/cranelift/codegen/src/isa/x64/inst/regs.rs index d3a06da70a03..fdf97b17c2c8 100644 --- a/cranelift/codegen/src/isa/x64/inst/regs.rs +++ b/cranelift/codegen/src/isa/x64/inst/regs.rs @@ -31,127 +31,127 @@ pub const ENC_R15: u8 = 15; // Constructors for Regs. -fn gpr(enc: u8) -> Reg { +const fn gpr(enc: u8) -> Reg { let preg = gpr_preg(enc); - Reg::from(VReg::new(preg.index(), RegClass::Int)) + Reg::from_virtual_reg(VReg::new(preg.index(), RegClass::Int)) } pub(crate) const fn gpr_preg(enc: u8) -> PReg { PReg::new(enc as usize, RegClass::Int) } -pub(crate) fn rsi() -> Reg { +pub(crate) const fn rsi() -> Reg { gpr(ENC_RSI) } -pub(crate) fn rdi() -> Reg { +pub(crate) const fn rdi() -> Reg { gpr(ENC_RDI) } -pub(crate) fn rax() -> Reg { +pub(crate) const fn rax() -> Reg { gpr(ENC_RAX) } -pub(crate) fn rcx() -> Reg { +pub(crate) const fn rcx() -> Reg { gpr(ENC_RCX) } -pub(crate) fn rdx() -> Reg { +pub(crate) const fn rdx() -> Reg { gpr(ENC_RDX) } -pub(crate) fn r8() -> Reg { +pub(crate) const fn r8() -> Reg { gpr(ENC_R8) } -pub(crate) fn r9() -> Reg { +pub(crate) const fn r9() -> Reg { gpr(ENC_R9) } -pub(crate) fn r10() -> Reg { +pub(crate) const fn r10() -> Reg { gpr(ENC_R10) } -pub(crate) fn r11() -> Reg { +pub(crate) const fn r11() -> Reg { gpr(ENC_R11) } -pub(crate) fn r12() -> Reg { +pub(crate) const fn r12() -> Reg { gpr(ENC_R12) } -pub(crate) fn r13() -> Reg { +pub(crate) const fn r13() -> Reg { gpr(ENC_R13) } -pub(crate) fn r14() -> Reg { +pub(crate) const fn r14() -> Reg { gpr(ENC_R14) } -pub(crate) fn rbx() -> Reg { +pub(crate) const fn rbx() -> Reg { gpr(ENC_RBX) } -pub(crate) fn r15() -> Reg { +pub(crate) const fn r15() -> Reg { gpr(ENC_R15) } -pub(crate) fn rsp() -> Reg { +pub(crate) const fn rsp() -> Reg { gpr(ENC_RSP) } -pub(crate) fn rbp() -> Reg { +pub(crate) const fn rbp() -> Reg { gpr(ENC_RBP) } /// The pinned register on this architecture. /// It must be the same as Spidermonkey's HeapReg, as found in this file. /// https://searchfox.org/mozilla-central/source/js/src/jit/x64/Assembler-x64.h#99 -pub(crate) fn pinned_reg() -> Reg { +pub(crate) const fn pinned_reg() -> Reg { r15() } -fn fpr(enc: u8) -> Reg { +const fn fpr(enc: u8) -> Reg { let preg = fpr_preg(enc); - Reg::from(VReg::new(preg.index(), RegClass::Float)) + Reg::from_virtual_reg(VReg::new(preg.index(), RegClass::Float)) } pub(crate) const fn fpr_preg(enc: u8) -> PReg { PReg::new(enc as usize, RegClass::Float) } -pub(crate) fn xmm0() -> Reg { +pub(crate) const fn xmm0() -> Reg { fpr(0) } -pub(crate) fn xmm1() -> Reg { +pub(crate) const fn xmm1() -> Reg { fpr(1) } -pub(crate) fn xmm2() -> Reg { +pub(crate) const fn xmm2() -> Reg { fpr(2) } -pub(crate) fn xmm3() -> Reg { +pub(crate) const fn xmm3() -> Reg { fpr(3) } -pub(crate) fn xmm4() -> Reg { +pub(crate) const fn xmm4() -> Reg { fpr(4) } -pub(crate) fn xmm5() -> Reg { +pub(crate) const fn xmm5() -> Reg { fpr(5) } -pub(crate) fn xmm6() -> Reg { +pub(crate) const fn xmm6() -> Reg { fpr(6) } -pub(crate) fn xmm7() -> Reg { +pub(crate) const fn xmm7() -> Reg { fpr(7) } -pub(crate) fn xmm8() -> Reg { +pub(crate) const fn xmm8() -> Reg { fpr(8) } -pub(crate) fn xmm9() -> Reg { +pub(crate) const fn xmm9() -> Reg { fpr(9) } -pub(crate) fn xmm10() -> Reg { +pub(crate) const fn xmm10() -> Reg { fpr(10) } -pub(crate) fn xmm11() -> Reg { +pub(crate) const fn xmm11() -> Reg { fpr(11) } -pub(crate) fn xmm12() -> Reg { +pub(crate) const fn xmm12() -> Reg { fpr(12) } -pub(crate) fn xmm13() -> Reg { +pub(crate) const fn xmm13() -> Reg { fpr(13) } -pub(crate) fn xmm14() -> Reg { +pub(crate) const fn xmm14() -> Reg { fpr(14) } -pub(crate) fn xmm15() -> Reg { +pub(crate) const fn xmm15() -> Reg { fpr(15) } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 02e8479bfeb3..5dba95032f4f 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -3492,6 +3492,15 @@ (rule (lower (return_call_indirect sig_ref callee args)) (gen_return_call_indirect sig_ref callee args)) +;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets) + (gen_try_call sig_ref extname dist et inputs targets)) + +(rule (lower_branch (try_call_indirect val inputs et) targets) + (if-let (exception_sig sig_ref) et) + (gen_try_call_indirect sig_ref val et inputs targets)) + ;; Rules for `stack_switch` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; currently, only the Basic model is supported diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d8b0d4ce9eaa..b2998feec508 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -185,7 +185,7 @@ fn emit_vm_call( outputs.push(retval_regs.only_reg().unwrap()); } - abi.emit_call(ctx); + abi.emit_call(ctx, None); Ok(outputs) } diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index c18c5284059a..7eba50977116 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -100,12 +100,14 @@ use crate::entity::SecondaryMap; use crate::ir::types::*; -use crate::ir::{ArgumentExtension, ArgumentPurpose, Signature}; +use crate::ir::{ArgumentExtension, ArgumentPurpose, ExceptionTable, ExceptionTag, Signature}; use crate::isa::TargetIsa; use crate::settings::ProbestackStrategy; use crate::CodegenError; use crate::{ir, isa}; use crate::{machinst::*, trace}; +use alloc::boxed::Box; +use cranelift_entity::packed_option::PackedOption; use regalloc2::{MachineEnv, PReg, PRegSet}; use rustc_hash::FxHashMap; use smallvec::smallvec; @@ -272,7 +274,7 @@ pub enum ArgsOrRets { /// Abstract location for a machine-specific ABI impl to translate into the /// appropriate addressing mode. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum StackAMode { /// Offset into the current frame's argument area. IncomingArg(i64, u32), @@ -582,7 +584,10 @@ pub trait ABIMachineSpec { /// Get all caller-save registers, that is, registers that we expect /// not to be saved across a call to a callee with the given ABI. - fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet; + fn get_regs_clobbered_by_call( + call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet; /// Get the needed extension mode, given the mode attached to the argument /// in the signature and the calling convention. The input (the attribute in @@ -599,6 +604,12 @@ pub trait ABIMachineSpec { /// return values. This is used to move stack-carried return /// values directly into spillslots if needed. fn retval_temp_reg(call_conv_of_callee: isa::CallConv) -> Writable; + + /// Get the exception payload registers, if any, for a calling + /// convention. + fn exception_payload_regs(_call_conv: isa::CallConv) -> &'static [Reg] { + &[] + } } /// Out-of-line data for calls, to keep the size of `Inst` down. @@ -620,6 +631,22 @@ pub struct CallInfo { /// caller, if any. (Used for popping stack arguments with the `tail` /// calling convention.) pub callee_pop_size: u32, + /// Information for a try-call, if this is one. We combine + /// handling of calls and try-calls as much as possible to share + /// argument/return logic; they mostly differ in the metadata that + /// they emit, which this information feeds into. + pub try_call_info: Option, +} + +/// Out-of-line information present on `try_call` instructions only: +/// information that is used to generate exception-handling tables and +/// link up to destination blocks properly. +#[derive(Clone, Debug)] +pub struct TryCallInfo { + /// The target to jump to on a normal returhn. + pub continuation: MachLabel, + /// Exception tags to catch and corresponding destination labels. + pub exception_dests: Box<[(PackedOption, MachLabel)]>, } impl CallInfo { @@ -634,6 +661,7 @@ impl CallInfo { caller_conv: call_conv, callee_conv: call_conv, callee_pop_size: 0, + try_call_info: None, } } @@ -647,6 +675,7 @@ impl CallInfo { caller_conv: self.caller_conv, callee_conv: self.callee_conv, callee_pop_size: self.callee_pop_size, + try_call_info: self.try_call_info, } } } @@ -1998,7 +2027,7 @@ pub struct CallRetPair { } /// A location to load a return-value from after a call completes. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum RetLocation { /// A physical register. Reg(Reg, Type), @@ -2395,7 +2424,11 @@ impl CallSite { /// /// This function should only be called once, as it is allowed to re-use /// parts of the `CallSite` object in emitting instructions. - pub fn emit_call(&mut self, ctx: &mut Lower) { + pub fn emit_call( + &mut self, + ctx: &mut Lower, + try_call_info: Option<(ExceptionTable, &[MachLabel])>, + ) { let word_type = M::word_type(); if let Some(i) = ctx.sigs()[self.sig].stack_ret_arg { let rd = ctx.alloc_tmp(word_type).only_reg().unwrap(); @@ -2408,21 +2441,7 @@ impl CallSite { } let uses = mem::take(&mut self.uses); - let defs = mem::take(&mut self.defs); - let clobbers = { - // Get clobbers: all caller-saves. These may include return value - // regs, which we will remove from the clobber set below. - let mut clobbers = ::get_regs_clobbered_by_call(ctx.sigs()[self.sig].call_conv); - - // Remove retval regs from clobbers. - for def in &defs { - if let RetLocation::Reg(preg, ..) = def.location { - clobbers.remove(PReg::from(preg.to_real_reg().unwrap())); - } - } - - clobbers - }; + let mut defs = mem::take(&mut self.defs); let sig = &ctx.sigs()[self.sig]; let callee_pop_size = if sig.call_conv() == isa::CallConv::Tail { @@ -2441,6 +2460,74 @@ impl CallSite { let tmp = ctx.alloc_tmp(word_type).only_reg().unwrap(); + let try_call_info = try_call_info.map(|(et, labels)| { + let exception_dests = ctx.dfg().exception_tables[et] + .catches() + .map(|(tag, _)| tag.into()) + .zip(labels.iter().cloned()) + .collect::>() + .into_boxed_slice(); + + // We need to update `defs` to contain the exception + // payload regs as well. We have two sources of info that + // we join: + // + // - The machine-specific ABI implementation `M`, which + // tells us the particular registers that payload values + // must be in + // - The passed-in lowering context, which gives us the + // vregs we must define. + // + // Note that payload values may need to end up in the same + // physical registers as ordinary return values; this is + // not a conflict, because we either get one or the + // other. For regalloc's purposes, we define both starting + // here at the callsite, but we can share one def in the + // `defs` list and alias one vreg to another. Thus we + // handle the two cases below for each payload register: + // overlaps a return value (and we alias to it) or not + // (and we add a def). + let pregs = M::exception_payload_regs(call_conv); + for (i, &preg) in pregs.iter().enumerate() { + let vreg = ctx.try_call_exception_defs(ctx.cur_inst())[i]; + if let Some(existing) = defs.iter().find(|def| match def.location { + RetLocation::Reg(r, _) => r == preg, + _ => false, + }) { + ctx.vregs_mut() + .set_vreg_alias(vreg.to_reg(), existing.vreg.to_reg()); + } else { + defs.push(CallRetPair { + vreg, + location: RetLocation::Reg(preg, M::word_type()), + }); + } + } + + TryCallInfo { + continuation: *labels.last().unwrap(), + exception_dests, + } + }); + + let clobbers = { + // Get clobbers: all caller-saves. These may include return value + // regs, which we will remove from the clobber set below. + let mut clobbers = ::get_regs_clobbered_by_call( + ctx.sigs()[self.sig].call_conv, + try_call_info.is_some(), + ); + + // Remove retval regs from clobbers. + for def in &defs { + if let RetLocation::Reg(preg, _) = def.location { + clobbers.remove(PReg::from(preg.to_real_reg().unwrap())); + } + } + + clobbers + }; + // Any adjustment to SP to account for required outgoing arguments/stack return values must // be done inside of the call pseudo-op, to ensure that SP is always in a consistent // state for all other instructions. For example, if a tail-call abi function is called @@ -2461,6 +2548,7 @@ impl CallSite { callee_conv: call_conv, caller_conv: self.caller_conv, callee_pop_size, + try_call_info, }, ) .into_iter() @@ -2502,8 +2590,11 @@ impl CallInfo { let temp = M::retval_temp_reg(self.callee_conv); // The temporary must be noted as clobbered. - debug_assert!(M::get_regs_clobbered_by_call(self.callee_conv) - .contains(PReg::from(temp.to_reg().to_real_reg().unwrap()))); + debug_assert!(M::get_regs_clobbered_by_call( + self.callee_conv, + self.try_call_info.is_some() + ) + .contains(PReg::from(temp.to_reg().to_real_reg().unwrap()))); for CallRetPair { vreg, location } in &self.defs { match location { diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 403f425e8ec1..2b1ac7ca07e3 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -181,6 +181,7 @@ use crate::trace; use crate::{ir, MachInstEmitState}; use crate::{timing, VCodeConstantData}; use cranelift_control::ControlPlane; +use cranelift_entity::packed_option::PackedOption; use cranelift_entity::{entity_impl, PrimaryMap}; use smallvec::SmallVec; use std::cmp::Ordering; @@ -258,6 +259,8 @@ pub struct MachBuffer { user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>, /// Any unwind info at a given location. unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, + /// Any exception handler targets at a given location. + exception_handlers: SmallVec<[(CodeOffset, PackedOption, MachLabel); 8]>, /// The current source location in progress (after `start_srcloc()` and /// before `end_srcloc()`). This is a (start_offset, src_loc) tuple. cur_srcloc: Option<(CodeOffset, RelSourceLoc)>, @@ -336,6 +339,7 @@ impl MachBufferFinalized { .collect(), user_stack_maps: self.user_stack_maps, unwind_info: self.unwind_info, + exception_handlers: self.exception_handlers, alignment: self.alignment, } } @@ -368,6 +372,8 @@ pub struct MachBufferFinalized { pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>, /// Any unwind info at a given location. pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, + /// Any exception handler targets at a given location. + pub exception_handlers: SmallVec<[(CodeOffset, PackedOption, CodeOffset); 8]>, /// The required alignment of this buffer. pub alignment: u32, } @@ -442,6 +448,7 @@ impl MachBuffer { srclocs: SmallVec::new(), user_stack_maps: SmallVec::new(), unwind_info: SmallVec::new(), + exception_handlers: SmallVec::new(), cur_srcloc: None, label_offsets: SmallVec::new(), label_aliases: SmallVec::new(), @@ -1519,6 +1526,12 @@ impl MachBuffer { }) .collect(); + let exception_handlers = self + .exception_handlers + .iter() + .map(|&(off, tag, target)| (off, tag, self.resolve_label_offset(target))) + .collect(); + let mut srclocs = self.srclocs; srclocs.sort_by_key(|entry| entry.start); @@ -1530,6 +1543,7 @@ impl MachBuffer { srclocs, user_stack_maps: self.user_stack_maps, unwind_info: self.unwind_info, + exception_handlers, alignment, } } @@ -1614,6 +1628,16 @@ impl MachBuffer { self.unwind_info.push((self.cur_offset(), unwind)); } + /// Add an exception handler record at the current offset. + pub fn add_exception_handler( + &mut self, + tag: PackedOption, + target: MachLabel, + ) { + self.exception_handlers + .push((self.cur_offset(), tag, target)); + } + /// Set the `SourceLoc` for code from this offset until the offset at the /// next call to `end_srcloc()`. /// Returns the current [CodeOffset] and [RelSourceLoc]. diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 8066d89d7748..e36a78eb92c2 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -356,6 +356,11 @@ macro_rules! isle_lower_prelude_methods { (funcdata.signature, funcdata.name.clone(), reloc_distance) } + #[inline] + fn exception_sig(&mut self, et: ExceptionTable) -> SigRef { + self.lower_ctx.dfg().exception_tables[et].signature() + } + #[inline] fn box_external_name(&mut self, extname: ExternalName) -> BoxExternalName { Box::new(extname) @@ -771,7 +776,13 @@ macro_rules! isle_prelude_caller_methods { sig.params.len() ); - crate::machinst::isle::gen_call_common(&mut self.lower_ctx, num_rets, caller, args) + crate::machinst::isle::gen_call_common( + &mut self.lower_ctx, + num_rets, + caller, + args, + None, + ) } fn gen_call_indirect( @@ -798,7 +809,13 @@ macro_rules! isle_prelude_caller_methods { sig.params.len() ); - crate::machinst::isle::gen_call_common(&mut self.lower_ctx, num_rets, caller, args) + crate::machinst::isle::gen_call_common( + &mut self.lower_ctx, + num_rets, + caller, + args, + None, + ) } fn gen_return_call( @@ -856,6 +873,70 @@ macro_rules! isle_prelude_caller_methods { InstOutput::new() } + + fn gen_try_call( + &mut self, + sig_ref: SigRef, + extname: ExternalName, + dist: RelocDistance, + et: ExceptionTable, + args: ValueSlice, + targets: &MachLabelSlice, + ) -> () { + let caller_conv = self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()); + let sigref = self.lower_ctx.dfg().exception_tables[et].signature(); + let sig = &self.lower_ctx.dfg().signatures[sigref]; + let num_rets = sig.returns.len(); + let caller = <$abicaller>::from_func( + self.lower_ctx.sigs(), + sig_ref, + &extname, + IsTailCall::No, + dist, + caller_conv, + self.backend.flags().clone(), + ); + + crate::machinst::isle::gen_call_common( + &mut self.lower_ctx, + num_rets, + caller, + args, + Some((et, targets)), + ); + } + + fn gen_try_call_indirect( + &mut self, + sigref: SigRef, + callee: Value, + et: ExceptionTable, + args: ValueSlice, + targets: &MachLabelSlice, + ) -> () { + let caller_conv = self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()); + let sig = &self.lower_ctx.dfg().signatures[sigref]; + let num_rets = sig.returns.len(); + + let callee = self.put_in_reg(callee); + + let caller = <$abicaller>::from_ptr( + self.lower_ctx.sigs(), + sigref, + callee, + IsTailCall::No, + caller_conv, + self.backend.flags().clone(), + ); + + crate::machinst::isle::gen_call_common( + &mut self.lower_ctx, + num_rets, + caller, + args, + Some((et, targets)), + ); + } }; } @@ -885,6 +966,7 @@ pub fn gen_call_common( num_rets: usize, mut caller: CallSite, args: ValueSlice, + try_call_info: Option<(ExceptionTable, &MachLabelSlice)>, ) -> InstOutput { gen_call_common_args(ctx, &mut caller, args); @@ -900,7 +982,20 @@ pub fn gen_call_common( outputs.push(retval_regs); } - caller.emit_call(ctx); + caller.emit_call(ctx, try_call_info); + + // If this is a try-call, alias return value vregs to the ones + // already allocated for the block-call arg defs. + if try_call_info.is_some() { + for i in 0..outputs.len() { + let result_regs = outputs[i]; + let def_regs = ctx.try_call_return_defs(ctx.cur_inst())[i]; + for (result_reg, def_reg) in result_regs.regs().iter().zip(def_regs.regs().iter()) { + ctx.vregs_mut() + .set_vreg_alias(def_reg.to_reg(), *result_reg); + } + } + } outputs } diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index cfe7fd9f90bf..516b41ee84b7 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -9,15 +9,15 @@ use crate::entity::SecondaryMap; use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit}; use crate::ir::pcc::{Fact, FactContext, PccError, PccResult}; use crate::ir::{ - ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function, - GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, RelSourceLoc, Type, - Value, ValueDef, ValueLabelAssignments, ValueLabelStart, + ArgumentPurpose, Block, BlockArg, Constant, ConstantData, DataFlowGraph, ExternalName, + Function, GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, + RelSourceLoc, Type, Value, ValueDef, ValueLabelAssignments, ValueLabelStart, }; use crate::machinst::valueregs::InvalidSentinel; use crate::machinst::{ - writable_value_regs, BackwardsInsnIndex, BlockIndex, BlockLoweringOrder, Callee, InsnIndex, - LoweredBlock, MachLabel, Reg, SigSet, VCode, VCodeBuilder, VCodeConstant, VCodeConstantData, - VCodeConstants, VCodeInst, ValueRegs, Writable, + writable_value_regs, ABIMachineSpec, BackwardsInsnIndex, BlockIndex, BlockLoweringOrder, + Callee, InsnIndex, LoweredBlock, MachLabel, Reg, SigSet, VCode, VCodeBuilder, VCodeConstant, + VCodeConstantData, VCodeConstants, VCodeInst, ValueRegs, Writable, }; use crate::settings::Flags; use crate::{trace, CodegenError, CodegenResult}; @@ -223,6 +223,14 @@ pub struct Lower<'func, I: VCodeInst> { /// Instructions collected for the CLIF inst in progress, in forward order. ir_insts: Vec, + /// Try-call block arg normal-return values, indexed by instruction. + try_call_rets: FxHashMap>; 2]>>, + + /// Try-call block arg exceptional-return payloads, indexed by + /// instruction. Payloads are carried in registers per the ABI and + /// can only be one register each. + try_call_payloads: FxHashMap; 2]>>, + /// The register to use for GetPinnedReg, if any, on this architecture. pinned_reg: Option, @@ -390,8 +398,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> { let mut vregs = VRegAllocator::with_capacity(f.dfg.num_values() * 2); let mut value_regs = SecondaryMap::with_default(ValueRegs::invalid()); + let mut try_call_rets = FxHashMap::default(); + let mut try_call_payloads = FxHashMap::default(); - // Assign a vreg to each block param and each inst result. + // Assign a vreg to each block param, each inst result, and + // each edge-defined block-call arg. for bb in f.layout.blocks() { for ¶m in f.dfg.block_params(bb) { let ty = f.dfg.value_type(param); @@ -417,6 +428,26 @@ impl<'func, I: VCodeInst> Lower<'func, I> { ); } } + + if let Some(et) = f.dfg.insts[inst].exception_table() { + let exdata = &f.dfg.exception_tables[et]; + let sig = &f.dfg.signatures[exdata.signature()]; + + let mut rets = smallvec![]; + for ty in sig.returns.iter().map(|ret| ret.value_type) { + rets.push(vregs.alloc(ty)?.map(|r| Writable::from_reg(r))); + } + try_call_rets.insert(inst, rets); + + let mut payloads = smallvec![]; + for &ty in sig + .call_conv + .exception_payload_types(I::ABIMachineSpec::word_type()) + { + payloads.push(Writable::from_reg(vregs.alloc(ty)?.only_reg().unwrap())); + } + try_call_payloads.insert(inst, payloads); + } } } @@ -492,6 +523,8 @@ impl<'func, I: VCodeInst> Lower<'func, I> { cur_scan_entry_color: None, cur_inst: None, ir_insts: vec![], + try_call_rets, + try_call_payloads, pinned_reg: None, flags, }) @@ -505,6 +538,10 @@ impl<'func, I: VCodeInst> Lower<'func, I> { self.vcode.sigs_mut() } + pub fn vregs_mut(&mut self) -> &mut VRegAllocator { + &mut self.vregs + } + fn gen_arg_setup(&mut self) { if let Some(entry_bb) = self.f.layout.entry_block() { trace!( @@ -970,13 +1007,26 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } }; - let block_call = - self.f.dfg.insts[branch_inst].branch_destination(&self.f.dfg.jump_tables)[succ_idx]; - let args = block_call.args_slice(&self.f.dfg.value_lists); - for &arg in args { - debug_assert!(self.f.dfg.value_is_real(arg)); - let regs = self.put_value_in_regs(arg); - buffer.extend_from_slice(regs.regs()); + let block_call = self.f.dfg.insts[branch_inst] + .branch_destination(&self.f.dfg.jump_tables, &self.f.dfg.exception_tables)[succ_idx]; + for arg in block_call.args(&self.f.dfg.value_lists) { + match arg { + BlockArg::Value(arg) => { + debug_assert!(self.f.dfg.value_is_real(arg)); + let regs = self.put_value_in_regs(arg); + buffer.extend_from_slice(regs.regs()); + } + BlockArg::TryCallRet(i) => { + let regs = self.try_call_rets.get(&branch_inst).unwrap()[i as usize] + .map(|r| r.to_reg()); + buffer.extend_from_slice(regs.regs()); + } + BlockArg::TryCallExn(i) => { + let reg = + self.try_call_payloads.get(&branch_inst).unwrap()[i as usize].to_reg(); + buffer.push(reg); + } + } } (succ, &buffer[..]) } @@ -1489,6 +1539,18 @@ impl<'func, I: VCodeInst> Lower<'func, I> { regs } + + /// Get the ValueRegs for the edge-defined values for special + /// try-call-return block arguments. + pub fn try_call_return_defs(&mut self, ir_inst: Inst) -> &[ValueRegs>] { + &self.try_call_rets.get(&ir_inst).unwrap()[..] + } + + /// Get the Regs for the edge-defined values for special + /// try-call-return exception payload arguments. + pub fn try_call_exception_defs(&mut self, ir_inst: Inst) -> &[Writable] { + &self.try_call_payloads.get(&ir_inst).unwrap()[..] + } } /// Codegen primitives: allocate temps, emit instructions, set result registers, @@ -1499,6 +1561,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> { writable_value_regs(self.vregs.alloc_with_deferred_error(ty)) } + /// Get the current root instruction that we are lowering. + pub fn cur_inst(&self) -> Inst { + self.cur_inst.unwrap() + } + /// Emit a machine instruction. pub fn emit(&mut self, mach_inst: I) { trace!("emit: {:?}", mach_inst); diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index fce309471ada..f7427bf72e4a 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -268,8 +268,7 @@ pub trait MachInstLabelUse: Clone + Copy + Debug + Eq { fn from_reloc(reloc: Reloc, addend: Addend) -> Option; } -/// Describes a block terminator (not call) in the vcode, when its branches -/// have not yet been finalized (so a branch may have two targets). +/// Describes a block terminator (not call) in the VCode. /// /// Actual targets are not included: the single-source-of-truth for /// those is the VCode itself, which holds, for each block, successors @@ -282,12 +281,8 @@ pub enum MachTerminator { Ret, /// A tail call. RetCall, - /// An unconditional branch to another block. - Uncond, - /// A conditional branch to one of two other blocks. - Cond, - /// An indirect branch with known possible targets. - Indirect, + /// A branch. + Branch, } /// A trait describing the ability to encode a MachInst into binary machine code. diff --git a/cranelift/codegen/src/machinst/reg.rs b/cranelift/codegen/src/machinst/reg.rs index 5a6ea1f9b326..a05a33743fc1 100644 --- a/cranelift/codegen/src/machinst/reg.rs +++ b/cranelift/codegen/src/machinst/reg.rs @@ -10,13 +10,9 @@ use regalloc2::{Operand, OperandConstraint, OperandKind, OperandPos, PReg, PRegS use serde_derive::{Deserialize, Serialize}; /// The first 192 vregs (64 int, 64 float, 64 vec) are "pinned" to -/// physical registers: this means that they are always constrained to -/// the corresponding register at all use/mod/def sites. -/// -/// Arbitrary vregs can also be constrained to physical registers at -/// particular use/def/mod sites, and this is preferable; but pinned -/// vregs allow us to migrate code that has been written using -/// RealRegs directly. +/// physical registers. These must not be passed into the regalloc, +/// but they are used to represent physical registers in the same +/// `Reg` type post-regalloc. const PINNED_VREGS: usize = 192; /// Convert a `VReg` to its pinned `PReg`, if any. @@ -28,6 +24,11 @@ pub fn pinned_vreg_to_preg(vreg: VReg) -> Option { } } +/// Convert a `PReg` to its pinned `VReg`. +pub const fn preg_to_pinned_vreg(preg: PReg) -> VReg { + VReg::new(preg.index(), preg.class()) +} + /// Give the first available vreg for generated code (i.e., after all /// pinned vregs). pub fn first_user_vreg_index() -> usize { @@ -51,6 +52,16 @@ const REG_SPILLSLOT_BIT: u32 = 0x8000_0000; const REG_SPILLSLOT_MASK: u32 = !REG_SPILLSLOT_BIT; impl Reg { + /// Const constructor: create a new Reg from a regalloc2 VReg. + pub const fn from_virtual_reg(vreg: regalloc2::VReg) -> Reg { + Reg(vreg.bits() as u32) + } + + /// Const constructor: create a new Reg from a regalloc2 PReg. + pub const fn from_real_reg(preg: regalloc2::PReg) -> Reg { + Reg(preg_to_pinned_vreg(preg).bits() as u32) + } + /// Get the physical register (`RealReg`), if this register is /// one. pub fn to_real_reg(self) -> Option { diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index dd6f48668b75..b00ccea75f6a 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -1335,13 +1335,13 @@ impl RegallocFunction for VCode { // We treat blocks terminated by an unconditional trap like a return for regalloc. MachTerminator::None => self.insts[insn.index()].is_trap(), MachTerminator::Ret | MachTerminator::RetCall => true, - MachTerminator::Uncond | MachTerminator::Cond | MachTerminator::Indirect => false, + MachTerminator::Branch => false, } } fn is_branch(&self, insn: InsnIndex) -> bool { match self.insts[insn.index()].is_term() { - MachTerminator::Cond | MachTerminator::Uncond | MachTerminator::Indirect => true, + MachTerminator::Branch => true, _ => false, } } diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isle index 860a1072b41d..2fb4144bb4fe 100644 --- a/cranelift/codegen/src/prelude_lower.isle +++ b/cranelift/codegen/src/prelude_lower.isle @@ -975,6 +975,10 @@ (decl func_ref_data (SigRef ExternalName RelocDistance) FuncRef) (extern extractor infallible func_ref_data func_ref_data) +;; Accessor for `ExceptionTable`. +(decl exception_sig (SigRef) ExceptionTable) +(extern extractor infallible exception_sig exception_sig) + ;; Accessor for `GlobalValue`. (decl symbol_value_data (ExternalName RelocDistance i64) GlobalValue) diff --git a/cranelift/codegen/src/remove_constant_phis.rs b/cranelift/codegen/src/remove_constant_phis.rs index bb2159c3bbeb..ccaa3973f875 100644 --- a/cranelift/codegen/src/remove_constant_phis.rs +++ b/cranelift/codegen/src/remove_constant_phis.rs @@ -3,7 +3,7 @@ use crate::dominator_tree::DominatorTree; use crate::ir; use crate::ir::Function; -use crate::ir::{Block, BlockCall, Inst, Value}; +use crate::ir::{Block, BlockArg, BlockCall, Inst, Value}; use crate::timing; use bumpalo::Bump; use cranelift_entity::SecondaryMap; @@ -116,7 +116,7 @@ struct OutEdge<'a> { /// The arguments to that block. /// /// These values can be from both groups A and B. - args: &'a [Value], + args: &'a [BlockArg], } impl<'a> OutEdge<'a> { @@ -132,10 +132,10 @@ impl<'a> OutEdge<'a> { branch_index: usize, block: BlockCall, ) -> Option { - let inst_var_args = block.args_slice(&dfg.value_lists); + let inst_var_args = block.args(&dfg.value_lists); // Skip edges without params. - if inst_var_args.is_empty() { + if inst_var_args.len() == 0 { return None; } @@ -144,9 +144,7 @@ impl<'a> OutEdge<'a> { branch_index: branch_index as u32, block: block.block(&dfg.value_lists), args: bump.alloc_slice_fill_iter( - inst_var_args - .iter() - .map(|value| dfg.resolve_aliases(*value)), + inst_var_args.map(|arg| arg.map_value(|value| dfg.resolve_aliases(value))), ), }) } @@ -239,7 +237,7 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) for inst in func.layout.block_insts(b) { for (ix, dest) in func.dfg.insts[inst] - .branch_destination(&func.dfg.jump_tables) + .branch_destination(&func.dfg.jump_tables, &func.dfg.exception_tables) .iter() .enumerate() { @@ -305,9 +303,12 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) // to be found in the solver state. If not, then it's a // real value defining point (not a phi), in which case // return it itself. - let actual_absval = match state.maybe_get(*actual) { - Some(pt) => *pt, - None => AbstractValue::One(*actual), + let actual_absval = match actual { + BlockArg::Value(actual) => match state.maybe_get(*actual) { + Some(pt) => *pt, + None => AbstractValue::One(*actual), + }, + _ => AbstractValue::Many, }; // And `join` the new value with the old. @@ -385,10 +386,11 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) } let dfg = &mut func.dfg; - let dests = dfg.insts[edge.inst].branch_destination_mut(&mut dfg.jump_tables); + let dests = dfg.insts[edge.inst] + .branch_destination_mut(&mut dfg.jump_tables, &mut dfg.exception_tables); let block = &mut dests[edge.branch_index as usize]; - old_actuals.extend(block.args_slice(&dfg.value_lists)); + old_actuals.extend(block.args(&dfg.value_lists)); // Check that the numbers of arguments make sense. let formals = &summaries[edge.block].formals; @@ -405,8 +407,7 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) // This leaks the value list from the old block, // https://github.com/bytecodealliance/wasmtime/issues/5451 for more information. let destination = block.block(&dfg.value_lists); - *block = BlockCall::new(destination, &old_actuals, &mut dfg.value_lists); - old_actuals.clear(); + *block = BlockCall::new(destination, old_actuals.drain(..), &mut dfg.value_lists); } } diff --git a/cranelift/codegen/src/traversals.rs b/cranelift/codegen/src/traversals.rs index b3234601bf46..5319cd23139b 100644 --- a/cranelift/codegen/src/traversals.rs +++ b/cranelift/codegen/src/traversals.rs @@ -194,7 +194,7 @@ mod tests { cur.insert_block(block1); let v1 = cur.ins().iconst(I32, 1); let v2 = cur.ins().iadd(v0, v1); - cur.ins().jump(block0, &[v2]); + cur.ins().jump(block0, &[v2.into()]); // block2: // return v0 diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs index 292e129e1dbb..1e82d727bb1b 100644 --- a/cranelift/codegen/src/verifier/mod.rs +++ b/cranelift/codegen/src/verifier/mod.rs @@ -69,7 +69,7 @@ use crate::entity::SparseSet; use crate::flowgraph::{BlockPredecessor, ControlFlowGraph}; use crate::ir::entities::AnyEntity; use crate::ir::instructions::{CallInfo, InstructionFormat, ResolvedConstraint}; -use crate::ir::{self, ArgumentExtension}; +use crate::ir::{self, ArgumentExtension, BlockArg, ExceptionTable}; use crate::ir::{ types, ArgumentPurpose, Block, Constant, DynamicStackSlot, FuncRef, Function, GlobalValue, Inst, JumpTable, MemFlags, MemoryTypeData, Opcode, SigRef, StackSlot, Type, Value, ValueDef, @@ -295,6 +295,13 @@ pub fn verify_context<'a, FOI: Into>>( verifier.run(errors) } +#[derive(Clone, Copy, Debug)] +enum BlockCallTargetType { + Normal, + ExNormalRet, + Exception, +} + struct Verifier<'a> { func: &'a Function, expected_cfg: ControlFlowGraph, @@ -594,6 +601,26 @@ impl<'a> Verifier<'a> { self.verify_sig_ref(inst, sig_ref, errors)?; self.verify_value_list(inst, args, errors)?; } + TryCall { + func_ref, + ref args, + exception, + .. + } => { + self.verify_func_ref(inst, func_ref, errors)?; + self.verify_value_list(inst, args, errors)?; + self.verify_exception_table(inst, exception, errors)?; + self.verify_exception_compatible_abi(inst, exception, errors)?; + } + TryCallIndirect { + ref args, + exception, + .. + } => { + self.verify_value_list(inst, args, errors)?; + self.verify_exception_table(inst, exception, errors)?; + self.verify_exception_compatible_abi(inst, exception, errors)?; + } FuncAddr { func_ref, .. } => { self.verify_func_ref(inst, func_ref, errors)?; } @@ -872,6 +899,56 @@ impl<'a> Verifier<'a> { } } + fn verify_exception_table( + &self, + inst: Inst, + et: ExceptionTable, + errors: &mut VerifierErrors, + ) -> VerifierStepResult { + // Verify that the exception table reference itself is valid. + if !self.func.stencil.dfg.exception_tables.is_valid(et) { + errors.nonfatal(( + inst, + self.context(inst), + format!("invalid exception table reference {et}"), + ))?; + } + + let pool = &self.func.stencil.dfg.value_lists; + let exdata = &self.func.stencil.dfg.exception_tables[et]; + + // Verify that the exception table's signature reference + // is valid. + self.verify_sig_ref(inst, exdata.signature(), errors)?; + + // Verify that the exception table's block references are valid. + for block in exdata.all_branches() { + self.verify_block(inst, block.block(pool), errors)?; + } + Ok(()) + } + + fn verify_exception_compatible_abi( + &self, + inst: Inst, + et: ExceptionTable, + errors: &mut VerifierErrors, + ) -> VerifierStepResult { + let callee_sig_ref = self.func.dfg.exception_tables[et].signature(); + let callee_sig = &self.func.dfg.signatures[callee_sig_ref]; + let callee_call_conv = callee_sig.call_conv; + if !callee_call_conv.supports_exceptions() { + errors.nonfatal(( + inst, + self.context(inst), + format!( + "calling convention `{callee_call_conv}` of callee does not support exceptions" + ), + ))?; + } + Ok(()) + } + fn verify_value( &self, loc_inst: Inst, @@ -1314,24 +1391,39 @@ impl<'a> Verifier<'a> { ) -> VerifierStepResult { match &self.func.dfg.insts[inst] { ir::InstructionData::Jump { destination, .. } => { - self.typecheck_block_call(inst, destination, errors)?; + self.typecheck_block_call(inst, destination, BlockCallTargetType::Normal, errors)?; } ir::InstructionData::Brif { blocks: [block_then, block_else], .. } => { - self.typecheck_block_call(inst, block_then, errors)?; - self.typecheck_block_call(inst, block_else, errors)?; + self.typecheck_block_call(inst, block_then, BlockCallTargetType::Normal, errors)?; + self.typecheck_block_call(inst, block_else, BlockCallTargetType::Normal, errors)?; } ir::InstructionData::BranchTable { table, .. } => { for block in self.func.stencil.dfg.jump_tables[*table].all_branches() { - self.typecheck_block_call(inst, block, errors)?; + self.typecheck_block_call(inst, block, BlockCallTargetType::Normal, errors)?; + } + } + ir::InstructionData::TryCall { exception, .. } + | ir::InstructionData::TryCallIndirect { exception, .. } => { + let exdata = &self.func.dfg.exception_tables[*exception]; + self.typecheck_block_call( + inst, + exdata.normal_return(), + BlockCallTargetType::ExNormalRet, + errors, + )?; + for (_tag, block) in exdata.catches() { + self.typecheck_block_call(inst, block, BlockCallTargetType::Exception, errors)?; } } inst => debug_assert!(!inst.opcode().is_branch()), } - match self.func.dfg.insts[inst].analyze_call(&self.func.dfg.value_lists) { + match self.func.dfg.insts[inst] + .analyze_call(&self.func.dfg.value_lists, &self.func.dfg.exception_tables) + { CallInfo::Direct(func_ref, args) => { let sig_ref = self.func.dfg.ext_funcs[func_ref].signature; let arg_types = self.func.dfg.signatures[sig_ref] @@ -1340,6 +1432,26 @@ impl<'a> Verifier<'a> { .map(|a| a.value_type); self.typecheck_variable_args_iterator(inst, arg_types, args, errors)?; } + CallInfo::DirectWithSig(func_ref, sig_ref, args) => { + let expected_sig_ref = self.func.dfg.ext_funcs[func_ref].signature; + let sigdata = &self.func.dfg.signatures; + // Compare signatures by value, not by ID -- any + // equivalent signature ID is acceptable. + if sigdata[sig_ref] != sigdata[expected_sig_ref] { + errors.nonfatal(( + inst, + self.context(inst), + format!( + "exception table signature {sig_ref} did not match function {func_ref}'s signature {expected_sig_ref}" + ), + ))?; + } + let arg_types = self.func.dfg.signatures[sig_ref] + .params + .iter() + .map(|a| a.value_type); + self.typecheck_variable_args_iterator(inst, arg_types, args, errors)?; + } CallInfo::Indirect(sig_ref, args) => { let arg_types = self.func.dfg.signatures[sig_ref] .params @@ -1352,27 +1464,143 @@ impl<'a> Verifier<'a> { Ok(()) } + fn pointer_type_or_error(&self, inst: Inst, errors: &mut VerifierErrors) -> Result { + // Ensure we have an ISA so we know what the pointer size is. + if let Some(isa) = self.isa { + Ok(isa.pointer_type()) + } else { + errors + .fatal(( + inst, + self.context(inst), + format!("need an ISA to validate correct pointer type"), + )) + // Will always return an `Err`, but the `Ok` type + // doesn't match, so map it. + .map(|_| Type::default()) + } + } + fn typecheck_block_call( &self, inst: Inst, block: &ir::BlockCall, + target_type: BlockCallTargetType, errors: &mut VerifierErrors, ) -> VerifierStepResult { let pool = &self.func.dfg.value_lists; - let iter = self - .func - .dfg - .block_params(block.block(pool)) - .iter() - .map(|&v| self.func.dfg.value_type(v)); - let args = block.args_slice(pool); - self.typecheck_variable_args_iterator(inst, iter, args, errors) + let block_params = self.func.dfg.block_params(block.block(pool)); + let args = block.args(pool); + if args.len() != block_params.len() { + return errors.nonfatal(( + inst, + self.context(inst), + format!( + "mismatched argument count for `{}`: got {}, expected {}", + self.func.dfg.display_inst(inst), + args.len(), + block_params.len(), + ), + )); + } + for (arg, param) in args.zip(block_params.iter()) { + let arg_ty = self.block_call_arg_ty(arg, inst, target_type, errors)?; + let param_ty = self.func.dfg.value_type(*param); + if arg_ty != param_ty { + errors.nonfatal(( + inst, + self.context(inst), + format!("arg {arg} has type {arg_ty}, expected {param_ty}"), + ))?; + } + } + Ok(()) + } + + fn block_call_arg_ty( + &self, + arg: BlockArg, + inst: Inst, + target_type: BlockCallTargetType, + errors: &mut VerifierErrors, + ) -> Result { + match arg { + BlockArg::Value(v) => Ok(self.func.dfg.value_type(v)), + BlockArg::TryCallRet(_) | BlockArg::TryCallExn(_) => { + // Get the invoked signature. + let et = match self.func.dfg.insts[inst].exception_table() { + Some(et) => et, + None => { + errors.fatal(( + inst, + self.context(inst), + format!( + "`retN` block argument in block-call not on `try_call` instruction" + ), + ))?; + unreachable!() + } + }; + let exdata = &self.func.dfg.exception_tables[et]; + let sig = &self.func.dfg.signatures[exdata.signature()]; + + match (arg, target_type) { + (BlockArg::TryCallRet(i), BlockCallTargetType::ExNormalRet) + if (i as usize) < sig.returns.len() => + { + Ok(sig.returns[i as usize].value_type) + } + (BlockArg::TryCallRet(_), BlockCallTargetType::ExNormalRet) => { + errors.fatal(( + inst, + self.context(inst), + format!("out-of-bounds `retN` block argument"), + ))?; + unreachable!() + } + (BlockArg::TryCallRet(_), _) => { + errors.fatal(( + inst, + self.context(inst), + format!("`retN` block argument used outside normal-return target of `try_call`"), + ))?; + unreachable!() + } + (BlockArg::TryCallExn(i), BlockCallTargetType::Exception) => { + match sig + .call_conv + .exception_payload_types(self.pointer_type_or_error(inst, errors)?) + .get(i as usize) + { + Some(ty) => Ok(*ty), + None => { + errors.fatal(( + inst, + self.context(inst), + format!("out-of-bounds `exnN` block argument"), + ))?; + unreachable!() + } + } + } + (BlockArg::TryCallExn(_), _) => { + errors.fatal(( + inst, + self.context(inst), + format!("`exnN` block argument used outside normal-return target of `try_call`"), + ))?; + unreachable!() + } + _ => unreachable!(), + } + } + } } - fn typecheck_variable_args_iterator>( + fn typecheck_variable_args_iterator( &self, inst: Inst, - iter: I, + iter: impl ExactSizeIterator, variable_args: &[Value], errors: &mut VerifierErrors, ) -> VerifierStepResult { diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs index 5c77aa679275..ef0e9fd41d40 100644 --- a/cranelift/codegen/src/write.rs +++ b/cranelift/codegen/src/write.rs @@ -389,6 +389,7 @@ fn write_instruction( pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt::Result { let pool = &dfg.value_lists; let jump_tables = &dfg.jump_tables; + let exception_tables = &dfg.exception_tables; use crate::ir::instructions::InstructionData::*; let ctrl_ty = dfg.ctrl_typevar(inst); match dfg.insts[inst] { @@ -479,6 +480,34 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt )?; write_user_stack_map_entries(w, dfg, inst) } + TryCall { + func_ref, + ref args, + exception, + .. + } => { + write!( + w, + " {}({}), {}", + func_ref, + DisplayValues(args.as_slice(pool)), + exception_tables[exception].display(pool), + ) + } + TryCallIndirect { + ref args, + exception, + .. + } => { + let args = args.as_slice(pool); + write!( + w, + " {}({}), {}", + args[0], + DisplayValues(&args[1..]), + exception_tables[exception].display(pool), + ) + } FuncAddr { func_ref, .. } => write!(w, " {func_ref}"), StackLoad { stack_slot, offset, .. diff --git a/cranelift/filetests/filetests/isa/aarch64/exceptions.clif b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif new file mode 100644 index 000000000000..604e9fbef506 --- /dev/null +++ b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif @@ -0,0 +1,249 @@ +test compile precise-output +target aarch64 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = colocated %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; stp fp, lr, [sp, #-16]! +; mov fp, sp +; stp x27, x28, [sp, #-16]! +; stp x25, x26, [sp, #-16]! +; stp x23, x24, [sp, #-16]! +; stp x21, x22, [sp, #-16]! +; stp x19, x20, [sp, #-16]! +; stp d14, d15, [sp, #-16]! +; stp d12, d13, [sp, #-16]! +; stp d10, d11, [sp, #-16]! +; stp d8, d9, [sp, #-16]! +; sub sp, sp, #16 +; block0: +; fmov d1, #1 +; mov x2, x0 +; str q1, [sp] +; bl 0; b MachLabel(1); catch [None: MachLabel(2)] +; block1: +; movz w0, #1 +; ldr q1, [sp] +; add sp, sp, #16 +; ldp d8, d9, [sp], #16 +; ldp d10, d11, [sp], #16 +; ldp d12, d13, [sp], #16 +; ldp d14, d15, [sp], #16 +; ldp x19, x20, [sp], #16 +; ldp x21, x22, [sp], #16 +; ldp x23, x24, [sp], #16 +; ldp x25, x26, [sp], #16 +; ldp x27, x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; block2: +; ldr q1, [sp] +; add w0, w0, #1 +; movi v0.2s, #0 +; add sp, sp, #16 +; ldp d8, d9, [sp], #16 +; ldp d10, d11, [sp], #16 +; ldp d12, d13, [sp], #16 +; ldp d14, d15, [sp], #16 +; ldp x19, x20, [sp], #16 +; ldp x21, x22, [sp], #16 +; ldp x23, x24, [sp], #16 +; ldp x25, x26, [sp], #16 +; ldp x27, x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; stp x29, x30, [sp, #-0x10]! +; mov x29, sp +; stp x27, x28, [sp, #-0x10]! +; stp x25, x26, [sp, #-0x10]! +; stp x23, x24, [sp, #-0x10]! +; stp x21, x22, [sp, #-0x10]! +; stp x19, x20, [sp, #-0x10]! +; stp d14, d15, [sp, #-0x10]! +; stp d12, d13, [sp, #-0x10]! +; stp d10, d11, [sp, #-0x10]! +; stp d8, d9, [sp, #-0x10]! +; sub sp, sp, #0x10 +; block1: ; offset 0x30 +; fmov d1, #1.00000000 +; mov x2, x0 +; stur q1, [sp] +; bl #0x3c ; reloc_external Call %g 0 +; block2: ; offset 0x40 +; mov w0, #1 +; ldur q1, [sp] +; add sp, sp, #0x10 +; ldp d8, d9, [sp], #0x10 +; ldp d10, d11, [sp], #0x10 +; ldp d12, d13, [sp], #0x10 +; ldp d14, d15, [sp], #0x10 +; ldp x19, x20, [sp], #0x10 +; ldp x21, x22, [sp], #0x10 +; ldp x23, x24, [sp], #0x10 +; ldp x25, x26, [sp], #0x10 +; ldp x27, x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret +; block3: ; offset 0x78 +; ldur q1, [sp] +; add w0, w0, #1 +; movi v0.2s, #0 +; add sp, sp, #0x10 +; ldp d8, d9, [sp], #0x10 +; ldp d10, d11, [sp], #0x10 +; ldp d12, d13, [sp], #0x10 +; ldp d14, d15, [sp], #0x10 +; ldp x19, x20, [sp], #0x10 +; ldp x21, x22, [sp], #0x10 +; ldp x23, x24, [sp], #0x10 +; ldp x25, x26, [sp], #0x10 +; ldp x27, x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret + +function %f2(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + v10 = func_addr.i64 fn0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; stp fp, lr, [sp, #-16]! +; mov fp, sp +; stp x27, x28, [sp, #-16]! +; stp x25, x26, [sp, #-16]! +; stp x23, x24, [sp, #-16]! +; stp x21, x22, [sp, #-16]! +; stp x19, x20, [sp, #-16]! +; stp d14, d15, [sp, #-16]! +; stp d12, d13, [sp, #-16]! +; stp d10, d11, [sp, #-16]! +; stp d8, d9, [sp, #-16]! +; sub sp, sp, #16 +; block0: +; fmov d1, #1 +; str q1, [sp] +; load_ext_name x11, TestCase(%g)+0 +; mov x2, x0 +; blr x11; b MachLabel(1); catch [None: MachLabel(2)] +; block1: +; movz w0, #1 +; ldr q1, [sp] +; add sp, sp, #16 +; ldp d8, d9, [sp], #16 +; ldp d10, d11, [sp], #16 +; ldp d12, d13, [sp], #16 +; ldp d14, d15, [sp], #16 +; ldp x19, x20, [sp], #16 +; ldp x21, x22, [sp], #16 +; ldp x23, x24, [sp], #16 +; ldp x25, x26, [sp], #16 +; ldp x27, x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; block2: +; ldr q1, [sp] +; add w0, w0, #1 +; movi v0.2s, #0 +; add sp, sp, #16 +; ldp d8, d9, [sp], #16 +; ldp d10, d11, [sp], #16 +; ldp d12, d13, [sp], #16 +; ldp d14, d15, [sp], #16 +; ldp x19, x20, [sp], #16 +; ldp x21, x22, [sp], #16 +; ldp x23, x24, [sp], #16 +; ldp x25, x26, [sp], #16 +; ldp x27, x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; stp x29, x30, [sp, #-0x10]! +; mov x29, sp +; stp x27, x28, [sp, #-0x10]! +; stp x25, x26, [sp, #-0x10]! +; stp x23, x24, [sp, #-0x10]! +; stp x21, x22, [sp, #-0x10]! +; stp x19, x20, [sp, #-0x10]! +; stp d14, d15, [sp, #-0x10]! +; stp d12, d13, [sp, #-0x10]! +; stp d10, d11, [sp, #-0x10]! +; stp d8, d9, [sp, #-0x10]! +; sub sp, sp, #0x10 +; block1: ; offset 0x30 +; fmov d1, #1.00000000 +; stur q1, [sp] +; ldr x11, #0x40 +; b #0x48 +; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0 +; .byte 0x00, 0x00, 0x00, 0x00 +; mov x2, x0 +; blr x11 +; block2: ; offset 0x50 +; mov w0, #1 +; ldur q1, [sp] +; add sp, sp, #0x10 +; ldp d8, d9, [sp], #0x10 +; ldp d10, d11, [sp], #0x10 +; ldp d12, d13, [sp], #0x10 +; ldp d14, d15, [sp], #0x10 +; ldp x19, x20, [sp], #0x10 +; ldp x21, x22, [sp], #0x10 +; ldp x23, x24, [sp], #0x10 +; ldp x25, x26, [sp], #0x10 +; ldp x27, x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret +; block3: ; offset 0x88 +; ldur q1, [sp] +; add w0, w0, #1 +; movi v0.2s, #0 +; add sp, sp, #0x10 +; ldp d8, d9, [sp], #0x10 +; ldp d10, d11, [sp], #0x10 +; ldp d12, d13, [sp], #0x10 +; ldp d14, d15, [sp], #0x10 +; ldp x19, x20, [sp], #0x10 +; ldp x21, x22, [sp], #0x10 +; ldp x23, x24, [sp], #0x10 +; ldp x25, x26, [sp], #0x10 +; ldp x27, x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley32/call.clif b/cranelift/filetests/filetests/isa/pulley32/call.clif index a2fa6dd632f4..75c058b3d695 100644 --- a/cranelift/filetests/filetests/isa/pulley32/call.clif +++ b/cranelift/filetests/filetests/isa/pulley32/call.clif @@ -16,7 +16,7 @@ block0: ; push_frame ; block0: ; xzero x2 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xone x0 ; pop_frame ; ret @@ -43,7 +43,7 @@ block0: ; push_frame ; block0: ; xzero x2 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xone x0 ; pop_frame ; ret @@ -75,7 +75,7 @@ block0: ; xone x4 ; xconst8 x5, 2 ; xconst8 x6, 3 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p3i), XReg(p4i), XReg(p5i), XReg(p6i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p3i), XReg(p4i), XReg(p5i), XReg(p6i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -103,7 +103,7 @@ block0: ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }], clobbers: PRegSet { bits: [65520, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }], clobbers: PRegSet { bits: [65520, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xadd64 x4, x0, x2 ; xadd64 x3, x1, x3 ; xadd64 x0, x4, x3 @@ -149,7 +149,7 @@ block0: ; xmov x11, x14 ; xmov x12, x14 ; xmov x13, x14 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame_restore 64, {} ; ret ; @@ -217,7 +217,7 @@ block0: ; push_frame_save 112, {x16, x17, x18, x19, x26, x27, x28, x29} ; block0: ; x12 = load_addr OutgoingArg(0) -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p12i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }, CallRetPair { vreg: Writable { reg: p4i }, location: Reg(p4i, types::I64) }, CallRetPair { vreg: Writable { reg: p5i }, location: Reg(p5i, types::I64) }, CallRetPair { vreg: Writable { reg: p6i }, location: Reg(p6i, types::I64) }, CallRetPair { vreg: Writable { reg: p7i }, location: Reg(p7i, types::I64) }, CallRetPair { vreg: Writable { reg: p8i }, location: Reg(p8i, types::I64) }, CallRetPair { vreg: Writable { reg: p9i }, location: Reg(p9i, types::I64) }, CallRetPair { vreg: Writable { reg: p10i }, location: Reg(p10i, types::I64) }, CallRetPair { vreg: Writable { reg: p11i }, location: Reg(p11i, types::I64) }, CallRetPair { vreg: Writable { reg: p12i }, location: Reg(p12i, types::I64) }, CallRetPair { vreg: Writable { reg: p13i }, location: Reg(p13i, types::I64) }, CallRetPair { vreg: Writable { reg: p14i }, location: Reg(p14i, types::I64) }, CallRetPair { vreg: Writable { reg: p27i }, location: Stack(OutgoingArg(0), types::I64) }, CallRetPair { vreg: Writable { reg: p19i }, location: Stack(OutgoingArg(8), types::I64) }, CallRetPair { vreg: Writable { reg: p29i }, location: Stack(OutgoingArg(16), types::I64) }, CallRetPair { vreg: Writable { reg: p16i }, location: Stack(OutgoingArg(24), types::I64) }, CallRetPair { vreg: Writable { reg: p17i }, location: Stack(OutgoingArg(32), types::I64) }, CallRetPair { vreg: Writable { reg: p18i }, location: Stack(OutgoingArg(40), types::I64) }], clobbers: PRegSet { bits: [32768, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p12i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }, CallRetPair { vreg: Writable { reg: p4i }, location: Reg(p4i, types::I64) }, CallRetPair { vreg: Writable { reg: p5i }, location: Reg(p5i, types::I64) }, CallRetPair { vreg: Writable { reg: p6i }, location: Reg(p6i, types::I64) }, CallRetPair { vreg: Writable { reg: p7i }, location: Reg(p7i, types::I64) }, CallRetPair { vreg: Writable { reg: p8i }, location: Reg(p8i, types::I64) }, CallRetPair { vreg: Writable { reg: p9i }, location: Reg(p9i, types::I64) }, CallRetPair { vreg: Writable { reg: p10i }, location: Reg(p10i, types::I64) }, CallRetPair { vreg: Writable { reg: p11i }, location: Reg(p11i, types::I64) }, CallRetPair { vreg: Writable { reg: p12i }, location: Reg(p12i, types::I64) }, CallRetPair { vreg: Writable { reg: p13i }, location: Reg(p13i, types::I64) }, CallRetPair { vreg: Writable { reg: p14i }, location: Reg(p14i, types::I64) }, CallRetPair { vreg: Writable { reg: p27i }, location: Stack(OutgoingArg(0), types::I64) }, CallRetPair { vreg: Writable { reg: p19i }, location: Stack(OutgoingArg(8), types::I64) }, CallRetPair { vreg: Writable { reg: p29i }, location: Stack(OutgoingArg(16), types::I64) }, CallRetPair { vreg: Writable { reg: p16i }, location: Stack(OutgoingArg(24), types::I64) }, CallRetPair { vreg: Writable { reg: p17i }, location: Stack(OutgoingArg(32), types::I64) }, CallRetPair { vreg: Writable { reg: p18i }, location: Stack(OutgoingArg(40), types::I64) }], clobbers: PRegSet { bits: [32768, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xadd64 x26, x0, x1 ; xadd64 x28, x2, x3 ; xadd64 x2, x4, x5 @@ -248,7 +248,6 @@ block0: ; push_frame_save 112, x16, x17, x18, x19, x26, x27, x28, x29 ; xmov x12, sp ; call1 x12, 0x0 // target = 0x8 -; jump 0x5 // target = 0x13 ; xload64le_o32 x27, sp, 0 ; xload64le_o32 x19, sp, 8 ; xload64le_o32 x29, sp, 16 @@ -292,7 +291,7 @@ block0(v0: i32): ; VCode: ; push_frame ; block0: -; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0 } +; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley32/exceptions.clif b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif new file mode 100644 index 000000000000..40f643708c87 --- /dev/null +++ b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif @@ -0,0 +1,297 @@ +test compile precise-output +target pulley32 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = colocated %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i32): + v8 = iadd_imm.i32 v6, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; push_frame_save 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; fstore64 sp+136, f16 // flags = notrap aligned +; fstore64 sp+128, f17 // flags = notrap aligned +; fstore64 sp+120, f18 // flags = notrap aligned +; fstore64 sp+112, f19 // flags = notrap aligned +; fstore64 sp+104, f20 // flags = notrap aligned +; fstore64 sp+96, f21 // flags = notrap aligned +; fstore64 sp+88, f22 // flags = notrap aligned +; fstore64 sp+80, f23 // flags = notrap aligned +; fstore64 sp+72, f24 // flags = notrap aligned +; fstore64 sp+64, f25 // flags = notrap aligned +; fstore64 sp+56, f26 // flags = notrap aligned +; fstore64 sp+48, f27 // flags = notrap aligned +; fstore64 sp+40, f28 // flags = notrap aligned +; fstore64 sp+32, f29 // flags = notrap aligned +; fstore64 sp+24, f30 // flags = notrap aligned +; fstore64 sp+16, f31 // flags = notrap aligned +; block0: +; fconst64 f1, 4607182418800017408 +; fstore64 Slot(0), f1 // flags = notrap aligned +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I32) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_dests: [(None, MachLabel(2))] }) }; jump MachLabel(1); catch [None: MachLabel(2)] +; block1: +; xone x0 +; f1 = fload64 Slot(0) // flags = notrap aligned +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; block2: +; f1 = fload64 Slot(0) // flags = notrap aligned +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; +; Disassembled: +; push_frame_save 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; fstore64le_o32 sp, 136, f16 +; fstore64le_o32 sp, 128, f17 +; fstore64le_o32 sp, 120, f18 +; fstore64le_o32 sp, 112, f19 +; fstore64le_o32 sp, 104, f20 +; fstore64le_o32 sp, 96, f21 +; fstore64le_o32 sp, 88, f22 +; fstore64le_o32 sp, 80, f23 +; fstore64le_o32 sp, 72, f24 +; fstore64le_o32 sp, 64, f25 +; fstore64le_o32 sp, 56, f26 +; fstore64le_o32 sp, 48, f27 +; fstore64le_o32 sp, 40, f28 +; fstore64le_o32 sp, 32, f29 +; fstore64le_o32 sp, 24, f30 +; fstore64le_o32 sp, 16, f31 +; fconst64 f1, 4607182418800017408 +; fstore64le_o32 sp, 0, f1 +; call 0x0 // target = 0xaa +; xone x0 +; fload64le_o32 f1, sp, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret +; fload64le_o32 f1, sp, 0 +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret + +function %f2(i32, i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32, v10: i32): + v2 = f64const 0x1.0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v7: i32): + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; push_frame_save 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; fstore64 sp+136, f16 // flags = notrap aligned +; fstore64 sp+128, f17 // flags = notrap aligned +; fstore64 sp+120, f18 // flags = notrap aligned +; fstore64 sp+112, f19 // flags = notrap aligned +; fstore64 sp+104, f20 // flags = notrap aligned +; fstore64 sp+96, f21 // flags = notrap aligned +; fstore64 sp+88, f22 // flags = notrap aligned +; fstore64 sp+80, f23 // flags = notrap aligned +; fstore64 sp+72, f24 // flags = notrap aligned +; fstore64 sp+64, f25 // flags = notrap aligned +; fstore64 sp+56, f26 // flags = notrap aligned +; fstore64 sp+48, f27 // flags = notrap aligned +; fstore64 sp+40, f28 // flags = notrap aligned +; fstore64 sp+32, f29 // flags = notrap aligned +; fstore64 sp+24, f30 // flags = notrap aligned +; fstore64 sp+16, f31 // flags = notrap aligned +; block0: +; fconst64 f1, 4607182418800017408 +; fstore64 Slot(0), f1 // flags = notrap aligned +; indirect_call x1, CallInfo { dest: XReg(p1i), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I32) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_dests: [(None, MachLabel(2))] }) }; jump MachLabel(1); catch [None: MachLabel(2)] +; block1: +; xone x0 +; f1 = fload64 Slot(0) // flags = notrap aligned +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; block2: +; f1 = fload64 Slot(0) // flags = notrap aligned +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; +; Disassembled: +; push_frame_save 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; fstore64le_o32 sp, 136, f16 +; fstore64le_o32 sp, 128, f17 +; fstore64le_o32 sp, 120, f18 +; fstore64le_o32 sp, 112, f19 +; fstore64le_o32 sp, 104, f20 +; fstore64le_o32 sp, 96, f21 +; fstore64le_o32 sp, 88, f22 +; fstore64le_o32 sp, 80, f23 +; fstore64le_o32 sp, 72, f24 +; fstore64le_o32 sp, 64, f25 +; fstore64le_o32 sp, 56, f26 +; fstore64le_o32 sp, 48, f27 +; fstore64le_o32 sp, 40, f28 +; fstore64le_o32 sp, 32, f29 +; fstore64le_o32 sp, 24, f30 +; fstore64le_o32 sp, 16, f31 +; fconst64 f1, 4607182418800017408 +; fstore64le_o32 sp, 0, f1 +; call_indirect x1 +; xone x0 +; fload64le_o32 f1, sp, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret +; fload64le_o32 f1, sp, 0 +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley32/extend.clif b/cranelift/filetests/filetests/isa/pulley32/extend.clif index d82485e18853..c21d320067fa 100644 --- a/cranelift/filetests/filetests/isa/pulley32/extend.clif +++ b/cranelift/filetests/filetests/isa/pulley32/extend.clif @@ -12,7 +12,7 @@ block0(v0: i8): ; push_frame ; block0: ; zext8 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -34,7 +34,7 @@ block0(v0: i16): ; push_frame ; block0: ; zext16 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -55,7 +55,7 @@ block0(v0: i32): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -75,7 +75,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -96,7 +96,7 @@ block0(v0: i8): ; push_frame ; block0: ; sext8 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -118,7 +118,7 @@ block0(v0: i16): ; push_frame ; block0: ; sext16 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -139,7 +139,7 @@ block0(v0: i32): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -159,7 +159,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley64/call.clif b/cranelift/filetests/filetests/isa/pulley64/call.clif index 64044b01860c..50c104c8543a 100644 --- a/cranelift/filetests/filetests/isa/pulley64/call.clif +++ b/cranelift/filetests/filetests/isa/pulley64/call.clif @@ -16,7 +16,7 @@ block0: ; push_frame ; block0: ; xzero x2 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xone x0 ; pop_frame ; ret @@ -43,7 +43,7 @@ block0: ; push_frame ; block0: ; xzero x2 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xone x0 ; pop_frame ; ret @@ -75,7 +75,7 @@ block0: ; xone x4 ; xconst8 x5, 2 ; xconst8 x6, 3 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p3i), XReg(p4i), XReg(p5i), XReg(p6i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p3i), XReg(p4i), XReg(p5i), XReg(p6i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -103,7 +103,7 @@ block0: ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }], clobbers: PRegSet { bits: [65520, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }], clobbers: PRegSet { bits: [65520, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xadd64 x4, x0, x2 ; xadd64 x3, x1, x3 ; xadd64 x0, x4, x3 @@ -149,7 +149,7 @@ block0: ; xmov x11, x14 ; xmov x12, x14 ; xmov x13, x14 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame_restore 64, {} ; ret ; @@ -217,7 +217,7 @@ block0: ; push_frame_save 112, {x16, x17, x18, x19, x26, x27, x28, x29} ; block0: ; x12 = load_addr OutgoingArg(0) -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p12i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }, CallRetPair { vreg: Writable { reg: p4i }, location: Reg(p4i, types::I64) }, CallRetPair { vreg: Writable { reg: p5i }, location: Reg(p5i, types::I64) }, CallRetPair { vreg: Writable { reg: p6i }, location: Reg(p6i, types::I64) }, CallRetPair { vreg: Writable { reg: p7i }, location: Reg(p7i, types::I64) }, CallRetPair { vreg: Writable { reg: p8i }, location: Reg(p8i, types::I64) }, CallRetPair { vreg: Writable { reg: p9i }, location: Reg(p9i, types::I64) }, CallRetPair { vreg: Writable { reg: p10i }, location: Reg(p10i, types::I64) }, CallRetPair { vreg: Writable { reg: p11i }, location: Reg(p11i, types::I64) }, CallRetPair { vreg: Writable { reg: p12i }, location: Reg(p12i, types::I64) }, CallRetPair { vreg: Writable { reg: p13i }, location: Reg(p13i, types::I64) }, CallRetPair { vreg: Writable { reg: p14i }, location: Reg(p14i, types::I64) }, CallRetPair { vreg: Writable { reg: p27i }, location: Stack(OutgoingArg(0), types::I64) }, CallRetPair { vreg: Writable { reg: p19i }, location: Stack(OutgoingArg(8), types::I64) }, CallRetPair { vreg: Writable { reg: p29i }, location: Stack(OutgoingArg(16), types::I64) }, CallRetPair { vreg: Writable { reg: p16i }, location: Stack(OutgoingArg(24), types::I64) }, CallRetPair { vreg: Writable { reg: p17i }, location: Stack(OutgoingArg(32), types::I64) }, CallRetPair { vreg: Writable { reg: p18i }, location: Stack(OutgoingArg(40), types::I64) }], clobbers: PRegSet { bits: [32768, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p12i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }, CallRetPair { vreg: Writable { reg: p4i }, location: Reg(p4i, types::I64) }, CallRetPair { vreg: Writable { reg: p5i }, location: Reg(p5i, types::I64) }, CallRetPair { vreg: Writable { reg: p6i }, location: Reg(p6i, types::I64) }, CallRetPair { vreg: Writable { reg: p7i }, location: Reg(p7i, types::I64) }, CallRetPair { vreg: Writable { reg: p8i }, location: Reg(p8i, types::I64) }, CallRetPair { vreg: Writable { reg: p9i }, location: Reg(p9i, types::I64) }, CallRetPair { vreg: Writable { reg: p10i }, location: Reg(p10i, types::I64) }, CallRetPair { vreg: Writable { reg: p11i }, location: Reg(p11i, types::I64) }, CallRetPair { vreg: Writable { reg: p12i }, location: Reg(p12i, types::I64) }, CallRetPair { vreg: Writable { reg: p13i }, location: Reg(p13i, types::I64) }, CallRetPair { vreg: Writable { reg: p14i }, location: Reg(p14i, types::I64) }, CallRetPair { vreg: Writable { reg: p27i }, location: Stack(OutgoingArg(0), types::I64) }, CallRetPair { vreg: Writable { reg: p19i }, location: Stack(OutgoingArg(8), types::I64) }, CallRetPair { vreg: Writable { reg: p29i }, location: Stack(OutgoingArg(16), types::I64) }, CallRetPair { vreg: Writable { reg: p16i }, location: Stack(OutgoingArg(24), types::I64) }, CallRetPair { vreg: Writable { reg: p17i }, location: Stack(OutgoingArg(32), types::I64) }, CallRetPair { vreg: Writable { reg: p18i }, location: Stack(OutgoingArg(40), types::I64) }], clobbers: PRegSet { bits: [32768, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xadd64 x26, x0, x1 ; xadd64 x28, x2, x3 ; xadd64 x2, x4, x5 @@ -248,7 +248,6 @@ block0: ; push_frame_save 112, x16, x17, x18, x19, x26, x27, x28, x29 ; xmov x12, sp ; call1 x12, 0x0 // target = 0x8 -; jump 0x5 // target = 0x13 ; xload64le_o32 x27, sp, 0 ; xload64le_o32 x19, sp, 8 ; xload64le_o32 x29, sp, 16 @@ -292,7 +291,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0 } +; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -342,7 +341,7 @@ block0: ; xmov x11, x14 ; xmov x12, x14 ; xmov x13, x14 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame_restore 80, {} ; ret ; @@ -388,7 +387,7 @@ block0(v0: i32): ; xstore64 sp+1000008, x20 // flags = notrap aligned ; block0: ; xmov x20, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xmov x5, x20 ; xadd32 x0, x5, x0 ; x20 = xload64 sp+1000008 // flags = notrap aligned diff --git a/cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif b/cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif index b81ab16c44c7..6e1b88a25a41 100644 --- a/cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif +++ b/cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif @@ -11,7 +11,7 @@ block0: ; VCode: ; push_frame ; block0: -; indirect_call_host CallInfo { dest: User(userextname0), uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: SystemV, caller_conv: Fast, callee_pop_size: 0 } +; indirect_call_host CallInfo { dest: User(userextname0), uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: SystemV, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley64/exceptions.clif b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif new file mode 100644 index 000000000000..2f03e53b1210 --- /dev/null +++ b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif @@ -0,0 +1,299 @@ +test compile precise-output +target pulley64 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = colocated %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; push_frame_save 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; fstore64 sp+136, f16 // flags = notrap aligned +; fstore64 sp+128, f17 // flags = notrap aligned +; fstore64 sp+120, f18 // flags = notrap aligned +; fstore64 sp+112, f19 // flags = notrap aligned +; fstore64 sp+104, f20 // flags = notrap aligned +; fstore64 sp+96, f21 // flags = notrap aligned +; fstore64 sp+88, f22 // flags = notrap aligned +; fstore64 sp+80, f23 // flags = notrap aligned +; fstore64 sp+72, f24 // flags = notrap aligned +; fstore64 sp+64, f25 // flags = notrap aligned +; fstore64 sp+56, f26 // flags = notrap aligned +; fstore64 sp+48, f27 // flags = notrap aligned +; fstore64 sp+40, f28 // flags = notrap aligned +; fstore64 sp+32, f29 // flags = notrap aligned +; fstore64 sp+24, f30 // flags = notrap aligned +; fstore64 sp+16, f31 // flags = notrap aligned +; block0: +; fconst64 f1, 4607182418800017408 +; fstore64 Slot(0), f1 // flags = notrap aligned +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_dests: [(None, MachLabel(2))] }) }; jump MachLabel(1); catch [None: MachLabel(2)] +; block1: +; xone x0 +; f1 = fload64 Slot(0) // flags = notrap aligned +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; block2: +; f1 = fload64 Slot(0) // flags = notrap aligned +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; +; Disassembled: +; push_frame_save 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; fstore64le_o32 sp, 136, f16 +; fstore64le_o32 sp, 128, f17 +; fstore64le_o32 sp, 120, f18 +; fstore64le_o32 sp, 112, f19 +; fstore64le_o32 sp, 104, f20 +; fstore64le_o32 sp, 96, f21 +; fstore64le_o32 sp, 88, f22 +; fstore64le_o32 sp, 80, f23 +; fstore64le_o32 sp, 72, f24 +; fstore64le_o32 sp, 64, f25 +; fstore64le_o32 sp, 56, f26 +; fstore64le_o32 sp, 48, f27 +; fstore64le_o32 sp, 40, f28 +; fstore64le_o32 sp, 32, f29 +; fstore64le_o32 sp, 24, f30 +; fstore64le_o32 sp, 16, f31 +; fconst64 f1, 4607182418800017408 +; fstore64le_o32 sp, 0, f1 +; call 0x0 // target = 0xaa +; xone x0 +; fload64le_o32 f1, sp, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret +; fload64le_o32 f1, sp, 0 +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret + +function %f2(i32, i64) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32, v10: i64): + v2 = f64const 0x1.0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; push_frame_save 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; fstore64 sp+136, f16 // flags = notrap aligned +; fstore64 sp+128, f17 // flags = notrap aligned +; fstore64 sp+120, f18 // flags = notrap aligned +; fstore64 sp+112, f19 // flags = notrap aligned +; fstore64 sp+104, f20 // flags = notrap aligned +; fstore64 sp+96, f21 // flags = notrap aligned +; fstore64 sp+88, f22 // flags = notrap aligned +; fstore64 sp+80, f23 // flags = notrap aligned +; fstore64 sp+72, f24 // flags = notrap aligned +; fstore64 sp+64, f25 // flags = notrap aligned +; fstore64 sp+56, f26 // flags = notrap aligned +; fstore64 sp+48, f27 // flags = notrap aligned +; fstore64 sp+40, f28 // flags = notrap aligned +; fstore64 sp+32, f29 // flags = notrap aligned +; fstore64 sp+24, f30 // flags = notrap aligned +; fstore64 sp+16, f31 // flags = notrap aligned +; block0: +; fconst64 f1, 4607182418800017408 +; fstore64 Slot(0), f1 // flags = notrap aligned +; indirect_call x1, CallInfo { dest: XReg(p1i), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_dests: [(None, MachLabel(2))] }) }; jump MachLabel(1); catch [None: MachLabel(2)] +; block1: +; xone x0 +; f1 = fload64 Slot(0) // flags = notrap aligned +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; block2: +; f1 = fload64 Slot(0) // flags = notrap aligned +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; +; Disassembled: +; push_frame_save 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; fstore64le_o32 sp, 136, f16 +; fstore64le_o32 sp, 128, f17 +; fstore64le_o32 sp, 120, f18 +; fstore64le_o32 sp, 112, f19 +; fstore64le_o32 sp, 104, f20 +; fstore64le_o32 sp, 96, f21 +; fstore64le_o32 sp, 88, f22 +; fstore64le_o32 sp, 80, f23 +; fstore64le_o32 sp, 72, f24 +; fstore64le_o32 sp, 64, f25 +; fstore64le_o32 sp, 56, f26 +; fstore64le_o32 sp, 48, f27 +; fstore64le_o32 sp, 40, f28 +; fstore64le_o32 sp, 32, f29 +; fstore64le_o32 sp, 24, f30 +; fstore64le_o32 sp, 16, f31 +; fconst64 f1, 4607182418800017408 +; fstore64le_o32 sp, 0, f1 +; call_indirect x1 +; xone x0 +; fload64le_o32 f1, sp, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret +; fload64le_o32 f1, sp, 0 +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley64/extend.clif b/cranelift/filetests/filetests/isa/pulley64/extend.clif index 0efbfb6a9a5a..90a64cf0bb9f 100644 --- a/cranelift/filetests/filetests/isa/pulley64/extend.clif +++ b/cranelift/filetests/filetests/isa/pulley64/extend.clif @@ -12,7 +12,7 @@ block0(v0: i8): ; push_frame ; block0: ; zext8 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -34,7 +34,7 @@ block0(v0: i16): ; push_frame ; block0: ; zext16 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -56,7 +56,7 @@ block0(v0: i32): ; push_frame ; block0: ; zext32 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -77,7 +77,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -98,7 +98,7 @@ block0(v0: i8): ; push_frame ; block0: ; sext8 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -120,7 +120,7 @@ block0(v0: i16): ; push_frame ; block0: ; sext16 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -142,7 +142,7 @@ block0(v0: i32): ; push_frame ; block0: ; sext32 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -163,7 +163,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif new file mode 100644 index 000000000000..89b2da54b2e7 --- /dev/null +++ b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif @@ -0,0 +1,447 @@ +test compile precise-output +target riscv64 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = colocated %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; addi sp,sp,-16 +; sd ra,8(sp) +; sd fp,0(sp) +; mv fp,sp +; addi sp,sp,-208 +; sd fp,200(sp) +; sd s1,192(sp) +; sd s2,184(sp) +; sd s3,176(sp) +; sd s4,168(sp) +; sd s5,160(sp) +; sd s6,152(sp) +; sd s7,144(sp) +; sd s8,136(sp) +; sd s9,128(sp) +; sd s10,120(sp) +; sd s11,112(sp) +; fsd fs0,104(sp) +; fsd fs2,96(sp) +; fsd fs3,88(sp) +; fsd fs4,80(sp) +; fsd fs5,72(sp) +; fsd fs6,64(sp) +; fsd fs7,56(sp) +; fsd fs8,48(sp) +; fsd fs9,40(sp) +; fsd fs10,32(sp) +; fsd fs11,24(sp) +; block0: +; lui a4,1023 +; slli a1,a4,40 +; fmv.d.x fa1,a1 +; fsd fa1,0(slot) +; call %g; j MachLabel(1); catch [None: MachLabel(2)] +; block1: +; li a0,1 +; fld fa1,0(slot) +; ld fp,200(sp) +; ld s1,192(sp) +; ld s2,184(sp) +; ld s3,176(sp) +; ld s4,168(sp) +; ld s5,160(sp) +; ld s6,152(sp) +; ld s7,144(sp) +; ld s8,136(sp) +; ld s9,128(sp) +; ld s10,120(sp) +; ld s11,112(sp) +; fld fs0,104(sp) +; fld fs2,96(sp) +; fld fs3,88(sp) +; fld fs4,80(sp) +; fld fs5,72(sp) +; fld fs6,64(sp) +; fld fs7,56(sp) +; fld fs8,48(sp) +; fld fs9,40(sp) +; fld fs10,32(sp) +; fld fs11,24(sp) +; addi sp,sp,208 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; block2: +; fld fa1,0(slot) +; addiw a0,a0,1 +; fmv.w.x fa0,zero +; ld fp,200(sp) +; ld s1,192(sp) +; ld s2,184(sp) +; ld s3,176(sp) +; ld s4,168(sp) +; ld s5,160(sp) +; ld s6,152(sp) +; ld s7,144(sp) +; ld s8,136(sp) +; ld s9,128(sp) +; ld s10,120(sp) +; ld s11,112(sp) +; fld fs0,104(sp) +; fld fs2,96(sp) +; fld fs3,88(sp) +; fld fs4,80(sp) +; fld fs5,72(sp) +; fld fs6,64(sp) +; fld fs7,56(sp) +; fld fs8,48(sp) +; fld fs9,40(sp) +; fld fs10,32(sp) +; fld fs11,24(sp) +; addi sp,sp,208 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; addi sp, sp, -0x10 +; sd ra, 8(sp) +; sd s0, 0(sp) +; mv s0, sp +; addi sp, sp, -0xd0 +; sd s0, 0xc8(sp) +; sd s1, 0xc0(sp) +; sd s2, 0xb8(sp) +; sd s3, 0xb0(sp) +; sd s4, 0xa8(sp) +; sd s5, 0xa0(sp) +; sd s6, 0x98(sp) +; sd s7, 0x90(sp) +; sd s8, 0x88(sp) +; sd s9, 0x80(sp) +; sd s10, 0x78(sp) +; sd s11, 0x70(sp) +; fsd fs0, 0x68(sp) +; fsd fs2, 0x60(sp) +; fsd fs3, 0x58(sp) +; fsd fs4, 0x50(sp) +; fsd fs5, 0x48(sp) +; fsd fs6, 0x40(sp) +; fsd fs7, 0x38(sp) +; fsd fs8, 0x30(sp) +; fsd fs9, 0x28(sp) +; fsd fs10, 0x20(sp) +; fsd fs11, 0x18(sp) +; block1: ; offset 0x70 +; lui a4, 0x3ff +; slli a1, a4, 0x28 +; fmv.d.x fa1, a1 +; fsd fa1, 0(sp) +; auipc ra, 0 ; reloc_external RiscvCallPlt %g 0 +; jalr ra +; block2: ; offset 0x88 +; addi a0, zero, 1 +; fld fa1, 0(sp) +; ld s0, 0xc8(sp) +; ld s1, 0xc0(sp) +; ld s2, 0xb8(sp) +; ld s3, 0xb0(sp) +; ld s4, 0xa8(sp) +; ld s5, 0xa0(sp) +; ld s6, 0x98(sp) +; ld s7, 0x90(sp) +; ld s8, 0x88(sp) +; ld s9, 0x80(sp) +; ld s10, 0x78(sp) +; ld s11, 0x70(sp) +; fld fs0, 0x68(sp) +; fld fs2, 0x60(sp) +; fld fs3, 0x58(sp) +; fld fs4, 0x50(sp) +; fld fs5, 0x48(sp) +; fld fs6, 0x40(sp) +; fld fs7, 0x38(sp) +; fld fs8, 0x30(sp) +; fld fs9, 0x28(sp) +; fld fs10, 0x20(sp) +; fld fs11, 0x18(sp) +; addi sp, sp, 0xd0 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret +; block3: ; offset 0x100 +; fld fa1, 0(sp) +; addiw a0, a0, 1 +; fmv.w.x fa0, zero +; ld s0, 0xc8(sp) +; ld s1, 0xc0(sp) +; ld s2, 0xb8(sp) +; ld s3, 0xb0(sp) +; ld s4, 0xa8(sp) +; ld s5, 0xa0(sp) +; ld s6, 0x98(sp) +; ld s7, 0x90(sp) +; ld s8, 0x88(sp) +; ld s9, 0x80(sp) +; ld s10, 0x78(sp) +; ld s11, 0x70(sp) +; fld fs0, 0x68(sp) +; fld fs2, 0x60(sp) +; fld fs3, 0x58(sp) +; fld fs4, 0x50(sp) +; fld fs5, 0x48(sp) +; fld fs6, 0x40(sp) +; fld fs7, 0x38(sp) +; fld fs8, 0x30(sp) +; fld fs9, 0x28(sp) +; fld fs10, 0x20(sp) +; fld fs11, 0x18(sp) +; addi sp, sp, 0xd0 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret + +function %f2(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + v10 = func_addr.i64 fn0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; addi sp,sp,-16 +; sd ra,8(sp) +; sd fp,0(sp) +; mv fp,sp +; addi sp,sp,-208 +; sd fp,200(sp) +; sd s1,192(sp) +; sd s2,184(sp) +; sd s3,176(sp) +; sd s4,168(sp) +; sd s5,160(sp) +; sd s6,152(sp) +; sd s7,144(sp) +; sd s8,136(sp) +; sd s9,128(sp) +; sd s10,120(sp) +; sd s11,112(sp) +; fsd fs0,104(sp) +; fsd fs2,96(sp) +; fsd fs3,88(sp) +; fsd fs4,80(sp) +; fsd fs5,72(sp) +; fsd fs6,64(sp) +; fsd fs7,56(sp) +; fsd fs8,48(sp) +; fsd fs9,40(sp) +; fsd fs10,32(sp) +; fsd fs11,24(sp) +; block0: +; lui a5,1023 +; slli a1,a5,40 +; fmv.d.x fa1,a1 +; fsd fa1,0(slot) +; load_sym a1,%g+0 +; callind a1; j MachLabel(1); catch [None: MachLabel(2)] +; block1: +; li a0,1 +; fld fa1,0(slot) +; ld fp,200(sp) +; ld s1,192(sp) +; ld s2,184(sp) +; ld s3,176(sp) +; ld s4,168(sp) +; ld s5,160(sp) +; ld s6,152(sp) +; ld s7,144(sp) +; ld s8,136(sp) +; ld s9,128(sp) +; ld s10,120(sp) +; ld s11,112(sp) +; fld fs0,104(sp) +; fld fs2,96(sp) +; fld fs3,88(sp) +; fld fs4,80(sp) +; fld fs5,72(sp) +; fld fs6,64(sp) +; fld fs7,56(sp) +; fld fs8,48(sp) +; fld fs9,40(sp) +; fld fs10,32(sp) +; fld fs11,24(sp) +; addi sp,sp,208 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; block2: +; fld fa1,0(slot) +; addiw a0,a0,1 +; fmv.w.x fa0,zero +; ld fp,200(sp) +; ld s1,192(sp) +; ld s2,184(sp) +; ld s3,176(sp) +; ld s4,168(sp) +; ld s5,160(sp) +; ld s6,152(sp) +; ld s7,144(sp) +; ld s8,136(sp) +; ld s9,128(sp) +; ld s10,120(sp) +; ld s11,112(sp) +; fld fs0,104(sp) +; fld fs2,96(sp) +; fld fs3,88(sp) +; fld fs4,80(sp) +; fld fs5,72(sp) +; fld fs6,64(sp) +; fld fs7,56(sp) +; fld fs8,48(sp) +; fld fs9,40(sp) +; fld fs10,32(sp) +; fld fs11,24(sp) +; addi sp,sp,208 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; addi sp, sp, -0x10 +; sd ra, 8(sp) +; sd s0, 0(sp) +; mv s0, sp +; addi sp, sp, -0xd0 +; sd s0, 0xc8(sp) +; sd s1, 0xc0(sp) +; sd s2, 0xb8(sp) +; sd s3, 0xb0(sp) +; sd s4, 0xa8(sp) +; sd s5, 0xa0(sp) +; sd s6, 0x98(sp) +; sd s7, 0x90(sp) +; sd s8, 0x88(sp) +; sd s9, 0x80(sp) +; sd s10, 0x78(sp) +; sd s11, 0x70(sp) +; fsd fs0, 0x68(sp) +; fsd fs2, 0x60(sp) +; fsd fs3, 0x58(sp) +; fsd fs4, 0x50(sp) +; fsd fs5, 0x48(sp) +; fsd fs6, 0x40(sp) +; fsd fs7, 0x38(sp) +; fsd fs8, 0x30(sp) +; fsd fs9, 0x28(sp) +; fsd fs10, 0x20(sp) +; fsd fs11, 0x18(sp) +; block1: ; offset 0x70 +; lui a5, 0x3ff +; slli a1, a5, 0x28 +; fmv.d.x fa1, a1 +; fsd fa1, 0(sp) +; auipc a1, 0 +; ld a1, 0xc(a1) +; j 0xc +; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0 +; .byte 0x00, 0x00, 0x00, 0x00 +; jalr a1 +; block2: ; offset 0x98 +; addi a0, zero, 1 +; fld fa1, 0(sp) +; ld s0, 0xc8(sp) +; ld s1, 0xc0(sp) +; ld s2, 0xb8(sp) +; ld s3, 0xb0(sp) +; ld s4, 0xa8(sp) +; ld s5, 0xa0(sp) +; ld s6, 0x98(sp) +; ld s7, 0x90(sp) +; ld s8, 0x88(sp) +; ld s9, 0x80(sp) +; ld s10, 0x78(sp) +; ld s11, 0x70(sp) +; fld fs0, 0x68(sp) +; fld fs2, 0x60(sp) +; fld fs3, 0x58(sp) +; fld fs4, 0x50(sp) +; fld fs5, 0x48(sp) +; fld fs6, 0x40(sp) +; fld fs7, 0x38(sp) +; fld fs8, 0x30(sp) +; fld fs9, 0x28(sp) +; fld fs10, 0x20(sp) +; fld fs11, 0x18(sp) +; addi sp, sp, 0xd0 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret +; block3: ; offset 0x110 +; fld fa1, 0(sp) +; addiw a0, a0, 1 +; fmv.w.x fa0, zero +; ld s0, 0xc8(sp) +; ld s1, 0xc0(sp) +; ld s2, 0xb8(sp) +; ld s3, 0xb0(sp) +; ld s4, 0xa8(sp) +; ld s5, 0xa0(sp) +; ld s6, 0x98(sp) +; ld s7, 0x90(sp) +; ld s8, 0x88(sp) +; ld s9, 0x80(sp) +; ld s10, 0x78(sp) +; ld s11, 0x70(sp) +; fld fs0, 0x68(sp) +; fld fs2, 0x60(sp) +; fld fs3, 0x58(sp) +; fld fs4, 0x50(sp) +; fld fs5, 0x48(sp) +; fld fs6, 0x40(sp) +; fld fs7, 0x38(sp) +; fld fs8, 0x30(sp) +; fld fs9, 0x28(sp) +; fld fs10, 0x20(sp) +; fld fs11, 0x18(sp) +; addi sp, sp, 0xd0 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret + diff --git a/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif b/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif index 219c0e01849d..2becbf9aff09 100644 --- a/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif +++ b/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif @@ -535,7 +535,6 @@ block0: ; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %tail_callee_stack_rets 0 ; .byte 0x00, 0x00, 0x00, 0x00 ; jalr a4 -; j 4 ; ld a2, 0(sp) ; sd a2, 0xc0(sp) ; ld a2, 8(sp) @@ -968,7 +967,6 @@ block0: ; ld a2, 0x1c0(sp) ; jalr t1 ; addi sp, sp, -0xa0 -; j 4 ; ld a2, 0xa0(sp) ; sd a2, 0x160(sp) ; ld a2, 0xa8(sp) diff --git a/cranelift/filetests/filetests/isa/x64/exceptions.clif b/cranelift/filetests/filetests/isa/x64/exceptions.clif new file mode 100644 index 000000000000..ed410980a859 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/exceptions.clif @@ -0,0 +1,371 @@ +test compile precise-output +target x86_64 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; subq %rsp, $64, %rsp +; movq %rbx, 16(%rsp) +; movq %r12, 24(%rsp) +; movq %r13, 32(%rsp) +; movq %r14, 40(%rsp) +; movq %r15, 48(%rsp) +; block0: +; movabsq $4607182418800017408, %rcx +; movq %rcx, %xmm1 +; movdqu %xmm1, rsp(0 + virtual offset) +; load_ext_name %g+0, %rdx +; call *%rdx; jmp MachLabel(1); catch [None: MachLabel(2)] +; block1: +; movl $1, %eax +; movdqu rsp(0 + virtual offset), %xmm1 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; block2: +; movdqu rsp(0 + virtual offset), %xmm1 +; lea 1(%rax), %eax +; uninit %xmm0 +; xorps %xmm0, %xmm0 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; subq $0x40, %rsp +; movq %rbx, 0x10(%rsp) +; movq %r12, 0x18(%rsp) +; movq %r13, 0x20(%rsp) +; movq %r14, 0x28(%rsp) +; movq %r15, 0x30(%rsp) +; block1: ; offset 0x21 +; movabsq $0x3ff0000000000000, %rcx +; movq %rcx, %xmm1 +; movdqu %xmm1, (%rsp) +; movabsq $0, %rdx ; reloc_external Abs8 %g 0 +; callq *%rdx +; block2: ; offset 0x41 +; movl $1, %eax +; movdqu (%rsp), %xmm1 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq +; block3: ; offset 0x6d +; movdqu (%rsp), %xmm1 +; addl $1, %eax +; xorps %xmm0, %xmm0 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq + + +function %f1(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + brif v1, block1, block2 + + block1: + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block3(ret0, v2), [ default: block4(exn0) ] + + block2: + v3 = iconst.i64 42 + v4 = f32const 0x1234.0 + v5 = f64const 0x5678.0 + brif v1, block4(v3), block3(v4, v5) + + block3(v6: f32, v7: f64): + v8 = iconst.i32 1 + return v8, v6, v7 + + block4(v9: i64): + v10 = ireduce.i32 v9 + v11 = iadd_imm.i32 v10, 1 + v12 = f32const 0x0.0 + v13 = bitcast.f64 v9 + return v11, v12, v13 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; subq %rsp, $64, %rsp +; movq %rbx, 16(%rsp) +; movq %r12, 24(%rsp) +; movq %r13, 32(%rsp) +; movq %r14, 40(%rsp) +; movq %r15, 48(%rsp) +; block0: +; testl %edi, %edi +; jnz label4; j label1 +; block1: +; movl $42, %eax +; movl $1167171584, %ecx +; movd %ecx, %xmm0 +; movabsq $4671813911303946240, %rcx +; movq %rcx, %xmm1 +; testl %edi, %edi +; jnz label2; j label3 +; block2: +; movq %rax, %r11 +; jmp label8 +; block3: +; movdqu %xmm1, rsp(0 + virtual offset) +; jmp label7 +; block4: +; movabsq $4607182418800017408, %r10 +; movq %r10, %xmm1 +; movdqu %xmm1, rsp(0 + virtual offset) +; load_ext_name %g+0, %r11 +; call *%r11; jmp MachLabel(6); catch [None: MachLabel(5)] +; block5: +; movq %rax, %rsi +; movq %rsi, %r11 +; jmp label8 +; block6: +; jmp label7 +; block7: +; movl $1, %eax +; movdqu rsp(0 + virtual offset), %xmm1 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; block8: +; lea 1(%r11), %eax +; movq %r11, %rsi +; uninit %xmm0 +; xorps %xmm0, %xmm0 +; movq %rsi, %xmm1 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; subq $0x40, %rsp +; movq %rbx, 0x10(%rsp) +; movq %r12, 0x18(%rsp) +; movq %r13, 0x20(%rsp) +; movq %r14, 0x28(%rsp) +; movq %r15, 0x30(%rsp) +; block1: ; offset 0x21 +; testl %edi, %edi +; jne 0x60 +; block2: ; offset 0x29 +; movl $0x2a, %eax +; movl $0x4591a000, %ecx +; movd %ecx, %xmm0 +; movabsq $0x40d59e0000000000, %rcx +; movq %rcx, %xmm1 +; testl %edi, %edi +; je 0x56 +; block3: ; offset 0x4e +; movq %rax, %r11 +; jmp 0xbd +; block4: ; offset 0x56 +; movdqu %xmm1, (%rsp) +; jmp 0x91 +; block5: ; offset 0x60 +; movabsq $0x3ff0000000000000, %r10 +; movq %r10, %xmm1 +; movdqu %xmm1, (%rsp) +; movabsq $0, %r11 ; reloc_external Abs8 %g 0 +; callq *%r11 +; jmp 0x91 +; block6: ; offset 0x86 +; movq %rax, %rsi +; movq %rsi, %r11 +; jmp 0xbd +; block7: ; offset 0x91 +; movl $1, %eax +; movdqu (%rsp), %xmm1 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq +; block8: ; offset 0xbd +; leal 1(%r11), %eax +; movq %r11, %rsi +; xorps %xmm0, %xmm0 +; movq %rsi, %xmm1 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq + +function %f2(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + v10 = func_addr.i64 fn0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; subq %rsp, $64, %rsp +; movq %rbx, 16(%rsp) +; movq %r12, 24(%rsp) +; movq %r13, 32(%rsp) +; movq %r14, 40(%rsp) +; movq %r15, 48(%rsp) +; block0: +; movabsq $4607182418800017408, %rcx +; movq %rcx, %xmm1 +; movdqu %xmm1, rsp(0 + virtual offset) +; load_ext_name %g+0, %rdx +; call *%rdx; jmp MachLabel(1); catch [None: MachLabel(2)] +; block1: +; movl $1, %eax +; movdqu rsp(0 + virtual offset), %xmm1 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; block2: +; movdqu rsp(0 + virtual offset), %xmm1 +; lea 1(%rax), %eax +; uninit %xmm0 +; xorps %xmm0, %xmm0 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; subq $0x40, %rsp +; movq %rbx, 0x10(%rsp) +; movq %r12, 0x18(%rsp) +; movq %r13, 0x20(%rsp) +; movq %r14, 0x28(%rsp) +; movq %r15, 0x30(%rsp) +; block1: ; offset 0x21 +; movabsq $0x3ff0000000000000, %rcx +; movq %rcx, %xmm1 +; movdqu %xmm1, (%rsp) +; movabsq $0, %rdx ; reloc_external Abs8 %g 0 +; callq *%rdx +; block2: ; offset 0x41 +; movl $1, %eax +; movdqu (%rsp), %xmm1 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq +; block3: ; offset 0x6d +; movdqu (%rsp), %xmm1 +; addl $1, %eax +; xorps %xmm0, %xmm0 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq + diff --git a/cranelift/filetests/filetests/runtests/try_call.clif b/cranelift/filetests/filetests/runtests/try_call.clif new file mode 100644 index 000000000000..860ed3c5b413 --- /dev/null +++ b/cranelift/filetests/filetests/runtests/try_call.clif @@ -0,0 +1,40 @@ +test run +target x86_64 +target aarch64 +target aarch64 sign_return_address +target aarch64 has_pauth sign_return_address +;; target s390x -- not yet +target riscv64 +target riscv64 has_c has_zcb + + +function %callee_i64(i64) -> i64 tail { +block0(v0: i64): + v1 = iadd_imm.i64 v0, 10 + return v1 +} + +function %call_i64(i64) -> i64 { + sig0 = (i64) -> i64 tail + fn0 = %callee_i64(i64) -> i64 tail + +block0(v0: i64): + try_call fn0(v0), sig0, block1(ret0), [] + +block1(v1: i64): + return v1 +} +; run: %call_i64(10) == 20 + +function %call_indirect_i64(i64) -> i64 { + sig0 = (i64) -> i64 tail + fn0 = %callee_i64(i64) -> i64 tail + +block0(v0: i64): + v1 = func_addr.i64 fn0 + try_call_indirect v1(v0), sig0, block1(ret0), [] + +block1(v2: i64): + return v2 +} +; run: %call_i64(10) == 20 diff --git a/cranelift/filetests/filetests/verifier/exceptions.clif b/cranelift/filetests/filetests/verifier/exceptions.clif new file mode 100644 index 000000000000..8378acda6abb --- /dev/null +++ b/cranelift/filetests/filetests/verifier/exceptions.clif @@ -0,0 +1,133 @@ +test verifier +target x86_64 + +function %f0(i32) -> i32 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(v2, exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(v6: i64): + v7 = iconst.i32 2 + return v7 + + block3(v8: f64, v9: i64): + v10 = iconst.i32 3 + return v10 +} + +;; Non-matching function return type: v3 on block1. +function %f1(i32) -> i32 { + sig0 = (i32) -> i32 tail + fn0 = %g(i32) -> i32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(exn0, v2) ] ; error: arg ret0 has type i32, expected f32 + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(v6: i64): + v7 = iconst.i32 2 + return v7 + + block3(v8: i64, v9: f64): + v10 = iconst.i32 3 + return v10 +} + +;; Non-matching exception payload type (not a pointer-width value): v6 on block2. +function %f2(i32) -> i32 { + sig0 = (i32) -> i32 tail + fn0 = %g(i32) -> i32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(exn0, v2) ] ; error: arg exn0 has type i64, expected i32 + + block1(v3: i32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(v6: i32): + v7 = iconst.i32 2 + return v7 + + block3(v8: i64, v9: f64): + v10 = iconst.i32 3 + return v10 +} + +;; Non-matching signature in exception table. +function %f3(i32) -> i32 { + sig0 = (i32) -> i32 tail + fn0 = %g(i64) -> i32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(exn0, v2) ] ; error: exception table signature sig0 did not match function fn0's signature sig1 + + block1(v3: i32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(v6: i64): + v7 = iconst.i32 2 + return v7 + + block3(v8: i64, v9: f64): + v10 = iconst.i32 3 + return v10 +} + +;; Too few blockparams (in explicit blockparam part). +function %f4(i32) -> i32 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(exn0, v2) ] ; error: mismatched argument count + + block1(v3: f32): + v5 = iconst.i32 1 + return v5 + + block2(v6: i64): + v7 = iconst.i32 2 + return v7 + + block3(v8: i64, v9: f64): + v10 = iconst.i32 3 + return v10 +} + +;; Non-supported calling convention. +function %f5(i32) -> i32 { + sig0 = (i32) -> f32 system_v + fn0 = %g(i32) -> f32 system_v + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(), default: block3(v2) ] ; error: calling convention `system_v` of callee does not support exceptions + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(): + v7 = iconst.i32 2 + return v7 + + block3(v8: f64): + v10 = iconst.i32 3 + return v10 +} diff --git a/cranelift/filetests/filetests/verifier/type_check.clif b/cranelift/filetests/filetests/verifier/type_check.clif index bbd737ca3475..c76f04fc7dc7 100644 --- a/cranelift/filetests/filetests/verifier/type_check.clif +++ b/cranelift/filetests/filetests/verifier/type_check.clif @@ -78,8 +78,8 @@ function %jump_args() { block0: v0 = iconst.i16 10 v3 = iconst.i64 20 - jump block1(v0, v3) ; error: arg 0 (v0) has type i16, expected i64 - ; error: arg 1 (v3) has type i64, expected i16 + jump block1(v0, v3) ; error: arg v0 has type i16, expected i64 + ; error: arg v3 has type i64, expected i16 block1(v10: i64, v11: i16): return } @@ -88,8 +88,8 @@ function %jump_args2() { block0: v0 = iconst.i16 10 v3 = iconst.i64 20 - brif v0, block1(v3, v0), block1(v0, v3) ; error: arg 0 (v0) has type i16, expected i64 - ; error: arg 1 (v3) has type i64, expected i16 + brif v0, block1(v3, v0), block1(v0, v3) ; error: arg v0 has type i16, expected i64 + ; error: arg v3 has type i64, expected i16 block1(v10: i64, v11: i16): return } @@ -99,9 +99,8 @@ block0: v0 = iconst.i16 10 v1 = iconst.i16 10 brif v0, block1(v1), block2(v1) - ; error: arg 0 (v1) has type i16, expected i64 + ; error: arg v1 has type i16, expected i64 ; error: mismatched argument count - ; error: arg 0 (v1) has type i16, expected f32 block1(v2: i64): return diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs index 927c8833b948..bf59233aee29 100644 --- a/cranelift/frontend/src/frontend.rs +++ b/cranelift/frontend/src/frontend.rs @@ -778,7 +778,9 @@ impl<'a> FunctionBuilder<'a> { /// other jump instructions. pub fn change_jump_destination(&mut self, inst: Inst, old_block: Block, new_block: Block) { let dfg = &mut self.func.dfg; - for block in dfg.insts[inst].branch_destination_mut(&mut dfg.jump_tables) { + for block in + dfg.insts[inst].branch_destination_mut(&mut dfg.jump_tables, &mut dfg.exception_tables) + { if block.block(&dfg.value_lists) == old_block { self.func_ctx.ssa.remove_block_predecessor(old_block, inst); block.set_block(new_block, &mut dfg.value_lists); diff --git a/cranelift/frontend/src/frontend/safepoints.rs b/cranelift/frontend/src/frontend/safepoints.rs index 561d7c338e28..09b6f0179660 100644 --- a/cranelift/frontend/src/frontend/safepoints.rs +++ b/cranelift/frontend/src/frontend/safepoints.rs @@ -840,7 +840,7 @@ mod tests { builder.declare_value_needs_stack_map(b); builder.switch_to_block(block0); builder.ins().call(func_ref, &[a]); - builder.ins().jump(block0, &[a, b]); + builder.ins().jump(block0, &[a.into(), b.into()]); builder.seal_all_blocks(); builder.finalize(); @@ -1434,7 +1434,7 @@ block2: let v2 = builder.ins().iconst(ir::types::I64, 2); builder.declare_value_needs_stack_map(v2); builder.ins().call(func_ref, &[]); - builder.ins().jump(block3, &[v1, v2]); + builder.ins().jump(block3, &[v1.into(), v2.into()]); builder.switch_to_block(block2); let v3 = builder.ins().iconst(ir::types::I64, 3); @@ -1442,7 +1442,7 @@ block2: let v4 = builder.ins().iconst(ir::types::I64, 4); builder.declare_value_needs_stack_map(v4); builder.ins().call(func_ref, &[]); - builder.ins().jump(block3, &[v3, v3]); + builder.ins().jump(block3, &[v3.into(), v3.into()]); builder.switch_to_block(block3); builder.append_block_param(block3, ir::types::I64); @@ -2338,7 +2338,7 @@ block4: builder.declare_value_needs_stack_map(v2); let v3 = builder.func.dfg.block_params(block0)[3]; - builder.ins().jump(block1, &[v3]); + builder.ins().jump(block1, &[v3.into()]); builder.switch_to_block(block1); let v4 = builder.append_block_param(block1, ir::types::I32); @@ -2352,7 +2352,7 @@ block4: builder.ins().call(bar_func_ref, &[v1]); builder.ins().call(foo_func_ref, &[]); let v5 = builder.ins().iadd_imm(v4, -1); - builder.ins().brif(v4, block1, &[v5], block3, &[]); + builder.ins().brif(v4, block1, &[v5.into()], block3, &[]); builder.switch_to_block(block3); builder.ins().call(foo_func_ref, &[]); @@ -2573,7 +2573,7 @@ block3: let v1 = builder.func.dfg.first_result(call_inst); builder.def_var(var_array, v1); let v2 = builder.ins().iconst(ir::types::I32, 0); - builder.ins().jump(block_array_init_loop_head, &[v2]); + builder.ins().jump(block_array_init_loop_head, &[v2.into()]); builder.switch_to_block(block_array_init_loop_head); let v3 = builder.append_block_param(block_array_init_loop_head, ir::types::I32); @@ -2594,7 +2594,7 @@ block3: builder.ins().call(array_init_elem, &[v1, v4]); let v6 = builder.ins().iconst(ir::types::I32, 1); let v7 = builder.ins().iadd(v4, v6); - builder.ins().jump(block_array_init_loop_head, &[v7]); + builder.ins().jump(block_array_init_loop_head, &[v7.into()]); builder.seal_block(block_array_init_loop_head); builder.switch_to_block(block_array_init_loop_done); @@ -2608,7 +2608,7 @@ block3: builder.ins().brif( v10, block_ref_test_done, - &[v9], + &[v9.into()], block_ref_test_non_null, &[], ); @@ -2620,15 +2620,19 @@ block3: let v12 = builder.ins().iconst(ir::types::I32, 0xbeefbeef); let v13 = builder.ins().icmp(ir::condcodes::IntCC::Equal, v11, v12); let v14 = builder.ins().iconst(ir::types::I32, 1); - builder - .ins() - .brif(v13, block_ref_test_done, &[v14], block_ref_test_slow, &[]); + builder.ins().brif( + v13, + block_ref_test_done, + &[v14.into()], + block_ref_test_slow, + &[], + ); builder.switch_to_block(block_ref_test_slow); builder.seal_block(block_ref_test_slow); let call_inst = builder.ins().call(ref_test, &[v8, v12]); let v15 = builder.func.dfg.first_result(call_inst); - builder.ins().jump(block_ref_test_done, &[v15]); + builder.ins().jump(block_ref_test_done, &[v15.into()]); builder.switch_to_block(block_ref_test_done); let v16 = builder.append_block_param(block_ref_test_done, ir::types::I32); diff --git a/cranelift/frontend/src/ssa.rs b/cranelift/frontend/src/ssa.rs index c29a8fd9bd80..9df4095780d6 100644 --- a/cranelift/frontend/src/ssa.rs +++ b/cranelift/frontend/src/ssa.rs @@ -551,7 +551,8 @@ impl SSABuilder { let pred = preds.get_mut(idx, &mut self.inst_pool).unwrap(); let branch = *pred; - let dests = dfg.insts[branch].branch_destination_mut(&mut dfg.jump_tables); + let dests = dfg.insts[branch] + .branch_destination_mut(&mut dfg.jump_tables, &mut dfg.exception_tables); assert!( !dests.is_empty(), "you have declared a non-branch instruction as a predecessor to a block!" @@ -768,9 +769,9 @@ mod tests { .. } => { assert_eq!(block_then.block(&func.dfg.value_lists), block2); - assert_eq!(block_then.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(block_then.args(&func.dfg.value_lists).len(), 0); assert_eq!(block_else.block(&func.dfg.value_lists), block1); - assert_eq!(block_else.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(block_else.args(&func.dfg.value_lists).len(), 0); } _ => assert!(false), }; @@ -780,9 +781,9 @@ mod tests { .. } => { assert_eq!(block_then.block(&func.dfg.value_lists), block2); - assert_eq!(block_then.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(block_then.args(&func.dfg.value_lists).len(), 0); assert_eq!(block_else.block(&func.dfg.value_lists), block1); - assert_eq!(block_else.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(block_else.args(&func.dfg.value_lists).len(), 0); } _ => assert!(false), }; @@ -791,7 +792,7 @@ mod tests { destination: dest, .. } => { assert_eq!(dest.block(&func.dfg.value_lists), block2); - assert_eq!(dest.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(dest.args(&func.dfg.value_lists).len(), 0); } _ => assert!(false), }; diff --git a/cranelift/fuzzgen/src/function_generator.rs b/cranelift/fuzzgen/src/function_generator.rs index 5e018d881a35..2b9c634f07c2 100644 --- a/cranelift/fuzzgen/src/function_generator.rs +++ b/cranelift/fuzzgen/src/function_generator.rs @@ -9,8 +9,9 @@ use cranelift::codegen::ir::instructions::{InstructionFormat, ResolvedConstraint use cranelift::codegen::ir::stackslot::StackSize; use cranelift::codegen::ir::{ - types::*, AliasRegion, AtomicRmwOp, Block, ConstantData, Endianness, ExternalName, FuncRef, - Function, LibCall, Opcode, SigRef, Signature, StackSlot, UserExternalName, UserFuncName, Value, + types::*, AliasRegion, AtomicRmwOp, Block, BlockArg, ConstantData, Endianness, ExternalName, + FuncRef, Function, LibCall, Opcode, SigRef, Signature, StackSlot, UserExternalName, + UserFuncName, Value, }; use cranelift::codegen::isa::CallConv; use cranelift::frontend::{FunctionBuilder, FunctionBuilderContext, Switch, Variable}; @@ -818,7 +819,9 @@ static OPCODE_SIGNATURES: LazyLock> = LazyLock::new(|| { | Opcode::Jump | Opcode::Return | Opcode::ReturnCall - | Opcode::ReturnCallIndirect => false, + | Opcode::ReturnCallIndirect + | Opcode::TryCall + | Opcode::TryCallIndirect => false, // Constants are generated outside of `generate_instructions` Opcode::Iconst => false, @@ -1106,7 +1109,9 @@ fn inserter_for_format(fmt: InstructionFormat) -> OpcodeInserter { InstructionFormat::BranchTable | InstructionFormat::Brif | InstructionFormat::Jump - | InstructionFormat::MultiAry => { + | InstructionFormat::MultiAry + | InstructionFormat::TryCall + | InstructionFormat::TryCallIndirect => { panic!("Control-flow instructions should be handled by 'insert_terminator': {fmt:?}") } } @@ -1447,9 +1452,13 @@ where &mut self, builder: &mut FunctionBuilder, block: Block, - ) -> Result> { + ) -> Result> { let (_, sig) = self.resources.blocks[block.as_u32() as usize].clone(); - self.generate_values_for_signature(builder, sig.iter().copied()) + Ok(self + .generate_values_for_signature(builder, sig.iter().copied())? + .into_iter() + .map(|val| BlockArg::Value(val)) + .collect::>()) } fn generate_values_for_signature>( diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs index 8e602fdb262a..3ff6bd11fc20 100644 --- a/cranelift/interpreter/src/step.rs +++ b/cranelift/interpreter/src/step.rs @@ -1,14 +1,15 @@ //! The [step] function interprets a single Cranelift instruction given its [State] and //! [InstructionContext]. use crate::address::{Address, AddressSize}; +use crate::frame::Frame; use crate::instruction::InstructionContext; use crate::state::{InterpreterFunctionRef, MemoryError, State}; use crate::value::{DataValueExt, ValueConversionKind, ValueError, ValueResult}; use cranelift_codegen::data_value::DataValue; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::{ - types, AbiParam, AtomicRmwOp, Block, BlockCall, Endianness, ExternalName, FuncRef, Function, - InstructionData, MemFlags, Opcode, TrapCode, Type, Value as ValueRef, + types, AbiParam, AtomicRmwOp, Block, BlockArg, BlockCall, Endianness, ExternalName, FuncRef, + Function, InstructionData, MemFlags, Opcode, TrapCode, Type, Value as ValueRef, }; use log::trace; use smallvec::{smallvec, SmallVec}; @@ -42,6 +43,19 @@ fn sum_unsigned(head: DataValue, tail: SmallVec<[DataValue; 1]>) -> ValueResult< acc.into_int_unsigned() } +/// Collect a list of block arguments. +fn collect_block_args( + frame: &Frame, + args: impl Iterator, +) -> SmallVec<[DataValue; 1]> { + args.into_iter() + .map(|n| match n { + BlockArg::Value(n) => frame.get(n).clone(), + _ => panic!("exceptions not supported"), + }) + .collect() +} + /// Interpret a single Cranelift instruction. Note that program traps and interpreter errors are /// distinct: a program trap results in `Ok(Flow::Trap(...))` whereas an interpretation error (e.g. /// the types of two values are incompatible) results in `Err(...)`. @@ -234,8 +248,10 @@ where // Retrieve an instruction's branch destination; expects the instruction to be a branch. let continue_at = |block: BlockCall| { - let branch_args = - state.collect_values(block.args_slice(&state.get_current_function().dfg.value_lists)); + let branch_args = collect_block_args( + state.current_frame(), + block.args(&state.get_current_function().dfg.value_lists), + ); Ok(ControlFlow::ContinueAt( block.block(&state.get_current_function().dfg.value_lists), branch_args, @@ -1286,6 +1302,9 @@ where Opcode::X86Pmaddubsw => unimplemented!("X86Pmaddubsw"), Opcode::X86Cvtt2dq => unimplemented!("X86Cvtt2dq"), Opcode::StackSwitch => unimplemented!("StackSwitch"), + + Opcode::TryCall => unimplemented!("TryCall"), + Opcode::TryCallIndirect => unimplemented!("TryCallIndirect"), }) } diff --git a/cranelift/reader/src/lexer.rs b/cranelift/reader/src/lexer.rs index 76b36e1920de..17b3f63c2240 100644 --- a/cranelift/reader/src/lexer.rs +++ b/cranelift/reader/src/lexer.rs @@ -13,43 +13,47 @@ use std::u16; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Token<'a> { Comment(&'a str), - LPar, // '(' - RPar, // ')' - LBrace, // '{' - RBrace, // '}' - LBracket, // '[' - RBracket, // ']' - Minus, // '-' - Plus, // '+' - Multiply, // '*' - Comma, // ',' - Dot, // '.' - Colon, // ':' - Equal, // '=' - Bang, // '!' - At, // '@' - Arrow, // '->' - Float(&'a str), // Floating point immediate - Integer(&'a str), // Integer immediate - Type(types::Type), // i32, f32, i32x4, ... - DynamicType(u32), // dt5 - Value(Value), // v12, v7 - Block(Block), // block3 - Cold, // cold (flag on block) - StackSlot(u32), // ss3 - DynamicStackSlot(u32), // dss4 - GlobalValue(u32), // gv3 - MemoryType(u32), // mt0 - Constant(u32), // const2 - FuncRef(u32), // fn2 - SigRef(u32), // sig2 - UserRef(u32), // u345 - UserNameRef(u32), // userextname345 - Name(&'a str), // %9arbitrary_alphanum, %x3, %0, %function ... - String(&'a str), // "arbitrary quoted string with no escape" ... - HexSequence(&'a str), // #89AF - Identifier(&'a str), // Unrecognized identifier (opcode, enumerator, ...) - SourceLoc(&'a str), // @00c7 + LPar, // '(' + RPar, // ')' + LBrace, // '{' + RBrace, // '}' + LBracket, // '[' + RBracket, // ']' + Minus, // '-' + Plus, // '+' + Multiply, // '*' + Comma, // ',' + Dot, // '.' + Colon, // ':' + Equal, // '=' + Bang, // '!' + At, // '@' + Arrow, // '->' + Float(&'a str), // Floating point immediate + Integer(&'a str), // Integer immediate + Type(types::Type), // i32, f32, i32x4, ... + DynamicType(u32), // dt5 + Value(Value), // v12, v7 + Block(Block), // block3 + Cold, // cold (flag on block) + StackSlot(u32), // ss3 + DynamicStackSlot(u32), // dss4 + GlobalValue(u32), // gv3 + MemoryType(u32), // mt0 + Constant(u32), // const2 + FuncRef(u32), // fn2 + SigRef(u32), // sig2 + UserRef(u32), // u345 + UserNameRef(u32), // userextname345 + ExceptionTableRef(u32), // ex123 + ExceptionTag(u32), // tag123 + TryCallRet(u32), // ret123 + TryCallExn(u32), // exn123 + Name(&'a str), // %9arbitrary_alphanum, %x3, %0, %function ... + String(&'a str), // "arbitrary quoted string with no escape" ... + HexSequence(&'a str), // #89AF + Identifier(&'a str), // Unrecognized identifier (opcode, enumerator, ...) + SourceLoc(&'a str), // @00c7 } /// A `Token` with an associated location. @@ -349,6 +353,10 @@ impl<'a> Lexer<'a> { "sig" => Some(Token::SigRef(number)), "u" => Some(Token::UserRef(number)), "userextname" => Some(Token::UserNameRef(number)), + "extable" => Some(Token::ExceptionTableRef(number)), + "tag" => Some(Token::ExceptionTag(number)), + "ret" => Some(Token::TryCallRet(number)), + "exn" => Some(Token::TryCallExn(number)), _ => None, } } diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs index a4160305a726..ab7288748fae 100644 --- a/cranelift/reader/src/parser.rs +++ b/cranelift/reader/src/parser.rs @@ -15,14 +15,15 @@ use cranelift_codegen::ir::immediates::{ }; use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs}; use cranelift_codegen::ir::pcc::{BaseExpr, Expr, Fact}; -use cranelift_codegen::ir::types; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{self, UserExternalNameRef}; + use cranelift_codegen::ir::{ - AbiParam, ArgumentExtension, ArgumentPurpose, Block, Constant, ConstantData, DynamicStackSlot, - DynamicStackSlotData, DynamicTypeData, ExtFuncData, ExternalName, FuncRef, Function, - GlobalValue, GlobalValueData, JumpTableData, MemFlags, MemoryTypeData, MemoryTypeField, Opcode, - SigRef, Signature, StackSlot, StackSlotData, StackSlotKind, UserFuncName, Value, + types, AbiParam, ArgumentExtension, ArgumentPurpose, Block, BlockArg, Constant, ConstantData, + DynamicStackSlot, DynamicStackSlotData, DynamicTypeData, ExtFuncData, ExternalName, FuncRef, + Function, GlobalValue, GlobalValueData, JumpTableData, MemFlags, MemoryTypeData, + MemoryTypeField, Opcode, SigRef, Signature, StackSlot, StackSlotData, StackSlotKind, + UserFuncName, Value, }; use cranelift_codegen::isa::{self, CallConv}; use cranelift_codegen::packed_option::ReservedValue; @@ -1890,7 +1891,7 @@ impl<'a> Parser<'a> { match self.token() { Some(Token::Block(dest)) => { self.consume(); - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; data.push(ctx.function.dfg.block_call(dest, &args)); loop { @@ -1899,7 +1900,7 @@ impl<'a> Parser<'a> { self.consume(); if let Some(Token::Block(dest)) = self.token() { self.consume(); - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; data.push(ctx.function.dfg.block_call(dest, &args)); } else { return err!(self.loc, "expected jump_table entry"); @@ -1923,6 +1924,75 @@ impl<'a> Parser<'a> { .push(JumpTableData::new(def, &data))) } + // Parse an exception-table decl. + // + // exception-table ::= * SigRef(sig) "," BlockCall "," "[" (exception-table-entry ( "," exception-table-entry )*)? "]" + // exception-table-entry ::= * ExceptionTag(tag) ":" BlockCall + // * "default" ":" BlockCall + fn parse_exception_table(&mut self, ctx: &mut Context) -> ParseResult { + let sig = self.match_sig("expected signature of called function")?; + self.match_token(Token::Comma, "expected comma after signature argument")?; + + let mut tags_and_targets = vec![]; + + let block_num = self.match_block("expected branch destination block")?; + let args = self.parse_opt_block_call_args()?; + let normal_return = ctx.function.dfg.block_call(block_num, &args); + + self.match_token( + Token::Comma, + "expected comma after normal-return destination", + )?; + + match self.token() { + Some(Token::LBracket) => { + self.consume(); + loop { + if let Some(Token::RBracket) = self.token() { + break; + } + + let tag = match self.token() { + Some(Token::ExceptionTag(tag)) => { + self.consume(); + Some(ir::ExceptionTag::from_u32(tag)) + } + Some(Token::Identifier("default")) => { + self.consume(); + None + } + _ => return err!(self.loc, "invalid token"), + }; + self.match_token(Token::Colon, "expected ':' after exception tag")?; + + let block_num = self.match_block("expected branch destination block")?; + let args = self.parse_opt_block_call_args()?; + let block_call = ctx.function.dfg.block_call(block_num, &args); + + tags_and_targets.push((tag, block_call)); + + if let Some(Token::Comma) = self.token() { + self.consume(); + } else { + break; + } + } + self.match_token(Token::RBracket, "expected closing bracket")?; + } + _ => {} + }; + + Ok(ctx + .function + .dfg + .exception_tables + .push(ir::ExceptionTableData::new( + sig, + normal_return, + tags_and_targets, + ))) + } + // Parse a constant decl. // // constant-decl ::= * Constant(c) "=" ty? "[" literal {"," literal} "]" @@ -2663,19 +2733,46 @@ impl<'a> Parser<'a> { Ok(args) } - // Parse an optional value list enclosed in parentheses. - fn parse_opt_value_list(&mut self) -> ParseResult { + /// Parse an optional list of block-call arguments enclosed in + /// parentheses. + fn parse_opt_block_call_args(&mut self) -> ParseResult> { if !self.optional(Token::LPar) { - return Ok(VariableArgs::new()); + return Ok(vec![]); } - let args = self.parse_value_list()?; + let mut args = vec![]; + while self.token() != Some(Token::RPar) { + args.push(self.parse_block_call_arg()?); + if self.token() == Some(Token::Comma) { + self.consume(); + } else { + break; + } + } self.match_token(Token::RPar, "expected ')' after arguments")?; Ok(args) } + fn parse_block_call_arg(&mut self) -> ParseResult { + match self.token() { + Some(Token::Value(v)) => { + self.consume(); + Ok(BlockArg::Value(v)) + } + Some(Token::TryCallRet(i)) => { + self.consume(); + Ok(BlockArg::TryCallRet(i)) + } + Some(Token::TryCallExn(i)) => { + self.consume(); + Ok(BlockArg::TryCallExn(i)) + } + tok => Err(self.error(&format!("unexpected token: {tok:?}"))), + } + } + /// Parse a CLIF run command. /// /// run-command ::= "run" [":" invocation comparison expected] @@ -2963,7 +3060,7 @@ impl<'a> Parser<'a> { InstructionFormat::Jump => { // Parse the destination block number. let block_num = self.match_block("expected jump destination block")?; - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; let destination = ctx.function.dfg.block_call(block_num, &args); InstructionData::Jump { opcode, @@ -2975,13 +3072,13 @@ impl<'a> Parser<'a> { self.match_token(Token::Comma, "expected ',' between operands")?; let block_then = { let block_num = self.match_block("expected branch then block")?; - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; ctx.function.dfg.block_call(block_num, &args) }; self.match_token(Token::Comma, "expected ',' between operands")?; let block_else = { let block_num = self.match_block("expected branch else block")?; - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; ctx.function.dfg.block_call(block_num, &args) }; InstructionData::Brif { @@ -2994,7 +3091,7 @@ impl<'a> Parser<'a> { let arg = self.match_value("expected SSA value operand")?; self.match_token(Token::Comma, "expected ',' between operands")?; let block_num = self.match_block("expected branch destination block")?; - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; let destination = ctx.function.dfg.block_call(block_num, &args); self.match_token(Token::Comma, "expected ',' between operands")?; let table = self.parse_jump_table(ctx, destination)?; @@ -3085,6 +3182,34 @@ impl<'a> Parser<'a> { args: args.into_value_list(&[callee], &mut ctx.function.dfg.value_lists), } } + InstructionFormat::TryCall => { + let func_ref = self.match_fn("expected function reference")?; + ctx.check_fn(func_ref, self.loc)?; + self.match_token(Token::LPar, "expected '(' before arguments")?; + let args = self.parse_value_list()?; + self.match_token(Token::RPar, "expected ')' after arguments")?; + self.match_token(Token::Comma, "expected ',' after argument list")?; + let exception = self.parse_exception_table(ctx)?; + InstructionData::TryCall { + opcode, + func_ref, + args: args.into_value_list(&[], &mut ctx.function.dfg.value_lists), + exception, + } + } + InstructionFormat::TryCallIndirect => { + let callee = self.match_value("expected SSA value callee operand")?; + self.match_token(Token::LPar, "expected '(' before arguments")?; + let args = self.parse_value_list()?; + self.match_token(Token::RPar, "expected ')' after arguments")?; + self.match_token(Token::Comma, "expected ',' after argument list")?; + let exception = self.parse_exception_table(ctx)?; + InstructionData::TryCallIndirect { + opcode, + args: args.into_value_list(&[callee], &mut ctx.function.dfg.value_lists), + exception, + } + } InstructionFormat::FuncAddr => { let func_ref = self.match_fn("expected function reference")?; ctx.check_fn(func_ref, self.loc)?; diff --git a/cranelift/src/bugpoint.rs b/cranelift/src/bugpoint.rs index 3c0b506afdbe..4bfb35cbd3dd 100644 --- a/cranelift/src/bugpoint.rs +++ b/cranelift/src/bugpoint.rs @@ -420,7 +420,9 @@ impl Mutator for ReplaceBlockParamWithConst { // Remove parameters in branching instructions that point to this block for pred in cfg.pred_iter(self.block) { let dfg = &mut func.dfg; - for branch in dfg.insts[pred.inst].branch_destination_mut(&mut dfg.jump_tables) { + for branch in dfg.insts[pred.inst] + .branch_destination_mut(&mut dfg.jump_tables, &mut dfg.exception_tables) + { if branch.block(&dfg.value_lists) == self.block { branch.remove(param_index, &mut dfg.value_lists); } @@ -708,7 +710,8 @@ impl Mutator for MergeBlocks { // If the branch instruction that lead us to this block wasn't an unconditional jump, then // we have a conditional jump sequence that we should not break. - let branch_dests = func.dfg.insts[pred.inst].branch_destination(&func.dfg.jump_tables); + let branch_dests = func.dfg.insts[pred.inst] + .branch_destination(&func.dfg.jump_tables, &func.dfg.exception_tables); if branch_dests.len() != 1 { return Some(( func, @@ -717,7 +720,9 @@ impl Mutator for MergeBlocks { )); } - let branch_args = branch_dests[0].args_slice(&func.dfg.value_lists).to_vec(); + let branch_args = branch_dests[0] + .args(&func.dfg.value_lists) + .collect::>(); // TODO: should we free the entity list associated with the block params? let block_params = func @@ -731,8 +736,10 @@ impl Mutator for MergeBlocks { // If there were any block parameters in block, then the last instruction in pred will // fill these parameters. Make the block params aliases of the terminator arguments. for (block_param, arg) in block_params.into_iter().zip(branch_args) { - if block_param != arg { - func.dfg.change_to_alias(block_param, arg); + if let Some(arg) = arg.as_value() { + if block_param != arg { + func.dfg.change_to_alias(block_param, arg); + } } } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index b339b7bf08aa..ffb9dc5ad305 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -916,9 +916,13 @@ impl<'module_environment> FuncEnvironment<'module_environment> { let result_param = builder.append_block_param(continuation_block, pointer_type); builder.set_cold_block(null_block); - builder - .ins() - .brif(value, continuation_block, &[value_masked], null_block, &[]); + builder.ins().brif( + value, + continuation_block, + &[value_masked.into()], + null_block, + &[], + ); builder.seal_block(null_block); builder.switch_to_block(null_block); @@ -931,7 +935,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> { let index = self.cast_index_to_i64(&mut builder.cursor(), index, index_type); let call_inst = builder.ins().call(lazy_init, &[vmctx, table_index, index]); let returned_entry = builder.func.dfg.inst_results(call_inst)[0]; - builder.ins().jump(continuation_block, &[returned_entry]); + builder + .ins() + .jump(continuation_block, &[returned_entry.into()]); builder.seal_block(continuation_block); builder.switch_to_block(continuation_block); diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs index 3f45454239f8..e871c810f0ec 100644 --- a/crates/cranelift/src/func_environ/gc/enabled.rs +++ b/crates/cranelift/src/func_environ/gc/enabled.rs @@ -571,7 +571,7 @@ fn emit_array_fill_impl( // Current block: jump to the loop header block with the first element's // address. - builder.ins().jump(loop_header_block, &[elem_addr]); + builder.ins().jump(loop_header_block, &[elem_addr.into()]); // Loop header block: check if we're done, then jump to either the continue // block or the loop body block. @@ -590,7 +590,9 @@ fn emit_array_fill_impl( log::trace!("emit_array_fill_impl: loop body"); emit_elem_write(func_env, builder, elem_addr)?; let next_elem_addr = builder.ins().iadd(elem_addr, elem_size); - builder.ins().jump(loop_header_block, &[next_elem_addr]); + builder + .ins() + .jump(loop_header_block, &[next_elem_addr.into()]); // Continue... builder.switch_to_block(continue_block); @@ -934,7 +936,7 @@ pub fn translate_ref_test( builder.ins().brif( is_null, continue_block, - &[result_when_is_null], + &[result_when_is_null.into()], non_null_block, &[], ); @@ -961,7 +963,7 @@ pub fn translate_ref_test( builder.ins().brif( is_i31, continue_block, - &[result_when_is_i31], + &[result_when_is_i31.into()], non_null_non_i31_block, &[], ); @@ -1070,7 +1072,7 @@ pub fn translate_ref_test( WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. }; - builder.ins().jump(continue_block, &[result]); + builder.ins().jump(continue_block, &[result.into()]); // Control flow join point with the result. builder.switch_to_block(continue_block); @@ -1450,9 +1452,13 @@ impl FuncEnvironment<'_> { log::trace!("is_subtype: fast path check for exact same types"); let same_ty = builder.ins().icmp(IntCC::Equal, a, b); let same_ty = builder.ins().uextend(ir::types::I32, same_ty); - builder - .ins() - .brif(same_ty, continue_block, &[same_ty], diff_tys_block, &[]); + builder.ins().brif( + same_ty, + continue_block, + &[same_ty.into()], + diff_tys_block, + &[], + ); // Different types block: fall back to the `is_subtype` libcall. builder.switch_to_block(diff_tys_block); @@ -1461,7 +1467,7 @@ impl FuncEnvironment<'_> { let vmctx = self.vmctx_val(&mut builder.cursor()); let call_inst = builder.ins().call(is_subtype, &[vmctx, a, b]); let result = builder.func.dfg.first_result(call_inst); - builder.ins().jump(continue_block, &[result]); + builder.ins().jump(continue_block, &[result.into()]); // Continue block: join point for the result. builder.switch_to_block(continue_block); diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index a6e6c3310b96..31b14449f682 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -81,10 +81,10 @@ use crate::translate::translation_utils::{ use crate::Reachability; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::Offset32; -use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{ self, AtomicRmwOp, ConstantData, InstBuilder, JumpTableData, MemFlags, Value, ValueLabel, }; +use cranelift_codegen::ir::{types::*, BlockArg}; use cranelift_codegen::packed_option::ReservedValue; use cranelift_frontend::{FunctionBuilder, Variable}; use itertools::Itertools; @@ -3897,28 +3897,21 @@ fn is_non_canonical_v128(ty: ir::Type) -> bool { /// actually necessary, and if not, the original slice is returned. Otherwise the cast values /// are returned in a slice that belongs to the caller-supplied `SmallVec`. fn canonicalise_v128_values<'a>( - tmp_canonicalised: &'a mut SmallVec<[ir::Value; 16]>, + tmp_canonicalised: &'a mut SmallVec<[BlockArg; 16]>, builder: &mut FunctionBuilder, values: &'a [ir::Value], -) -> &'a [ir::Value] { +) -> &'a [BlockArg] { debug_assert!(tmp_canonicalised.is_empty()); - // First figure out if any of the parameters need to be cast. Mostly they don't need to be. - let any_non_canonical = values - .iter() - .any(|v| is_non_canonical_v128(builder.func.dfg.value_type(*v))); - // Hopefully we take this exit most of the time, hence doing no heap allocation. - if !any_non_canonical { - return values; - } - // Otherwise we'll have to cast, and push the resulting `Value`s into `canonicalised`. + // Cast, and push the resulting `Value`s into `canonicalised`. for v in values { - tmp_canonicalised.push(if is_non_canonical_v128(builder.func.dfg.value_type(*v)) { + let value = if is_non_canonical_v128(builder.func.dfg.value_type(*v)) { let mut flags = MemFlags::new(); flags.set_endianness(ir::Endianness::Little); builder.ins().bitcast(I8X16, flags, *v) } else { *v - }); + }; + tmp_canonicalised.push(BlockArg::from(value)); } tmp_canonicalised.as_slice() } @@ -3931,7 +3924,7 @@ fn canonicalise_then_jump( destination: ir::Block, params: &[ir::Value], ) -> ir::Inst { - let mut tmp_canonicalised = SmallVec::<[ir::Value; 16]>::new(); + let mut tmp_canonicalised = SmallVec::<[_; 16]>::new(); let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params); builder.ins().jump(destination, canonicalised) } @@ -3945,10 +3938,10 @@ fn canonicalise_brif( block_else: ir::Block, params_else: &[ir::Value], ) -> ir::Inst { - let mut tmp_canonicalised_then = SmallVec::<[ir::Value; 16]>::new(); + let mut tmp_canonicalised_then = SmallVec::<[_; 16]>::new(); let canonicalised_then = canonicalise_v128_values(&mut tmp_canonicalised_then, builder, params_then); - let mut tmp_canonicalised_else = SmallVec::<[ir::Value; 16]>::new(); + let mut tmp_canonicalised_else = SmallVec::<[_; 16]>::new(); let canonicalised_else = canonicalise_v128_values(&mut tmp_canonicalised_else, builder, params_else); builder.ins().brif( diff --git a/fuzz/fuzz_targets/misc.rs b/fuzz/fuzz_targets/misc.rs index f4fd1f3c88ea..087c1913428f 100644 --- a/fuzz/fuzz_targets/misc.rs +++ b/fuzz/fuzz_targets/misc.rs @@ -158,7 +158,9 @@ fn dominator_tree(mut data: Unstructured<'_>) -> Result<()> { } else { let block_calls = children .iter() - .map(|&block| BlockCall::new(block, &[], &mut cursor.func.dfg.value_lists)) + .map(|&block| { + BlockCall::new(block, core::iter::empty(), &mut cursor.func.dfg.value_lists) + }) .collect::>(); let data = JumpTableData::new(block_calls[0], &block_calls[1..]);