From 4beeab03bded863770cc1d9282497e7de0f477fa Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Fri, 4 Apr 2025 16:51:17 -0700 Subject: [PATCH 1/2] Cranelift: initial try_call / try_call_indirect (exception) support. This PR adds `try_call` and `try_call_indirect` instructions, and lowerings on four of five ISAs (x86-64, aarch64, riscv64, pulley; s390x has its own non-shared ABI code that will need separate work). It extends CLIF to support these instructions as new kinds of branches, and extends block-calls to accept `retN` and `exnN` block-call args that carry the normal return values or exception payloads (respectively) into the appropriate successor blocks. It wires up the "normal return path" so that it continues to work. It updates the ABI so that unwinding is possible without an initial register state at throw: specifically, as per our RFC, all registers are clobbered. It also includes metadata in the `MachBuffer` that describes exception-catch destinations. However, no unwinder exists to interpret these catch-destinations yet, so they are untested. --- cranelift/codegen/meta/src/gen_inst.rs | 12 +- cranelift/codegen/meta/src/shared/entities.rs | 5 + cranelift/codegen/meta/src/shared/formats.rs | 14 + .../codegen/meta/src/shared/instructions.rs | 69 +++++ cranelift/codegen/src/dominator_tree.rs | 2 +- .../codegen/src/dominator_tree/simple.rs | 2 +- cranelift/codegen/src/inst_predicates.rs | 10 + cranelift/codegen/src/ir/builder.rs | 2 +- cranelift/codegen/src/ir/dfg.rs | 86 ++++-- cranelift/codegen/src/ir/entities.rs | 58 ++++ cranelift/codegen/src/ir/exception_table.rs | 193 +++++++++++++ cranelift/codegen/src/ir/function.rs | 6 +- cranelift/codegen/src/ir/instructions.rs | 213 ++++++++++++-- cranelift/codegen/src/ir/jumptable.rs | 25 +- cranelift/codegen/src/ir/mod.rs | 12 +- cranelift/codegen/src/isa/aarch64/abi.rs | 83 +++++- cranelift/codegen/src/isa/aarch64/inst.isle | 6 + .../codegen/src/isa/aarch64/inst/emit.rs | 34 +++ cranelift/codegen/src/isa/aarch64/inst/mod.rs | 46 ++- .../codegen/src/isa/aarch64/inst/regs.rs | 4 +- cranelift/codegen/src/isa/aarch64/lower.isle | 7 + cranelift/codegen/src/isa/call_conv.rs | 33 ++- .../codegen/src/isa/pulley_shared/abi.rs | 117 +++++++- .../codegen/src/isa/pulley_shared/inst.isle | 6 + .../src/isa/pulley_shared/inst/emit.rs | 44 ++- .../codegen/src/isa/pulley_shared/inst/mod.rs | 37 ++- .../codegen/src/isa/pulley_shared/lower.isle | 7 + cranelift/codegen/src/isa/riscv64/abi.rs | 120 +++++++- cranelift/codegen/src/isa/riscv64/inst.isle | 13 + .../codegen/src/isa/riscv64/inst/emit.rs | 44 ++- cranelift/codegen/src/isa/riscv64/inst/mod.rs | 39 ++- .../codegen/src/isa/riscv64/inst/regs.rs | 8 +- cranelift/codegen/src/isa/s390x/abi.rs | 6 +- cranelift/codegen/src/isa/s390x/inst/mod.rs | 15 +- cranelift/codegen/src/isa/s390x/lower/isle.rs | 6 +- cranelift/codegen/src/isa/x64/abi.rs | 17 +- cranelift/codegen/src/isa/x64/inst.isle | 6 + cranelift/codegen/src/isa/x64/inst/emit.rs | 32 +++ .../codegen/src/isa/x64/inst/emit_tests.rs | 2 +- cranelift/codegen/src/isa/x64/inst/mod.rs | 42 ++- cranelift/codegen/src/isa/x64/inst/regs.rs | 74 ++--- cranelift/codegen/src/isa/x64/lower.isle | 9 + cranelift/codegen/src/isa/x64/lower.rs | 2 +- cranelift/codegen/src/machinst/abi.rs | 135 +++++++-- cranelift/codegen/src/machinst/buffer.rs | 24 ++ cranelift/codegen/src/machinst/isle.rs | 80 +++++- cranelift/codegen/src/machinst/lower.rs | 95 ++++++- cranelift/codegen/src/machinst/mod.rs | 11 +- cranelift/codegen/src/machinst/reg.rs | 25 +- cranelift/codegen/src/machinst/vcode.rs | 4 +- cranelift/codegen/src/prelude_lower.isle | 4 + cranelift/codegen/src/remove_constant_phis.rs | 31 +- cranelift/codegen/src/traversals.rs | 2 +- cranelift/codegen/src/verifier/mod.rs | 260 +++++++++++++++-- cranelift/codegen/src/write.rs | 29 ++ .../filetests/isa/aarch64/exceptions.clif | 123 ++++++++ .../filetests/isa/pulley32/call.clif | 15 +- .../filetests/isa/pulley32/exceptions.clif | 150 ++++++++++ .../filetests/isa/pulley32/extend.clif | 16 +- .../filetests/isa/pulley64/call.clif | 19 +- .../isa/pulley64/call_indirect_host.clif | 2 +- .../filetests/isa/pulley64/exceptions.clif | 151 ++++++++++ .../filetests/isa/pulley64/extend.clif | 16 +- .../filetests/isa/riscv64/exceptions.clif | 222 +++++++++++++++ .../filetests/isa/riscv64/tail-call-conv.clif | 2 - .../filetests/isa/x64/exceptions.clif | 267 ++++++++++++++++++ .../filetests/runtests/try_call.clif | 27 ++ .../filetests/verifier/exceptions.clif | 133 +++++++++ .../filetests/verifier/type_check.clif | 11 +- cranelift/frontend/src/frontend.rs | 4 +- cranelift/frontend/src/frontend/safepoints.rs | 28 +- cranelift/frontend/src/ssa.rs | 13 +- cranelift/fuzzgen/src/function_generator.rs | 21 +- cranelift/interpreter/src/step.rs | 27 +- cranelift/reader/src/lexer.rs | 82 +++--- cranelift/reader/src/parser.rs | 155 +++++++++- cranelift/src/bugpoint.rs | 17 +- crates/cranelift/src/func_environ.rs | 14 +- .../cranelift/src/func_environ/gc/enabled.rs | 24 +- .../src/translate/code_translator.rs | 27 +- fuzz/fuzz_targets/misc.rs | 4 +- 81 files changed, 3415 insertions(+), 423 deletions(-) create mode 100644 cranelift/codegen/src/ir/exception_table.rs create mode 100644 cranelift/filetests/filetests/isa/aarch64/exceptions.clif create mode 100644 cranelift/filetests/filetests/isa/pulley32/exceptions.clif create mode 100644 cranelift/filetests/filetests/isa/pulley64/exceptions.clif create mode 100644 cranelift/filetests/filetests/isa/riscv64/exceptions.clif create mode 100644 cranelift/filetests/filetests/isa/x64/exceptions.clif create mode 100644 cranelift/filetests/filetests/runtests/try_call.clif create mode 100644 cranelift/filetests/filetests/verifier/exceptions.clif diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs index f26b326f93ff..f7bd0ed3df71 100644 --- a/cranelift/codegen/meta/src/gen_inst.rs +++ b/cranelift/codegen/meta/src/gen_inst.rs @@ -336,7 +336,7 @@ fn gen_instruction_data_impl(formats: &[Rc], fmt: &mut Format fmtln!(fmt, "::core::hash::Hash::hash(&{len}, state);"); fmt.add_block(&format!("for &block in {blocks}"), |fmt| { fmtln!(fmt, "::core::hash::Hash::hash(&block.block(pool), state);"); - fmt.add_block("for &arg in block.args_slice(pool)", |fmt| { + fmt.add_block("for arg in block.args(pool)", |fmt| { fmtln!(fmt, "::core::hash::Hash::hash(&arg, state);"); }); }); @@ -964,6 +964,7 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo let mut tmpl_types = Vec::new(); let mut into_args = Vec::new(); let mut block_args = Vec::new(); + let mut lifetime_param = None; for op in &inst.operands_in { if op.kind.is_block() { args.push(format!("{}_label: {}", op.name, "ir::Block")); @@ -972,7 +973,14 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo op.name, "Destination basic block" )); - args.push(format!("{}_args: {}", op.name, "&[Value]")); + let lifetime = *lifetime_param.get_or_insert_with(|| { + tmpl_types.insert(0, "'a".to_string()); + "'a" + }); + args.push(format!( + "{}_args: impl IntoIterator", + op.name, lifetime, + )); args_doc.push(format!("- {}_args: {}", op.name, "Block arguments")); block_args.push(op); diff --git a/cranelift/codegen/meta/src/shared/entities.rs b/cranelift/codegen/meta/src/shared/entities.rs index 3df9ecaa171f..cc3cc9c399b5 100644 --- a/cranelift/codegen/meta/src/shared/entities.rs +++ b/cranelift/codegen/meta/src/shared/entities.rs @@ -43,6 +43,9 @@ pub(crate) struct EntityRefs { /// A reference to a jump table declared in the function preamble. pub(crate) jump_table: OperandKind, + /// A reference to an exception table declared in the function preamble. + pub(crate) exception_table: OperandKind, + /// A variable-sized list of value operands. Use for Block and function call arguments. pub(crate) varargs: OperandKind, } @@ -84,6 +87,8 @@ impl EntityRefs { jump_table: new("table", "ir::JumpTable", "A jump table."), + exception_table: new("exception", "ir::ExceptionTable", "An exception table."), + varargs: OperandKind::new( "", "&[Value]", diff --git a/cranelift/codegen/meta/src/shared/formats.rs b/cranelift/codegen/meta/src/shared/formats.rs index 86d54338d63d..4ff60bef627c 100644 --- a/cranelift/codegen/meta/src/shared/formats.rs +++ b/cranelift/codegen/meta/src/shared/formats.rs @@ -12,6 +12,8 @@ pub(crate) struct Formats { pub(crate) brif: Rc, pub(crate) call: Rc, pub(crate) call_indirect: Rc, + pub(crate) try_call: Rc, + pub(crate) try_call_indirect: Rc, pub(crate) cond_trap: Rc, pub(crate) float_compare: Rc, pub(crate) func_addr: Rc, @@ -133,6 +135,18 @@ impl Formats { .varargs() .build(), + try_call: Builder::new("TryCall") + .imm(&entities.func_ref) + .varargs() + .imm(&&entities.exception_table) + .build(), + + try_call_indirect: Builder::new("TryCallIndirect") + .value() + .varargs() + .imm(&&entities.exception_table) + .build(), + func_addr: Builder::new("FuncAddr").imm(&entities.func_ref).build(), atomic_rmw: Builder::new("AtomicRmw") diff --git a/cranelift/codegen/meta/src/shared/instructions.rs b/cranelift/codegen/meta/src/shared/instructions.rs index 81cf8d5b59ee..6a86e3e9bede 100644 --- a/cranelift/codegen/meta/src/shared/instructions.rs +++ b/cranelift/codegen/meta/src/shared/instructions.rs @@ -307,6 +307,75 @@ fn define_control_flow( .with_doc("function to call, declared by `function`")]) .operands_out(vec![Operand::new("addr", iAddr)]), ); + + ig.push( + Inst::new( + "try_call", + r#" + Call a function, catching the specified exceptions. + + Call the function pointed to by `callee` with the given arguments. On + normal return, branch to the first target, with function returns + available as `retN` block arguments. On exceptional return, + look up the thrown exception tag in the provided exception table; + if the tag matches one of the targets, branch to the matching + target with the exception payloads available as `exnN` block arguments. + If no tag matches, then propagate the exception up the stack. + + It is the Cranelift embedder's responsibility to define the meaning + of tags: they are accepted by this instruction and passed through + to unwind metadata tables in Cranelift's output. Actual unwinding is + outside the purview of the core Cranelift compiler. + + Payload values on exception are passed in fixed register(s) that are + defined by the platform and ABI. See the documentation on `CallConv` + for details. + "#, + &formats.try_call, + ) + .operands_in(vec![ + Operand::new("callee", &entities.func_ref) + .with_doc("function to call, declared by `function`"), + Operand::new("args", &entities.varargs).with_doc("call arguments"), + Operand::new("ET", &entities.exception_table).with_doc("exception table"), + ]) + .call() + .branches(), + ); + + ig.push( + Inst::new( + "try_call_indirect", + r#" + Call a function, catching the specified exceptions. + + Call the function pointed to by `callee` with the given arguments. On + normal return, branch to the first target, with function returns + available as `retN` block arguments. On exceptional return, + look up the thrown exception tag in the provided exception table; + if the tag matches one of the targets, branch to the matching + target with the exception payloads available as `exnN` block arguments. + If no tag matches, then propagate the exception up the stack. + + It is the Cranelift embedder's responsibility to define the meaning + of tags: they are accepted by this instruction and passed through + to unwind metadata tables in Cranelift's output. Actual unwinding is + outside the purview of the core Cranelift compiler. + + Payload values on exception are passed in fixed register(s) that are + defined by the platform and ABI. See the documentation on `CallConv` + for details. + "#, + &formats.try_call_indirect, + ) + .operands_in(vec![ + Operand::new("callee", iAddr).with_doc("address of function to call"), + Operand::new("args", &entities.varargs).with_doc("call arguments"), + Operand::new("ET", &entities.exception_table).with_doc("exception table"), + ]) + .call() + .branches(), + ); } #[inline(never)] diff --git a/cranelift/codegen/src/dominator_tree.rs b/cranelift/codegen/src/dominator_tree.rs index 92f7c143a48b..4441a4909d4e 100644 --- a/cranelift/codegen/src/dominator_tree.rs +++ b/cranelift/codegen/src/dominator_tree.rs @@ -644,7 +644,7 @@ mod tests { cur.insert_block(block1); let v1 = cur.ins().iconst(I32, 1); let v2 = cur.ins().iadd(v0, v1); - cur.ins().jump(block0, &[v2]); + cur.ins().jump(block0, &[v2.into()]); cur.insert_block(block2); cur.ins().return_(&[v0]); diff --git a/cranelift/codegen/src/dominator_tree/simple.rs b/cranelift/codegen/src/dominator_tree/simple.rs index d7117cbc073b..853158bc80b1 100644 --- a/cranelift/codegen/src/dominator_tree/simple.rs +++ b/cranelift/codegen/src/dominator_tree/simple.rs @@ -365,7 +365,7 @@ mod tests { cur.insert_block(block1); let v1 = cur.ins().iconst(I32, 1); let v2 = cur.ins().iadd(v0, v1); - cur.ins().jump(block0, &[v2]); + cur.ins().jump(block0, &[v2.into()]); cur.insert_block(block2); cur.ins().return_(&[v0]); diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs index 7a345264cc4c..7be7af565780 100644 --- a/cranelift/codegen/src/inst_predicates.rs +++ b/cranelift/codegen/src/inst_predicates.rs @@ -200,6 +200,16 @@ pub(crate) fn visit_block_succs( } } + ir::InstructionData::TryCall { exception, .. } + | ir::InstructionData::TryCallIndirect { exception, .. } => { + let pool = &f.dfg.value_lists; + let exdata = &f.stencil.dfg.exception_tables[*exception]; + + for dest in exdata.all_branches() { + visit(inst, dest.block(pool), false); + } + } + inst => debug_assert!(!inst.opcode().is_branch()), } } diff --git a/cranelift/codegen/src/ir/builder.rs b/cranelift/codegen/src/ir/builder.rs index 6032f07f5212..24d9beaee660 100644 --- a/cranelift/codegen/src/ir/builder.rs +++ b/cranelift/codegen/src/ir/builder.rs @@ -6,8 +6,8 @@ use crate::ir; use crate::ir::instructions::InstructionFormat; use crate::ir::types; +use crate::ir::{BlockArg, Inst, Opcode, Type, Value}; use crate::ir::{DataFlowGraph, InstructionData}; -use crate::ir::{Inst, Opcode, Type, Value}; /// Base trait for instruction builders. /// diff --git a/cranelift/codegen/src/ir/dfg.rs b/cranelift/codegen/src/ir/dfg.rs index 593d4cdb4a33..7ee5938768d1 100644 --- a/cranelift/codegen/src/ir/dfg.rs +++ b/cranelift/codegen/src/ir/dfg.rs @@ -8,9 +8,9 @@ use crate::ir::instructions::{CallInfo, InstructionData}; use crate::ir::pcc::Fact; use crate::ir::user_stack_maps::{UserStackMapEntry, UserStackMapEntryVec}; use crate::ir::{ - types, Block, BlockCall, ConstantData, ConstantPool, DynamicType, ExtFuncData, FuncRef, - Immediate, Inst, JumpTables, RelSourceLoc, SigRef, Signature, Type, Value, - ValueLabelAssignments, ValueList, ValueListPool, + types, Block, BlockArg, BlockCall, ConstantData, ConstantPool, DynamicType, ExceptionTables, + ExtFuncData, FuncRef, Immediate, Inst, JumpTables, RelSourceLoc, SigRef, Signature, Type, + Value, ValueLabelAssignments, ValueList, ValueListPool, }; use crate::packed_option::ReservedValue; use crate::write::write_operands; @@ -158,6 +158,9 @@ pub struct DataFlowGraph { /// Jump tables used in this function. pub jump_tables: JumpTables, + + /// Exception tables used in this function. + pub exception_tables: ExceptionTables, } impl DataFlowGraph { @@ -178,6 +181,7 @@ impl DataFlowGraph { constants: ConstantPool::new(), immediates: PrimaryMap::new(), jump_tables: JumpTables::new(), + exception_tables: ExceptionTables::new(), } } @@ -226,8 +230,12 @@ impl DataFlowGraph { } /// Make a BlockCall, bundling together the block and its arguments. - pub fn block_call(&mut self, block: Block, args: &[Value]) -> BlockCall { - BlockCall::new(block, args, &mut self.value_lists) + pub fn block_call<'a>( + &mut self, + block: Block, + args: impl IntoIterator, + ) -> BlockCall { + BlockCall::new(block, args.into_iter().copied(), &mut self.value_lists) } /// Get the total number of values. @@ -437,13 +445,18 @@ impl DataFlowGraph { // Rewrite InstructionData in `self.insts`. for inst in self.insts.0.values_mut() { - inst.map_values(&mut self.value_lists, &mut self.jump_tables, |arg| { - if let ValueData::Alias { original, .. } = self.values[arg].into() { - original - } else { - arg - } - }); + inst.map_values( + &mut self.value_lists, + &mut self.jump_tables, + &mut self.exception_tables, + |arg| { + if let ValueData::Alias { original, .. } = self.values[arg].into() { + original + } else { + arg + } + }, + ); } // - `results` and block-params in `blocks` are not aliases, by @@ -843,15 +856,16 @@ impl DataFlowGraph { &'dfg self, inst: Inst, ) -> impl DoubleEndedIterator + 'dfg { - self.inst_args(inst) - .iter() - .chain( - self.insts[inst] - .branch_destination(&self.jump_tables) - .into_iter() - .flat_map(|branch| branch.args_slice(&self.value_lists).iter()), - ) - .copied() + self.inst_args(inst).iter().copied().chain( + self.insts[inst] + .branch_destination(&self.jump_tables, &self.exception_tables) + .into_iter() + .flat_map(|branch| { + branch + .args(&self.value_lists) + .filter_map(|arg| arg.as_value()) + }), + ) } /// Map a function over the values of the instruction. @@ -859,7 +873,12 @@ impl DataFlowGraph { where F: FnMut(Value) -> Value, { - self.insts[inst].map_values(&mut self.value_lists, &mut self.jump_tables, body); + self.insts[inst].map_values( + &mut self.value_lists, + &mut self.jump_tables, + &mut self.exception_tables, + body, + ); } /// Overwrite the instruction's value references with values from the iterator. @@ -869,9 +888,12 @@ impl DataFlowGraph { where I: Iterator, { - self.insts[inst].map_values(&mut self.value_lists, &mut self.jump_tables, |_| { - values.next().unwrap() - }); + self.insts[inst].map_values( + &mut self.value_lists, + &mut self.jump_tables, + &mut self.exception_tables, + |_| values.next().unwrap(), + ); } /// Get all value arguments on `inst` as a slice. @@ -1078,18 +1100,22 @@ impl DataFlowGraph { /// Get the call signature of a direct or indirect call instruction. /// Returns `None` if `inst` is not a call instruction. pub fn call_signature(&self, inst: Inst) -> Option { - match self.insts[inst].analyze_call(&self.value_lists) { + match self.insts[inst].analyze_call(&self.value_lists, &self.exception_tables) { CallInfo::NotACall => None, CallInfo::Direct(f, _) => Some(self.ext_funcs[f].signature), + CallInfo::DirectWithSig(_, s, _) => Some(s), CallInfo::Indirect(s, _) => Some(s), } } - /// Like `call_signature` but returns none for tail call instructions. - fn non_tail_call_signature(&self, inst: Inst) -> Option { + /// Like `call_signature` but returns none for tail call + /// instructions and try-call (exception-handling invoke) + /// instructions. + fn non_tail_call_or_try_call_signature(&self, inst: Inst) -> Option { let sig = self.call_signature(inst)?; match self.insts[inst].opcode() { ir::Opcode::ReturnCall | ir::Opcode::ReturnCallIndirect => None, + ir::Opcode::TryCall | ir::Opcode::TryCallIndirect => None, _ => Some(sig), } } @@ -1097,7 +1123,7 @@ impl DataFlowGraph { // Only for use by the verifier. Everyone else should just use // `dfg.inst_results(inst).len()`. pub(crate) fn num_expected_results_for_verifier(&self, inst: Inst) -> usize { - match self.non_tail_call_signature(inst) { + match self.non_tail_call_or_try_call_signature(inst) { Some(sig) => self.signatures[sig].returns.len(), None => { let constraints = self.insts[inst].opcode().constraints(); @@ -1112,7 +1138,7 @@ impl DataFlowGraph { inst: Inst, ctrl_typevar: Type, ) -> impl iter::ExactSizeIterator + 'a { - return match self.non_tail_call_signature(inst) { + return match self.non_tail_call_or_try_call_signature(inst) { Some(sig) => InstResultTypes::Signature(self, sig, 0), None => { let constraints = self.insts[inst].opcode().constraints(); diff --git a/cranelift/codegen/src/ir/entities.rs b/cranelift/codegen/src/ir/entities.rs index 005007471b35..4712bbad2425 100644 --- a/cranelift/codegen/src/ir/entities.rs +++ b/cranelift/codegen/src/ir/entities.rs @@ -384,6 +384,55 @@ impl SigRef { } } +/// An opaque exception tag. +/// +/// Exception tags are used to denote the identity of an exception for +/// matching by catch-handlers in exception tables. +/// +/// The index space is arbitrary and is given meaning only by the +/// embedder of Cranelift. Cranelift will carry through these tags +/// from exception tables to the handler metadata produced as output +/// (for use by the embedder's unwinder). +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ExceptionTag(u32); +entity_impl!(ExceptionTag, "tag"); + +impl ExceptionTag { + /// Create a new exception tag from its arbitrary index. + /// + /// This method is for use by the parser. + pub fn with_number(n: u32) -> Option { + if n < u32::MAX { + Some(Self(n)) + } else { + None + } + } +} + +/// An opaque reference to an exception table. +/// +/// `ExceptionTable`s are used for describing exception catch handlers on +/// `try_call` and `try_call_indirect` instructions. +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ExceptionTable(u32); +entity_impl!(ExceptionTable, "extable"); + +impl ExceptionTable { + /// Create a new exception table reference from its number. + /// + /// This method is for use by the parser. + pub fn with_number(n: u32) -> Option { + if n < u32::MAX { + Some(Self(n)) + } else { + None + } + } +} + /// An opaque reference to any of the entities defined in this module that can appear in CLIF IR. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] @@ -414,6 +463,8 @@ pub enum AnyEntity { FuncRef(FuncRef), /// A function call signature. SigRef(SigRef), + /// An exception table. + ExceptionTable(ExceptionTable), /// A function's stack limit StackLimit, } @@ -434,6 +485,7 @@ impl fmt::Display for AnyEntity { Self::Constant(r) => r.fmt(f), Self::FuncRef(r) => r.fmt(f), Self::SigRef(r) => r.fmt(f), + Self::ExceptionTable(r) => r.fmt(f), Self::StackLimit => write!(f, "stack_limit"), } } @@ -517,6 +569,12 @@ impl From for AnyEntity { } } +impl From for AnyEntity { + fn from(r: ExceptionTable) -> Self { + Self::ExceptionTable(r) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/cranelift/codegen/src/ir/exception_table.rs b/cranelift/codegen/src/ir/exception_table.rs new file mode 100644 index 000000000000..a5a23c5fde5c --- /dev/null +++ b/cranelift/codegen/src/ir/exception_table.rs @@ -0,0 +1,193 @@ +//! Exception tables: catch handlers on `try_call` instructions. +//! +//! An exception table describes where execution flows after returning +//! from `try_call`. It contains both the "normal" destination -- the +//! block to branch to when the function returns without throwing an +//! exception -- and any "catch" destinations associated with +//! particular exception tags. Each target indicates the arguments to +//! pass to the block that receives control. +//! +//! Like other side-tables (e.g., jump tables), each exception table +//! must be used by only one instruction. Sharing is not permitted +//! because it can complicate transforms (how does one change the +//! table used by only one instruction if others also use it?). +//! +//! In order to allow the `try_call` instruction itself to remain +//! small, the exception table also contains the signature ID of the +//! called function. + +use crate::ir::entities::{ExceptionTag, SigRef}; +use crate::ir::instructions::ValueListPool; +use crate::ir::BlockCall; +use alloc::vec::Vec; +use core::fmt::{self, Display, Formatter}; +use cranelift_entity::packed_option::PackedOption; +#[cfg(feature = "enable-serde")] +use serde_derive::{Deserialize, Serialize}; + +/// Contents of an exception table. +/// +/// The "no exception" target for is stored as the last element of the +/// underlying vector. It can be accessed through the `normal_return` +/// and `normal_return_mut` functions. Exceptional catch clauses may +/// be iterated using the `catches` and `catches_mut` functions. All +/// targets may be iterated over using the `all_targets` and +/// `all_targets_mut` functions. +#[derive(Debug, Clone, PartialEq, Hash)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ExceptionTableData { + /// All BlockCalls packed together. This is necessary because the + /// rest of the compiler expects to be able to grab a slice of + /// branch targets for any branch instruction. The last BlockCall + /// is the normal-return destination, and the rest correspond to + /// the tags in `tags` below. Thus, we have the invariant that + /// `targets.len() == tags.len() + 1`. + targets: Vec, + + /// Tags corresponding to targets other than the first one. + /// + /// A tag value of `None` indicates a catch-all handler. The + /// catch-all handler matches only if no other handler matches, + /// regardless of the order in this vector. + /// + /// `tags[i]` corresponds to `targets[i]`. Note that there will be + /// one more `targets` element than `tags` because the last + /// element in `targets` is the normal-return path. + tags: Vec>, + + /// The signature of the function whose invocation is associated + /// with this handler table. + sig: SigRef, +} + +impl ExceptionTableData { + /// Create new exception-table data. + /// + /// This data represents the destinations upon return from + /// `try_call` or `try_call_indirect` instruction. There are two + /// possibilities: "normal return" (no exception thrown), or an + /// exceptional return corresponding to one of the listed + /// exception tags. + /// + /// The given tags are passed through to the metadata provided + /// alongside the provided function body, and Cranelift itself + /// does not implement an unwinder; thus, the meaning of the tags + /// is ultimately up to the embedder of Cranelift. The tags are + /// wrapped in `Option` to allow encoding a "catch-all" handler. + /// + /// The BlockCalls must have signatures that match the targeted + /// blocks, as usual. These calls are allowed to use + /// `BlockArg::TryCallRet` in the normal-return case, with types + /// corresponding to the signature's return values, and + /// `BlockArg::TryCallExn` in the exceptional-return cases, with + /// types corresponding to native machine words and an arity + /// corresponding to the number of payload values that the calling + /// convention and platform support. (See [`isa::CallConv`] for + /// more details.) + pub fn new( + sig: SigRef, + normal_return: BlockCall, + tags_and_targets: impl IntoIterator, BlockCall)>, + ) -> Self { + let mut targets = vec![]; + let mut tags = vec![]; + for (tag, target) in tags_and_targets { + tags.push(tag.into()); + targets.push(target); + } + targets.push(normal_return); + + ExceptionTableData { targets, tags, sig } + } + + /// Return a value that can display the contents of this exception + /// table. + pub fn display<'a>(&'a self, pool: &'a ValueListPool) -> DisplayExceptionTable<'a> { + DisplayExceptionTable { table: self, pool } + } + + /// Get the default target for the non-exceptional return case. + pub fn normal_return(&self) -> &BlockCall { + self.targets.last().unwrap() + } + + /// Get the default target for the non-exceptional return case. + pub fn normal_return_mut(&mut self) -> &mut BlockCall { + self.targets.last_mut().unwrap() + } + + /// Get the targets for exceptional return cases, together with + /// their tags. + pub fn catches(&self) -> impl Iterator, &BlockCall)> + '_ { + self.tags + .iter() + .map(|tag| tag.expand()) + // Skips the last entry of `targets` (the normal return) + // because `tags` is one element shorter. + .zip(self.targets.iter()) + } + + /// Get the targets for exceptional return cases, together with + /// their tags. + pub fn catches_mut( + &mut self, + ) -> impl Iterator, &mut BlockCall)> + '_ { + self.tags + .iter() + .map(|tag| tag.expand()) + // Skips the last entry of `targets` (the normal return) + // because `tags` is one element shorter. + .zip(self.targets.iter_mut()) + } + + /// Get all branch targets. + pub fn all_branches(&self) -> &[BlockCall] { + &self.targets[..] + } + + /// Get all branch targets. + pub fn all_branches_mut(&mut self) -> &mut [BlockCall] { + &mut self.targets[..] + } + + /// Get the signature of the function called with this exception + /// table. + pub fn signature(&self) -> SigRef { + self.sig + } +} + +/// A wrapper for the context required to display a +/// [ExceptionTableData]. +pub struct DisplayExceptionTable<'a> { + table: &'a ExceptionTableData, + pool: &'a ValueListPool, +} + +impl<'a> Display for DisplayExceptionTable<'a> { + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + write!( + fmt, + "{}, {}, [", + self.table.sig, + self.table.normal_return().display(self.pool) + )?; + let mut first = true; + for (tag, block_call) in self.table.catches() { + if first { + write!(fmt, " ")?; + first = false; + } else { + write!(fmt, ", ")?; + } + if let Some(tag) = tag { + write!(fmt, "{}: {}", tag, block_call.display(self.pool))?; + } else { + write!(fmt, "default: {}", block_call.display(self.pool))?; + } + } + let space = if first { "" } else { " " }; + write!(fmt, "{space}]")?; + Ok(()) + } +} diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index ebf7f08cb85c..ec3e5bb5de34 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -282,7 +282,9 @@ impl FunctionStencil { /// Rewrite the branch destination to `new_dest` if the destination matches `old_dest`. /// Does nothing if called with a non-jump or non-branch instruction. pub fn rewrite_branch_destination(&mut self, inst: Inst, old_dest: Block, new_dest: Block) { - for dest in self.dfg.insts[inst].branch_destination_mut(&mut self.dfg.jump_tables) { + for dest in self.dfg.insts[inst] + .branch_destination_mut(&mut self.dfg.jump_tables, &mut self.dfg.exception_tables) + { if dest.block(&self.dfg.value_lists) == old_dest { dest.set_block(new_dest, &mut self.dfg.value_lists) } @@ -312,7 +314,7 @@ impl FunctionStencil { pub fn block_successors(&self, block: Block) -> impl DoubleEndedIterator + '_ { self.layout.last_inst(block).into_iter().flat_map(|inst| { self.dfg.insts[inst] - .branch_destination(&self.dfg.jump_tables) + .branch_destination(&self.dfg.jump_tables, &self.dfg.exception_tables) .iter() .map(|block| block.block(&self.dfg.value_lists)) }) diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs index 8aa8d7cb0e5c..17f2c6500689 100644 --- a/cranelift/codegen/src/ir/instructions.rs +++ b/cranelift/codegen/src/ir/instructions.rs @@ -21,7 +21,8 @@ use crate::ir::{ self, condcodes::{FloatCC, IntCC}, trapcode::TrapCode, - types, Block, FuncRef, MemFlags, SigRef, StackSlot, Type, Value, + types, Block, ExceptionTable, ExceptionTables, FuncRef, MemFlags, SigRef, StackSlot, Type, + Value, }; /// Some instructions use an external list of argument values because there is not enough space in @@ -34,6 +35,8 @@ pub type ValueListPool = entity::ListPool; /// A pair of a Block and its arguments, stored in a single EntityList internally. /// +/// Block arguments are semantically a `BlockArg`. +/// /// NOTE: We don't expose either value_to_block or block_to_value outside of this module because /// this operation is not generally safe. However, as the two share the same underlying layout, /// they can be stored in the same value pool. @@ -69,10 +72,14 @@ impl BlockCall { } /// Construct a BlockCall with the given block and arguments. - pub fn new(block: Block, args: &[Value], pool: &mut ValueListPool) -> Self { + pub fn new( + block: Block, + args: impl Iterator, + pool: &mut ValueListPool, + ) -> Self { let mut values = ValueList::default(); values.push(Self::block_to_value(block), pool); - values.extend(args.iter().copied(), pool); + values.extend(args.map(|arg| arg.encode_as_value()), pool); Self { values } } @@ -88,18 +95,36 @@ impl BlockCall { } /// Append an argument to the block args. - pub fn append_argument(&mut self, arg: Value, pool: &mut ValueListPool) { - self.values.push(arg, pool); + pub fn append_argument(&mut self, arg: impl Into, pool: &mut ValueListPool) { + self.values.push(arg.into().encode_as_value(), pool); } - /// Return a slice for the arguments of this block. - pub fn args_slice<'a>(&self, pool: &'a ValueListPool) -> &'a [Value] { - &self.values.as_slice(pool)[1..] + /// Return the length of the argument list. + pub fn len(&self, pool: &ValueListPool) -> usize { + self.values.len(pool) - 1 } - /// Return a slice for the arguments of this block. - pub fn args_slice_mut<'a>(&'a mut self, pool: &'a mut ValueListPool) -> &'a mut [Value] { - &mut self.values.as_mut_slice(pool)[1..] + /// Return an iterator over the arguments of this block. + pub fn args<'a>( + &self, + pool: &'a ValueListPool, + ) -> impl ExactSizeIterator + DoubleEndedIterator + use<'a> + { + self.values.as_slice(pool)[1..] + .iter() + .map(|value| BlockArg::decode_from_value(*value)) + } + + /// Traverse the arguments with a closure that can mutate them. + pub fn update_args BlockArg>( + &mut self, + pool: &mut ValueListPool, + mut f: F, + ) { + for raw in self.values.as_mut_slice(pool)[1..].iter_mut() { + let new = f(BlockArg::decode_from_value(*raw)); + *raw = new.encode_as_value(); + } } /// Remove the argument at ix from the argument list. @@ -113,11 +138,17 @@ impl BlockCall { } /// Appends multiple elements to the arguments. - pub fn extend(&mut self, elements: I, pool: &mut ValueListPool) + pub fn extend(&mut self, elements: I, pool: &mut ValueListPool) where - I: IntoIterator, + I: IntoIterator, + T: Into, { - self.values.extend(elements, pool) + self.values.extend( + elements + .into_iter() + .map(|elem| elem.into().encode_as_value()), + pool, + ) } /// Return a value that can display this block call. @@ -144,10 +175,9 @@ pub struct DisplayBlockCall<'a> { impl<'a> Display for DisplayBlockCall<'a> { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "{}", self.block.block(&self.pool))?; - let args = self.block.args_slice(&self.pool); - if !args.is_empty() { + if self.block.len(self.pool) > 0 { write!(f, "(")?; - for (ix, arg) in args.iter().enumerate() { + for (ix, arg) in self.block.args(self.pool).enumerate() { if ix > 0 { write!(f, ", ")?; } @@ -159,6 +189,94 @@ impl<'a> Display for DisplayBlockCall<'a> { } } +/// A `BlockArg` is a sum type of `Value`, `TryCallRet`, and +/// `TryCallExn`. The latter two are values that are generated "on the +/// edge" out of a `try_call` instruction into a successor block. We +/// use special arguments rather than special values for these because +/// they are not definable as SSA values at a certain program point -- +/// only when the `BlockCall` is executed. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum BlockArg { + /// An ordinary value, usable at the branch instruction using this + /// `BlockArg`, whose value is passed as an argument. + Value(Value), + + /// A return value of a `try_call`'s called function. Signatures + /// allow multiple return values, so this carries an index. This + /// may be used only on the normal (non-exceptional) `BlockCall` + /// out of a `try_call` or `try_call_indirect` instruction. + TryCallRet(u32), + + /// An exception payload value of a `try_call`. Some ABIs may + /// allow multiple payload values, so this carries an index. Its + /// type is defined by the ABI of the called function. This may be + /// used only on an exceptional `BlockCall` out of a `try_call` or + /// `try_call_indirect` instruction. + TryCallExn(u32), +} + +impl BlockArg { + /// Encode this block argument as a `Value` for storage in the + /// value pool. Internal to `BlockCall`, must not be used + /// elsewhere to avoid exposing the raw bit encoding. + fn encode_as_value(&self) -> Value { + let (tag, payload) = match *self { + BlockArg::Value(v) => (0, v.as_bits()), + BlockArg::TryCallRet(i) => (1, i), + BlockArg::TryCallExn(i) => (2, i), + }; + assert!(payload < (1 << 30)); + let raw = (tag << 30) | payload; + Value::from_bits(raw) + } + + /// Decode a raw `Value` encoding of this block argument. + fn decode_from_value(v: Value) -> Self { + let raw = v.as_u32(); + let tag = raw >> 30; + let payload = raw & ((1 << 30) - 1); + match tag { + 0 => BlockArg::Value(Value::from_bits(payload)), + 1 => BlockArg::TryCallRet(payload), + 2 => BlockArg::TryCallExn(payload), + _ => unreachable!(), + } + } + + /// Return this argument as a `Value`, if it is one, or `None` + /// otherwise. + pub fn as_value(&self) -> Option { + match *self { + BlockArg::Value(v) => Some(v), + _ => None, + } + } + + /// Update the contained value, if any. + pub fn map_value Value>(&self, mut f: F) -> Self { + match *self { + BlockArg::Value(v) => BlockArg::Value(f(v)), + other => other, + } + } +} + +impl Display for BlockArg { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + BlockArg::Value(v) => write!(f, "{v}"), + BlockArg::TryCallRet(i) => write!(f, "ret{i}"), + BlockArg::TryCallExn(i) => write!(f, "exn{i}"), + } + } +} + +impl From for BlockArg { + fn from(value: Value) -> BlockArg { + BlockArg::Value(value) + } +} + // Include code generated by `cranelift-codegen/meta/src/gen_inst.rs`. This file contains: // // - The `pub enum InstructionFormat` enum with all the instruction formats. @@ -305,11 +423,18 @@ impl InstructionData { /// Get the destinations of this instruction, if it's a branch. /// /// `br_table` returns the empty slice. - pub fn branch_destination<'a>(&'a self, jump_tables: &'a ir::JumpTables) -> &'a [BlockCall] { + pub fn branch_destination<'a>( + &'a self, + jump_tables: &'a ir::JumpTables, + exception_tables: &'a ir::ExceptionTables, + ) -> &'a [BlockCall] { match self { Self::Jump { destination, .. } => std::slice::from_ref(destination), Self::Brif { blocks, .. } => blocks.as_slice(), Self::BranchTable { table, .. } => jump_tables.get(*table).unwrap().all_branches(), + Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => { + exception_tables.get(*exception).unwrap().all_branches() + } _ => { debug_assert!(!self.opcode().is_branch()); &[] @@ -323,6 +448,7 @@ impl InstructionData { pub fn branch_destination_mut<'a>( &'a mut self, jump_tables: &'a mut ir::JumpTables, + exception_tables: &'a mut ir::ExceptionTables, ) -> &'a mut [BlockCall] { match self { Self::Jump { destination, .. } => std::slice::from_mut(destination), @@ -330,6 +456,12 @@ impl InstructionData { Self::BranchTable { table, .. } => { jump_tables.get_mut(*table).unwrap().all_branches_mut() } + Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => { + exception_tables + .get_mut(*exception) + .unwrap() + .all_branches_mut() + } _ => { debug_assert!(!self.opcode().is_branch()); &mut [] @@ -343,16 +475,15 @@ impl InstructionData { &mut self, pool: &mut ValueListPool, jump_tables: &mut ir::JumpTables, + exception_tables: &mut ir::ExceptionTables, mut f: impl FnMut(Value) -> Value, ) { for arg in self.arguments_mut(pool) { *arg = f(*arg); } - for block in self.branch_destination_mut(jump_tables) { - for arg in block.args_slice_mut(pool) { - *arg = f(*arg); - } + for block in self.branch_destination_mut(jump_tables, exception_tables) { + block.update_args(pool, |arg| arg.map_value(|val| f(val))); } } @@ -437,7 +568,11 @@ impl InstructionData { /// Return information about a call instruction. /// /// Any instruction that can call another function reveals its call signature here. - pub fn analyze_call<'a>(&'a self, pool: &'a ValueListPool) -> CallInfo<'a> { + pub fn analyze_call<'a>( + &'a self, + pool: &'a ValueListPool, + exception_tables: &ExceptionTables, + ) -> CallInfo<'a> { match *self { Self::Call { func_ref, ref args, .. @@ -445,6 +580,23 @@ impl InstructionData { Self::CallIndirect { sig_ref, ref args, .. } => CallInfo::Indirect(sig_ref, &args.as_slice(pool)[1..]), + Self::TryCall { + func_ref, + ref args, + exception, + .. + } => { + let exdata = &exception_tables[exception]; + CallInfo::DirectWithSig(func_ref, exdata.signature(), args.as_slice(pool)) + } + Self::TryCallIndirect { + exception, + ref args, + .. + } => { + let exdata = &exception_tables[exception]; + CallInfo::Indirect(exdata.signature(), args.as_slice(pool)) + } Self::Ternary { opcode: Opcode::StackSwitch, .. @@ -495,6 +647,16 @@ impl InstructionData { _ => {} } } + + /// Get the exception table, if any, associated with this instruction. + pub fn exception_table(&self) -> Option { + match self { + Self::TryCall { exception, .. } | Self::TryCallIndirect { exception, .. } => { + Some(*exception) + } + _ => None, + } + } } /// Information about call instructions. @@ -508,6 +670,11 @@ pub enum CallInfo<'a> { /// This is an indirect call with the specified signature. See `DataFlowGraph.signatures`. Indirect(SigRef, &'a [Value]), + + /// This is a direct call to an external function declared in the + /// preamble, but the signature is also known by other means: + /// e.g., from an exception table entry. + DirectWithSig(FuncRef, SigRef, &'a [Value]), } /// Value type constraints for a given opcode. diff --git a/cranelift/codegen/src/ir/jumptable.rs b/cranelift/codegen/src/ir/jumptable.rs index 8e1e15c7d621..bca05c21b09f 100644 --- a/cranelift/codegen/src/ir/jumptable.rs +++ b/cranelift/codegen/src/ir/jumptable.rs @@ -113,13 +113,14 @@ mod tests { use super::JumpTableData; use crate::entity::EntityRef; use crate::ir::instructions::ValueListPool; - use crate::ir::{Block, BlockCall, Value}; + use crate::ir::{Block, BlockArg, BlockCall, Value}; + use alloc::vec::Vec; use std::string::ToString; #[test] fn empty() { let mut pool = ValueListPool::default(); - let def = BlockCall::new(Block::new(0), &[], &mut pool); + let def = BlockCall::new(Block::new(0), core::iter::empty(), &mut pool); let jt = JumpTableData::new(def, &[]); @@ -145,10 +146,10 @@ mod tests { let e1 = Block::new(1); let e2 = Block::new(2); - let def = BlockCall::new(e0, &[], &mut pool); - let b1 = BlockCall::new(e1, &[v0], &mut pool); - let b2 = BlockCall::new(e2, &[], &mut pool); - let b3 = BlockCall::new(e1, &[v1], &mut pool); + let def = BlockCall::new(e0, core::iter::empty(), &mut pool); + let b1 = BlockCall::new(e1, core::iter::once(v0.into()), &mut pool); + let b2 = BlockCall::new(e2, core::iter::empty(), &mut pool); + let b3 = BlockCall::new(e1, core::iter::once(v1.into()), &mut pool); let jt = JumpTableData::new(def, &[b1, b2, b3]); @@ -161,8 +162,14 @@ mod tests { assert_eq!(jt.all_branches(), [def, b1, b2, b3]); assert_eq!(jt.as_slice(), [b1, b2, b3]); - assert_eq!(jt.as_slice()[0].args_slice(&pool), [v0]); - assert_eq!(jt.as_slice()[1].args_slice(&pool), []); - assert_eq!(jt.as_slice()[2].args_slice(&pool), [v1]); + assert_eq!( + jt.as_slice()[0].args(&pool).collect::>(), + [BlockArg::Value(v0)] + ); + assert_eq!(jt.as_slice()[1].args(&pool).collect::>(), []); + assert_eq!( + jt.as_slice()[2].args(&pool).collect::>(), + [BlockArg::Value(v1)] + ); } } diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index e6f082d70a8d..ee052a3c92af 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -7,6 +7,7 @@ pub mod constant; pub mod dfg; pub mod dynamic_type; pub mod entities; +mod exception_table; mod extfunc; mod extname; pub mod function; @@ -38,9 +39,11 @@ pub use crate::ir::constant::{ConstantData, ConstantPool}; pub use crate::ir::dfg::{BlockData, DataFlowGraph, ValueDef}; pub use crate::ir::dynamic_type::{dynamic_to_fixed, DynamicTypeData, DynamicTypes}; pub use crate::ir::entities::{ - Block, Constant, DynamicStackSlot, DynamicType, FuncRef, GlobalValue, Immediate, Inst, - JumpTable, MemoryType, SigRef, StackSlot, UserExternalNameRef, Value, + Block, Constant, DynamicStackSlot, DynamicType, ExceptionTable, ExceptionTag, FuncRef, + GlobalValue, Immediate, Inst, JumpTable, MemoryType, SigRef, StackSlot, UserExternalNameRef, + Value, }; +pub use crate::ir::exception_table::ExceptionTableData; pub use crate::ir::extfunc::{ AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature, }; @@ -48,7 +51,7 @@ pub use crate::ir::extname::{ExternalName, UserExternalName, UserFuncName}; pub use crate::ir::function::Function; pub use crate::ir::globalvalue::GlobalValueData; pub use crate::ir::instructions::{ - BlockCall, InstructionData, Opcode, ValueList, ValueListPool, VariableArgs, + BlockArg, BlockCall, InstructionData, Opcode, ValueList, ValueListPool, VariableArgs, }; pub use crate::ir::jumptable::JumpTableData; pub use crate::ir::known_symbol::KnownSymbol; @@ -72,6 +75,9 @@ use crate::entity::{entity_impl, PrimaryMap, SecondaryMap}; /// Map of jump tables. pub type JumpTables = PrimaryMap; +/// Map of exception tables. +pub type ExceptionTables = PrimaryMap; + /// Source locations for instructions. pub(crate) type SourceLocs = SecondaryMap; diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 1c5eb7b96b70..178e44cd0fc9 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -1090,10 +1090,11 @@ impl ABIMachineSpec for AArch64MachineDeps { } ], defs: smallvec![], - clobbers: Self::get_regs_clobbered_by_call(call_conv), + clobbers: Self::get_regs_clobbered_by_call(call_conv, false), caller_conv: call_conv, callee_conv: call_conv, callee_pop_size: 0, + try_call_info: None, }), }); insts @@ -1123,9 +1124,10 @@ impl ABIMachineSpec for AArch64MachineDeps { } } - fn get_regs_clobbered_by_call(call_conv: isa::CallConv) -> PRegSet { + fn get_regs_clobbered_by_call(call_conv: isa::CallConv, is_exception: bool) -> PRegSet { match call_conv { isa::CallConv::Winch => WINCH_CLOBBERS, + _ if is_exception => ALL_CLOBBERS, _ => DEFAULT_AAPCS_CLOBBERS, } } @@ -1200,6 +1202,14 @@ impl ABIMachineSpec for AArch64MachineDeps { // retval. regs::writable_xreg(9) } + + fn exception_payload_regs(call_conv: isa::CallConv) -> &'static [Reg] { + const PAYLOAD_REGS: &'static [Reg] = &[regs::xreg(0), regs::xreg(1)]; + match call_conv { + isa::CallConv::SystemV | isa::CallConv::Tail => PAYLOAD_REGS, + _ => &[], + } + } } impl AArch64MachineDeps { @@ -1519,8 +1529,77 @@ const fn winch_clobbers() -> PRegSet { .with(vreg_preg(31)) } +const fn all_clobbers() -> PRegSet { + PRegSet::empty() + // integer registers: x0 to x28 inclusive. (x29 is FP, x30 is + // LR, x31 is SP/ZR.) + .with(xreg_preg(0)) + .with(xreg_preg(1)) + .with(xreg_preg(2)) + .with(xreg_preg(3)) + .with(xreg_preg(4)) + .with(xreg_preg(5)) + .with(xreg_preg(6)) + .with(xreg_preg(7)) + .with(xreg_preg(8)) + .with(xreg_preg(9)) + .with(xreg_preg(10)) + .with(xreg_preg(11)) + .with(xreg_preg(12)) + .with(xreg_preg(13)) + .with(xreg_preg(14)) + .with(xreg_preg(15)) + .with(xreg_preg(16)) + .with(xreg_preg(17)) + .with(xreg_preg(18)) + .with(xreg_preg(19)) + .with(xreg_preg(20)) + .with(xreg_preg(21)) + .with(xreg_preg(22)) + .with(xreg_preg(23)) + .with(xreg_preg(24)) + .with(xreg_preg(25)) + .with(xreg_preg(26)) + .with(xreg_preg(27)) + .with(xreg_preg(28)) + // vector registers: v0 to v31 inclusive. + .with(vreg_preg(0)) + .with(vreg_preg(1)) + .with(vreg_preg(2)) + .with(vreg_preg(3)) + .with(vreg_preg(4)) + .with(vreg_preg(5)) + .with(vreg_preg(6)) + .with(vreg_preg(7)) + .with(vreg_preg(8)) + .with(vreg_preg(9)) + .with(vreg_preg(10)) + .with(vreg_preg(11)) + .with(vreg_preg(12)) + .with(vreg_preg(13)) + .with(vreg_preg(14)) + .with(vreg_preg(15)) + .with(vreg_preg(16)) + .with(vreg_preg(17)) + .with(vreg_preg(18)) + .with(vreg_preg(19)) + .with(vreg_preg(20)) + .with(vreg_preg(21)) + .with(vreg_preg(22)) + .with(vreg_preg(23)) + .with(vreg_preg(24)) + .with(vreg_preg(25)) + .with(vreg_preg(26)) + .with(vreg_preg(27)) + .with(vreg_preg(28)) + .with(vreg_preg(29)) + .with(vreg_preg(30)) + .with(vreg_preg(31)) +} + const DEFAULT_AAPCS_CLOBBERS: PRegSet = default_aapcs_clobbers(); const WINCH_CLOBBERS: PRegSet = winch_clobbers(); +const ALL_CLOBBERS: PRegSet = all_clobbers(); fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { fn preg(r: Reg) -> PReg { diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 7b82b1ce791c..b81124aab612 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -4414,6 +4414,12 @@ (decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) (extern constructor gen_call_indirect gen_call_indirect) +(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call gen_try_call) + +(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call_indirect gen_try_call_indirect) + ;; Helpers for pinned register manipulation. (decl write_pinned_reg (Reg) SideEffectNoResult) diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 0d6e11e32f80..c9f84066da8d 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -2944,6 +2944,14 @@ impl MachInstEmit for Inst { } sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + if info.callee_pop_size > 0 { let callee_pop_size = i32::try_from(info.callee_pop_size).expect("callee popped more than 2GB"); @@ -2959,6 +2967,15 @@ impl MachInstEmit for Inst { |needed_space| Some(Inst::EmitIsland { needed_space }), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = Inst::Jump { + dest: BranchTarget::Label(try_call.continuation), + }; + jmp.emit(sink, emit_info, state); + } + // We produce an island above if needed, so disable // the worst-case-size check in this case. start_off = sink.cur_offset(); @@ -2974,6 +2991,14 @@ impl MachInstEmit for Inst { } sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + if info.callee_pop_size > 0 { let callee_pop_size = i32::try_from(info.callee_pop_size).expect("callee popped more than 2GB"); @@ -2989,6 +3014,15 @@ impl MachInstEmit for Inst { |needed_space| Some(Inst::EmitIsland { needed_space }), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = Inst::Jump { + dest: BranchTarget::Label(try_call.continuation), + }; + jmp.emit(sink, emit_info, state); + } + // We produce an island above if needed, so disable // the worst-case-size check in this case. start_off = sink.cur_offset(); diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs index 84dae78933f5..f8abf42d0cc8 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs @@ -932,7 +932,7 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { Inst::MachOTlsGetAddr { rd, .. } => { collector.reg_fixed_def(rd, regs::xreg(0)); let mut clobbers = - AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::AppleAarch64); + AArch64MachineDeps::get_regs_clobbered_by_call(CallConv::AppleAarch64, false); clobbers.remove(regs::xreg_preg(0)); collector.reg_clobbers(clobbers); } @@ -979,6 +979,8 @@ impl MachInst for Inst { fn is_included_in_clobbers(&self) -> bool { let (caller, callee) = match self { Inst::Args { .. } => return false, + Inst::Call { info } if info.try_call_info.is_some() => return true, + Inst::CallInd { info } if info.try_call_info.is_some() => return true, Inst::Call { info } => (info.caller_conv, info.callee_conv), Inst::CallInd { info } => (info.caller_conv, info.callee_conv), _ => return true, @@ -995,8 +997,8 @@ impl MachInst for Inst { // // See the note in [crate::isa::aarch64::abi::is_caller_save_reg] for // more information on this ABI-implementation hack. - let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller); - let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee); + let caller_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(caller, false); + let callee_clobbers = AArch64MachineDeps::get_regs_clobbered_by_call(callee, false); let mut all_clobbers = caller_clobbers; all_clobbers.union_from(callee_clobbers); @@ -1021,11 +1023,13 @@ impl MachInst for Inst { match self { &Inst::Rets { .. } => MachTerminator::Ret, &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall, - &Inst::Jump { .. } => MachTerminator::Uncond, - &Inst::CondBr { .. } => MachTerminator::Cond, - &Inst::TestBitAndBranch { .. } => MachTerminator::Cond, - &Inst::IndirectBr { .. } => MachTerminator::Indirect, - &Inst::JTSequence { .. } => MachTerminator::Indirect, + &Inst::Jump { .. } => MachTerminator::Branch, + &Inst::CondBr { .. } => MachTerminator::Branch, + &Inst::TestBitAndBranch { .. } => MachTerminator::Branch, + &Inst::IndirectBr { .. } => MachTerminator::Branch, + &Inst::JTSequence { .. } => MachTerminator::Branch, + &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, + &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, _ => MachTerminator::None, } } @@ -1200,6 +1204,16 @@ fn mem_finalize_for_show(mem: &AMode, access_ty: Type, state: &EmitState) -> (St (mem_str, mem) } +fn pretty_print_try_call(info: &TryCallInfo) -> String { + let dests = info + .exception_dests + .iter() + .map(|(tag, label)| format!("{tag:?}: {label:?}")) + .collect::>() + .join(", "); + format!("; b {:?}; catch [{dests}]", info.continuation) +} + impl Inst { fn print_with_state(&self, state: &mut EmitState) -> String { fn op_name(alu_op: ALUOp) -> &'static str { @@ -2565,10 +2579,22 @@ impl Inst { format!("{op} {rd}, {rn}") } } - &Inst::Call { .. } => format!("bl 0"), + &Inst::Call { ref info } => { + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("bl 0{try_call}") + } &Inst::CallInd { ref info } => { let rn = pretty_print_reg(info.dest); - format!("blr {rn}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("blr {rn}{try_call}") } &Inst::ReturnCall { ref info } => { let mut s = format!( diff --git a/cranelift/codegen/src/isa/aarch64/inst/regs.rs b/cranelift/codegen/src/isa/aarch64/inst/regs.rs index fd1abb7fffb4..f5a6eb526daf 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/regs.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/regs.rs @@ -20,8 +20,8 @@ pub const PINNED_REG: u8 = 21; /// Get a reference to an X-register (integer register). Do not use /// this for xsp / xzr; we have two special registers for those. -pub fn xreg(num: u8) -> Reg { - Reg::from(xreg_preg(num)) +pub const fn xreg(num: u8) -> Reg { + Reg::from_real_reg(xreg_preg(num)) } /// Get the given X-register as a PReg. diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 3c02482118ab..65e0d20e6d2c 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -2511,6 +2511,13 @@ (rule (lower (call_indirect sig_ref val inputs)) (gen_call_indirect sig_ref val inputs)) +(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets) + (gen_try_call sig_ref extname dist et inputs targets)) + +(rule (lower_branch (try_call_indirect val inputs et) targets) + (if-let (exception_sig sig_ref) et) + (gen_try_call_indirect sig_ref val et inputs targets)) + ;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; N.B.: the Ret itself is generated by the ABI. diff --git a/cranelift/codegen/src/isa/call_conv.rs b/cranelift/codegen/src/isa/call_conv.rs index cb21d67be3c5..cb869c7ca4f9 100644 --- a/cranelift/codegen/src/isa/call_conv.rs +++ b/cranelift/codegen/src/isa/call_conv.rs @@ -1,3 +1,5 @@ +use crate::ir::types; +use crate::ir::Type; use crate::settings::{self, LibcallCallConv}; use core::fmt; use core::str; @@ -14,7 +16,16 @@ pub enum CallConv { Fast, /// Smallest caller code size, not ABI-stable. Cold, - /// Supports tail calls, not ABI-stable. + /// Supports tail calls, not ABI-stable except for exception + /// payload registers. + /// + /// On exception resume, a caller to a `tail`-convention function + /// assumes that the exception payload values are in the following + /// registers (per platform): + /// - x86-64: rax, rdx + /// - aarch64: x0, x1 + /// - riscv64: a0, a1 + /// - pulley{32,64}: x0, x1 // // Currently, this is basically sys-v except that callees pop stack // arguments, rather than callers. Expected to change even more in the @@ -69,6 +80,26 @@ impl CallConv { _ => false, } } + + /// Does this calling convention support exceptions? + pub fn supports_exceptions(&self) -> bool { + match self { + CallConv::Tail => true, + _ => false, + } + } + + /// What types do the exception payload value(s) have? + pub fn exception_payload_types(&self, pointer_ty: Type) -> &[Type] { + match self { + CallConv::Tail => match pointer_ty { + types::I32 => &[types::I32, types::I32], + types::I64 => &[types::I64, types::I64], + _ => unreachable!(), + }, + _ => &[], + } + } } impl fmt::Display for CallConv { diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs index 0f4a4efbe296..ab09eeec4692 100644 --- a/cranelift/codegen/src/isa/pulley_shared/abi.rs +++ b/cranelift/codegen/src/isa/pulley_shared/abi.rs @@ -532,8 +532,15 @@ where MACHINE_ENV.get_or_init(create_reg_environment) } - fn get_regs_clobbered_by_call(_call_conv_of_callee: isa::CallConv) -> PRegSet { - DEFAULT_CLOBBERS + fn get_regs_clobbered_by_call( + _call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet { + if is_exception { + ALL_CLOBBERS + } else { + DEFAULT_CLOBBERS + } } fn compute_frame_layout( @@ -600,6 +607,14 @@ where // retval. Writable::from_reg(regs::x_reg(15)) } + + fn exception_payload_regs(_call_conv: isa::CallConv) -> &'static [Reg] { + const PAYLOAD_REGS: &'static [Reg] = &[ + Reg::from_real_reg(regs::px_reg(0)), + Reg::from_real_reg(regs::px_reg(1)), + ]; + PAYLOAD_REGS + } } /// Different styles of management of fp/lr and clobbered registers. @@ -934,6 +949,104 @@ const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() .with(pv_reg(30)) .with(pv_reg(31)); +const ALL_CLOBBERS: PRegSet = PRegSet::empty() + .with(px_reg(0)) + .with(px_reg(1)) + .with(px_reg(2)) + .with(px_reg(3)) + .with(px_reg(4)) + .with(px_reg(5)) + .with(px_reg(6)) + .with(px_reg(7)) + .with(px_reg(8)) + .with(px_reg(9)) + .with(px_reg(10)) + .with(px_reg(11)) + .with(px_reg(12)) + .with(px_reg(13)) + .with(px_reg(14)) + .with(px_reg(15)) + .with(px_reg(16)) + .with(px_reg(17)) + .with(px_reg(18)) + .with(px_reg(19)) + .with(px_reg(20)) + .with(px_reg(21)) + .with(px_reg(22)) + .with(px_reg(23)) + .with(px_reg(24)) + .with(px_reg(25)) + .with(px_reg(26)) + .with(px_reg(27)) + .with(px_reg(28)) + .with(px_reg(29)) + .with(px_reg(30)) + .with(px_reg(31)) + .with(pf_reg(0)) + .with(pf_reg(1)) + .with(pf_reg(2)) + .with(pf_reg(3)) + .with(pf_reg(4)) + .with(pf_reg(5)) + .with(pf_reg(6)) + .with(pf_reg(7)) + .with(pf_reg(8)) + .with(pf_reg(9)) + .with(pf_reg(10)) + .with(pf_reg(11)) + .with(pf_reg(12)) + .with(pf_reg(13)) + .with(pf_reg(14)) + .with(pf_reg(15)) + .with(pf_reg(16)) + .with(pf_reg(17)) + .with(pf_reg(18)) + .with(pf_reg(19)) + .with(pf_reg(20)) + .with(pf_reg(21)) + .with(pf_reg(22)) + .with(pf_reg(23)) + .with(pf_reg(24)) + .with(pf_reg(25)) + .with(pf_reg(26)) + .with(pf_reg(27)) + .with(pf_reg(28)) + .with(pf_reg(29)) + .with(pf_reg(30)) + .with(pf_reg(31)) + .with(pv_reg(0)) + .with(pv_reg(1)) + .with(pv_reg(2)) + .with(pv_reg(3)) + .with(pv_reg(4)) + .with(pv_reg(5)) + .with(pv_reg(6)) + .with(pv_reg(7)) + .with(pv_reg(8)) + .with(pv_reg(9)) + .with(pv_reg(10)) + .with(pv_reg(11)) + .with(pv_reg(12)) + .with(pv_reg(13)) + .with(pv_reg(14)) + .with(pv_reg(15)) + .with(pv_reg(16)) + .with(pv_reg(17)) + .with(pv_reg(18)) + .with(pv_reg(19)) + .with(pv_reg(20)) + .with(pv_reg(21)) + .with(pv_reg(22)) + .with(pv_reg(23)) + .with(pv_reg(24)) + .with(pv_reg(25)) + .with(pv_reg(26)) + .with(pv_reg(27)) + .with(pv_reg(28)) + .with(pv_reg(29)) + .with(pv_reg(30)) + .with(pv_reg(31)); + fn create_reg_environment() -> MachineEnv { // Prefer caller-saved registers over callee-saved registers, because that // way we don't need to emit code to save and restore them if we don't diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle index bdca3c8d978f..2387d7c0ea35 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst.isle +++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle @@ -714,6 +714,12 @@ (decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) (extern constructor gen_call_indirect gen_call_indirect) +(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call gen_try_call) + +(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call_indirect gen_try_call_indirect) + ;;;; Helpers for Sign extension ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Sign extend a `Value` to at least 32-bit diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index 1a5095e6fe21..d0b741467758 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -184,6 +184,14 @@ fn pulley_emit

( } sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + let adjust = -i32::try_from(info.callee_pop_size).unwrap(); for i in PulleyMachineDeps::

::gen_sp_reg_adjust(adjust) { >::from(i).emit(sink, emit_info, state); @@ -196,6 +204,15 @@ fn pulley_emit

( |space_needed| Some(>::from(Inst::EmitIsland { space_needed })), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = InstAndKind::

::from(Inst::Jump { + label: try_call.continuation, + }); + jmp.emit(sink, emit_info, state); + } + // We produce an island above if needed, so disable // the worst-case-size check in this case. *start_offset = sink.cur_offset(); @@ -211,6 +228,14 @@ fn pulley_emit

( sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + let adjust = -i32::try_from(info.callee_pop_size).unwrap(); for i in PulleyMachineDeps::

::gen_sp_reg_adjust(adjust) { >::from(i).emit(sink, emit_info, state); @@ -223,6 +248,15 @@ fn pulley_emit

( |space_needed| Some(>::from(Inst::EmitIsland { space_needed })), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = InstAndKind::

::from(Inst::Jump { + label: try_call.continuation, + }); + jmp.emit(sink, emit_info, state); + } + // We produce an island above if needed, so disable // the worst-case-size check in this case. *start_offset = sink.cur_offset(); @@ -541,10 +575,12 @@ fn pulley_emit

( } Inst::EmitIsland { space_needed } => { - let label = sink.get_label(); - >::from(Inst::Jump { label }).emit(sink, emit_info, state); - sink.emit_island(space_needed + 8, &mut state.ctrl_plane); - sink.bind_label(label, &mut state.ctrl_plane); + if sink.island_needed(*space_needed) { + let label = sink.get_label(); + >::from(Inst::Jump { label }).emit(sink, emit_info, state); + sink.emit_island(space_needed + 8, &mut state.ctrl_plane); + sink.bind_label(label, &mut state.ctrl_plane); + } } } } diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs index 1956b2367867..49d3180c5687 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs @@ -10,6 +10,7 @@ use crate::isa::FunctionAlignment; use crate::{machinst::*, trace}; use crate::{settings, CodegenError, CodegenResult}; use alloc::string::{String, ToString}; +use alloc::vec::Vec; use regalloc2::RegClass; use smallvec::SmallVec; @@ -446,15 +447,17 @@ where } fn is_term(&self) -> MachTerminator { - match self.inst { + match &self.inst { Inst::Raw { raw: RawInst::Ret { .. }, } | Inst::Rets { .. } => MachTerminator::Ret, - Inst::Jump { .. } => MachTerminator::Uncond, - Inst::BrIf { .. } => MachTerminator::Cond, - Inst::BrTable { .. } => MachTerminator::Indirect, - Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => MachTerminator::Indirect, + Inst::Jump { .. } => MachTerminator::Branch, + Inst::BrIf { .. } => MachTerminator::Branch, + Inst::BrTable { .. } => MachTerminator::Branch, + Inst::ReturnCall { .. } | Inst::ReturnIndirectCall { .. } => MachTerminator::RetCall, + Inst::Call { info } if info.try_call_info.is_some() => MachTerminator::Branch, + Inst::IndirectCall { info } if info.try_call_info.is_some() => MachTerminator::Branch, _ => MachTerminator::None, } } @@ -584,6 +587,16 @@ pub fn reg_name(reg: Reg) -> String { } } +fn pretty_print_try_call(info: &TryCallInfo) -> String { + let dests = info + .exception_dests + .iter() + .map(|(tag, label)| format!("{tag:?}: {label:?}")) + .collect::>() + .join(", "); + format!("; jump {:?}; catch [{dests}]", info.continuation) +} + impl Inst { fn print_with_state

(&self, _state: &mut EmitState

) -> String where @@ -636,12 +649,22 @@ impl Inst { } Inst::Call { info } => { - format!("call {info:?}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("call {info:?}{try_call}") } Inst::IndirectCall { info } => { let callee = format_reg(*info.dest); - format!("indirect_call {callee}, {info:?}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("indirect_call {callee}, {info:?}{try_call}") } Inst::ReturnCall { info } => { diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle index 8371f9c60b91..7e4a421c10bb 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower.isle +++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle @@ -146,6 +146,13 @@ (rule (lower (call_indirect sig_ref val inputs)) (gen_call_indirect sig_ref val inputs)) +(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets) + (gen_try_call sig_ref extname dist et inputs targets)) + +(rule (lower_branch (try_call_indirect val inputs et) targets) + (if-let (exception_sig sig_ref) et) + (gen_try_call_indirect sig_ref val et inputs targets)) + ;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (return_call (func_ref_data sig_ref extname dist) args)) diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs index 5364b78dd0d2..5df160884887 100644 --- a/cranelift/codegen/src/isa/riscv64/abi.rs +++ b/cranelift/codegen/src/isa/riscv64/abi.rs @@ -610,10 +610,11 @@ impl ABIMachineSpec for Riscv64MachineDeps { } ], defs: smallvec![], - clobbers: Self::get_regs_clobbered_by_call(call_conv), + clobbers: Self::get_regs_clobbered_by_call(call_conv, false), caller_conv: call_conv, callee_conv: call_conv, callee_pop_size: 0, + try_call_info: None, }), }); insts @@ -637,8 +638,15 @@ impl ABIMachineSpec for Riscv64MachineDeps { MACHINE_ENV.get_or_init(create_reg_environment) } - fn get_regs_clobbered_by_call(_call_conv_of_callee: isa::CallConv) -> PRegSet { - DEFAULT_CLOBBERS + fn get_regs_clobbered_by_call( + _call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet { + if is_exception { + ALL_CLOBBERS + } else { + DEFAULT_CLOBBERS + } } fn compute_frame_layout( @@ -727,6 +735,14 @@ impl ABIMachineSpec for Riscv64MachineDeps { // retval. Writable::from_reg(regs::x_reg(12)) } + + fn exception_payload_regs(call_conv: isa::CallConv) -> &'static [Reg] { + const PAYLOAD_REGS: &'static [Reg] = &[regs::a0(), regs::a1()]; + match call_conv { + isa::CallConv::SystemV | isa::CallConv::Tail => PAYLOAD_REGS, + _ => &[], + } + } } impl Riscv64ABICallSite { @@ -902,6 +918,104 @@ const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() .with(pv_reg(30)) .with(pv_reg(31)); +const ALL_CLOBBERS: PRegSet = PRegSet::empty() + // Specials: x0 is the zero register; x1 is the return address; x2 is SP. + .with(px_reg(3)) + .with(px_reg(4)) + .with(px_reg(5)) + .with(px_reg(6)) + .with(px_reg(7)) + .with(px_reg(8)) + .with(px_reg(9)) + .with(px_reg(10)) + .with(px_reg(11)) + .with(px_reg(12)) + .with(px_reg(13)) + .with(px_reg(14)) + .with(px_reg(15)) + .with(px_reg(16)) + .with(px_reg(17)) + .with(px_reg(18)) + .with(px_reg(19)) + .with(px_reg(20)) + .with(px_reg(21)) + .with(px_reg(22)) + .with(px_reg(23)) + .with(px_reg(24)) + .with(px_reg(25)) + .with(px_reg(26)) + .with(px_reg(27)) + .with(px_reg(28)) + .with(px_reg(29)) + .with(px_reg(30)) + .with(px_reg(31)) + // F Regs + .with(pf_reg(0)) + .with(pf_reg(1)) + .with(pf_reg(2)) + .with(pf_reg(3)) + .with(pf_reg(4)) + .with(pf_reg(5)) + .with(pf_reg(6)) + .with(pf_reg(7)) + .with(pf_reg(8)) + .with(pf_reg(9)) + .with(pf_reg(10)) + .with(pf_reg(11)) + .with(pf_reg(12)) + .with(pf_reg(13)) + .with(pf_reg(14)) + .with(pf_reg(15)) + .with(pf_reg(16)) + .with(pf_reg(17)) + .with(pf_reg(18)) + .with(pf_reg(19)) + .with(pf_reg(20)) + .with(pf_reg(21)) + .with(pf_reg(22)) + .with(pf_reg(23)) + .with(pf_reg(24)) + .with(pf_reg(25)) + .with(pf_reg(26)) + .with(pf_reg(27)) + .with(pf_reg(28)) + .with(pf_reg(29)) + .with(pf_reg(30)) + .with(pf_reg(31)) + // V Regs + .with(pv_reg(0)) + .with(pv_reg(1)) + .with(pv_reg(2)) + .with(pv_reg(3)) + .with(pv_reg(4)) + .with(pv_reg(5)) + .with(pv_reg(6)) + .with(pv_reg(7)) + .with(pv_reg(8)) + .with(pv_reg(9)) + .with(pv_reg(10)) + .with(pv_reg(11)) + .with(pv_reg(12)) + .with(pv_reg(13)) + .with(pv_reg(14)) + .with(pv_reg(15)) + .with(pv_reg(16)) + .with(pv_reg(17)) + .with(pv_reg(18)) + .with(pv_reg(19)) + .with(pv_reg(20)) + .with(pv_reg(21)) + .with(pv_reg(22)) + .with(pv_reg(23)) + .with(pv_reg(24)) + .with(pv_reg(25)) + .with(pv_reg(26)) + .with(pv_reg(27)) + .with(pv_reg(28)) + .with(pv_reg(29)) + .with(pv_reg(30)) + .with(pv_reg(31)); + fn create_reg_environment() -> MachineEnv { // Some C Extension instructions can only use a subset of the registers. // x8 - x15, f8 - f15, v8 - v15 so we should prefer to use those since diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle index 7d6b5c96adac..e9004fdcd2d1 100644 --- a/cranelift/codegen/src/isa/riscv64/inst.isle +++ b/cranelift/codegen/src/isa/riscv64/inst.isle @@ -2913,6 +2913,13 @@ (rule (lower_branch (br_table index _) targets) (lower_br_table index targets)) +(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets) + (gen_try_call sig_ref extname dist et inputs targets)) + +(rule (lower_branch (try_call_indirect val inputs et) targets) + (if-let (exception_sig sig_ref) et) + (gen_try_call_indirect sig_ref val et inputs targets)) + (decl load_ra () Reg) (extern constructor load_ra load_ra) @@ -3017,6 +3024,12 @@ (decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) (extern constructor gen_call_indirect gen_call_indirect) +(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call gen_try_call) + +(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call_indirect gen_try_call_indirect) + ;;; this is trying to imitate aarch64 `madd` instruction. (decl madd (XReg XReg XReg) XReg) (rule diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs index 36e72db5c2e8..a2883f0fc711 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs @@ -1128,6 +1128,14 @@ impl Inst { sink.push_user_stack_map(state, offset, s); } + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap(); if callee_pop_size > 0 { for inst in Riscv64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) { @@ -1142,6 +1150,15 @@ impl Inst { |needed_space| Some(Inst::EmitIsland { needed_space }), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = Inst::Jal { + label: try_call.continuation, + }; + jmp.emit(sink, emit_info, state); + } + *start_off = sink.cur_offset(); } &Inst::CallInd { ref info } => { @@ -1159,6 +1176,14 @@ impl Inst { sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap(); if callee_pop_size > 0 { for inst in Riscv64MachineDeps::gen_sp_reg_adjust(-callee_pop_size) { @@ -1173,6 +1198,15 @@ impl Inst { |needed_space| Some(Inst::EmitIsland { needed_space }), ); + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = info.try_call_info.as_ref() { + let jmp = Inst::Jal { + label: try_call.continuation, + }; + jmp.emit(sink, emit_info, state); + } + *start_off = sink.cur_offset(); } @@ -2598,10 +2632,12 @@ impl Inst { } Inst::EmitIsland { needed_space } => { - let jump_around_label = sink.get_label(); - Inst::gen_jump(jump_around_label).emit(sink, emit_info, state); - sink.emit_island(needed_space + 4, &mut state.ctrl_plane); - sink.bind_label(jump_around_label, &mut state.ctrl_plane); + if sink.island_needed(*needed_space) { + let jump_around_label = sink.get_label(); + Inst::gen_jump(jump_around_label).emit(sink, emit_info, state); + sink.emit_island(needed_space + 4, &mut state.ctrl_plane); + sink.bind_label(jump_around_label, &mut state.ctrl_plane); + } } } } diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs index d75e0c1c827c..05e38bbf7a91 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs @@ -390,7 +390,8 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { Inst::ElfTlsGetAddr { rd, .. } => { // x10 is a0 which is both the first argument and the first return value. collector.reg_fixed_def(rd, a0()); - let mut clobbers = Riscv64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV); + let mut clobbers = + Riscv64MachineDeps::get_regs_clobbered_by_call(CallConv::SystemV, false); clobbers.remove(px_reg(10)); collector.reg_clobbers(clobbers); } @@ -751,12 +752,14 @@ impl MachInst for Inst { fn is_term(&self) -> MachTerminator { match self { - &Inst::Jal { .. } => MachTerminator::Uncond, - &Inst::CondBr { .. } => MachTerminator::Cond, - &Inst::Jalr { .. } => MachTerminator::Uncond, + &Inst::Jal { .. } => MachTerminator::Branch, + &Inst::CondBr { .. } => MachTerminator::Branch, + &Inst::Jalr { .. } => MachTerminator::Branch, &Inst::Rets { .. } => MachTerminator::Ret, - &Inst::BrTable { .. } => MachTerminator::Indirect, + &Inst::BrTable { .. } => MachTerminator::Branch, &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall, + &Inst::Call { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, + &Inst::CallInd { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, _ => MachTerminator::None, } } @@ -874,6 +877,16 @@ pub fn reg_name(reg: Reg) -> String { } } +fn pretty_print_try_call(info: &TryCallInfo) -> String { + let dests = info + .exception_dests + .iter() + .map(|(tag, label)| format!("{tag:?}: {label:?}")) + .collect::>() + .join(", "); + format!("; j {:?}; catch [{dests}]", info.continuation) +} + impl Inst { fn print_with_state(&self, _state: &mut EmitState) -> String { let format_reg = |reg: Reg| -> String { reg_name(reg) }; @@ -1305,10 +1318,22 @@ impl Inst { format!("slli {rd},{rn},{shift_bits}; {op} {rd},{rd},{shift_bits}") }; } - &MInst::Call { ref info } => format!("call {}", info.dest.display(None)), + &MInst::Call { ref info } => { + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("call {}{try_call}", info.dest.display(None)) + } &MInst::CallInd { ref info } => { let rd = format_reg(info.dest); - format!("callind {rd}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("callind {rd}{try_call}") } &MInst::ReturnCall { ref info } => { let mut s = format!( diff --git a/cranelift/codegen/src/isa/riscv64/inst/regs.rs b/cranelift/codegen/src/isa/riscv64/inst/regs.rs index ffdc484a00c7..1231567f8703 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/regs.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/regs.rs @@ -10,14 +10,14 @@ use regalloc2::{PReg, RegClass, VReg}; // first argument of function call #[inline] -pub fn a0() -> Reg { +pub const fn a0() -> Reg { x_reg(10) } // second argument of function call #[inline] #[allow(dead_code)] -pub fn a1() -> Reg { +pub const fn a1() -> Reg { x_reg(11) } @@ -135,10 +135,10 @@ pub fn writable_spilltmp_reg2() -> Writable { } #[inline] -pub fn x_reg(enc: usize) -> Reg { +pub const fn x_reg(enc: usize) -> Reg { let p_reg = PReg::new(enc, RegClass::Int); let v_reg = VReg::new(p_reg.index(), p_reg.class()); - Reg::from(v_reg) + Reg::from_virtual_reg(v_reg) } pub const fn px_reg(enc: usize) -> PReg { PReg::new(enc, RegClass::Int) diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs index 34999f1d79c1..7dd2c53c7c66 100644 --- a/cranelift/codegen/src/isa/s390x/abi.rs +++ b/cranelift/codegen/src/isa/s390x/abi.rs @@ -902,7 +902,11 @@ impl ABIMachineSpec for S390xMachineDeps { } } - fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet { + fn get_regs_clobbered_by_call( + call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet { + assert!(!is_exception); match call_conv_of_callee { isa::CallConv::Tail => TAIL_CLOBBERS, _ => SYSV_CLOBBERS, diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs index a8fc78cd86bc..c4d71dea30e5 100644 --- a/cranelift/codegen/src/isa/s390x/inst/mod.rs +++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs @@ -914,7 +914,8 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor false, - &Inst::Call { ref info, .. } => info.caller_conv != info.callee_conv, + &Inst::Call { ref info, .. } => { + info.caller_conv != info.callee_conv || info.try_call_info.is_some() + } &Inst::ElfTlsGetOffset { .. } => false, _ => true, } @@ -1069,10 +1072,10 @@ impl MachInst for Inst { match self { &Inst::Rets { .. } => MachTerminator::Ret, &Inst::ReturnCall { .. } => MachTerminator::RetCall, - &Inst::Jump { .. } => MachTerminator::Uncond, - &Inst::CondBr { .. } => MachTerminator::Cond, - &Inst::IndirectBr { .. } => MachTerminator::Indirect, - &Inst::JTSequence { .. } => MachTerminator::Indirect, + &Inst::Jump { .. } => MachTerminator::Branch, + &Inst::CondBr { .. } => MachTerminator::Branch, + &Inst::IndirectBr { .. } => MachTerminator::Branch, + &Inst::JTSequence { .. } => MachTerminator::Branch, _ => MachTerminator::None, } } diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index 4226c8f9832d..842d514af2d4 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -278,7 +278,10 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { let sig_data = &self.lower_ctx.sigs()[abi]; // Get clobbers: all caller-saves. These may include return value // regs, which we will remove from the clobber set later. - let clobbers = S390xMachineDeps::get_regs_clobbered_by_call(sig_data.call_conv()); + let clobbers = S390xMachineDeps::get_regs_clobbered_by_call( + sig_data.call_conv(), + /* is_exception = */ false, + ); let callee_pop_size = if sig_data.call_conv() == CallConv::Tail { sig_data.sized_stack_arg_space() as u32 } else { @@ -292,6 +295,7 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> { callee_pop_size, caller_conv: self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()), callee_conv: self.lower_ctx.sigs()[abi].call_conv(), + try_call_info: None, }); (info, outputs) } diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index c1bbff697931..c6ed91cc4f86 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -875,10 +875,11 @@ impl ABIMachineSpec for X64ABIMachineSpec { }, ], defs: smallvec![], - clobbers: Self::get_regs_clobbered_by_call(call_conv), + clobbers: Self::get_regs_clobbered_by_call(call_conv, false), callee_pop_size, callee_conv: call_conv, caller_conv: call_conv, + try_call_info: None, }))); insts } @@ -906,10 +907,14 @@ impl ABIMachineSpec for X64ABIMachineSpec { } } - fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet { + fn get_regs_clobbered_by_call( + call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet { match call_conv_of_callee { CallConv::Winch => ALL_CLOBBERS, CallConv::WindowsFastcall => WINDOWS_CLOBBERS, + _ if is_exception => ALL_CLOBBERS, _ => SYSV_CLOBBERS, } } @@ -981,6 +986,14 @@ impl ABIMachineSpec for X64ABIMachineSpec { // supported calling conventions. Writable::from_reg(regs::r11()) } + + fn exception_payload_regs(call_conv: isa::CallConv) -> &'static [Reg] { + const PAYLOAD_REGS: &'static [Reg] = &[regs::rax(), regs::rdx()]; + match call_conv { + isa::CallConv::SystemV | isa::CallConv::Tail => PAYLOAD_REGS, + _ => &[], + } + } } impl X64CallSite { diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index c8338adae6d3..5e3d21024168 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -2477,6 +2477,12 @@ (decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) (extern constructor gen_call_indirect gen_call_indirect) +(decl gen_try_call (SigRef ExternalName RelocDistance ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call gen_try_call) + +(decl gen_try_call_indirect (SigRef Value ExceptionTable ValueSlice MachLabelSlice) Unit) +(extern constructor gen_try_call_indirect gen_try_call_indirect) + ;;;; Helpers for emitting stack switches ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl x64_stack_switch_basic (Gpr Gpr Gpr) Gpr) diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 01f8e281e065..b086c215c92d 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1613,6 +1613,14 @@ pub(crate) fn emit( sink.put4(0); sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = call_info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + // Reclaim the outgoing argument area that was released by the callee, to ensure that // StackAMode values are always computed from a consistent SP. if call_info.callee_pop_size > 0 { @@ -1631,6 +1639,15 @@ pub(crate) fn emit( |inst| inst.emit(sink, info, state), |_space_needed| None, ); + + // If this is a try-call, jump to the continuation + // (normal-return) block. + if let Some(try_call) = call_info.try_call_info.as_ref() { + let jmp = Inst::JmpKnown { + dst: try_call.continuation, + }; + jmp.emit(sink, info, state); + } } Inst::ReturnCallKnown { info: call_info } => { @@ -1702,6 +1719,14 @@ pub(crate) fn emit( sink.add_call_site(); + // Add exception info, if any, at this point (which will + // be the return address on stack). + if let Some(try_call) = call_info.try_call_info.as_ref() { + for &(tag, label) in &try_call.exception_dests { + sink.add_exception_handler(tag, label); + } + } + // Reclaim the outgoing argument area that was released by the callee, to ensure that // StackAMode values are always computed from a consistent SP. if call_info.callee_pop_size > 0 { @@ -1720,6 +1745,13 @@ pub(crate) fn emit( |inst| inst.emit(sink, info, state), |_space_needed| None, ); + + if let Some(try_call) = call_info.try_call_info.as_ref() { + let jmp = Inst::JmpKnown { + dst: try_call.continuation, + }; + jmp.emit(sink, info, state); + } } Inst::Args { .. } => {} diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 79740b9d5165..d95b88fe3752 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -5123,7 +5123,7 @@ fn test_x64_emit() { for (insn, expected_encoding, expected_printing) in insns { // Check the printed text is as expected. let actual_printing = insn.pretty_print_inst(&mut Default::default()); - assert_eq!(expected_printing, actual_printing); + assert_eq!(expected_printing, actual_printing.trim()); let mut buffer = MachBuffer::new(); insn.emit(&mut buffer, &emit_info, &mut Default::default()); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 5ee15133c24b..6d6d3ac3efba 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -11,6 +11,7 @@ use crate::isa::{CallConv, FunctionAlignment}; use crate::{machinst::*, trace}; use crate::{settings, CodegenError, CodegenResult}; use alloc::boxed::Box; +use alloc::vec::Vec; use smallvec::{smallvec, SmallVec}; use std::fmt::{self, Write}; use std::string::{String, ToString}; @@ -1645,13 +1646,23 @@ impl PrettyPrint for Inst { Inst::CallKnown { info } => { let op = ljustify("call".to_string()); - format!("{op} {:?}", info.dest) + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("{op} {:?}{try_call}", info.dest) } Inst::CallUnknown { info } => { let dest = info.dest.pretty_print(8); let op = ljustify("call".to_string()); - format!("{op} *{dest}") + let try_call = info + .try_call_info + .as_ref() + .map(|tci| pretty_print_try_call(tci)) + .unwrap_or_default(); + format!("{op} *{dest}{try_call}") } Inst::ReturnCallKnown { info } => { @@ -1972,6 +1983,16 @@ impl PrettyPrint for Inst { } } +fn pretty_print_try_call(info: &TryCallInfo) -> String { + let dests = info + .exception_dests + .iter() + .map(|(tag, label)| format!("{tag:?}: {label:?}")) + .collect::>() + .join(", "); + format!("; jmp {:?}; catch [{dests}]", info.continuation) +} + impl fmt::Debug for Inst { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { write!(fmt, "{}", self.pretty_print_inst(&mut Default::default())) @@ -2468,7 +2489,7 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { // TODO(https://github.com/bytecodealliance/regalloc2/issues/145): // This shouldn't be a fixed register constraint. r10 is caller-saved, so this // should be safe to use. - collector.reg_fixed_use(reg, regs::r10()) + collector.reg_fixed_use(reg, regs::r10()); } _ => dest.get_operands(collector), } @@ -2696,7 +2717,8 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { // pseudoinstruction (and relocation that it emits) is specific to // ELF systems; other x86-64 targets with other conventions (i.e., // Windows) use different TLS strategies. - let mut clobbers = X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV); + let mut clobbers = + X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV, false); clobbers.remove(regs::gpr_preg(regs::ENC_RAX)); collector.reg_clobbers(clobbers); } @@ -2795,10 +2817,14 @@ impl MachInst for Inst { &Self::ReturnCallKnown { .. } | &Self::ReturnCallUnknown { .. } => { MachTerminator::RetCall } - &Self::JmpKnown { .. } => MachTerminator::Uncond, - &Self::JmpCond { .. } => MachTerminator::Cond, - &Self::JmpCondOr { .. } => MachTerminator::Cond, - &Self::JmpTableSeq { .. } => MachTerminator::Indirect, + &Self::JmpKnown { .. } => MachTerminator::Branch, + &Self::JmpCond { .. } => MachTerminator::Branch, + &Self::JmpCondOr { .. } => MachTerminator::Branch, + &Self::JmpTableSeq { .. } => MachTerminator::Branch, + &Self::CallKnown { ref info } if info.try_call_info.is_some() => MachTerminator::Branch, + &Self::CallUnknown { ref info } if info.try_call_info.is_some() => { + MachTerminator::Branch + } // All other cases are boring. _ => MachTerminator::None, } diff --git a/cranelift/codegen/src/isa/x64/inst/regs.rs b/cranelift/codegen/src/isa/x64/inst/regs.rs index d3a06da70a03..fdf97b17c2c8 100644 --- a/cranelift/codegen/src/isa/x64/inst/regs.rs +++ b/cranelift/codegen/src/isa/x64/inst/regs.rs @@ -31,127 +31,127 @@ pub const ENC_R15: u8 = 15; // Constructors for Regs. -fn gpr(enc: u8) -> Reg { +const fn gpr(enc: u8) -> Reg { let preg = gpr_preg(enc); - Reg::from(VReg::new(preg.index(), RegClass::Int)) + Reg::from_virtual_reg(VReg::new(preg.index(), RegClass::Int)) } pub(crate) const fn gpr_preg(enc: u8) -> PReg { PReg::new(enc as usize, RegClass::Int) } -pub(crate) fn rsi() -> Reg { +pub(crate) const fn rsi() -> Reg { gpr(ENC_RSI) } -pub(crate) fn rdi() -> Reg { +pub(crate) const fn rdi() -> Reg { gpr(ENC_RDI) } -pub(crate) fn rax() -> Reg { +pub(crate) const fn rax() -> Reg { gpr(ENC_RAX) } -pub(crate) fn rcx() -> Reg { +pub(crate) const fn rcx() -> Reg { gpr(ENC_RCX) } -pub(crate) fn rdx() -> Reg { +pub(crate) const fn rdx() -> Reg { gpr(ENC_RDX) } -pub(crate) fn r8() -> Reg { +pub(crate) const fn r8() -> Reg { gpr(ENC_R8) } -pub(crate) fn r9() -> Reg { +pub(crate) const fn r9() -> Reg { gpr(ENC_R9) } -pub(crate) fn r10() -> Reg { +pub(crate) const fn r10() -> Reg { gpr(ENC_R10) } -pub(crate) fn r11() -> Reg { +pub(crate) const fn r11() -> Reg { gpr(ENC_R11) } -pub(crate) fn r12() -> Reg { +pub(crate) const fn r12() -> Reg { gpr(ENC_R12) } -pub(crate) fn r13() -> Reg { +pub(crate) const fn r13() -> Reg { gpr(ENC_R13) } -pub(crate) fn r14() -> Reg { +pub(crate) const fn r14() -> Reg { gpr(ENC_R14) } -pub(crate) fn rbx() -> Reg { +pub(crate) const fn rbx() -> Reg { gpr(ENC_RBX) } -pub(crate) fn r15() -> Reg { +pub(crate) const fn r15() -> Reg { gpr(ENC_R15) } -pub(crate) fn rsp() -> Reg { +pub(crate) const fn rsp() -> Reg { gpr(ENC_RSP) } -pub(crate) fn rbp() -> Reg { +pub(crate) const fn rbp() -> Reg { gpr(ENC_RBP) } /// The pinned register on this architecture. /// It must be the same as Spidermonkey's HeapReg, as found in this file. /// https://searchfox.org/mozilla-central/source/js/src/jit/x64/Assembler-x64.h#99 -pub(crate) fn pinned_reg() -> Reg { +pub(crate) const fn pinned_reg() -> Reg { r15() } -fn fpr(enc: u8) -> Reg { +const fn fpr(enc: u8) -> Reg { let preg = fpr_preg(enc); - Reg::from(VReg::new(preg.index(), RegClass::Float)) + Reg::from_virtual_reg(VReg::new(preg.index(), RegClass::Float)) } pub(crate) const fn fpr_preg(enc: u8) -> PReg { PReg::new(enc as usize, RegClass::Float) } -pub(crate) fn xmm0() -> Reg { +pub(crate) const fn xmm0() -> Reg { fpr(0) } -pub(crate) fn xmm1() -> Reg { +pub(crate) const fn xmm1() -> Reg { fpr(1) } -pub(crate) fn xmm2() -> Reg { +pub(crate) const fn xmm2() -> Reg { fpr(2) } -pub(crate) fn xmm3() -> Reg { +pub(crate) const fn xmm3() -> Reg { fpr(3) } -pub(crate) fn xmm4() -> Reg { +pub(crate) const fn xmm4() -> Reg { fpr(4) } -pub(crate) fn xmm5() -> Reg { +pub(crate) const fn xmm5() -> Reg { fpr(5) } -pub(crate) fn xmm6() -> Reg { +pub(crate) const fn xmm6() -> Reg { fpr(6) } -pub(crate) fn xmm7() -> Reg { +pub(crate) const fn xmm7() -> Reg { fpr(7) } -pub(crate) fn xmm8() -> Reg { +pub(crate) const fn xmm8() -> Reg { fpr(8) } -pub(crate) fn xmm9() -> Reg { +pub(crate) const fn xmm9() -> Reg { fpr(9) } -pub(crate) fn xmm10() -> Reg { +pub(crate) const fn xmm10() -> Reg { fpr(10) } -pub(crate) fn xmm11() -> Reg { +pub(crate) const fn xmm11() -> Reg { fpr(11) } -pub(crate) fn xmm12() -> Reg { +pub(crate) const fn xmm12() -> Reg { fpr(12) } -pub(crate) fn xmm13() -> Reg { +pub(crate) const fn xmm13() -> Reg { fpr(13) } -pub(crate) fn xmm14() -> Reg { +pub(crate) const fn xmm14() -> Reg { fpr(14) } -pub(crate) fn xmm15() -> Reg { +pub(crate) const fn xmm15() -> Reg { fpr(15) } diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 02e8479bfeb3..5dba95032f4f 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -3492,6 +3492,15 @@ (rule (lower (return_call_indirect sig_ref callee args)) (gen_return_call_indirect sig_ref callee args)) +;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower_branch (try_call (func_ref_data sig_ref extname dist) inputs et) targets) + (gen_try_call sig_ref extname dist et inputs targets)) + +(rule (lower_branch (try_call_indirect val inputs et) targets) + (if-let (exception_sig sig_ref) et) + (gen_try_call_indirect sig_ref val et inputs targets)) + ;; Rules for `stack_switch` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; currently, only the Basic model is supported diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d8b0d4ce9eaa..b2998feec508 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -185,7 +185,7 @@ fn emit_vm_call( outputs.push(retval_regs.only_reg().unwrap()); } - abi.emit_call(ctx); + abi.emit_call(ctx, None); Ok(outputs) } diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index c18c5284059a..7eba50977116 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -100,12 +100,14 @@ use crate::entity::SecondaryMap; use crate::ir::types::*; -use crate::ir::{ArgumentExtension, ArgumentPurpose, Signature}; +use crate::ir::{ArgumentExtension, ArgumentPurpose, ExceptionTable, ExceptionTag, Signature}; use crate::isa::TargetIsa; use crate::settings::ProbestackStrategy; use crate::CodegenError; use crate::{ir, isa}; use crate::{machinst::*, trace}; +use alloc::boxed::Box; +use cranelift_entity::packed_option::PackedOption; use regalloc2::{MachineEnv, PReg, PRegSet}; use rustc_hash::FxHashMap; use smallvec::smallvec; @@ -272,7 +274,7 @@ pub enum ArgsOrRets { /// Abstract location for a machine-specific ABI impl to translate into the /// appropriate addressing mode. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum StackAMode { /// Offset into the current frame's argument area. IncomingArg(i64, u32), @@ -582,7 +584,10 @@ pub trait ABIMachineSpec { /// Get all caller-save registers, that is, registers that we expect /// not to be saved across a call to a callee with the given ABI. - fn get_regs_clobbered_by_call(call_conv_of_callee: isa::CallConv) -> PRegSet; + fn get_regs_clobbered_by_call( + call_conv_of_callee: isa::CallConv, + is_exception: bool, + ) -> PRegSet; /// Get the needed extension mode, given the mode attached to the argument /// in the signature and the calling convention. The input (the attribute in @@ -599,6 +604,12 @@ pub trait ABIMachineSpec { /// return values. This is used to move stack-carried return /// values directly into spillslots if needed. fn retval_temp_reg(call_conv_of_callee: isa::CallConv) -> Writable; + + /// Get the exception payload registers, if any, for a calling + /// convention. + fn exception_payload_regs(_call_conv: isa::CallConv) -> &'static [Reg] { + &[] + } } /// Out-of-line data for calls, to keep the size of `Inst` down. @@ -620,6 +631,22 @@ pub struct CallInfo { /// caller, if any. (Used for popping stack arguments with the `tail` /// calling convention.) pub callee_pop_size: u32, + /// Information for a try-call, if this is one. We combine + /// handling of calls and try-calls as much as possible to share + /// argument/return logic; they mostly differ in the metadata that + /// they emit, which this information feeds into. + pub try_call_info: Option, +} + +/// Out-of-line information present on `try_call` instructions only: +/// information that is used to generate exception-handling tables and +/// link up to destination blocks properly. +#[derive(Clone, Debug)] +pub struct TryCallInfo { + /// The target to jump to on a normal returhn. + pub continuation: MachLabel, + /// Exception tags to catch and corresponding destination labels. + pub exception_dests: Box<[(PackedOption, MachLabel)]>, } impl CallInfo { @@ -634,6 +661,7 @@ impl CallInfo { caller_conv: call_conv, callee_conv: call_conv, callee_pop_size: 0, + try_call_info: None, } } @@ -647,6 +675,7 @@ impl CallInfo { caller_conv: self.caller_conv, callee_conv: self.callee_conv, callee_pop_size: self.callee_pop_size, + try_call_info: self.try_call_info, } } } @@ -1998,7 +2027,7 @@ pub struct CallRetPair { } /// A location to load a return-value from after a call completes. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum RetLocation { /// A physical register. Reg(Reg, Type), @@ -2395,7 +2424,11 @@ impl CallSite { /// /// This function should only be called once, as it is allowed to re-use /// parts of the `CallSite` object in emitting instructions. - pub fn emit_call(&mut self, ctx: &mut Lower) { + pub fn emit_call( + &mut self, + ctx: &mut Lower, + try_call_info: Option<(ExceptionTable, &[MachLabel])>, + ) { let word_type = M::word_type(); if let Some(i) = ctx.sigs()[self.sig].stack_ret_arg { let rd = ctx.alloc_tmp(word_type).only_reg().unwrap(); @@ -2408,21 +2441,7 @@ impl CallSite { } let uses = mem::take(&mut self.uses); - let defs = mem::take(&mut self.defs); - let clobbers = { - // Get clobbers: all caller-saves. These may include return value - // regs, which we will remove from the clobber set below. - let mut clobbers = ::get_regs_clobbered_by_call(ctx.sigs()[self.sig].call_conv); - - // Remove retval regs from clobbers. - for def in &defs { - if let RetLocation::Reg(preg, ..) = def.location { - clobbers.remove(PReg::from(preg.to_real_reg().unwrap())); - } - } - - clobbers - }; + let mut defs = mem::take(&mut self.defs); let sig = &ctx.sigs()[self.sig]; let callee_pop_size = if sig.call_conv() == isa::CallConv::Tail { @@ -2441,6 +2460,74 @@ impl CallSite { let tmp = ctx.alloc_tmp(word_type).only_reg().unwrap(); + let try_call_info = try_call_info.map(|(et, labels)| { + let exception_dests = ctx.dfg().exception_tables[et] + .catches() + .map(|(tag, _)| tag.into()) + .zip(labels.iter().cloned()) + .collect::>() + .into_boxed_slice(); + + // We need to update `defs` to contain the exception + // payload regs as well. We have two sources of info that + // we join: + // + // - The machine-specific ABI implementation `M`, which + // tells us the particular registers that payload values + // must be in + // - The passed-in lowering context, which gives us the + // vregs we must define. + // + // Note that payload values may need to end up in the same + // physical registers as ordinary return values; this is + // not a conflict, because we either get one or the + // other. For regalloc's purposes, we define both starting + // here at the callsite, but we can share one def in the + // `defs` list and alias one vreg to another. Thus we + // handle the two cases below for each payload register: + // overlaps a return value (and we alias to it) or not + // (and we add a def). + let pregs = M::exception_payload_regs(call_conv); + for (i, &preg) in pregs.iter().enumerate() { + let vreg = ctx.try_call_exception_defs(ctx.cur_inst())[i]; + if let Some(existing) = defs.iter().find(|def| match def.location { + RetLocation::Reg(r, _) => r == preg, + _ => false, + }) { + ctx.vregs_mut() + .set_vreg_alias(vreg.to_reg(), existing.vreg.to_reg()); + } else { + defs.push(CallRetPair { + vreg, + location: RetLocation::Reg(preg, M::word_type()), + }); + } + } + + TryCallInfo { + continuation: *labels.last().unwrap(), + exception_dests, + } + }); + + let clobbers = { + // Get clobbers: all caller-saves. These may include return value + // regs, which we will remove from the clobber set below. + let mut clobbers = ::get_regs_clobbered_by_call( + ctx.sigs()[self.sig].call_conv, + try_call_info.is_some(), + ); + + // Remove retval regs from clobbers. + for def in &defs { + if let RetLocation::Reg(preg, _) = def.location { + clobbers.remove(PReg::from(preg.to_real_reg().unwrap())); + } + } + + clobbers + }; + // Any adjustment to SP to account for required outgoing arguments/stack return values must // be done inside of the call pseudo-op, to ensure that SP is always in a consistent // state for all other instructions. For example, if a tail-call abi function is called @@ -2461,6 +2548,7 @@ impl CallSite { callee_conv: call_conv, caller_conv: self.caller_conv, callee_pop_size, + try_call_info, }, ) .into_iter() @@ -2502,8 +2590,11 @@ impl CallInfo { let temp = M::retval_temp_reg(self.callee_conv); // The temporary must be noted as clobbered. - debug_assert!(M::get_regs_clobbered_by_call(self.callee_conv) - .contains(PReg::from(temp.to_reg().to_real_reg().unwrap()))); + debug_assert!(M::get_regs_clobbered_by_call( + self.callee_conv, + self.try_call_info.is_some() + ) + .contains(PReg::from(temp.to_reg().to_real_reg().unwrap()))); for CallRetPair { vreg, location } in &self.defs { match location { diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 403f425e8ec1..2b1ac7ca07e3 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -181,6 +181,7 @@ use crate::trace; use crate::{ir, MachInstEmitState}; use crate::{timing, VCodeConstantData}; use cranelift_control::ControlPlane; +use cranelift_entity::packed_option::PackedOption; use cranelift_entity::{entity_impl, PrimaryMap}; use smallvec::SmallVec; use std::cmp::Ordering; @@ -258,6 +259,8 @@ pub struct MachBuffer { user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>, /// Any unwind info at a given location. unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, + /// Any exception handler targets at a given location. + exception_handlers: SmallVec<[(CodeOffset, PackedOption, MachLabel); 8]>, /// The current source location in progress (after `start_srcloc()` and /// before `end_srcloc()`). This is a (start_offset, src_loc) tuple. cur_srcloc: Option<(CodeOffset, RelSourceLoc)>, @@ -336,6 +339,7 @@ impl MachBufferFinalized { .collect(), user_stack_maps: self.user_stack_maps, unwind_info: self.unwind_info, + exception_handlers: self.exception_handlers, alignment: self.alignment, } } @@ -368,6 +372,8 @@ pub struct MachBufferFinalized { pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>, /// Any unwind info at a given location. pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, + /// Any exception handler targets at a given location. + pub exception_handlers: SmallVec<[(CodeOffset, PackedOption, CodeOffset); 8]>, /// The required alignment of this buffer. pub alignment: u32, } @@ -442,6 +448,7 @@ impl MachBuffer { srclocs: SmallVec::new(), user_stack_maps: SmallVec::new(), unwind_info: SmallVec::new(), + exception_handlers: SmallVec::new(), cur_srcloc: None, label_offsets: SmallVec::new(), label_aliases: SmallVec::new(), @@ -1519,6 +1526,12 @@ impl MachBuffer { }) .collect(); + let exception_handlers = self + .exception_handlers + .iter() + .map(|&(off, tag, target)| (off, tag, self.resolve_label_offset(target))) + .collect(); + let mut srclocs = self.srclocs; srclocs.sort_by_key(|entry| entry.start); @@ -1530,6 +1543,7 @@ impl MachBuffer { srclocs, user_stack_maps: self.user_stack_maps, unwind_info: self.unwind_info, + exception_handlers, alignment, } } @@ -1614,6 +1628,16 @@ impl MachBuffer { self.unwind_info.push((self.cur_offset(), unwind)); } + /// Add an exception handler record at the current offset. + pub fn add_exception_handler( + &mut self, + tag: PackedOption, + target: MachLabel, + ) { + self.exception_handlers + .push((self.cur_offset(), tag, target)); + } + /// Set the `SourceLoc` for code from this offset until the offset at the /// next call to `end_srcloc()`. /// Returns the current [CodeOffset] and [RelSourceLoc]. diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 8066d89d7748..ee1faa7503d6 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -356,6 +356,11 @@ macro_rules! isle_lower_prelude_methods { (funcdata.signature, funcdata.name.clone(), reloc_distance) } + #[inline] + fn exception_sig(&mut self, et: ExceptionTable) -> SigRef { + self.lower_ctx.dfg().exception_tables[et].signature() + } + #[inline] fn box_external_name(&mut self, extname: ExternalName) -> BoxExternalName { Box::new(extname) @@ -771,7 +776,13 @@ macro_rules! isle_prelude_caller_methods { sig.params.len() ); - crate::machinst::isle::gen_call_common(&mut self.lower_ctx, num_rets, caller, args) + crate::machinst::isle::gen_call_common( + &mut self.lower_ctx, + num_rets, + caller, + args, + None, + ) } fn gen_call_indirect( @@ -798,7 +809,13 @@ macro_rules! isle_prelude_caller_methods { sig.params.len() ); - crate::machinst::isle::gen_call_common(&mut self.lower_ctx, num_rets, caller, args) + crate::machinst::isle::gen_call_common( + &mut self.lower_ctx, + num_rets, + caller, + args, + None, + ) } fn gen_return_call( @@ -856,6 +873,49 @@ macro_rules! isle_prelude_caller_methods { InstOutput::new() } + + fn gen_try_call( + &mut self, + sig_ref: SigRef, + extname: ExternalName, + dist: RelocDistance, + et: ExceptionTable, + args: ValueSlice, + targets: &MachLabelSlice, + ) -> () { + let caller_conv = self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()); + let sigref = self.lower_ctx.dfg().exception_tables[et].signature(); + let sig = &self.lower_ctx.dfg().signatures[sigref]; + let num_rets = sig.returns.len(); + let caller = <$abicaller>::from_func( + self.lower_ctx.sigs(), + sig_ref, + &extname, + IsTailCall::No, + dist, + caller_conv, + self.backend.flags().clone(), + ); + + crate::machinst::isle::gen_call_common( + &mut self.lower_ctx, + num_rets, + caller, + args, + Some((et, targets)), + ); + } + + fn gen_try_call_indirect( + &mut self, + _sig_ref: SigRef, + _callee: Value, + _et: ExceptionTable, + _args: ValueSlice, + _targets: &MachLabelSlice, + ) -> () { + todo!() + } }; } @@ -885,6 +945,7 @@ pub fn gen_call_common( num_rets: usize, mut caller: CallSite, args: ValueSlice, + try_call_info: Option<(ExceptionTable, &MachLabelSlice)>, ) -> InstOutput { gen_call_common_args(ctx, &mut caller, args); @@ -900,7 +961,20 @@ pub fn gen_call_common( outputs.push(retval_regs); } - caller.emit_call(ctx); + caller.emit_call(ctx, try_call_info); + + // If this is a try-call, alias return value vregs to the ones + // already allocated for the block-call arg defs. + if try_call_info.is_some() { + for i in 0..outputs.len() { + let result_regs = outputs[i]; + let def_regs = ctx.try_call_return_defs(ctx.cur_inst())[i]; + for (result_reg, def_reg) in result_regs.regs().iter().zip(def_regs.regs().iter()) { + ctx.vregs_mut() + .set_vreg_alias(def_reg.to_reg(), *result_reg); + } + } + } outputs } diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index cfe7fd9f90bf..516b41ee84b7 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -9,15 +9,15 @@ use crate::entity::SecondaryMap; use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit}; use crate::ir::pcc::{Fact, FactContext, PccError, PccResult}; use crate::ir::{ - ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function, - GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, RelSourceLoc, Type, - Value, ValueDef, ValueLabelAssignments, ValueLabelStart, + ArgumentPurpose, Block, BlockArg, Constant, ConstantData, DataFlowGraph, ExternalName, + Function, GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, + RelSourceLoc, Type, Value, ValueDef, ValueLabelAssignments, ValueLabelStart, }; use crate::machinst::valueregs::InvalidSentinel; use crate::machinst::{ - writable_value_regs, BackwardsInsnIndex, BlockIndex, BlockLoweringOrder, Callee, InsnIndex, - LoweredBlock, MachLabel, Reg, SigSet, VCode, VCodeBuilder, VCodeConstant, VCodeConstantData, - VCodeConstants, VCodeInst, ValueRegs, Writable, + writable_value_regs, ABIMachineSpec, BackwardsInsnIndex, BlockIndex, BlockLoweringOrder, + Callee, InsnIndex, LoweredBlock, MachLabel, Reg, SigSet, VCode, VCodeBuilder, VCodeConstant, + VCodeConstantData, VCodeConstants, VCodeInst, ValueRegs, Writable, }; use crate::settings::Flags; use crate::{trace, CodegenError, CodegenResult}; @@ -223,6 +223,14 @@ pub struct Lower<'func, I: VCodeInst> { /// Instructions collected for the CLIF inst in progress, in forward order. ir_insts: Vec, + /// Try-call block arg normal-return values, indexed by instruction. + try_call_rets: FxHashMap>; 2]>>, + + /// Try-call block arg exceptional-return payloads, indexed by + /// instruction. Payloads are carried in registers per the ABI and + /// can only be one register each. + try_call_payloads: FxHashMap; 2]>>, + /// The register to use for GetPinnedReg, if any, on this architecture. pinned_reg: Option, @@ -390,8 +398,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> { let mut vregs = VRegAllocator::with_capacity(f.dfg.num_values() * 2); let mut value_regs = SecondaryMap::with_default(ValueRegs::invalid()); + let mut try_call_rets = FxHashMap::default(); + let mut try_call_payloads = FxHashMap::default(); - // Assign a vreg to each block param and each inst result. + // Assign a vreg to each block param, each inst result, and + // each edge-defined block-call arg. for bb in f.layout.blocks() { for ¶m in f.dfg.block_params(bb) { let ty = f.dfg.value_type(param); @@ -417,6 +428,26 @@ impl<'func, I: VCodeInst> Lower<'func, I> { ); } } + + if let Some(et) = f.dfg.insts[inst].exception_table() { + let exdata = &f.dfg.exception_tables[et]; + let sig = &f.dfg.signatures[exdata.signature()]; + + let mut rets = smallvec![]; + for ty in sig.returns.iter().map(|ret| ret.value_type) { + rets.push(vregs.alloc(ty)?.map(|r| Writable::from_reg(r))); + } + try_call_rets.insert(inst, rets); + + let mut payloads = smallvec![]; + for &ty in sig + .call_conv + .exception_payload_types(I::ABIMachineSpec::word_type()) + { + payloads.push(Writable::from_reg(vregs.alloc(ty)?.only_reg().unwrap())); + } + try_call_payloads.insert(inst, payloads); + } } } @@ -492,6 +523,8 @@ impl<'func, I: VCodeInst> Lower<'func, I> { cur_scan_entry_color: None, cur_inst: None, ir_insts: vec![], + try_call_rets, + try_call_payloads, pinned_reg: None, flags, }) @@ -505,6 +538,10 @@ impl<'func, I: VCodeInst> Lower<'func, I> { self.vcode.sigs_mut() } + pub fn vregs_mut(&mut self) -> &mut VRegAllocator { + &mut self.vregs + } + fn gen_arg_setup(&mut self) { if let Some(entry_bb) = self.f.layout.entry_block() { trace!( @@ -970,13 +1007,26 @@ impl<'func, I: VCodeInst> Lower<'func, I> { } }; - let block_call = - self.f.dfg.insts[branch_inst].branch_destination(&self.f.dfg.jump_tables)[succ_idx]; - let args = block_call.args_slice(&self.f.dfg.value_lists); - for &arg in args { - debug_assert!(self.f.dfg.value_is_real(arg)); - let regs = self.put_value_in_regs(arg); - buffer.extend_from_slice(regs.regs()); + let block_call = self.f.dfg.insts[branch_inst] + .branch_destination(&self.f.dfg.jump_tables, &self.f.dfg.exception_tables)[succ_idx]; + for arg in block_call.args(&self.f.dfg.value_lists) { + match arg { + BlockArg::Value(arg) => { + debug_assert!(self.f.dfg.value_is_real(arg)); + let regs = self.put_value_in_regs(arg); + buffer.extend_from_slice(regs.regs()); + } + BlockArg::TryCallRet(i) => { + let regs = self.try_call_rets.get(&branch_inst).unwrap()[i as usize] + .map(|r| r.to_reg()); + buffer.extend_from_slice(regs.regs()); + } + BlockArg::TryCallExn(i) => { + let reg = + self.try_call_payloads.get(&branch_inst).unwrap()[i as usize].to_reg(); + buffer.push(reg); + } + } } (succ, &buffer[..]) } @@ -1489,6 +1539,18 @@ impl<'func, I: VCodeInst> Lower<'func, I> { regs } + + /// Get the ValueRegs for the edge-defined values for special + /// try-call-return block arguments. + pub fn try_call_return_defs(&mut self, ir_inst: Inst) -> &[ValueRegs>] { + &self.try_call_rets.get(&ir_inst).unwrap()[..] + } + + /// Get the Regs for the edge-defined values for special + /// try-call-return exception payload arguments. + pub fn try_call_exception_defs(&mut self, ir_inst: Inst) -> &[Writable] { + &self.try_call_payloads.get(&ir_inst).unwrap()[..] + } } /// Codegen primitives: allocate temps, emit instructions, set result registers, @@ -1499,6 +1561,11 @@ impl<'func, I: VCodeInst> Lower<'func, I> { writable_value_regs(self.vregs.alloc_with_deferred_error(ty)) } + /// Get the current root instruction that we are lowering. + pub fn cur_inst(&self) -> Inst { + self.cur_inst.unwrap() + } + /// Emit a machine instruction. pub fn emit(&mut self, mach_inst: I) { trace!("emit: {:?}", mach_inst); diff --git a/cranelift/codegen/src/machinst/mod.rs b/cranelift/codegen/src/machinst/mod.rs index fce309471ada..f7427bf72e4a 100644 --- a/cranelift/codegen/src/machinst/mod.rs +++ b/cranelift/codegen/src/machinst/mod.rs @@ -268,8 +268,7 @@ pub trait MachInstLabelUse: Clone + Copy + Debug + Eq { fn from_reloc(reloc: Reloc, addend: Addend) -> Option; } -/// Describes a block terminator (not call) in the vcode, when its branches -/// have not yet been finalized (so a branch may have two targets). +/// Describes a block terminator (not call) in the VCode. /// /// Actual targets are not included: the single-source-of-truth for /// those is the VCode itself, which holds, for each block, successors @@ -282,12 +281,8 @@ pub enum MachTerminator { Ret, /// A tail call. RetCall, - /// An unconditional branch to another block. - Uncond, - /// A conditional branch to one of two other blocks. - Cond, - /// An indirect branch with known possible targets. - Indirect, + /// A branch. + Branch, } /// A trait describing the ability to encode a MachInst into binary machine code. diff --git a/cranelift/codegen/src/machinst/reg.rs b/cranelift/codegen/src/machinst/reg.rs index 5a6ea1f9b326..a05a33743fc1 100644 --- a/cranelift/codegen/src/machinst/reg.rs +++ b/cranelift/codegen/src/machinst/reg.rs @@ -10,13 +10,9 @@ use regalloc2::{Operand, OperandConstraint, OperandKind, OperandPos, PReg, PRegS use serde_derive::{Deserialize, Serialize}; /// The first 192 vregs (64 int, 64 float, 64 vec) are "pinned" to -/// physical registers: this means that they are always constrained to -/// the corresponding register at all use/mod/def sites. -/// -/// Arbitrary vregs can also be constrained to physical registers at -/// particular use/def/mod sites, and this is preferable; but pinned -/// vregs allow us to migrate code that has been written using -/// RealRegs directly. +/// physical registers. These must not be passed into the regalloc, +/// but they are used to represent physical registers in the same +/// `Reg` type post-regalloc. const PINNED_VREGS: usize = 192; /// Convert a `VReg` to its pinned `PReg`, if any. @@ -28,6 +24,11 @@ pub fn pinned_vreg_to_preg(vreg: VReg) -> Option { } } +/// Convert a `PReg` to its pinned `VReg`. +pub const fn preg_to_pinned_vreg(preg: PReg) -> VReg { + VReg::new(preg.index(), preg.class()) +} + /// Give the first available vreg for generated code (i.e., after all /// pinned vregs). pub fn first_user_vreg_index() -> usize { @@ -51,6 +52,16 @@ const REG_SPILLSLOT_BIT: u32 = 0x8000_0000; const REG_SPILLSLOT_MASK: u32 = !REG_SPILLSLOT_BIT; impl Reg { + /// Const constructor: create a new Reg from a regalloc2 VReg. + pub const fn from_virtual_reg(vreg: regalloc2::VReg) -> Reg { + Reg(vreg.bits() as u32) + } + + /// Const constructor: create a new Reg from a regalloc2 PReg. + pub const fn from_real_reg(preg: regalloc2::PReg) -> Reg { + Reg(preg_to_pinned_vreg(preg).bits() as u32) + } + /// Get the physical register (`RealReg`), if this register is /// one. pub fn to_real_reg(self) -> Option { diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs index dd6f48668b75..b00ccea75f6a 100644 --- a/cranelift/codegen/src/machinst/vcode.rs +++ b/cranelift/codegen/src/machinst/vcode.rs @@ -1335,13 +1335,13 @@ impl RegallocFunction for VCode { // We treat blocks terminated by an unconditional trap like a return for regalloc. MachTerminator::None => self.insts[insn.index()].is_trap(), MachTerminator::Ret | MachTerminator::RetCall => true, - MachTerminator::Uncond | MachTerminator::Cond | MachTerminator::Indirect => false, + MachTerminator::Branch => false, } } fn is_branch(&self, insn: InsnIndex) -> bool { match self.insts[insn.index()].is_term() { - MachTerminator::Cond | MachTerminator::Uncond | MachTerminator::Indirect => true, + MachTerminator::Branch => true, _ => false, } } diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isle index 860a1072b41d..2fb4144bb4fe 100644 --- a/cranelift/codegen/src/prelude_lower.isle +++ b/cranelift/codegen/src/prelude_lower.isle @@ -975,6 +975,10 @@ (decl func_ref_data (SigRef ExternalName RelocDistance) FuncRef) (extern extractor infallible func_ref_data func_ref_data) +;; Accessor for `ExceptionTable`. +(decl exception_sig (SigRef) ExceptionTable) +(extern extractor infallible exception_sig exception_sig) + ;; Accessor for `GlobalValue`. (decl symbol_value_data (ExternalName RelocDistance i64) GlobalValue) diff --git a/cranelift/codegen/src/remove_constant_phis.rs b/cranelift/codegen/src/remove_constant_phis.rs index bb2159c3bbeb..ccaa3973f875 100644 --- a/cranelift/codegen/src/remove_constant_phis.rs +++ b/cranelift/codegen/src/remove_constant_phis.rs @@ -3,7 +3,7 @@ use crate::dominator_tree::DominatorTree; use crate::ir; use crate::ir::Function; -use crate::ir::{Block, BlockCall, Inst, Value}; +use crate::ir::{Block, BlockArg, BlockCall, Inst, Value}; use crate::timing; use bumpalo::Bump; use cranelift_entity::SecondaryMap; @@ -116,7 +116,7 @@ struct OutEdge<'a> { /// The arguments to that block. /// /// These values can be from both groups A and B. - args: &'a [Value], + args: &'a [BlockArg], } impl<'a> OutEdge<'a> { @@ -132,10 +132,10 @@ impl<'a> OutEdge<'a> { branch_index: usize, block: BlockCall, ) -> Option { - let inst_var_args = block.args_slice(&dfg.value_lists); + let inst_var_args = block.args(&dfg.value_lists); // Skip edges without params. - if inst_var_args.is_empty() { + if inst_var_args.len() == 0 { return None; } @@ -144,9 +144,7 @@ impl<'a> OutEdge<'a> { branch_index: branch_index as u32, block: block.block(&dfg.value_lists), args: bump.alloc_slice_fill_iter( - inst_var_args - .iter() - .map(|value| dfg.resolve_aliases(*value)), + inst_var_args.map(|arg| arg.map_value(|value| dfg.resolve_aliases(value))), ), }) } @@ -239,7 +237,7 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) for inst in func.layout.block_insts(b) { for (ix, dest) in func.dfg.insts[inst] - .branch_destination(&func.dfg.jump_tables) + .branch_destination(&func.dfg.jump_tables, &func.dfg.exception_tables) .iter() .enumerate() { @@ -305,9 +303,12 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) // to be found in the solver state. If not, then it's a // real value defining point (not a phi), in which case // return it itself. - let actual_absval = match state.maybe_get(*actual) { - Some(pt) => *pt, - None => AbstractValue::One(*actual), + let actual_absval = match actual { + BlockArg::Value(actual) => match state.maybe_get(*actual) { + Some(pt) => *pt, + None => AbstractValue::One(*actual), + }, + _ => AbstractValue::Many, }; // And `join` the new value with the old. @@ -385,10 +386,11 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) } let dfg = &mut func.dfg; - let dests = dfg.insts[edge.inst].branch_destination_mut(&mut dfg.jump_tables); + let dests = dfg.insts[edge.inst] + .branch_destination_mut(&mut dfg.jump_tables, &mut dfg.exception_tables); let block = &mut dests[edge.branch_index as usize]; - old_actuals.extend(block.args_slice(&dfg.value_lists)); + old_actuals.extend(block.args(&dfg.value_lists)); // Check that the numbers of arguments make sense. let formals = &summaries[edge.block].formals; @@ -405,8 +407,7 @@ pub fn do_remove_constant_phis(func: &mut Function, domtree: &mut DominatorTree) // This leaks the value list from the old block, // https://github.com/bytecodealliance/wasmtime/issues/5451 for more information. let destination = block.block(&dfg.value_lists); - *block = BlockCall::new(destination, &old_actuals, &mut dfg.value_lists); - old_actuals.clear(); + *block = BlockCall::new(destination, old_actuals.drain(..), &mut dfg.value_lists); } } diff --git a/cranelift/codegen/src/traversals.rs b/cranelift/codegen/src/traversals.rs index b3234601bf46..5319cd23139b 100644 --- a/cranelift/codegen/src/traversals.rs +++ b/cranelift/codegen/src/traversals.rs @@ -194,7 +194,7 @@ mod tests { cur.insert_block(block1); let v1 = cur.ins().iconst(I32, 1); let v2 = cur.ins().iadd(v0, v1); - cur.ins().jump(block0, &[v2]); + cur.ins().jump(block0, &[v2.into()]); // block2: // return v0 diff --git a/cranelift/codegen/src/verifier/mod.rs b/cranelift/codegen/src/verifier/mod.rs index 292e129e1dbb..1e82d727bb1b 100644 --- a/cranelift/codegen/src/verifier/mod.rs +++ b/cranelift/codegen/src/verifier/mod.rs @@ -69,7 +69,7 @@ use crate::entity::SparseSet; use crate::flowgraph::{BlockPredecessor, ControlFlowGraph}; use crate::ir::entities::AnyEntity; use crate::ir::instructions::{CallInfo, InstructionFormat, ResolvedConstraint}; -use crate::ir::{self, ArgumentExtension}; +use crate::ir::{self, ArgumentExtension, BlockArg, ExceptionTable}; use crate::ir::{ types, ArgumentPurpose, Block, Constant, DynamicStackSlot, FuncRef, Function, GlobalValue, Inst, JumpTable, MemFlags, MemoryTypeData, Opcode, SigRef, StackSlot, Type, Value, ValueDef, @@ -295,6 +295,13 @@ pub fn verify_context<'a, FOI: Into>>( verifier.run(errors) } +#[derive(Clone, Copy, Debug)] +enum BlockCallTargetType { + Normal, + ExNormalRet, + Exception, +} + struct Verifier<'a> { func: &'a Function, expected_cfg: ControlFlowGraph, @@ -594,6 +601,26 @@ impl<'a> Verifier<'a> { self.verify_sig_ref(inst, sig_ref, errors)?; self.verify_value_list(inst, args, errors)?; } + TryCall { + func_ref, + ref args, + exception, + .. + } => { + self.verify_func_ref(inst, func_ref, errors)?; + self.verify_value_list(inst, args, errors)?; + self.verify_exception_table(inst, exception, errors)?; + self.verify_exception_compatible_abi(inst, exception, errors)?; + } + TryCallIndirect { + ref args, + exception, + .. + } => { + self.verify_value_list(inst, args, errors)?; + self.verify_exception_table(inst, exception, errors)?; + self.verify_exception_compatible_abi(inst, exception, errors)?; + } FuncAddr { func_ref, .. } => { self.verify_func_ref(inst, func_ref, errors)?; } @@ -872,6 +899,56 @@ impl<'a> Verifier<'a> { } } + fn verify_exception_table( + &self, + inst: Inst, + et: ExceptionTable, + errors: &mut VerifierErrors, + ) -> VerifierStepResult { + // Verify that the exception table reference itself is valid. + if !self.func.stencil.dfg.exception_tables.is_valid(et) { + errors.nonfatal(( + inst, + self.context(inst), + format!("invalid exception table reference {et}"), + ))?; + } + + let pool = &self.func.stencil.dfg.value_lists; + let exdata = &self.func.stencil.dfg.exception_tables[et]; + + // Verify that the exception table's signature reference + // is valid. + self.verify_sig_ref(inst, exdata.signature(), errors)?; + + // Verify that the exception table's block references are valid. + for block in exdata.all_branches() { + self.verify_block(inst, block.block(pool), errors)?; + } + Ok(()) + } + + fn verify_exception_compatible_abi( + &self, + inst: Inst, + et: ExceptionTable, + errors: &mut VerifierErrors, + ) -> VerifierStepResult { + let callee_sig_ref = self.func.dfg.exception_tables[et].signature(); + let callee_sig = &self.func.dfg.signatures[callee_sig_ref]; + let callee_call_conv = callee_sig.call_conv; + if !callee_call_conv.supports_exceptions() { + errors.nonfatal(( + inst, + self.context(inst), + format!( + "calling convention `{callee_call_conv}` of callee does not support exceptions" + ), + ))?; + } + Ok(()) + } + fn verify_value( &self, loc_inst: Inst, @@ -1314,24 +1391,39 @@ impl<'a> Verifier<'a> { ) -> VerifierStepResult { match &self.func.dfg.insts[inst] { ir::InstructionData::Jump { destination, .. } => { - self.typecheck_block_call(inst, destination, errors)?; + self.typecheck_block_call(inst, destination, BlockCallTargetType::Normal, errors)?; } ir::InstructionData::Brif { blocks: [block_then, block_else], .. } => { - self.typecheck_block_call(inst, block_then, errors)?; - self.typecheck_block_call(inst, block_else, errors)?; + self.typecheck_block_call(inst, block_then, BlockCallTargetType::Normal, errors)?; + self.typecheck_block_call(inst, block_else, BlockCallTargetType::Normal, errors)?; } ir::InstructionData::BranchTable { table, .. } => { for block in self.func.stencil.dfg.jump_tables[*table].all_branches() { - self.typecheck_block_call(inst, block, errors)?; + self.typecheck_block_call(inst, block, BlockCallTargetType::Normal, errors)?; + } + } + ir::InstructionData::TryCall { exception, .. } + | ir::InstructionData::TryCallIndirect { exception, .. } => { + let exdata = &self.func.dfg.exception_tables[*exception]; + self.typecheck_block_call( + inst, + exdata.normal_return(), + BlockCallTargetType::ExNormalRet, + errors, + )?; + for (_tag, block) in exdata.catches() { + self.typecheck_block_call(inst, block, BlockCallTargetType::Exception, errors)?; } } inst => debug_assert!(!inst.opcode().is_branch()), } - match self.func.dfg.insts[inst].analyze_call(&self.func.dfg.value_lists) { + match self.func.dfg.insts[inst] + .analyze_call(&self.func.dfg.value_lists, &self.func.dfg.exception_tables) + { CallInfo::Direct(func_ref, args) => { let sig_ref = self.func.dfg.ext_funcs[func_ref].signature; let arg_types = self.func.dfg.signatures[sig_ref] @@ -1340,6 +1432,26 @@ impl<'a> Verifier<'a> { .map(|a| a.value_type); self.typecheck_variable_args_iterator(inst, arg_types, args, errors)?; } + CallInfo::DirectWithSig(func_ref, sig_ref, args) => { + let expected_sig_ref = self.func.dfg.ext_funcs[func_ref].signature; + let sigdata = &self.func.dfg.signatures; + // Compare signatures by value, not by ID -- any + // equivalent signature ID is acceptable. + if sigdata[sig_ref] != sigdata[expected_sig_ref] { + errors.nonfatal(( + inst, + self.context(inst), + format!( + "exception table signature {sig_ref} did not match function {func_ref}'s signature {expected_sig_ref}" + ), + ))?; + } + let arg_types = self.func.dfg.signatures[sig_ref] + .params + .iter() + .map(|a| a.value_type); + self.typecheck_variable_args_iterator(inst, arg_types, args, errors)?; + } CallInfo::Indirect(sig_ref, args) => { let arg_types = self.func.dfg.signatures[sig_ref] .params @@ -1352,27 +1464,143 @@ impl<'a> Verifier<'a> { Ok(()) } + fn pointer_type_or_error(&self, inst: Inst, errors: &mut VerifierErrors) -> Result { + // Ensure we have an ISA so we know what the pointer size is. + if let Some(isa) = self.isa { + Ok(isa.pointer_type()) + } else { + errors + .fatal(( + inst, + self.context(inst), + format!("need an ISA to validate correct pointer type"), + )) + // Will always return an `Err`, but the `Ok` type + // doesn't match, so map it. + .map(|_| Type::default()) + } + } + fn typecheck_block_call( &self, inst: Inst, block: &ir::BlockCall, + target_type: BlockCallTargetType, errors: &mut VerifierErrors, ) -> VerifierStepResult { let pool = &self.func.dfg.value_lists; - let iter = self - .func - .dfg - .block_params(block.block(pool)) - .iter() - .map(|&v| self.func.dfg.value_type(v)); - let args = block.args_slice(pool); - self.typecheck_variable_args_iterator(inst, iter, args, errors) + let block_params = self.func.dfg.block_params(block.block(pool)); + let args = block.args(pool); + if args.len() != block_params.len() { + return errors.nonfatal(( + inst, + self.context(inst), + format!( + "mismatched argument count for `{}`: got {}, expected {}", + self.func.dfg.display_inst(inst), + args.len(), + block_params.len(), + ), + )); + } + for (arg, param) in args.zip(block_params.iter()) { + let arg_ty = self.block_call_arg_ty(arg, inst, target_type, errors)?; + let param_ty = self.func.dfg.value_type(*param); + if arg_ty != param_ty { + errors.nonfatal(( + inst, + self.context(inst), + format!("arg {arg} has type {arg_ty}, expected {param_ty}"), + ))?; + } + } + Ok(()) + } + + fn block_call_arg_ty( + &self, + arg: BlockArg, + inst: Inst, + target_type: BlockCallTargetType, + errors: &mut VerifierErrors, + ) -> Result { + match arg { + BlockArg::Value(v) => Ok(self.func.dfg.value_type(v)), + BlockArg::TryCallRet(_) | BlockArg::TryCallExn(_) => { + // Get the invoked signature. + let et = match self.func.dfg.insts[inst].exception_table() { + Some(et) => et, + None => { + errors.fatal(( + inst, + self.context(inst), + format!( + "`retN` block argument in block-call not on `try_call` instruction" + ), + ))?; + unreachable!() + } + }; + let exdata = &self.func.dfg.exception_tables[et]; + let sig = &self.func.dfg.signatures[exdata.signature()]; + + match (arg, target_type) { + (BlockArg::TryCallRet(i), BlockCallTargetType::ExNormalRet) + if (i as usize) < sig.returns.len() => + { + Ok(sig.returns[i as usize].value_type) + } + (BlockArg::TryCallRet(_), BlockCallTargetType::ExNormalRet) => { + errors.fatal(( + inst, + self.context(inst), + format!("out-of-bounds `retN` block argument"), + ))?; + unreachable!() + } + (BlockArg::TryCallRet(_), _) => { + errors.fatal(( + inst, + self.context(inst), + format!("`retN` block argument used outside normal-return target of `try_call`"), + ))?; + unreachable!() + } + (BlockArg::TryCallExn(i), BlockCallTargetType::Exception) => { + match sig + .call_conv + .exception_payload_types(self.pointer_type_or_error(inst, errors)?) + .get(i as usize) + { + Some(ty) => Ok(*ty), + None => { + errors.fatal(( + inst, + self.context(inst), + format!("out-of-bounds `exnN` block argument"), + ))?; + unreachable!() + } + } + } + (BlockArg::TryCallExn(_), _) => { + errors.fatal(( + inst, + self.context(inst), + format!("`exnN` block argument used outside normal-return target of `try_call`"), + ))?; + unreachable!() + } + _ => unreachable!(), + } + } + } } - fn typecheck_variable_args_iterator>( + fn typecheck_variable_args_iterator( &self, inst: Inst, - iter: I, + iter: impl ExactSizeIterator, variable_args: &[Value], errors: &mut VerifierErrors, ) -> VerifierStepResult { diff --git a/cranelift/codegen/src/write.rs b/cranelift/codegen/src/write.rs index 5c77aa679275..ef0e9fd41d40 100644 --- a/cranelift/codegen/src/write.rs +++ b/cranelift/codegen/src/write.rs @@ -389,6 +389,7 @@ fn write_instruction( pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt::Result { let pool = &dfg.value_lists; let jump_tables = &dfg.jump_tables; + let exception_tables = &dfg.exception_tables; use crate::ir::instructions::InstructionData::*; let ctrl_ty = dfg.ctrl_typevar(inst); match dfg.insts[inst] { @@ -479,6 +480,34 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt )?; write_user_stack_map_entries(w, dfg, inst) } + TryCall { + func_ref, + ref args, + exception, + .. + } => { + write!( + w, + " {}({}), {}", + func_ref, + DisplayValues(args.as_slice(pool)), + exception_tables[exception].display(pool), + ) + } + TryCallIndirect { + ref args, + exception, + .. + } => { + let args = args.as_slice(pool); + write!( + w, + " {}({}), {}", + args[0], + DisplayValues(&args[1..]), + exception_tables[exception].display(pool), + ) + } FuncAddr { func_ref, .. } => write!(w, " {func_ref}"), StackLoad { stack_slot, offset, .. diff --git a/cranelift/filetests/filetests/isa/aarch64/exceptions.clif b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif new file mode 100644 index 000000000000..67913660dd4b --- /dev/null +++ b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif @@ -0,0 +1,123 @@ +test compile precise-output +target aarch64 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = colocated %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; stp fp, lr, [sp, #-16]! +; mov fp, sp +; stp x27, x28, [sp, #-16]! +; stp x25, x26, [sp, #-16]! +; stp x23, x24, [sp, #-16]! +; stp x21, x22, [sp, #-16]! +; stp x19, x20, [sp, #-16]! +; stp d14, d15, [sp, #-16]! +; stp d12, d13, [sp, #-16]! +; stp d10, d11, [sp, #-16]! +; stp d8, d9, [sp, #-16]! +; sub sp, sp, #16 +; block0: +; fmov d1, #1 +; mov x2, x0 +; str q1, [sp] +; bl 0; b MachLabel(1); catch [None: MachLabel(2)] +; block1: +; movz w0, #1 +; ldr q1, [sp] +; add sp, sp, #16 +; ldp d8, d9, [sp], #16 +; ldp d10, d11, [sp], #16 +; ldp d12, d13, [sp], #16 +; ldp d14, d15, [sp], #16 +; ldp x19, x20, [sp], #16 +; ldp x21, x22, [sp], #16 +; ldp x23, x24, [sp], #16 +; ldp x25, x26, [sp], #16 +; ldp x27, x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; block2: +; ldr q1, [sp] +; add w0, w0, #1 +; movi v0.2s, #0 +; add sp, sp, #16 +; ldp d8, d9, [sp], #16 +; ldp d10, d11, [sp], #16 +; ldp d12, d13, [sp], #16 +; ldp d14, d15, [sp], #16 +; ldp x19, x20, [sp], #16 +; ldp x21, x22, [sp], #16 +; ldp x23, x24, [sp], #16 +; ldp x25, x26, [sp], #16 +; ldp x27, x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; stp x29, x30, [sp, #-0x10]! +; mov x29, sp +; stp x27, x28, [sp, #-0x10]! +; stp x25, x26, [sp, #-0x10]! +; stp x23, x24, [sp, #-0x10]! +; stp x21, x22, [sp, #-0x10]! +; stp x19, x20, [sp, #-0x10]! +; stp d14, d15, [sp, #-0x10]! +; stp d12, d13, [sp, #-0x10]! +; stp d10, d11, [sp, #-0x10]! +; stp d8, d9, [sp, #-0x10]! +; sub sp, sp, #0x10 +; block1: ; offset 0x30 +; fmov d1, #1.00000000 +; mov x2, x0 +; stur q1, [sp] +; bl #0x3c ; reloc_external Call %g 0 +; block2: ; offset 0x40 +; mov w0, #1 +; ldur q1, [sp] +; add sp, sp, #0x10 +; ldp d8, d9, [sp], #0x10 +; ldp d10, d11, [sp], #0x10 +; ldp d12, d13, [sp], #0x10 +; ldp d14, d15, [sp], #0x10 +; ldp x19, x20, [sp], #0x10 +; ldp x21, x22, [sp], #0x10 +; ldp x23, x24, [sp], #0x10 +; ldp x25, x26, [sp], #0x10 +; ldp x27, x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret +; block3: ; offset 0x78 +; ldur q1, [sp] +; add w0, w0, #1 +; movi v0.2s, #0 +; add sp, sp, #0x10 +; ldp d8, d9, [sp], #0x10 +; ldp d10, d11, [sp], #0x10 +; ldp d12, d13, [sp], #0x10 +; ldp d14, d15, [sp], #0x10 +; ldp x19, x20, [sp], #0x10 +; ldp x21, x22, [sp], #0x10 +; ldp x23, x24, [sp], #0x10 +; ldp x25, x26, [sp], #0x10 +; ldp x27, x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley32/call.clif b/cranelift/filetests/filetests/isa/pulley32/call.clif index a2fa6dd632f4..75c058b3d695 100644 --- a/cranelift/filetests/filetests/isa/pulley32/call.clif +++ b/cranelift/filetests/filetests/isa/pulley32/call.clif @@ -16,7 +16,7 @@ block0: ; push_frame ; block0: ; xzero x2 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xone x0 ; pop_frame ; ret @@ -43,7 +43,7 @@ block0: ; push_frame ; block0: ; xzero x2 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xone x0 ; pop_frame ; ret @@ -75,7 +75,7 @@ block0: ; xone x4 ; xconst8 x5, 2 ; xconst8 x6, 3 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p3i), XReg(p4i), XReg(p5i), XReg(p6i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p3i), XReg(p4i), XReg(p5i), XReg(p6i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -103,7 +103,7 @@ block0: ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }], clobbers: PRegSet { bits: [65520, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }], clobbers: PRegSet { bits: [65520, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xadd64 x4, x0, x2 ; xadd64 x3, x1, x3 ; xadd64 x0, x4, x3 @@ -149,7 +149,7 @@ block0: ; xmov x11, x14 ; xmov x12, x14 ; xmov x13, x14 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame_restore 64, {} ; ret ; @@ -217,7 +217,7 @@ block0: ; push_frame_save 112, {x16, x17, x18, x19, x26, x27, x28, x29} ; block0: ; x12 = load_addr OutgoingArg(0) -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p12i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }, CallRetPair { vreg: Writable { reg: p4i }, location: Reg(p4i, types::I64) }, CallRetPair { vreg: Writable { reg: p5i }, location: Reg(p5i, types::I64) }, CallRetPair { vreg: Writable { reg: p6i }, location: Reg(p6i, types::I64) }, CallRetPair { vreg: Writable { reg: p7i }, location: Reg(p7i, types::I64) }, CallRetPair { vreg: Writable { reg: p8i }, location: Reg(p8i, types::I64) }, CallRetPair { vreg: Writable { reg: p9i }, location: Reg(p9i, types::I64) }, CallRetPair { vreg: Writable { reg: p10i }, location: Reg(p10i, types::I64) }, CallRetPair { vreg: Writable { reg: p11i }, location: Reg(p11i, types::I64) }, CallRetPair { vreg: Writable { reg: p12i }, location: Reg(p12i, types::I64) }, CallRetPair { vreg: Writable { reg: p13i }, location: Reg(p13i, types::I64) }, CallRetPair { vreg: Writable { reg: p14i }, location: Reg(p14i, types::I64) }, CallRetPair { vreg: Writable { reg: p27i }, location: Stack(OutgoingArg(0), types::I64) }, CallRetPair { vreg: Writable { reg: p19i }, location: Stack(OutgoingArg(8), types::I64) }, CallRetPair { vreg: Writable { reg: p29i }, location: Stack(OutgoingArg(16), types::I64) }, CallRetPair { vreg: Writable { reg: p16i }, location: Stack(OutgoingArg(24), types::I64) }, CallRetPair { vreg: Writable { reg: p17i }, location: Stack(OutgoingArg(32), types::I64) }, CallRetPair { vreg: Writable { reg: p18i }, location: Stack(OutgoingArg(40), types::I64) }], clobbers: PRegSet { bits: [32768, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p12i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }, CallRetPair { vreg: Writable { reg: p4i }, location: Reg(p4i, types::I64) }, CallRetPair { vreg: Writable { reg: p5i }, location: Reg(p5i, types::I64) }, CallRetPair { vreg: Writable { reg: p6i }, location: Reg(p6i, types::I64) }, CallRetPair { vreg: Writable { reg: p7i }, location: Reg(p7i, types::I64) }, CallRetPair { vreg: Writable { reg: p8i }, location: Reg(p8i, types::I64) }, CallRetPair { vreg: Writable { reg: p9i }, location: Reg(p9i, types::I64) }, CallRetPair { vreg: Writable { reg: p10i }, location: Reg(p10i, types::I64) }, CallRetPair { vreg: Writable { reg: p11i }, location: Reg(p11i, types::I64) }, CallRetPair { vreg: Writable { reg: p12i }, location: Reg(p12i, types::I64) }, CallRetPair { vreg: Writable { reg: p13i }, location: Reg(p13i, types::I64) }, CallRetPair { vreg: Writable { reg: p14i }, location: Reg(p14i, types::I64) }, CallRetPair { vreg: Writable { reg: p27i }, location: Stack(OutgoingArg(0), types::I64) }, CallRetPair { vreg: Writable { reg: p19i }, location: Stack(OutgoingArg(8), types::I64) }, CallRetPair { vreg: Writable { reg: p29i }, location: Stack(OutgoingArg(16), types::I64) }, CallRetPair { vreg: Writable { reg: p16i }, location: Stack(OutgoingArg(24), types::I64) }, CallRetPair { vreg: Writable { reg: p17i }, location: Stack(OutgoingArg(32), types::I64) }, CallRetPair { vreg: Writable { reg: p18i }, location: Stack(OutgoingArg(40), types::I64) }], clobbers: PRegSet { bits: [32768, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xadd64 x26, x0, x1 ; xadd64 x28, x2, x3 ; xadd64 x2, x4, x5 @@ -248,7 +248,6 @@ block0: ; push_frame_save 112, x16, x17, x18, x19, x26, x27, x28, x29 ; xmov x12, sp ; call1 x12, 0x0 // target = 0x8 -; jump 0x5 // target = 0x13 ; xload64le_o32 x27, sp, 0 ; xload64le_o32 x19, sp, 8 ; xload64le_o32 x29, sp, 16 @@ -292,7 +291,7 @@ block0(v0: i32): ; VCode: ; push_frame ; block0: -; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0 } +; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley32/exceptions.clif b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif new file mode 100644 index 000000000000..833c20fb1da1 --- /dev/null +++ b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif @@ -0,0 +1,150 @@ +test compile precise-output +target pulley32 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = colocated %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i32): + v8 = iadd_imm.i32 v6, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; push_frame_save 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; fstore64 sp+136, f16 // flags = notrap aligned +; fstore64 sp+128, f17 // flags = notrap aligned +; fstore64 sp+120, f18 // flags = notrap aligned +; fstore64 sp+112, f19 // flags = notrap aligned +; fstore64 sp+104, f20 // flags = notrap aligned +; fstore64 sp+96, f21 // flags = notrap aligned +; fstore64 sp+88, f22 // flags = notrap aligned +; fstore64 sp+80, f23 // flags = notrap aligned +; fstore64 sp+72, f24 // flags = notrap aligned +; fstore64 sp+64, f25 // flags = notrap aligned +; fstore64 sp+56, f26 // flags = notrap aligned +; fstore64 sp+48, f27 // flags = notrap aligned +; fstore64 sp+40, f28 // flags = notrap aligned +; fstore64 sp+32, f29 // flags = notrap aligned +; fstore64 sp+24, f30 // flags = notrap aligned +; fstore64 sp+16, f31 // flags = notrap aligned +; block0: +; fconst64 f1, 4607182418800017408 +; fstore64 Slot(0), f1 // flags = notrap aligned +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I32) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_dests: [(None, MachLabel(2))] }) }; jump MachLabel(1); catch [None: MachLabel(2)] +; block1: +; xone x0 +; f1 = fload64 Slot(0) // flags = notrap aligned +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; block2: +; f1 = fload64 Slot(0) // flags = notrap aligned +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; +; Disassembled: +; push_frame_save 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; fstore64le_o32 sp, 136, f16 +; fstore64le_o32 sp, 128, f17 +; fstore64le_o32 sp, 120, f18 +; fstore64le_o32 sp, 112, f19 +; fstore64le_o32 sp, 104, f20 +; fstore64le_o32 sp, 96, f21 +; fstore64le_o32 sp, 88, f22 +; fstore64le_o32 sp, 80, f23 +; fstore64le_o32 sp, 72, f24 +; fstore64le_o32 sp, 64, f25 +; fstore64le_o32 sp, 56, f26 +; fstore64le_o32 sp, 48, f27 +; fstore64le_o32 sp, 40, f28 +; fstore64le_o32 sp, 32, f29 +; fstore64le_o32 sp, 24, f30 +; fstore64le_o32 sp, 16, f31 +; fconst64 f1, 4607182418800017408 +; fstore64le_o32 sp, 0, f1 +; call 0x0 // target = 0xaa +; xone x0 +; fload64le_o32 f1, sp, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret +; fload64le_o32 f1, sp, 0 +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley32/extend.clif b/cranelift/filetests/filetests/isa/pulley32/extend.clif index d82485e18853..c21d320067fa 100644 --- a/cranelift/filetests/filetests/isa/pulley32/extend.clif +++ b/cranelift/filetests/filetests/isa/pulley32/extend.clif @@ -12,7 +12,7 @@ block0(v0: i8): ; push_frame ; block0: ; zext8 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -34,7 +34,7 @@ block0(v0: i16): ; push_frame ; block0: ; zext16 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -55,7 +55,7 @@ block0(v0: i32): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -75,7 +75,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -96,7 +96,7 @@ block0(v0: i8): ; push_frame ; block0: ; sext8 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -118,7 +118,7 @@ block0(v0: i16): ; push_frame ; block0: ; sext16 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -139,7 +139,7 @@ block0(v0: i32): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -159,7 +159,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley64/call.clif b/cranelift/filetests/filetests/isa/pulley64/call.clif index 64044b01860c..50c104c8543a 100644 --- a/cranelift/filetests/filetests/isa/pulley64/call.clif +++ b/cranelift/filetests/filetests/isa/pulley64/call.clif @@ -16,7 +16,7 @@ block0: ; push_frame ; block0: ; xzero x2 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xone x0 ; pop_frame ; ret @@ -43,7 +43,7 @@ block0: ; push_frame ; block0: ; xzero x2 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xone x0 ; pop_frame ; ret @@ -75,7 +75,7 @@ block0: ; xone x4 ; xconst8 x5, 2 ; xconst8 x6, 3 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p3i), XReg(p4i), XReg(p5i), XReg(p6i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p3i), XReg(p4i), XReg(p5i), XReg(p6i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -103,7 +103,7 @@ block0: ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }], clobbers: PRegSet { bits: [65520, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }], clobbers: PRegSet { bits: [65520, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xadd64 x4, x0, x2 ; xadd64 x3, x1, x3 ; xadd64 x0, x4, x3 @@ -149,7 +149,7 @@ block0: ; xmov x11, x14 ; xmov x12, x14 ; xmov x13, x14 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame_restore 64, {} ; ret ; @@ -217,7 +217,7 @@ block0: ; push_frame_save 112, {x16, x17, x18, x19, x26, x27, x28, x29} ; block0: ; x12 = load_addr OutgoingArg(0) -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p12i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }, CallRetPair { vreg: Writable { reg: p4i }, location: Reg(p4i, types::I64) }, CallRetPair { vreg: Writable { reg: p5i }, location: Reg(p5i, types::I64) }, CallRetPair { vreg: Writable { reg: p6i }, location: Reg(p6i, types::I64) }, CallRetPair { vreg: Writable { reg: p7i }, location: Reg(p7i, types::I64) }, CallRetPair { vreg: Writable { reg: p8i }, location: Reg(p8i, types::I64) }, CallRetPair { vreg: Writable { reg: p9i }, location: Reg(p9i, types::I64) }, CallRetPair { vreg: Writable { reg: p10i }, location: Reg(p10i, types::I64) }, CallRetPair { vreg: Writable { reg: p11i }, location: Reg(p11i, types::I64) }, CallRetPair { vreg: Writable { reg: p12i }, location: Reg(p12i, types::I64) }, CallRetPair { vreg: Writable { reg: p13i }, location: Reg(p13i, types::I64) }, CallRetPair { vreg: Writable { reg: p14i }, location: Reg(p14i, types::I64) }, CallRetPair { vreg: Writable { reg: p27i }, location: Stack(OutgoingArg(0), types::I64) }, CallRetPair { vreg: Writable { reg: p19i }, location: Stack(OutgoingArg(8), types::I64) }, CallRetPair { vreg: Writable { reg: p29i }, location: Stack(OutgoingArg(16), types::I64) }, CallRetPair { vreg: Writable { reg: p16i }, location: Stack(OutgoingArg(24), types::I64) }, CallRetPair { vreg: Writable { reg: p17i }, location: Stack(OutgoingArg(32), types::I64) }, CallRetPair { vreg: Writable { reg: p18i }, location: Stack(OutgoingArg(40), types::I64) }], clobbers: PRegSet { bits: [32768, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p12i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }, CallRetPair { vreg: Writable { reg: p2i }, location: Reg(p2i, types::I64) }, CallRetPair { vreg: Writable { reg: p3i }, location: Reg(p3i, types::I64) }, CallRetPair { vreg: Writable { reg: p4i }, location: Reg(p4i, types::I64) }, CallRetPair { vreg: Writable { reg: p5i }, location: Reg(p5i, types::I64) }, CallRetPair { vreg: Writable { reg: p6i }, location: Reg(p6i, types::I64) }, CallRetPair { vreg: Writable { reg: p7i }, location: Reg(p7i, types::I64) }, CallRetPair { vreg: Writable { reg: p8i }, location: Reg(p8i, types::I64) }, CallRetPair { vreg: Writable { reg: p9i }, location: Reg(p9i, types::I64) }, CallRetPair { vreg: Writable { reg: p10i }, location: Reg(p10i, types::I64) }, CallRetPair { vreg: Writable { reg: p11i }, location: Reg(p11i, types::I64) }, CallRetPair { vreg: Writable { reg: p12i }, location: Reg(p12i, types::I64) }, CallRetPair { vreg: Writable { reg: p13i }, location: Reg(p13i, types::I64) }, CallRetPair { vreg: Writable { reg: p14i }, location: Reg(p14i, types::I64) }, CallRetPair { vreg: Writable { reg: p27i }, location: Stack(OutgoingArg(0), types::I64) }, CallRetPair { vreg: Writable { reg: p19i }, location: Stack(OutgoingArg(8), types::I64) }, CallRetPair { vreg: Writable { reg: p29i }, location: Stack(OutgoingArg(16), types::I64) }, CallRetPair { vreg: Writable { reg: p16i }, location: Stack(OutgoingArg(24), types::I64) }, CallRetPair { vreg: Writable { reg: p17i }, location: Stack(OutgoingArg(32), types::I64) }, CallRetPair { vreg: Writable { reg: p18i }, location: Stack(OutgoingArg(40), types::I64) }], clobbers: PRegSet { bits: [32768, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xadd64 x26, x0, x1 ; xadd64 x28, x2, x3 ; xadd64 x2, x4, x5 @@ -248,7 +248,6 @@ block0: ; push_frame_save 112, x16, x17, x18, x19, x26, x27, x28, x29 ; xmov x12, sp ; call1 x12, 0x0 // target = 0x8 -; jump 0x5 // target = 0x13 ; xload64le_o32 x27, sp, 0 ; xload64le_o32 x19, sp, 8 ; xload64le_o32 x29, sp, 16 @@ -292,7 +291,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0 } +; indirect_call x0, CallInfo { dest: XReg(p0i), uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -342,7 +341,7 @@ block0: ; xmov x11, x14 ; xmov x12, x14 ; xmov x13, x14 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p14i), XReg(p14i), XReg(p14i), XReg(p14i)] }, uses: [CallArgPair { vreg: p4i, preg: p4i }, CallArgPair { vreg: p5i, preg: p5i }, CallArgPair { vreg: p6i, preg: p6i }, CallArgPair { vreg: p7i, preg: p7i }, CallArgPair { vreg: p8i, preg: p8i }, CallArgPair { vreg: p9i, preg: p9i }, CallArgPair { vreg: p10i, preg: p10i }, CallArgPair { vreg: p11i, preg: p11i }, CallArgPair { vreg: p12i, preg: p12i }, CallArgPair { vreg: p13i, preg: p13i }, CallArgPair { vreg: p14i, preg: p14i }], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame_restore 80, {} ; ret ; @@ -388,7 +387,7 @@ block0(v0: i32): ; xstore64 sp+1000008, x20 // flags = notrap aligned ; block0: ; xmov x20, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }], clobbers: PRegSet { bits: [65534, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; xmov x5, x20 ; xadd32 x0, x5, x0 ; x20 = xload64 sp+1000008 // flags = notrap aligned diff --git a/cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif b/cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif index b81ab16c44c7..6e1b88a25a41 100644 --- a/cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif +++ b/cranelift/filetests/filetests/isa/pulley64/call_indirect_host.clif @@ -11,7 +11,7 @@ block0: ; VCode: ; push_frame ; block0: -; indirect_call_host CallInfo { dest: User(userextname0), uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: SystemV, caller_conv: Fast, callee_pop_size: 0 } +; indirect_call_host CallInfo { dest: User(userextname0), uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: SystemV, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/pulley64/exceptions.clif b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif new file mode 100644 index 000000000000..fdccf3515dc0 --- /dev/null +++ b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif @@ -0,0 +1,151 @@ +test compile precise-output +target pulley64 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = colocated %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; push_frame_save 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; fstore64 sp+136, f16 // flags = notrap aligned +; fstore64 sp+128, f17 // flags = notrap aligned +; fstore64 sp+120, f18 // flags = notrap aligned +; fstore64 sp+112, f19 // flags = notrap aligned +; fstore64 sp+104, f20 // flags = notrap aligned +; fstore64 sp+96, f21 // flags = notrap aligned +; fstore64 sp+88, f22 // flags = notrap aligned +; fstore64 sp+80, f23 // flags = notrap aligned +; fstore64 sp+72, f24 // flags = notrap aligned +; fstore64 sp+64, f25 // flags = notrap aligned +; fstore64 sp+56, f26 // flags = notrap aligned +; fstore64 sp+48, f27 // flags = notrap aligned +; fstore64 sp+40, f28 // flags = notrap aligned +; fstore64 sp+32, f29 // flags = notrap aligned +; fstore64 sp+24, f30 // flags = notrap aligned +; fstore64 sp+16, f31 // flags = notrap aligned +; block0: +; fconst64 f1, 4607182418800017408 +; fstore64 Slot(0), f1 // flags = notrap aligned +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_dests: [(None, MachLabel(2))] }) }; jump MachLabel(1); catch [None: MachLabel(2)] +; block1: +; xone x0 +; f1 = fload64 Slot(0) // flags = notrap aligned +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; block2: +; f1 = fload64 Slot(0) // flags = notrap aligned +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; +; Disassembled: +; push_frame_save 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; fstore64le_o32 sp, 136, f16 +; fstore64le_o32 sp, 128, f17 +; fstore64le_o32 sp, 120, f18 +; fstore64le_o32 sp, 112, f19 +; fstore64le_o32 sp, 104, f20 +; fstore64le_o32 sp, 96, f21 +; fstore64le_o32 sp, 88, f22 +; fstore64le_o32 sp, 80, f23 +; fstore64le_o32 sp, 72, f24 +; fstore64le_o32 sp, 64, f25 +; fstore64le_o32 sp, 56, f26 +; fstore64le_o32 sp, 48, f27 +; fstore64le_o32 sp, 40, f28 +; fstore64le_o32 sp, 32, f29 +; fstore64le_o32 sp, 24, f30 +; fstore64le_o32 sp, 16, f31 +; fconst64 f1, 4607182418800017408 +; fstore64le_o32 sp, 0, f1 +; call 0x0 // target = 0xaa +; xone x0 +; fload64le_o32 f1, sp, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret +; fload64le_o32 f1, sp, 0 +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley64/extend.clif b/cranelift/filetests/filetests/isa/pulley64/extend.clif index 0efbfb6a9a5a..90a64cf0bb9f 100644 --- a/cranelift/filetests/filetests/isa/pulley64/extend.clif +++ b/cranelift/filetests/filetests/isa/pulley64/extend.clif @@ -12,7 +12,7 @@ block0(v0: i8): ; push_frame ; block0: ; zext8 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -34,7 +34,7 @@ block0(v0: i16): ; push_frame ; block0: ; zext16 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -56,7 +56,7 @@ block0(v0: i32): ; push_frame ; block0: ; zext32 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -77,7 +77,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -98,7 +98,7 @@ block0(v0: i8): ; push_frame ; block0: ; sext8 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -120,7 +120,7 @@ block0(v0: i16): ; push_frame ; block0: ; sext16 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -142,7 +142,7 @@ block0(v0: i32): ; push_frame ; block0: ; sext32 x2, x0 -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p2i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; @@ -163,7 +163,7 @@ block0(v0: i64): ; VCode: ; push_frame ; block0: -; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0 } +; call CallInfo { dest: PulleyCall { name: TestCase(%g), args: [XReg(p0i)] }, uses: [], defs: [], clobbers: PRegSet { bits: [65535, 65535, 4294967295, 0] }, callee_conv: Fast, caller_conv: Fast, callee_pop_size: 0, try_call_info: None } ; pop_frame ; ret ; diff --git a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif new file mode 100644 index 000000000000..084ebb5339a2 --- /dev/null +++ b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif @@ -0,0 +1,222 @@ +test compile precise-output +target riscv64 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = colocated %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; addi sp,sp,-16 +; sd ra,8(sp) +; sd fp,0(sp) +; mv fp,sp +; addi sp,sp,-208 +; sd fp,200(sp) +; sd s1,192(sp) +; sd s2,184(sp) +; sd s3,176(sp) +; sd s4,168(sp) +; sd s5,160(sp) +; sd s6,152(sp) +; sd s7,144(sp) +; sd s8,136(sp) +; sd s9,128(sp) +; sd s10,120(sp) +; sd s11,112(sp) +; fsd fs0,104(sp) +; fsd fs2,96(sp) +; fsd fs3,88(sp) +; fsd fs4,80(sp) +; fsd fs5,72(sp) +; fsd fs6,64(sp) +; fsd fs7,56(sp) +; fsd fs8,48(sp) +; fsd fs9,40(sp) +; fsd fs10,32(sp) +; fsd fs11,24(sp) +; block0: +; lui a4,1023 +; slli a1,a4,40 +; fmv.d.x fa1,a1 +; fsd fa1,0(slot) +; call %g; j MachLabel(1); catch [None: MachLabel(2)] +; block1: +; li a0,1 +; fld fa1,0(slot) +; ld fp,200(sp) +; ld s1,192(sp) +; ld s2,184(sp) +; ld s3,176(sp) +; ld s4,168(sp) +; ld s5,160(sp) +; ld s6,152(sp) +; ld s7,144(sp) +; ld s8,136(sp) +; ld s9,128(sp) +; ld s10,120(sp) +; ld s11,112(sp) +; fld fs0,104(sp) +; fld fs2,96(sp) +; fld fs3,88(sp) +; fld fs4,80(sp) +; fld fs5,72(sp) +; fld fs6,64(sp) +; fld fs7,56(sp) +; fld fs8,48(sp) +; fld fs9,40(sp) +; fld fs10,32(sp) +; fld fs11,24(sp) +; addi sp,sp,208 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; block2: +; fld fa1,0(slot) +; addiw a0,a0,1 +; fmv.w.x fa0,zero +; ld fp,200(sp) +; ld s1,192(sp) +; ld s2,184(sp) +; ld s3,176(sp) +; ld s4,168(sp) +; ld s5,160(sp) +; ld s6,152(sp) +; ld s7,144(sp) +; ld s8,136(sp) +; ld s9,128(sp) +; ld s10,120(sp) +; ld s11,112(sp) +; fld fs0,104(sp) +; fld fs2,96(sp) +; fld fs3,88(sp) +; fld fs4,80(sp) +; fld fs5,72(sp) +; fld fs6,64(sp) +; fld fs7,56(sp) +; fld fs8,48(sp) +; fld fs9,40(sp) +; fld fs10,32(sp) +; fld fs11,24(sp) +; addi sp,sp,208 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; addi sp, sp, -0x10 +; sd ra, 8(sp) +; sd s0, 0(sp) +; mv s0, sp +; addi sp, sp, -0xd0 +; sd s0, 0xc8(sp) +; sd s1, 0xc0(sp) +; sd s2, 0xb8(sp) +; sd s3, 0xb0(sp) +; sd s4, 0xa8(sp) +; sd s5, 0xa0(sp) +; sd s6, 0x98(sp) +; sd s7, 0x90(sp) +; sd s8, 0x88(sp) +; sd s9, 0x80(sp) +; sd s10, 0x78(sp) +; sd s11, 0x70(sp) +; fsd fs0, 0x68(sp) +; fsd fs2, 0x60(sp) +; fsd fs3, 0x58(sp) +; fsd fs4, 0x50(sp) +; fsd fs5, 0x48(sp) +; fsd fs6, 0x40(sp) +; fsd fs7, 0x38(sp) +; fsd fs8, 0x30(sp) +; fsd fs9, 0x28(sp) +; fsd fs10, 0x20(sp) +; fsd fs11, 0x18(sp) +; block1: ; offset 0x70 +; lui a4, 0x3ff +; slli a1, a4, 0x28 +; fmv.d.x fa1, a1 +; fsd fa1, 0(sp) +; auipc ra, 0 ; reloc_external RiscvCallPlt %g 0 +; jalr ra +; block2: ; offset 0x88 +; addi a0, zero, 1 +; fld fa1, 0(sp) +; ld s0, 0xc8(sp) +; ld s1, 0xc0(sp) +; ld s2, 0xb8(sp) +; ld s3, 0xb0(sp) +; ld s4, 0xa8(sp) +; ld s5, 0xa0(sp) +; ld s6, 0x98(sp) +; ld s7, 0x90(sp) +; ld s8, 0x88(sp) +; ld s9, 0x80(sp) +; ld s10, 0x78(sp) +; ld s11, 0x70(sp) +; fld fs0, 0x68(sp) +; fld fs2, 0x60(sp) +; fld fs3, 0x58(sp) +; fld fs4, 0x50(sp) +; fld fs5, 0x48(sp) +; fld fs6, 0x40(sp) +; fld fs7, 0x38(sp) +; fld fs8, 0x30(sp) +; fld fs9, 0x28(sp) +; fld fs10, 0x20(sp) +; fld fs11, 0x18(sp) +; addi sp, sp, 0xd0 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret +; block3: ; offset 0x100 +; fld fa1, 0(sp) +; addiw a0, a0, 1 +; fmv.w.x fa0, zero +; ld s0, 0xc8(sp) +; ld s1, 0xc0(sp) +; ld s2, 0xb8(sp) +; ld s3, 0xb0(sp) +; ld s4, 0xa8(sp) +; ld s5, 0xa0(sp) +; ld s6, 0x98(sp) +; ld s7, 0x90(sp) +; ld s8, 0x88(sp) +; ld s9, 0x80(sp) +; ld s10, 0x78(sp) +; ld s11, 0x70(sp) +; fld fs0, 0x68(sp) +; fld fs2, 0x60(sp) +; fld fs3, 0x58(sp) +; fld fs4, 0x50(sp) +; fld fs5, 0x48(sp) +; fld fs6, 0x40(sp) +; fld fs7, 0x38(sp) +; fld fs8, 0x30(sp) +; fld fs9, 0x28(sp) +; fld fs10, 0x20(sp) +; fld fs11, 0x18(sp) +; addi sp, sp, 0xd0 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret + diff --git a/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif b/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif index 219c0e01849d..2becbf9aff09 100644 --- a/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif +++ b/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif @@ -535,7 +535,6 @@ block0: ; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %tail_callee_stack_rets 0 ; .byte 0x00, 0x00, 0x00, 0x00 ; jalr a4 -; j 4 ; ld a2, 0(sp) ; sd a2, 0xc0(sp) ; ld a2, 8(sp) @@ -968,7 +967,6 @@ block0: ; ld a2, 0x1c0(sp) ; jalr t1 ; addi sp, sp, -0xa0 -; j 4 ; ld a2, 0xa0(sp) ; sd a2, 0x160(sp) ; ld a2, 0xa8(sp) diff --git a/cranelift/filetests/filetests/isa/x64/exceptions.clif b/cranelift/filetests/filetests/isa/x64/exceptions.clif new file mode 100644 index 000000000000..7b1e5ac23e8b --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/exceptions.clif @@ -0,0 +1,267 @@ +test compile precise-output +target x86_64 + +function %f0(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; subq %rsp, $64, %rsp +; movq %rbx, 16(%rsp) +; movq %r12, 24(%rsp) +; movq %r13, 32(%rsp) +; movq %r14, 40(%rsp) +; movq %r15, 48(%rsp) +; block0: +; movabsq $4607182418800017408, %rcx +; movq %rcx, %xmm1 +; movdqu %xmm1, rsp(0 + virtual offset) +; load_ext_name %g+0, %rdx +; call *%rdx; jmp MachLabel(1); catch [None: MachLabel(2)] +; block1: +; movl $1, %eax +; movdqu rsp(0 + virtual offset), %xmm1 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; block2: +; movdqu rsp(0 + virtual offset), %xmm1 +; lea 1(%rax), %eax +; uninit %xmm0 +; xorps %xmm0, %xmm0 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; subq $0x40, %rsp +; movq %rbx, 0x10(%rsp) +; movq %r12, 0x18(%rsp) +; movq %r13, 0x20(%rsp) +; movq %r14, 0x28(%rsp) +; movq %r15, 0x30(%rsp) +; block1: ; offset 0x21 +; movabsq $0x3ff0000000000000, %rcx +; movq %rcx, %xmm1 +; movdqu %xmm1, (%rsp) +; movabsq $0, %rdx ; reloc_external Abs8 %g 0 +; callq *%rdx +; block2: ; offset 0x41 +; movl $1, %eax +; movdqu (%rsp), %xmm1 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq +; block3: ; offset 0x6d +; movdqu (%rsp), %xmm1 +; addl $1, %eax +; xorps %xmm0, %xmm0 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq + + +function %f1(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + brif v1, block1, block2 + + block1: + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block3(ret0, v2), [ default: block4(exn0) ] + + block2: + v3 = iconst.i64 42 + v4 = f32const 0x1234.0 + v5 = f64const 0x5678.0 + brif v1, block4(v3), block3(v4, v5) + + block3(v6: f32, v7: f64): + v8 = iconst.i32 1 + return v8, v6, v7 + + block4(v9: i64): + v10 = ireduce.i32 v9 + v11 = iadd_imm.i32 v10, 1 + v12 = f32const 0x0.0 + v13 = bitcast.f64 v9 + return v11, v12, v13 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; subq %rsp, $64, %rsp +; movq %rbx, 16(%rsp) +; movq %r12, 24(%rsp) +; movq %r13, 32(%rsp) +; movq %r14, 40(%rsp) +; movq %r15, 48(%rsp) +; block0: +; testl %edi, %edi +; jnz label4; j label1 +; block1: +; movl $42, %eax +; movl $1167171584, %ecx +; movd %ecx, %xmm0 +; movabsq $4671813911303946240, %rcx +; movq %rcx, %xmm1 +; testl %edi, %edi +; jnz label2; j label3 +; block2: +; movq %rax, %r11 +; jmp label8 +; block3: +; movdqu %xmm1, rsp(0 + virtual offset) +; jmp label7 +; block4: +; movabsq $4607182418800017408, %r10 +; movq %r10, %xmm1 +; movdqu %xmm1, rsp(0 + virtual offset) +; load_ext_name %g+0, %r11 +; call *%r11; jmp MachLabel(6); catch [None: MachLabel(5)] +; block5: +; movq %rax, %rsi +; movq %rsi, %r11 +; jmp label8 +; block6: +; jmp label7 +; block7: +; movl $1, %eax +; movdqu rsp(0 + virtual offset), %xmm1 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; block8: +; lea 1(%r11), %eax +; movq %r11, %rsi +; uninit %xmm0 +; xorps %xmm0, %xmm0 +; movq %rsi, %xmm1 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; subq $0x40, %rsp +; movq %rbx, 0x10(%rsp) +; movq %r12, 0x18(%rsp) +; movq %r13, 0x20(%rsp) +; movq %r14, 0x28(%rsp) +; movq %r15, 0x30(%rsp) +; block1: ; offset 0x21 +; testl %edi, %edi +; jne 0x60 +; block2: ; offset 0x29 +; movl $0x2a, %eax +; movl $0x4591a000, %ecx +; movd %ecx, %xmm0 +; movabsq $0x40d59e0000000000, %rcx +; movq %rcx, %xmm1 +; testl %edi, %edi +; je 0x56 +; block3: ; offset 0x4e +; movq %rax, %r11 +; jmp 0xbd +; block4: ; offset 0x56 +; movdqu %xmm1, (%rsp) +; jmp 0x91 +; block5: ; offset 0x60 +; movabsq $0x3ff0000000000000, %r10 +; movq %r10, %xmm1 +; movdqu %xmm1, (%rsp) +; movabsq $0, %r11 ; reloc_external Abs8 %g 0 +; callq *%r11 +; jmp 0x91 +; block6: ; offset 0x86 +; movq %rax, %rsi +; movq %rsi, %r11 +; jmp 0xbd +; block7: ; offset 0x91 +; movl $1, %eax +; movdqu (%rsp), %xmm1 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq +; block8: ; offset 0xbd +; leal 1(%r11), %eax +; movq %r11, %rsi +; xorps %xmm0, %xmm0 +; movq %rsi, %xmm1 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq + diff --git a/cranelift/filetests/filetests/runtests/try_call.clif b/cranelift/filetests/filetests/runtests/try_call.clif new file mode 100644 index 000000000000..fe6a9fc286ee --- /dev/null +++ b/cranelift/filetests/filetests/runtests/try_call.clif @@ -0,0 +1,27 @@ +test run +target x86_64 +target aarch64 +target aarch64 sign_return_address +target aarch64 has_pauth sign_return_address +;; target s390x -- not yet +target riscv64 +target riscv64 has_c has_zcb + + +function %callee_i64(i64) -> i64 tail { +block0(v0: i64): + v1 = iadd_imm.i64 v0, 10 + return v1 +} + +function %call_i64(i64) -> i64 { + sig0 = (i64) -> i64 tail + fn0 = %callee_i64(i64) -> i64 tail + +block0(v0: i64): + try_call fn0(v0), sig0, block1(ret0), [] + +block1(v1: i64): + return v1 +} +; run: %call_i64(10) == 20 diff --git a/cranelift/filetests/filetests/verifier/exceptions.clif b/cranelift/filetests/filetests/verifier/exceptions.clif new file mode 100644 index 000000000000..8378acda6abb --- /dev/null +++ b/cranelift/filetests/filetests/verifier/exceptions.clif @@ -0,0 +1,133 @@ +test verifier +target x86_64 + +function %f0(i32) -> i32 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(v2, exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(v6: i64): + v7 = iconst.i32 2 + return v7 + + block3(v8: f64, v9: i64): + v10 = iconst.i32 3 + return v10 +} + +;; Non-matching function return type: v3 on block1. +function %f1(i32) -> i32 { + sig0 = (i32) -> i32 tail + fn0 = %g(i32) -> i32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(exn0, v2) ] ; error: arg ret0 has type i32, expected f32 + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(v6: i64): + v7 = iconst.i32 2 + return v7 + + block3(v8: i64, v9: f64): + v10 = iconst.i32 3 + return v10 +} + +;; Non-matching exception payload type (not a pointer-width value): v6 on block2. +function %f2(i32) -> i32 { + sig0 = (i32) -> i32 tail + fn0 = %g(i32) -> i32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(exn0, v2) ] ; error: arg exn0 has type i64, expected i32 + + block1(v3: i32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(v6: i32): + v7 = iconst.i32 2 + return v7 + + block3(v8: i64, v9: f64): + v10 = iconst.i32 3 + return v10 +} + +;; Non-matching signature in exception table. +function %f3(i32) -> i32 { + sig0 = (i32) -> i32 tail + fn0 = %g(i64) -> i32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(exn0, v2) ] ; error: exception table signature sig0 did not match function fn0's signature sig1 + + block1(v3: i32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(v6: i64): + v7 = iconst.i32 2 + return v7 + + block3(v8: i64, v9: f64): + v10 = iconst.i32 3 + return v10 +} + +;; Too few blockparams (in explicit blockparam part). +function %f4(i32) -> i32 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(exn0), default: block3(exn0, v2) ] ; error: mismatched argument count + + block1(v3: f32): + v5 = iconst.i32 1 + return v5 + + block2(v6: i64): + v7 = iconst.i32 2 + return v7 + + block3(v8: i64, v9: f64): + v10 = iconst.i32 3 + return v10 +} + +;; Non-supported calling convention. +function %f5(i32) -> i32 { + sig0 = (i32) -> f32 system_v + fn0 = %g(i32) -> f32 system_v + + block0(v1: i32): + v2 = f64const 0x1.0 + try_call fn0(v1), sig0, block1(ret0, v2), [ tag1: block2(), default: block3(v2) ] ; error: calling convention `system_v` of callee does not support exceptions + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5 + + block2(): + v7 = iconst.i32 2 + return v7 + + block3(v8: f64): + v10 = iconst.i32 3 + return v10 +} diff --git a/cranelift/filetests/filetests/verifier/type_check.clif b/cranelift/filetests/filetests/verifier/type_check.clif index bbd737ca3475..c76f04fc7dc7 100644 --- a/cranelift/filetests/filetests/verifier/type_check.clif +++ b/cranelift/filetests/filetests/verifier/type_check.clif @@ -78,8 +78,8 @@ function %jump_args() { block0: v0 = iconst.i16 10 v3 = iconst.i64 20 - jump block1(v0, v3) ; error: arg 0 (v0) has type i16, expected i64 - ; error: arg 1 (v3) has type i64, expected i16 + jump block1(v0, v3) ; error: arg v0 has type i16, expected i64 + ; error: arg v3 has type i64, expected i16 block1(v10: i64, v11: i16): return } @@ -88,8 +88,8 @@ function %jump_args2() { block0: v0 = iconst.i16 10 v3 = iconst.i64 20 - brif v0, block1(v3, v0), block1(v0, v3) ; error: arg 0 (v0) has type i16, expected i64 - ; error: arg 1 (v3) has type i64, expected i16 + brif v0, block1(v3, v0), block1(v0, v3) ; error: arg v0 has type i16, expected i64 + ; error: arg v3 has type i64, expected i16 block1(v10: i64, v11: i16): return } @@ -99,9 +99,8 @@ block0: v0 = iconst.i16 10 v1 = iconst.i16 10 brif v0, block1(v1), block2(v1) - ; error: arg 0 (v1) has type i16, expected i64 + ; error: arg v1 has type i16, expected i64 ; error: mismatched argument count - ; error: arg 0 (v1) has type i16, expected f32 block1(v2: i64): return diff --git a/cranelift/frontend/src/frontend.rs b/cranelift/frontend/src/frontend.rs index 927c8833b948..bf59233aee29 100644 --- a/cranelift/frontend/src/frontend.rs +++ b/cranelift/frontend/src/frontend.rs @@ -778,7 +778,9 @@ impl<'a> FunctionBuilder<'a> { /// other jump instructions. pub fn change_jump_destination(&mut self, inst: Inst, old_block: Block, new_block: Block) { let dfg = &mut self.func.dfg; - for block in dfg.insts[inst].branch_destination_mut(&mut dfg.jump_tables) { + for block in + dfg.insts[inst].branch_destination_mut(&mut dfg.jump_tables, &mut dfg.exception_tables) + { if block.block(&dfg.value_lists) == old_block { self.func_ctx.ssa.remove_block_predecessor(old_block, inst); block.set_block(new_block, &mut dfg.value_lists); diff --git a/cranelift/frontend/src/frontend/safepoints.rs b/cranelift/frontend/src/frontend/safepoints.rs index 561d7c338e28..09b6f0179660 100644 --- a/cranelift/frontend/src/frontend/safepoints.rs +++ b/cranelift/frontend/src/frontend/safepoints.rs @@ -840,7 +840,7 @@ mod tests { builder.declare_value_needs_stack_map(b); builder.switch_to_block(block0); builder.ins().call(func_ref, &[a]); - builder.ins().jump(block0, &[a, b]); + builder.ins().jump(block0, &[a.into(), b.into()]); builder.seal_all_blocks(); builder.finalize(); @@ -1434,7 +1434,7 @@ block2: let v2 = builder.ins().iconst(ir::types::I64, 2); builder.declare_value_needs_stack_map(v2); builder.ins().call(func_ref, &[]); - builder.ins().jump(block3, &[v1, v2]); + builder.ins().jump(block3, &[v1.into(), v2.into()]); builder.switch_to_block(block2); let v3 = builder.ins().iconst(ir::types::I64, 3); @@ -1442,7 +1442,7 @@ block2: let v4 = builder.ins().iconst(ir::types::I64, 4); builder.declare_value_needs_stack_map(v4); builder.ins().call(func_ref, &[]); - builder.ins().jump(block3, &[v3, v3]); + builder.ins().jump(block3, &[v3.into(), v3.into()]); builder.switch_to_block(block3); builder.append_block_param(block3, ir::types::I64); @@ -2338,7 +2338,7 @@ block4: builder.declare_value_needs_stack_map(v2); let v3 = builder.func.dfg.block_params(block0)[3]; - builder.ins().jump(block1, &[v3]); + builder.ins().jump(block1, &[v3.into()]); builder.switch_to_block(block1); let v4 = builder.append_block_param(block1, ir::types::I32); @@ -2352,7 +2352,7 @@ block4: builder.ins().call(bar_func_ref, &[v1]); builder.ins().call(foo_func_ref, &[]); let v5 = builder.ins().iadd_imm(v4, -1); - builder.ins().brif(v4, block1, &[v5], block3, &[]); + builder.ins().brif(v4, block1, &[v5.into()], block3, &[]); builder.switch_to_block(block3); builder.ins().call(foo_func_ref, &[]); @@ -2573,7 +2573,7 @@ block3: let v1 = builder.func.dfg.first_result(call_inst); builder.def_var(var_array, v1); let v2 = builder.ins().iconst(ir::types::I32, 0); - builder.ins().jump(block_array_init_loop_head, &[v2]); + builder.ins().jump(block_array_init_loop_head, &[v2.into()]); builder.switch_to_block(block_array_init_loop_head); let v3 = builder.append_block_param(block_array_init_loop_head, ir::types::I32); @@ -2594,7 +2594,7 @@ block3: builder.ins().call(array_init_elem, &[v1, v4]); let v6 = builder.ins().iconst(ir::types::I32, 1); let v7 = builder.ins().iadd(v4, v6); - builder.ins().jump(block_array_init_loop_head, &[v7]); + builder.ins().jump(block_array_init_loop_head, &[v7.into()]); builder.seal_block(block_array_init_loop_head); builder.switch_to_block(block_array_init_loop_done); @@ -2608,7 +2608,7 @@ block3: builder.ins().brif( v10, block_ref_test_done, - &[v9], + &[v9.into()], block_ref_test_non_null, &[], ); @@ -2620,15 +2620,19 @@ block3: let v12 = builder.ins().iconst(ir::types::I32, 0xbeefbeef); let v13 = builder.ins().icmp(ir::condcodes::IntCC::Equal, v11, v12); let v14 = builder.ins().iconst(ir::types::I32, 1); - builder - .ins() - .brif(v13, block_ref_test_done, &[v14], block_ref_test_slow, &[]); + builder.ins().brif( + v13, + block_ref_test_done, + &[v14.into()], + block_ref_test_slow, + &[], + ); builder.switch_to_block(block_ref_test_slow); builder.seal_block(block_ref_test_slow); let call_inst = builder.ins().call(ref_test, &[v8, v12]); let v15 = builder.func.dfg.first_result(call_inst); - builder.ins().jump(block_ref_test_done, &[v15]); + builder.ins().jump(block_ref_test_done, &[v15.into()]); builder.switch_to_block(block_ref_test_done); let v16 = builder.append_block_param(block_ref_test_done, ir::types::I32); diff --git a/cranelift/frontend/src/ssa.rs b/cranelift/frontend/src/ssa.rs index c29a8fd9bd80..9df4095780d6 100644 --- a/cranelift/frontend/src/ssa.rs +++ b/cranelift/frontend/src/ssa.rs @@ -551,7 +551,8 @@ impl SSABuilder { let pred = preds.get_mut(idx, &mut self.inst_pool).unwrap(); let branch = *pred; - let dests = dfg.insts[branch].branch_destination_mut(&mut dfg.jump_tables); + let dests = dfg.insts[branch] + .branch_destination_mut(&mut dfg.jump_tables, &mut dfg.exception_tables); assert!( !dests.is_empty(), "you have declared a non-branch instruction as a predecessor to a block!" @@ -768,9 +769,9 @@ mod tests { .. } => { assert_eq!(block_then.block(&func.dfg.value_lists), block2); - assert_eq!(block_then.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(block_then.args(&func.dfg.value_lists).len(), 0); assert_eq!(block_else.block(&func.dfg.value_lists), block1); - assert_eq!(block_else.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(block_else.args(&func.dfg.value_lists).len(), 0); } _ => assert!(false), }; @@ -780,9 +781,9 @@ mod tests { .. } => { assert_eq!(block_then.block(&func.dfg.value_lists), block2); - assert_eq!(block_then.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(block_then.args(&func.dfg.value_lists).len(), 0); assert_eq!(block_else.block(&func.dfg.value_lists), block1); - assert_eq!(block_else.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(block_else.args(&func.dfg.value_lists).len(), 0); } _ => assert!(false), }; @@ -791,7 +792,7 @@ mod tests { destination: dest, .. } => { assert_eq!(dest.block(&func.dfg.value_lists), block2); - assert_eq!(dest.args_slice(&func.dfg.value_lists).len(), 0); + assert_eq!(dest.args(&func.dfg.value_lists).len(), 0); } _ => assert!(false), }; diff --git a/cranelift/fuzzgen/src/function_generator.rs b/cranelift/fuzzgen/src/function_generator.rs index 5e018d881a35..2b9c634f07c2 100644 --- a/cranelift/fuzzgen/src/function_generator.rs +++ b/cranelift/fuzzgen/src/function_generator.rs @@ -9,8 +9,9 @@ use cranelift::codegen::ir::instructions::{InstructionFormat, ResolvedConstraint use cranelift::codegen::ir::stackslot::StackSize; use cranelift::codegen::ir::{ - types::*, AliasRegion, AtomicRmwOp, Block, ConstantData, Endianness, ExternalName, FuncRef, - Function, LibCall, Opcode, SigRef, Signature, StackSlot, UserExternalName, UserFuncName, Value, + types::*, AliasRegion, AtomicRmwOp, Block, BlockArg, ConstantData, Endianness, ExternalName, + FuncRef, Function, LibCall, Opcode, SigRef, Signature, StackSlot, UserExternalName, + UserFuncName, Value, }; use cranelift::codegen::isa::CallConv; use cranelift::frontend::{FunctionBuilder, FunctionBuilderContext, Switch, Variable}; @@ -818,7 +819,9 @@ static OPCODE_SIGNATURES: LazyLock> = LazyLock::new(|| { | Opcode::Jump | Opcode::Return | Opcode::ReturnCall - | Opcode::ReturnCallIndirect => false, + | Opcode::ReturnCallIndirect + | Opcode::TryCall + | Opcode::TryCallIndirect => false, // Constants are generated outside of `generate_instructions` Opcode::Iconst => false, @@ -1106,7 +1109,9 @@ fn inserter_for_format(fmt: InstructionFormat) -> OpcodeInserter { InstructionFormat::BranchTable | InstructionFormat::Brif | InstructionFormat::Jump - | InstructionFormat::MultiAry => { + | InstructionFormat::MultiAry + | InstructionFormat::TryCall + | InstructionFormat::TryCallIndirect => { panic!("Control-flow instructions should be handled by 'insert_terminator': {fmt:?}") } } @@ -1447,9 +1452,13 @@ where &mut self, builder: &mut FunctionBuilder, block: Block, - ) -> Result> { + ) -> Result> { let (_, sig) = self.resources.blocks[block.as_u32() as usize].clone(); - self.generate_values_for_signature(builder, sig.iter().copied()) + Ok(self + .generate_values_for_signature(builder, sig.iter().copied())? + .into_iter() + .map(|val| BlockArg::Value(val)) + .collect::>()) } fn generate_values_for_signature>( diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs index 8e602fdb262a..3ff6bd11fc20 100644 --- a/cranelift/interpreter/src/step.rs +++ b/cranelift/interpreter/src/step.rs @@ -1,14 +1,15 @@ //! The [step] function interprets a single Cranelift instruction given its [State] and //! [InstructionContext]. use crate::address::{Address, AddressSize}; +use crate::frame::Frame; use crate::instruction::InstructionContext; use crate::state::{InterpreterFunctionRef, MemoryError, State}; use crate::value::{DataValueExt, ValueConversionKind, ValueError, ValueResult}; use cranelift_codegen::data_value::DataValue; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::{ - types, AbiParam, AtomicRmwOp, Block, BlockCall, Endianness, ExternalName, FuncRef, Function, - InstructionData, MemFlags, Opcode, TrapCode, Type, Value as ValueRef, + types, AbiParam, AtomicRmwOp, Block, BlockArg, BlockCall, Endianness, ExternalName, FuncRef, + Function, InstructionData, MemFlags, Opcode, TrapCode, Type, Value as ValueRef, }; use log::trace; use smallvec::{smallvec, SmallVec}; @@ -42,6 +43,19 @@ fn sum_unsigned(head: DataValue, tail: SmallVec<[DataValue; 1]>) -> ValueResult< acc.into_int_unsigned() } +/// Collect a list of block arguments. +fn collect_block_args( + frame: &Frame, + args: impl Iterator, +) -> SmallVec<[DataValue; 1]> { + args.into_iter() + .map(|n| match n { + BlockArg::Value(n) => frame.get(n).clone(), + _ => panic!("exceptions not supported"), + }) + .collect() +} + /// Interpret a single Cranelift instruction. Note that program traps and interpreter errors are /// distinct: a program trap results in `Ok(Flow::Trap(...))` whereas an interpretation error (e.g. /// the types of two values are incompatible) results in `Err(...)`. @@ -234,8 +248,10 @@ where // Retrieve an instruction's branch destination; expects the instruction to be a branch. let continue_at = |block: BlockCall| { - let branch_args = - state.collect_values(block.args_slice(&state.get_current_function().dfg.value_lists)); + let branch_args = collect_block_args( + state.current_frame(), + block.args(&state.get_current_function().dfg.value_lists), + ); Ok(ControlFlow::ContinueAt( block.block(&state.get_current_function().dfg.value_lists), branch_args, @@ -1286,6 +1302,9 @@ where Opcode::X86Pmaddubsw => unimplemented!("X86Pmaddubsw"), Opcode::X86Cvtt2dq => unimplemented!("X86Cvtt2dq"), Opcode::StackSwitch => unimplemented!("StackSwitch"), + + Opcode::TryCall => unimplemented!("TryCall"), + Opcode::TryCallIndirect => unimplemented!("TryCallIndirect"), }) } diff --git a/cranelift/reader/src/lexer.rs b/cranelift/reader/src/lexer.rs index 76b36e1920de..17b3f63c2240 100644 --- a/cranelift/reader/src/lexer.rs +++ b/cranelift/reader/src/lexer.rs @@ -13,43 +13,47 @@ use std::u16; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Token<'a> { Comment(&'a str), - LPar, // '(' - RPar, // ')' - LBrace, // '{' - RBrace, // '}' - LBracket, // '[' - RBracket, // ']' - Minus, // '-' - Plus, // '+' - Multiply, // '*' - Comma, // ',' - Dot, // '.' - Colon, // ':' - Equal, // '=' - Bang, // '!' - At, // '@' - Arrow, // '->' - Float(&'a str), // Floating point immediate - Integer(&'a str), // Integer immediate - Type(types::Type), // i32, f32, i32x4, ... - DynamicType(u32), // dt5 - Value(Value), // v12, v7 - Block(Block), // block3 - Cold, // cold (flag on block) - StackSlot(u32), // ss3 - DynamicStackSlot(u32), // dss4 - GlobalValue(u32), // gv3 - MemoryType(u32), // mt0 - Constant(u32), // const2 - FuncRef(u32), // fn2 - SigRef(u32), // sig2 - UserRef(u32), // u345 - UserNameRef(u32), // userextname345 - Name(&'a str), // %9arbitrary_alphanum, %x3, %0, %function ... - String(&'a str), // "arbitrary quoted string with no escape" ... - HexSequence(&'a str), // #89AF - Identifier(&'a str), // Unrecognized identifier (opcode, enumerator, ...) - SourceLoc(&'a str), // @00c7 + LPar, // '(' + RPar, // ')' + LBrace, // '{' + RBrace, // '}' + LBracket, // '[' + RBracket, // ']' + Minus, // '-' + Plus, // '+' + Multiply, // '*' + Comma, // ',' + Dot, // '.' + Colon, // ':' + Equal, // '=' + Bang, // '!' + At, // '@' + Arrow, // '->' + Float(&'a str), // Floating point immediate + Integer(&'a str), // Integer immediate + Type(types::Type), // i32, f32, i32x4, ... + DynamicType(u32), // dt5 + Value(Value), // v12, v7 + Block(Block), // block3 + Cold, // cold (flag on block) + StackSlot(u32), // ss3 + DynamicStackSlot(u32), // dss4 + GlobalValue(u32), // gv3 + MemoryType(u32), // mt0 + Constant(u32), // const2 + FuncRef(u32), // fn2 + SigRef(u32), // sig2 + UserRef(u32), // u345 + UserNameRef(u32), // userextname345 + ExceptionTableRef(u32), // ex123 + ExceptionTag(u32), // tag123 + TryCallRet(u32), // ret123 + TryCallExn(u32), // exn123 + Name(&'a str), // %9arbitrary_alphanum, %x3, %0, %function ... + String(&'a str), // "arbitrary quoted string with no escape" ... + HexSequence(&'a str), // #89AF + Identifier(&'a str), // Unrecognized identifier (opcode, enumerator, ...) + SourceLoc(&'a str), // @00c7 } /// A `Token` with an associated location. @@ -349,6 +353,10 @@ impl<'a> Lexer<'a> { "sig" => Some(Token::SigRef(number)), "u" => Some(Token::UserRef(number)), "userextname" => Some(Token::UserNameRef(number)), + "extable" => Some(Token::ExceptionTableRef(number)), + "tag" => Some(Token::ExceptionTag(number)), + "ret" => Some(Token::TryCallRet(number)), + "exn" => Some(Token::TryCallExn(number)), _ => None, } } diff --git a/cranelift/reader/src/parser.rs b/cranelift/reader/src/parser.rs index a4160305a726..ab7288748fae 100644 --- a/cranelift/reader/src/parser.rs +++ b/cranelift/reader/src/parser.rs @@ -15,14 +15,15 @@ use cranelift_codegen::ir::immediates::{ }; use cranelift_codegen::ir::instructions::{InstructionData, InstructionFormat, VariableArgs}; use cranelift_codegen::ir::pcc::{BaseExpr, Expr, Fact}; -use cranelift_codegen::ir::types; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{self, UserExternalNameRef}; + use cranelift_codegen::ir::{ - AbiParam, ArgumentExtension, ArgumentPurpose, Block, Constant, ConstantData, DynamicStackSlot, - DynamicStackSlotData, DynamicTypeData, ExtFuncData, ExternalName, FuncRef, Function, - GlobalValue, GlobalValueData, JumpTableData, MemFlags, MemoryTypeData, MemoryTypeField, Opcode, - SigRef, Signature, StackSlot, StackSlotData, StackSlotKind, UserFuncName, Value, + types, AbiParam, ArgumentExtension, ArgumentPurpose, Block, BlockArg, Constant, ConstantData, + DynamicStackSlot, DynamicStackSlotData, DynamicTypeData, ExtFuncData, ExternalName, FuncRef, + Function, GlobalValue, GlobalValueData, JumpTableData, MemFlags, MemoryTypeData, + MemoryTypeField, Opcode, SigRef, Signature, StackSlot, StackSlotData, StackSlotKind, + UserFuncName, Value, }; use cranelift_codegen::isa::{self, CallConv}; use cranelift_codegen::packed_option::ReservedValue; @@ -1890,7 +1891,7 @@ impl<'a> Parser<'a> { match self.token() { Some(Token::Block(dest)) => { self.consume(); - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; data.push(ctx.function.dfg.block_call(dest, &args)); loop { @@ -1899,7 +1900,7 @@ impl<'a> Parser<'a> { self.consume(); if let Some(Token::Block(dest)) = self.token() { self.consume(); - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; data.push(ctx.function.dfg.block_call(dest, &args)); } else { return err!(self.loc, "expected jump_table entry"); @@ -1923,6 +1924,75 @@ impl<'a> Parser<'a> { .push(JumpTableData::new(def, &data))) } + // Parse an exception-table decl. + // + // exception-table ::= * SigRef(sig) "," BlockCall "," "[" (exception-table-entry ( "," exception-table-entry )*)? "]" + // exception-table-entry ::= * ExceptionTag(tag) ":" BlockCall + // * "default" ":" BlockCall + fn parse_exception_table(&mut self, ctx: &mut Context) -> ParseResult { + let sig = self.match_sig("expected signature of called function")?; + self.match_token(Token::Comma, "expected comma after signature argument")?; + + let mut tags_and_targets = vec![]; + + let block_num = self.match_block("expected branch destination block")?; + let args = self.parse_opt_block_call_args()?; + let normal_return = ctx.function.dfg.block_call(block_num, &args); + + self.match_token( + Token::Comma, + "expected comma after normal-return destination", + )?; + + match self.token() { + Some(Token::LBracket) => { + self.consume(); + loop { + if let Some(Token::RBracket) = self.token() { + break; + } + + let tag = match self.token() { + Some(Token::ExceptionTag(tag)) => { + self.consume(); + Some(ir::ExceptionTag::from_u32(tag)) + } + Some(Token::Identifier("default")) => { + self.consume(); + None + } + _ => return err!(self.loc, "invalid token"), + }; + self.match_token(Token::Colon, "expected ':' after exception tag")?; + + let block_num = self.match_block("expected branch destination block")?; + let args = self.parse_opt_block_call_args()?; + let block_call = ctx.function.dfg.block_call(block_num, &args); + + tags_and_targets.push((tag, block_call)); + + if let Some(Token::Comma) = self.token() { + self.consume(); + } else { + break; + } + } + self.match_token(Token::RBracket, "expected closing bracket")?; + } + _ => {} + }; + + Ok(ctx + .function + .dfg + .exception_tables + .push(ir::ExceptionTableData::new( + sig, + normal_return, + tags_and_targets, + ))) + } + // Parse a constant decl. // // constant-decl ::= * Constant(c) "=" ty? "[" literal {"," literal} "]" @@ -2663,19 +2733,46 @@ impl<'a> Parser<'a> { Ok(args) } - // Parse an optional value list enclosed in parentheses. - fn parse_opt_value_list(&mut self) -> ParseResult { + /// Parse an optional list of block-call arguments enclosed in + /// parentheses. + fn parse_opt_block_call_args(&mut self) -> ParseResult> { if !self.optional(Token::LPar) { - return Ok(VariableArgs::new()); + return Ok(vec![]); } - let args = self.parse_value_list()?; + let mut args = vec![]; + while self.token() != Some(Token::RPar) { + args.push(self.parse_block_call_arg()?); + if self.token() == Some(Token::Comma) { + self.consume(); + } else { + break; + } + } self.match_token(Token::RPar, "expected ')' after arguments")?; Ok(args) } + fn parse_block_call_arg(&mut self) -> ParseResult { + match self.token() { + Some(Token::Value(v)) => { + self.consume(); + Ok(BlockArg::Value(v)) + } + Some(Token::TryCallRet(i)) => { + self.consume(); + Ok(BlockArg::TryCallRet(i)) + } + Some(Token::TryCallExn(i)) => { + self.consume(); + Ok(BlockArg::TryCallExn(i)) + } + tok => Err(self.error(&format!("unexpected token: {tok:?}"))), + } + } + /// Parse a CLIF run command. /// /// run-command ::= "run" [":" invocation comparison expected] @@ -2963,7 +3060,7 @@ impl<'a> Parser<'a> { InstructionFormat::Jump => { // Parse the destination block number. let block_num = self.match_block("expected jump destination block")?; - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; let destination = ctx.function.dfg.block_call(block_num, &args); InstructionData::Jump { opcode, @@ -2975,13 +3072,13 @@ impl<'a> Parser<'a> { self.match_token(Token::Comma, "expected ',' between operands")?; let block_then = { let block_num = self.match_block("expected branch then block")?; - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; ctx.function.dfg.block_call(block_num, &args) }; self.match_token(Token::Comma, "expected ',' between operands")?; let block_else = { let block_num = self.match_block("expected branch else block")?; - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; ctx.function.dfg.block_call(block_num, &args) }; InstructionData::Brif { @@ -2994,7 +3091,7 @@ impl<'a> Parser<'a> { let arg = self.match_value("expected SSA value operand")?; self.match_token(Token::Comma, "expected ',' between operands")?; let block_num = self.match_block("expected branch destination block")?; - let args = self.parse_opt_value_list()?; + let args = self.parse_opt_block_call_args()?; let destination = ctx.function.dfg.block_call(block_num, &args); self.match_token(Token::Comma, "expected ',' between operands")?; let table = self.parse_jump_table(ctx, destination)?; @@ -3085,6 +3182,34 @@ impl<'a> Parser<'a> { args: args.into_value_list(&[callee], &mut ctx.function.dfg.value_lists), } } + InstructionFormat::TryCall => { + let func_ref = self.match_fn("expected function reference")?; + ctx.check_fn(func_ref, self.loc)?; + self.match_token(Token::LPar, "expected '(' before arguments")?; + let args = self.parse_value_list()?; + self.match_token(Token::RPar, "expected ')' after arguments")?; + self.match_token(Token::Comma, "expected ',' after argument list")?; + let exception = self.parse_exception_table(ctx)?; + InstructionData::TryCall { + opcode, + func_ref, + args: args.into_value_list(&[], &mut ctx.function.dfg.value_lists), + exception, + } + } + InstructionFormat::TryCallIndirect => { + let callee = self.match_value("expected SSA value callee operand")?; + self.match_token(Token::LPar, "expected '(' before arguments")?; + let args = self.parse_value_list()?; + self.match_token(Token::RPar, "expected ')' after arguments")?; + self.match_token(Token::Comma, "expected ',' after argument list")?; + let exception = self.parse_exception_table(ctx)?; + InstructionData::TryCallIndirect { + opcode, + args: args.into_value_list(&[callee], &mut ctx.function.dfg.value_lists), + exception, + } + } InstructionFormat::FuncAddr => { let func_ref = self.match_fn("expected function reference")?; ctx.check_fn(func_ref, self.loc)?; diff --git a/cranelift/src/bugpoint.rs b/cranelift/src/bugpoint.rs index 3c0b506afdbe..4bfb35cbd3dd 100644 --- a/cranelift/src/bugpoint.rs +++ b/cranelift/src/bugpoint.rs @@ -420,7 +420,9 @@ impl Mutator for ReplaceBlockParamWithConst { // Remove parameters in branching instructions that point to this block for pred in cfg.pred_iter(self.block) { let dfg = &mut func.dfg; - for branch in dfg.insts[pred.inst].branch_destination_mut(&mut dfg.jump_tables) { + for branch in dfg.insts[pred.inst] + .branch_destination_mut(&mut dfg.jump_tables, &mut dfg.exception_tables) + { if branch.block(&dfg.value_lists) == self.block { branch.remove(param_index, &mut dfg.value_lists); } @@ -708,7 +710,8 @@ impl Mutator for MergeBlocks { // If the branch instruction that lead us to this block wasn't an unconditional jump, then // we have a conditional jump sequence that we should not break. - let branch_dests = func.dfg.insts[pred.inst].branch_destination(&func.dfg.jump_tables); + let branch_dests = func.dfg.insts[pred.inst] + .branch_destination(&func.dfg.jump_tables, &func.dfg.exception_tables); if branch_dests.len() != 1 { return Some(( func, @@ -717,7 +720,9 @@ impl Mutator for MergeBlocks { )); } - let branch_args = branch_dests[0].args_slice(&func.dfg.value_lists).to_vec(); + let branch_args = branch_dests[0] + .args(&func.dfg.value_lists) + .collect::>(); // TODO: should we free the entity list associated with the block params? let block_params = func @@ -731,8 +736,10 @@ impl Mutator for MergeBlocks { // If there were any block parameters in block, then the last instruction in pred will // fill these parameters. Make the block params aliases of the terminator arguments. for (block_param, arg) in block_params.into_iter().zip(branch_args) { - if block_param != arg { - func.dfg.change_to_alias(block_param, arg); + if let Some(arg) = arg.as_value() { + if block_param != arg { + func.dfg.change_to_alias(block_param, arg); + } } } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index b339b7bf08aa..ffb9dc5ad305 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -916,9 +916,13 @@ impl<'module_environment> FuncEnvironment<'module_environment> { let result_param = builder.append_block_param(continuation_block, pointer_type); builder.set_cold_block(null_block); - builder - .ins() - .brif(value, continuation_block, &[value_masked], null_block, &[]); + builder.ins().brif( + value, + continuation_block, + &[value_masked.into()], + null_block, + &[], + ); builder.seal_block(null_block); builder.switch_to_block(null_block); @@ -931,7 +935,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> { let index = self.cast_index_to_i64(&mut builder.cursor(), index, index_type); let call_inst = builder.ins().call(lazy_init, &[vmctx, table_index, index]); let returned_entry = builder.func.dfg.inst_results(call_inst)[0]; - builder.ins().jump(continuation_block, &[returned_entry]); + builder + .ins() + .jump(continuation_block, &[returned_entry.into()]); builder.seal_block(continuation_block); builder.switch_to_block(continuation_block); diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs index 3f45454239f8..e871c810f0ec 100644 --- a/crates/cranelift/src/func_environ/gc/enabled.rs +++ b/crates/cranelift/src/func_environ/gc/enabled.rs @@ -571,7 +571,7 @@ fn emit_array_fill_impl( // Current block: jump to the loop header block with the first element's // address. - builder.ins().jump(loop_header_block, &[elem_addr]); + builder.ins().jump(loop_header_block, &[elem_addr.into()]); // Loop header block: check if we're done, then jump to either the continue // block or the loop body block. @@ -590,7 +590,9 @@ fn emit_array_fill_impl( log::trace!("emit_array_fill_impl: loop body"); emit_elem_write(func_env, builder, elem_addr)?; let next_elem_addr = builder.ins().iadd(elem_addr, elem_size); - builder.ins().jump(loop_header_block, &[next_elem_addr]); + builder + .ins() + .jump(loop_header_block, &[next_elem_addr.into()]); // Continue... builder.switch_to_block(continue_block); @@ -934,7 +936,7 @@ pub fn translate_ref_test( builder.ins().brif( is_null, continue_block, - &[result_when_is_null], + &[result_when_is_null.into()], non_null_block, &[], ); @@ -961,7 +963,7 @@ pub fn translate_ref_test( builder.ins().brif( is_i31, continue_block, - &[result_when_is_i31], + &[result_when_is_i31.into()], non_null_non_i31_block, &[], ); @@ -1070,7 +1072,7 @@ pub fn translate_ref_test( WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. }; - builder.ins().jump(continue_block, &[result]); + builder.ins().jump(continue_block, &[result.into()]); // Control flow join point with the result. builder.switch_to_block(continue_block); @@ -1450,9 +1452,13 @@ impl FuncEnvironment<'_> { log::trace!("is_subtype: fast path check for exact same types"); let same_ty = builder.ins().icmp(IntCC::Equal, a, b); let same_ty = builder.ins().uextend(ir::types::I32, same_ty); - builder - .ins() - .brif(same_ty, continue_block, &[same_ty], diff_tys_block, &[]); + builder.ins().brif( + same_ty, + continue_block, + &[same_ty.into()], + diff_tys_block, + &[], + ); // Different types block: fall back to the `is_subtype` libcall. builder.switch_to_block(diff_tys_block); @@ -1461,7 +1467,7 @@ impl FuncEnvironment<'_> { let vmctx = self.vmctx_val(&mut builder.cursor()); let call_inst = builder.ins().call(is_subtype, &[vmctx, a, b]); let result = builder.func.dfg.first_result(call_inst); - builder.ins().jump(continue_block, &[result]); + builder.ins().jump(continue_block, &[result.into()]); // Continue block: join point for the result. builder.switch_to_block(continue_block); diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index a6e6c3310b96..31b14449f682 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -81,10 +81,10 @@ use crate::translate::translation_utils::{ use crate::Reachability; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::Offset32; -use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{ self, AtomicRmwOp, ConstantData, InstBuilder, JumpTableData, MemFlags, Value, ValueLabel, }; +use cranelift_codegen::ir::{types::*, BlockArg}; use cranelift_codegen::packed_option::ReservedValue; use cranelift_frontend::{FunctionBuilder, Variable}; use itertools::Itertools; @@ -3897,28 +3897,21 @@ fn is_non_canonical_v128(ty: ir::Type) -> bool { /// actually necessary, and if not, the original slice is returned. Otherwise the cast values /// are returned in a slice that belongs to the caller-supplied `SmallVec`. fn canonicalise_v128_values<'a>( - tmp_canonicalised: &'a mut SmallVec<[ir::Value; 16]>, + tmp_canonicalised: &'a mut SmallVec<[BlockArg; 16]>, builder: &mut FunctionBuilder, values: &'a [ir::Value], -) -> &'a [ir::Value] { +) -> &'a [BlockArg] { debug_assert!(tmp_canonicalised.is_empty()); - // First figure out if any of the parameters need to be cast. Mostly they don't need to be. - let any_non_canonical = values - .iter() - .any(|v| is_non_canonical_v128(builder.func.dfg.value_type(*v))); - // Hopefully we take this exit most of the time, hence doing no heap allocation. - if !any_non_canonical { - return values; - } - // Otherwise we'll have to cast, and push the resulting `Value`s into `canonicalised`. + // Cast, and push the resulting `Value`s into `canonicalised`. for v in values { - tmp_canonicalised.push(if is_non_canonical_v128(builder.func.dfg.value_type(*v)) { + let value = if is_non_canonical_v128(builder.func.dfg.value_type(*v)) { let mut flags = MemFlags::new(); flags.set_endianness(ir::Endianness::Little); builder.ins().bitcast(I8X16, flags, *v) } else { *v - }); + }; + tmp_canonicalised.push(BlockArg::from(value)); } tmp_canonicalised.as_slice() } @@ -3931,7 +3924,7 @@ fn canonicalise_then_jump( destination: ir::Block, params: &[ir::Value], ) -> ir::Inst { - let mut tmp_canonicalised = SmallVec::<[ir::Value; 16]>::new(); + let mut tmp_canonicalised = SmallVec::<[_; 16]>::new(); let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params); builder.ins().jump(destination, canonicalised) } @@ -3945,10 +3938,10 @@ fn canonicalise_brif( block_else: ir::Block, params_else: &[ir::Value], ) -> ir::Inst { - let mut tmp_canonicalised_then = SmallVec::<[ir::Value; 16]>::new(); + let mut tmp_canonicalised_then = SmallVec::<[_; 16]>::new(); let canonicalised_then = canonicalise_v128_values(&mut tmp_canonicalised_then, builder, params_then); - let mut tmp_canonicalised_else = SmallVec::<[ir::Value; 16]>::new(); + let mut tmp_canonicalised_else = SmallVec::<[_; 16]>::new(); let canonicalised_else = canonicalise_v128_values(&mut tmp_canonicalised_else, builder, params_else); builder.ins().brif( diff --git a/fuzz/fuzz_targets/misc.rs b/fuzz/fuzz_targets/misc.rs index f4fd1f3c88ea..087c1913428f 100644 --- a/fuzz/fuzz_targets/misc.rs +++ b/fuzz/fuzz_targets/misc.rs @@ -158,7 +158,9 @@ fn dominator_tree(mut data: Unstructured<'_>) -> Result<()> { } else { let block_calls = children .iter() - .map(|&block| BlockCall::new(block, &[], &mut cursor.func.dfg.value_lists)) + .map(|&block| { + BlockCall::new(block, core::iter::empty(), &mut cursor.func.dfg.value_lists) + }) .collect::>(); let data = JumpTableData::new(block_calls[0], &block_calls[1..]); From a40502259644bc569316f76479ba63b017392527 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Mon, 7 Apr 2025 16:23:03 -0700 Subject: [PATCH 2/2] Add try_call_indirect lowering as well. --- cranelift/codegen/src/ir/instructions.rs | 2 +- cranelift/codegen/src/machinst/isle.rs | 33 ++- .../filetests/isa/aarch64/exceptions.clif | 126 ++++++++++ .../filetests/isa/pulley32/exceptions.clif | 147 ++++++++++++ .../filetests/isa/pulley64/exceptions.clif | 148 ++++++++++++ .../filetests/isa/riscv64/exceptions.clif | 225 ++++++++++++++++++ .../filetests/isa/x64/exceptions.clif | 104 ++++++++ .../filetests/runtests/try_call.clif | 13 + 8 files changed, 791 insertions(+), 7 deletions(-) diff --git a/cranelift/codegen/src/ir/instructions.rs b/cranelift/codegen/src/ir/instructions.rs index 17f2c6500689..66f093c504c6 100644 --- a/cranelift/codegen/src/ir/instructions.rs +++ b/cranelift/codegen/src/ir/instructions.rs @@ -595,7 +595,7 @@ impl InstructionData { .. } => { let exdata = &exception_tables[exception]; - CallInfo::Indirect(exdata.signature(), args.as_slice(pool)) + CallInfo::Indirect(exdata.signature(), &args.as_slice(pool)[1..]) } Self::Ternary { opcode: Opcode::StackSwitch, diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index ee1faa7503d6..e36a78eb92c2 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -908,13 +908,34 @@ macro_rules! isle_prelude_caller_methods { fn gen_try_call_indirect( &mut self, - _sig_ref: SigRef, - _callee: Value, - _et: ExceptionTable, - _args: ValueSlice, - _targets: &MachLabelSlice, + sigref: SigRef, + callee: Value, + et: ExceptionTable, + args: ValueSlice, + targets: &MachLabelSlice, ) -> () { - todo!() + let caller_conv = self.lower_ctx.abi().call_conv(self.lower_ctx.sigs()); + let sig = &self.lower_ctx.dfg().signatures[sigref]; + let num_rets = sig.returns.len(); + + let callee = self.put_in_reg(callee); + + let caller = <$abicaller>::from_ptr( + self.lower_ctx.sigs(), + sigref, + callee, + IsTailCall::No, + caller_conv, + self.backend.flags().clone(), + ); + + crate::machinst::isle::gen_call_common( + &mut self.lower_ctx, + num_rets, + caller, + args, + Some((et, targets)), + ); } }; } diff --git a/cranelift/filetests/filetests/isa/aarch64/exceptions.clif b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif index 67913660dd4b..604e9fbef506 100644 --- a/cranelift/filetests/filetests/isa/aarch64/exceptions.clif +++ b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif @@ -121,3 +121,129 @@ function %f0(i32) -> i32, f32, f64 { ; ldp x29, x30, [sp], #0x10 ; ret +function %f2(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + v10 = func_addr.i64 fn0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; stp fp, lr, [sp, #-16]! +; mov fp, sp +; stp x27, x28, [sp, #-16]! +; stp x25, x26, [sp, #-16]! +; stp x23, x24, [sp, #-16]! +; stp x21, x22, [sp, #-16]! +; stp x19, x20, [sp, #-16]! +; stp d14, d15, [sp, #-16]! +; stp d12, d13, [sp, #-16]! +; stp d10, d11, [sp, #-16]! +; stp d8, d9, [sp, #-16]! +; sub sp, sp, #16 +; block0: +; fmov d1, #1 +; str q1, [sp] +; load_ext_name x11, TestCase(%g)+0 +; mov x2, x0 +; blr x11; b MachLabel(1); catch [None: MachLabel(2)] +; block1: +; movz w0, #1 +; ldr q1, [sp] +; add sp, sp, #16 +; ldp d8, d9, [sp], #16 +; ldp d10, d11, [sp], #16 +; ldp d12, d13, [sp], #16 +; ldp d14, d15, [sp], #16 +; ldp x19, x20, [sp], #16 +; ldp x21, x22, [sp], #16 +; ldp x23, x24, [sp], #16 +; ldp x25, x26, [sp], #16 +; ldp x27, x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; block2: +; ldr q1, [sp] +; add w0, w0, #1 +; movi v0.2s, #0 +; add sp, sp, #16 +; ldp d8, d9, [sp], #16 +; ldp d10, d11, [sp], #16 +; ldp d12, d13, [sp], #16 +; ldp d14, d15, [sp], #16 +; ldp x19, x20, [sp], #16 +; ldp x21, x22, [sp], #16 +; ldp x23, x24, [sp], #16 +; ldp x25, x26, [sp], #16 +; ldp x27, x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; stp x29, x30, [sp, #-0x10]! +; mov x29, sp +; stp x27, x28, [sp, #-0x10]! +; stp x25, x26, [sp, #-0x10]! +; stp x23, x24, [sp, #-0x10]! +; stp x21, x22, [sp, #-0x10]! +; stp x19, x20, [sp, #-0x10]! +; stp d14, d15, [sp, #-0x10]! +; stp d12, d13, [sp, #-0x10]! +; stp d10, d11, [sp, #-0x10]! +; stp d8, d9, [sp, #-0x10]! +; sub sp, sp, #0x10 +; block1: ; offset 0x30 +; fmov d1, #1.00000000 +; stur q1, [sp] +; ldr x11, #0x40 +; b #0x48 +; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0 +; .byte 0x00, 0x00, 0x00, 0x00 +; mov x2, x0 +; blr x11 +; block2: ; offset 0x50 +; mov w0, #1 +; ldur q1, [sp] +; add sp, sp, #0x10 +; ldp d8, d9, [sp], #0x10 +; ldp d10, d11, [sp], #0x10 +; ldp d12, d13, [sp], #0x10 +; ldp d14, d15, [sp], #0x10 +; ldp x19, x20, [sp], #0x10 +; ldp x21, x22, [sp], #0x10 +; ldp x23, x24, [sp], #0x10 +; ldp x25, x26, [sp], #0x10 +; ldp x27, x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret +; block3: ; offset 0x88 +; ldur q1, [sp] +; add w0, w0, #1 +; movi v0.2s, #0 +; add sp, sp, #0x10 +; ldp d8, d9, [sp], #0x10 +; ldp d10, d11, [sp], #0x10 +; ldp d12, d13, [sp], #0x10 +; ldp d14, d15, [sp], #0x10 +; ldp x19, x20, [sp], #0x10 +; ldp x21, x22, [sp], #0x10 +; ldp x23, x24, [sp], #0x10 +; ldp x25, x26, [sp], #0x10 +; ldp x27, x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley32/exceptions.clif b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif index 833c20fb1da1..40f643708c87 100644 --- a/cranelift/filetests/filetests/isa/pulley32/exceptions.clif +++ b/cranelift/filetests/filetests/isa/pulley32/exceptions.clif @@ -148,3 +148,150 @@ function %f0(i32) -> i32, f32, f64 { ; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 ; ret +function %f2(i32, i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32, v10: i32): + v2 = f64const 0x1.0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v7: i32): + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; push_frame_save 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; fstore64 sp+136, f16 // flags = notrap aligned +; fstore64 sp+128, f17 // flags = notrap aligned +; fstore64 sp+120, f18 // flags = notrap aligned +; fstore64 sp+112, f19 // flags = notrap aligned +; fstore64 sp+104, f20 // flags = notrap aligned +; fstore64 sp+96, f21 // flags = notrap aligned +; fstore64 sp+88, f22 // flags = notrap aligned +; fstore64 sp+80, f23 // flags = notrap aligned +; fstore64 sp+72, f24 // flags = notrap aligned +; fstore64 sp+64, f25 // flags = notrap aligned +; fstore64 sp+56, f26 // flags = notrap aligned +; fstore64 sp+48, f27 // flags = notrap aligned +; fstore64 sp+40, f28 // flags = notrap aligned +; fstore64 sp+32, f29 // flags = notrap aligned +; fstore64 sp+24, f30 // flags = notrap aligned +; fstore64 sp+16, f31 // flags = notrap aligned +; block0: +; fconst64 f1, 4607182418800017408 +; fstore64 Slot(0), f1 // flags = notrap aligned +; indirect_call x1, CallInfo { dest: XReg(p1i), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I32) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I32) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_dests: [(None, MachLabel(2))] }) }; jump MachLabel(1); catch [None: MachLabel(2)] +; block1: +; xone x0 +; f1 = fload64 Slot(0) // flags = notrap aligned +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; block2: +; f1 = fload64 Slot(0) // flags = notrap aligned +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; +; Disassembled: +; push_frame_save 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; fstore64le_o32 sp, 136, f16 +; fstore64le_o32 sp, 128, f17 +; fstore64le_o32 sp, 120, f18 +; fstore64le_o32 sp, 112, f19 +; fstore64le_o32 sp, 104, f20 +; fstore64le_o32 sp, 96, f21 +; fstore64le_o32 sp, 88, f22 +; fstore64le_o32 sp, 80, f23 +; fstore64le_o32 sp, 72, f24 +; fstore64le_o32 sp, 64, f25 +; fstore64le_o32 sp, 56, f26 +; fstore64le_o32 sp, 48, f27 +; fstore64le_o32 sp, 40, f28 +; fstore64le_o32 sp, 32, f29 +; fstore64le_o32 sp, 24, f30 +; fstore64le_o32 sp, 16, f31 +; fconst64 f1, 4607182418800017408 +; fstore64le_o32 sp, 0, f1 +; call_indirect x1 +; xone x0 +; fload64le_o32 f1, sp, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret +; fload64le_o32 f1, sp, 0 +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley64/exceptions.clif b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif index fdccf3515dc0..2f03e53b1210 100644 --- a/cranelift/filetests/filetests/isa/pulley64/exceptions.clif +++ b/cranelift/filetests/filetests/isa/pulley64/exceptions.clif @@ -149,3 +149,151 @@ function %f0(i32) -> i32, f32, f64 { ; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 ; ret +function %f2(i32, i64) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32, v10: i64): + v2 = f64const 0x1.0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; push_frame_save 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; fstore64 sp+136, f16 // flags = notrap aligned +; fstore64 sp+128, f17 // flags = notrap aligned +; fstore64 sp+120, f18 // flags = notrap aligned +; fstore64 sp+112, f19 // flags = notrap aligned +; fstore64 sp+104, f20 // flags = notrap aligned +; fstore64 sp+96, f21 // flags = notrap aligned +; fstore64 sp+88, f22 // flags = notrap aligned +; fstore64 sp+80, f23 // flags = notrap aligned +; fstore64 sp+72, f24 // flags = notrap aligned +; fstore64 sp+64, f25 // flags = notrap aligned +; fstore64 sp+56, f26 // flags = notrap aligned +; fstore64 sp+48, f27 // flags = notrap aligned +; fstore64 sp+40, f28 // flags = notrap aligned +; fstore64 sp+32, f29 // flags = notrap aligned +; fstore64 sp+24, f30 // flags = notrap aligned +; fstore64 sp+16, f31 // flags = notrap aligned +; block0: +; fconst64 f1, 4607182418800017408 +; fstore64 Slot(0), f1 // flags = notrap aligned +; indirect_call x1, CallInfo { dest: XReg(p1i), uses: [CallArgPair { vreg: p0i, preg: p0i }], defs: [CallRetPair { vreg: Writable { reg: p0f }, location: Reg(p0f, types::F32) }, CallRetPair { vreg: Writable { reg: p0i }, location: Reg(p0i, types::I64) }, CallRetPair { vreg: Writable { reg: p1i }, location: Reg(p1i, types::I64) }], clobbers: PRegSet { bits: [4294967292, 4294967294, 4294967295, 0] }, callee_conv: Tail, caller_conv: Fast, callee_pop_size: 0, try_call_info: Some(TryCallInfo { continuation: MachLabel(1), exception_dests: [(None, MachLabel(2))] }) }; jump MachLabel(1); catch [None: MachLabel(2)] +; block1: +; xone x0 +; f1 = fload64 Slot(0) // flags = notrap aligned +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; block2: +; f1 = fload64 Slot(0) // flags = notrap aligned +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; f16 = fload64 sp+136 // flags = notrap aligned +; f17 = fload64 sp+128 // flags = notrap aligned +; f18 = fload64 sp+120 // flags = notrap aligned +; f19 = fload64 sp+112 // flags = notrap aligned +; f20 = fload64 sp+104 // flags = notrap aligned +; f21 = fload64 sp+96 // flags = notrap aligned +; f22 = fload64 sp+88 // flags = notrap aligned +; f23 = fload64 sp+80 // flags = notrap aligned +; f24 = fload64 sp+72 // flags = notrap aligned +; f25 = fload64 sp+64 // flags = notrap aligned +; f26 = fload64 sp+56 // flags = notrap aligned +; f27 = fload64 sp+48 // flags = notrap aligned +; f28 = fload64 sp+40 // flags = notrap aligned +; f29 = fload64 sp+32 // flags = notrap aligned +; f30 = fload64 sp+24 // flags = notrap aligned +; f31 = fload64 sp+16 // flags = notrap aligned +; pop_frame_restore 272, {x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0} +; ret +; +; Disassembled: +; push_frame_save 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; fstore64le_o32 sp, 136, f16 +; fstore64le_o32 sp, 128, f17 +; fstore64le_o32 sp, 120, f18 +; fstore64le_o32 sp, 112, f19 +; fstore64le_o32 sp, 104, f20 +; fstore64le_o32 sp, 96, f21 +; fstore64le_o32 sp, 88, f22 +; fstore64le_o32 sp, 80, f23 +; fstore64le_o32 sp, 72, f24 +; fstore64le_o32 sp, 64, f25 +; fstore64le_o32 sp, 56, f26 +; fstore64le_o32 sp, 48, f27 +; fstore64le_o32 sp, 40, f28 +; fstore64le_o32 sp, 32, f29 +; fstore64le_o32 sp, 24, f30 +; fstore64le_o32 sp, 16, f31 +; fconst64 f1, 4607182418800017408 +; fstore64le_o32 sp, 0, f1 +; call_indirect x1 +; xone x0 +; fload64le_o32 f1, sp, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret +; fload64le_o32 f1, sp, 0 +; xadd32_u8 x0, x0, 1 +; fconst32 f0, 0 +; fload64le_o32 f16, sp, 136 +; fload64le_o32 f17, sp, 128 +; fload64le_o32 f18, sp, 120 +; fload64le_o32 f19, sp, 112 +; fload64le_o32 f20, sp, 104 +; fload64le_o32 f21, sp, 96 +; fload64le_o32 f22, sp, 88 +; fload64le_o32 f23, sp, 80 +; fload64le_o32 f24, sp, 72 +; fload64le_o32 f25, sp, 64 +; fload64le_o32 f26, sp, 56 +; fload64le_o32 f27, sp, 48 +; fload64le_o32 f28, sp, 40 +; fload64le_o32 f29, sp, 32 +; fload64le_o32 f30, sp, 24 +; fload64le_o32 f31, sp, 16 +; pop_frame_restore 272, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0 +; ret + diff --git a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif index 084ebb5339a2..89b2da54b2e7 100644 --- a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif +++ b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif @@ -220,3 +220,228 @@ function %f0(i32) -> i32, f32, f64 { ; addi sp, sp, 0x10 ; ret +function %f2(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + v10 = func_addr.i64 fn0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; addi sp,sp,-16 +; sd ra,8(sp) +; sd fp,0(sp) +; mv fp,sp +; addi sp,sp,-208 +; sd fp,200(sp) +; sd s1,192(sp) +; sd s2,184(sp) +; sd s3,176(sp) +; sd s4,168(sp) +; sd s5,160(sp) +; sd s6,152(sp) +; sd s7,144(sp) +; sd s8,136(sp) +; sd s9,128(sp) +; sd s10,120(sp) +; sd s11,112(sp) +; fsd fs0,104(sp) +; fsd fs2,96(sp) +; fsd fs3,88(sp) +; fsd fs4,80(sp) +; fsd fs5,72(sp) +; fsd fs6,64(sp) +; fsd fs7,56(sp) +; fsd fs8,48(sp) +; fsd fs9,40(sp) +; fsd fs10,32(sp) +; fsd fs11,24(sp) +; block0: +; lui a5,1023 +; slli a1,a5,40 +; fmv.d.x fa1,a1 +; fsd fa1,0(slot) +; load_sym a1,%g+0 +; callind a1; j MachLabel(1); catch [None: MachLabel(2)] +; block1: +; li a0,1 +; fld fa1,0(slot) +; ld fp,200(sp) +; ld s1,192(sp) +; ld s2,184(sp) +; ld s3,176(sp) +; ld s4,168(sp) +; ld s5,160(sp) +; ld s6,152(sp) +; ld s7,144(sp) +; ld s8,136(sp) +; ld s9,128(sp) +; ld s10,120(sp) +; ld s11,112(sp) +; fld fs0,104(sp) +; fld fs2,96(sp) +; fld fs3,88(sp) +; fld fs4,80(sp) +; fld fs5,72(sp) +; fld fs6,64(sp) +; fld fs7,56(sp) +; fld fs8,48(sp) +; fld fs9,40(sp) +; fld fs10,32(sp) +; fld fs11,24(sp) +; addi sp,sp,208 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; block2: +; fld fa1,0(slot) +; addiw a0,a0,1 +; fmv.w.x fa0,zero +; ld fp,200(sp) +; ld s1,192(sp) +; ld s2,184(sp) +; ld s3,176(sp) +; ld s4,168(sp) +; ld s5,160(sp) +; ld s6,152(sp) +; ld s7,144(sp) +; ld s8,136(sp) +; ld s9,128(sp) +; ld s10,120(sp) +; ld s11,112(sp) +; fld fs0,104(sp) +; fld fs2,96(sp) +; fld fs3,88(sp) +; fld fs4,80(sp) +; fld fs5,72(sp) +; fld fs6,64(sp) +; fld fs7,56(sp) +; fld fs8,48(sp) +; fld fs9,40(sp) +; fld fs10,32(sp) +; fld fs11,24(sp) +; addi sp,sp,208 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; addi sp, sp, -0x10 +; sd ra, 8(sp) +; sd s0, 0(sp) +; mv s0, sp +; addi sp, sp, -0xd0 +; sd s0, 0xc8(sp) +; sd s1, 0xc0(sp) +; sd s2, 0xb8(sp) +; sd s3, 0xb0(sp) +; sd s4, 0xa8(sp) +; sd s5, 0xa0(sp) +; sd s6, 0x98(sp) +; sd s7, 0x90(sp) +; sd s8, 0x88(sp) +; sd s9, 0x80(sp) +; sd s10, 0x78(sp) +; sd s11, 0x70(sp) +; fsd fs0, 0x68(sp) +; fsd fs2, 0x60(sp) +; fsd fs3, 0x58(sp) +; fsd fs4, 0x50(sp) +; fsd fs5, 0x48(sp) +; fsd fs6, 0x40(sp) +; fsd fs7, 0x38(sp) +; fsd fs8, 0x30(sp) +; fsd fs9, 0x28(sp) +; fsd fs10, 0x20(sp) +; fsd fs11, 0x18(sp) +; block1: ; offset 0x70 +; lui a5, 0x3ff +; slli a1, a5, 0x28 +; fmv.d.x fa1, a1 +; fsd fa1, 0(sp) +; auipc a1, 0 +; ld a1, 0xc(a1) +; j 0xc +; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0 +; .byte 0x00, 0x00, 0x00, 0x00 +; jalr a1 +; block2: ; offset 0x98 +; addi a0, zero, 1 +; fld fa1, 0(sp) +; ld s0, 0xc8(sp) +; ld s1, 0xc0(sp) +; ld s2, 0xb8(sp) +; ld s3, 0xb0(sp) +; ld s4, 0xa8(sp) +; ld s5, 0xa0(sp) +; ld s6, 0x98(sp) +; ld s7, 0x90(sp) +; ld s8, 0x88(sp) +; ld s9, 0x80(sp) +; ld s10, 0x78(sp) +; ld s11, 0x70(sp) +; fld fs0, 0x68(sp) +; fld fs2, 0x60(sp) +; fld fs3, 0x58(sp) +; fld fs4, 0x50(sp) +; fld fs5, 0x48(sp) +; fld fs6, 0x40(sp) +; fld fs7, 0x38(sp) +; fld fs8, 0x30(sp) +; fld fs9, 0x28(sp) +; fld fs10, 0x20(sp) +; fld fs11, 0x18(sp) +; addi sp, sp, 0xd0 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret +; block3: ; offset 0x110 +; fld fa1, 0(sp) +; addiw a0, a0, 1 +; fmv.w.x fa0, zero +; ld s0, 0xc8(sp) +; ld s1, 0xc0(sp) +; ld s2, 0xb8(sp) +; ld s3, 0xb0(sp) +; ld s4, 0xa8(sp) +; ld s5, 0xa0(sp) +; ld s6, 0x98(sp) +; ld s7, 0x90(sp) +; ld s8, 0x88(sp) +; ld s9, 0x80(sp) +; ld s10, 0x78(sp) +; ld s11, 0x70(sp) +; fld fs0, 0x68(sp) +; fld fs2, 0x60(sp) +; fld fs3, 0x58(sp) +; fld fs4, 0x50(sp) +; fld fs5, 0x48(sp) +; fld fs6, 0x40(sp) +; fld fs7, 0x38(sp) +; fld fs8, 0x30(sp) +; fld fs9, 0x28(sp) +; fld fs10, 0x20(sp) +; fld fs11, 0x18(sp) +; addi sp, sp, 0xd0 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret + diff --git a/cranelift/filetests/filetests/isa/x64/exceptions.clif b/cranelift/filetests/filetests/isa/x64/exceptions.clif index 7b1e5ac23e8b..ed410980a859 100644 --- a/cranelift/filetests/filetests/isa/x64/exceptions.clif +++ b/cranelift/filetests/filetests/isa/x64/exceptions.clif @@ -265,3 +265,107 @@ function %f1(i32) -> i32, f32, f64 { ; popq %rbp ; retq +function %f2(i32) -> i32, f32, f64 { + sig0 = (i32) -> f32 tail + fn0 = %g(i32) -> f32 tail + + block0(v1: i32): + v2 = f64const 0x1.0 + v10 = func_addr.i64 fn0 + try_call_indirect v10(v1), sig0, block1(ret0, v2), [ default: block2(exn0) ] + + block1(v3: f32, v4: f64): + v5 = iconst.i32 1 + return v5, v3, v4 + + block2(v6: i64): + v7 = ireduce.i32 v6 + v8 = iadd_imm.i32 v7, 1 + v9 = f32const 0x0.0 + return v8, v9, v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; subq %rsp, $64, %rsp +; movq %rbx, 16(%rsp) +; movq %r12, 24(%rsp) +; movq %r13, 32(%rsp) +; movq %r14, 40(%rsp) +; movq %r15, 48(%rsp) +; block0: +; movabsq $4607182418800017408, %rcx +; movq %rcx, %xmm1 +; movdqu %xmm1, rsp(0 + virtual offset) +; load_ext_name %g+0, %rdx +; call *%rdx; jmp MachLabel(1); catch [None: MachLabel(2)] +; block1: +; movl $1, %eax +; movdqu rsp(0 + virtual offset), %xmm1 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; block2: +; movdqu rsp(0 + virtual offset), %xmm1 +; lea 1(%rax), %eax +; uninit %xmm0 +; xorps %xmm0, %xmm0 +; movq 16(%rsp), %rbx +; movq 24(%rsp), %r12 +; movq 32(%rsp), %r13 +; movq 40(%rsp), %r14 +; movq 48(%rsp), %r15 +; addq %rsp, $64, %rsp +; movq %rbp, %rsp +; popq %rbp +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; subq $0x40, %rsp +; movq %rbx, 0x10(%rsp) +; movq %r12, 0x18(%rsp) +; movq %r13, 0x20(%rsp) +; movq %r14, 0x28(%rsp) +; movq %r15, 0x30(%rsp) +; block1: ; offset 0x21 +; movabsq $0x3ff0000000000000, %rcx +; movq %rcx, %xmm1 +; movdqu %xmm1, (%rsp) +; movabsq $0, %rdx ; reloc_external Abs8 %g 0 +; callq *%rdx +; block2: ; offset 0x41 +; movl $1, %eax +; movdqu (%rsp), %xmm1 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq +; block3: ; offset 0x6d +; movdqu (%rsp), %xmm1 +; addl $1, %eax +; xorps %xmm0, %xmm0 +; movq 0x10(%rsp), %rbx +; movq 0x18(%rsp), %r12 +; movq 0x20(%rsp), %r13 +; movq 0x28(%rsp), %r14 +; movq 0x30(%rsp), %r15 +; addq $0x40, %rsp +; movq %rbp, %rsp +; popq %rbp +; retq + diff --git a/cranelift/filetests/filetests/runtests/try_call.clif b/cranelift/filetests/filetests/runtests/try_call.clif index fe6a9fc286ee..860ed3c5b413 100644 --- a/cranelift/filetests/filetests/runtests/try_call.clif +++ b/cranelift/filetests/filetests/runtests/try_call.clif @@ -25,3 +25,16 @@ block1(v1: i64): return v1 } ; run: %call_i64(10) == 20 + +function %call_indirect_i64(i64) -> i64 { + sig0 = (i64) -> i64 tail + fn0 = %callee_i64(i64) -> i64 tail + +block0(v0: i64): + v1 = func_addr.i64 fn0 + try_call_indirect v1(v0), sig0, block1(ret0), [] + +block1(v2: i64): + return v2 +} +; run: %call_i64(10) == 20