From 40a317eb1e91e11f55bd0a2cba52ad2cabd783fa Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Tue, 3 Jun 2025 22:49:54 -0700 Subject: [PATCH] Cranelift: implement an "unwinder" crate and exception throws in filetests. This commit introduces the next major piece of machinery (after the previously-landed `try_call` support) that we will eventually use to implement Wasm exceptions in Wasmtime. In particular, it implements a generic unwinder as a new crate that supports (i) walking a stack produced by Cranelift code, (ii) serializing Cranelift exception metadata to compact tables (in a way very similar to address maps in Wasmtime, so they will be mappable directly from disk), (iii) using these serialized tables to find handlers during a stack-walk, and (iv) jumping to handlers (i.e., actually unwinding). This crate is currently used in the filetests runner, and will next be used in Wasmtime. The commit first performs code-motion: it moves stack-walking code from Wasmtime to `cranelift-unwinder`. This itself has no functional effect, but isolates the code that understands contiguous sequences of Cranelift frames ("activations") from that which is specific to Wasmtime's activation delimiters and metadata. It then implements a compact exception-table format. This format uses the `object` crate's mechanisms for directly referencing in-memory arrays of little-endian `u32`s in a way that will allow us to find handlers when mapping exception metadata directly from an ELF section in a `.cwasm` (for example). The format consists of four sorted `u32` arrays in a way that allows us to look up a callsite first, then search its sorted array of handler offsets by tags. It next implements the actual unwind control flow: it contains an assembly stub for each supported architecture that transfers control to a PC, SP, and FP value "up the stack", with payload values placed in the payload registers we have defined per our exception ABI in Cranelift. Finally, it puts these pieces together in the filetest runner. Note that the runtest does a lot "by hand": we don't have entry and exit trampolines as we do in Wasmtime, so the filetest contains three functions, with the middle one invoking the "throw hostcall" and entry and exit trampolines around it grabbing the appropriate entry/exit FPs and exit PC. The dance to call back to host code is also somewhat delicate, as we haven't done this before. The `JITModule`'s linking + relocation support does not seem sufficient to properly define a symbol, so instead we scan for `func_addr` instructions referencing a well-known name (`__cranelift_throw`) and replace them with `iconst`s with the function address at runtime, baking it in. This is somewhat ugly, but it works. All of these filetest-specific details will be handled much more nicely in the Wasmtime version of this functionality, as we have proper abstractions for entry/exit trampolines and hostcalls. --- Cargo.lock | 24 +- Cargo.toml | 1 + cranelift/codegen/src/ir/extname.rs | 5 + cranelift/codegen/src/lib.rs | 4 +- cranelift/filetests/Cargo.toml | 3 +- .../filetests/filetests/runtests/throw.clif | 50 +++ cranelift/filetests/src/function_runner.rs | 137 +++++++- cranelift/filetests/src/test_run.rs | 2 +- cranelift/jit/Cargo.toml | 3 + cranelift/jit/src/backend.rs | 75 ++++- cranelift/jit/src/compiled_blob.rs | 2 + cranelift/src/run.rs | 2 +- crates/unwinder/Cargo.toml | 29 ++ crates/unwinder/LICENSE | 220 +++++++++++++ crates/unwinder/README.md | 13 + .../vm => unwinder/src}/arch/aarch64.rs | 31 ++ crates/unwinder/src/arch/mod.rs | 109 +++++++ crates/unwinder/src/arch/riscv64.rs | 57 ++++ crates/unwinder/src/arch/s390x.rs | 59 ++++ .../runtime/vm => unwinder/src}/arch/x86.rs | 31 ++ crates/unwinder/src/exception_table.rs | 300 ++++++++++++++++++ crates/unwinder/src/lib.rs | 18 ++ crates/unwinder/src/stackwalk.rs | 175 ++++++++++ crates/unwinder/src/throw.rs | 80 +++++ crates/wasmtime/Cargo.toml | 8 +- crates/wasmtime/src/runtime/store.rs | 2 +- crates/wasmtime/src/runtime/vm.rs | 10 +- crates/wasmtime/src/runtime/vm/arch/mod.rs | 45 --- .../wasmtime/src/runtime/vm/arch/riscv64.rs | 26 -- crates/wasmtime/src/runtime/vm/arch/s390x.rs | 22 -- .../src/runtime/vm/arch/unsupported.rs | 28 -- crates/wasmtime/src/runtime/vm/interpreter.rs | 36 +++ .../src/runtime/vm/interpreter_disabled.rs | 5 + .../wasmtime/src/runtime/vm/traphandlers.rs | 2 +- .../src/runtime/vm/traphandlers/backtrace.rs | 131 +------- crates/wasmtime/src/runtime/vm/unwind.rs | 59 ---- fuzz/fuzz_targets/cranelift-fuzzgen.rs | 12 +- scripts/publish.rs | 3 + 38 files changed, 1472 insertions(+), 347 deletions(-) create mode 100644 cranelift/filetests/filetests/runtests/throw.clif create mode 100644 crates/unwinder/Cargo.toml create mode 100644 crates/unwinder/LICENSE create mode 100644 crates/unwinder/README.md rename crates/{wasmtime/src/runtime/vm => unwinder/src}/arch/aarch64.rs (77%) create mode 100644 crates/unwinder/src/arch/mod.rs create mode 100644 crates/unwinder/src/arch/riscv64.rs create mode 100644 crates/unwinder/src/arch/s390x.rs rename crates/{wasmtime/src/runtime/vm => unwinder/src}/arch/x86.rs (57%) create mode 100644 crates/unwinder/src/exception_table.rs create mode 100644 crates/unwinder/src/lib.rs create mode 100644 crates/unwinder/src/stackwalk.rs create mode 100644 crates/unwinder/src/throw.rs delete mode 100644 crates/wasmtime/src/runtime/vm/arch/mod.rs delete mode 100644 crates/wasmtime/src/runtime/vm/arch/riscv64.rs delete mode 100644 crates/wasmtime/src/runtime/vm/arch/s390x.rs delete mode 100644 crates/wasmtime/src/runtime/vm/arch/unsupported.rs delete mode 100644 crates/wasmtime/src/runtime/vm/unwind.rs diff --git a/Cargo.lock b/Cargo.lock index f4de6613af55..bb416f937c2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -796,6 +796,7 @@ dependencies = [ "target-lexicon", "thiserror 2.0.12", "toml", + "wasmtime-unwinder", "wat", ] @@ -864,6 +865,7 @@ dependencies = [ "region", "target-lexicon", "wasmtime-jit-icache-coherence", + "wasmtime-unwinder", "windows-sys 0.59.0", ] @@ -2628,15 +2630,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "psm" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871372391786ccec00d3c5d3d6608905b3d4db263639cfe075d3b60a736d115a" -dependencies = [ - "cc", -] - [[package]] name = "pulley-interpreter" version = "35.0.0" @@ -4173,7 +4166,6 @@ dependencies = [ "once_cell", "postcard", "proptest", - "psm", "pulley-interpreter", "rand", "rayon", @@ -4202,6 +4194,7 @@ dependencies = [ "wasmtime-math", "wasmtime-slab", "wasmtime-test-util", + "wasmtime-unwinder", "wasmtime-versioned-export-macros", "wasmtime-winch", "wasmtime-wmemcheck", @@ -4612,6 +4605,17 @@ dependencies = [ "wasmtime-environ", ] +[[package]] +name = "wasmtime-unwinder" +version = "35.0.0" +dependencies = [ + "anyhow", + "cfg-if", + "cranelift-codegen", + "log", + "object", +] + [[package]] name = "wasmtime-versioned-export-macros" version = "35.0.0" diff --git a/Cargo.toml b/Cargo.toml index b69b5d95c929..683f79141a4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -249,6 +249,7 @@ wasmtime-fuzzing = { path = "crates/fuzzing" } wasmtime-jit-icache-coherence = { path = "crates/jit-icache-coherence", version = "=35.0.0" } wasmtime-wit-bindgen = { path = "crates/wit-bindgen", version = "=35.0.0" } wasmtime-math = { path = "crates/math", version = "=35.0.0" } +wasmtime-unwinder = { path = "crates/unwinder", version = "=35.0.0" } test-programs-artifacts = { path = 'crates/test-programs/artifacts' } wasmtime-test-util = { path = "crates/test-util" } diff --git a/cranelift/codegen/src/ir/extname.rs b/cranelift/codegen/src/ir/extname.rs index 452652baf28a..f29d854b1a44 100644 --- a/cranelift/codegen/src/ir/extname.rs +++ b/cranelift/codegen/src/ir/extname.rs @@ -113,6 +113,11 @@ impl TestcaseName { pub(crate) fn new>(v: T) -> Self { Self(v.as_ref().into()) } + + /// Get the raw test case name as bytes. + pub fn raw(&self) -> &[u8] { + &self.0 + } } /// The name of an external is either a reference to a user-defined symbol diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index af7a11477317..f45e9a9b39be 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -59,8 +59,8 @@ pub mod write; pub use crate::entity::packed_option; pub use crate::machinst::buffer::{ - FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc, MachTextSectionBuilder, - MachTrap, OpenPatchRegion, PatchRegion, + FinalizedMachCallSite, FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc, + MachTextSectionBuilder, MachTrap, OpenPatchRegion, PatchRegion, }; pub use crate::machinst::{ CallInfo, CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, diff --git a/cranelift/filetests/Cargo.toml b/cranelift/filetests/Cargo.toml index 7736ff247903..c70a15237a4b 100644 --- a/cranelift/filetests/Cargo.toml +++ b/cranelift/filetests/Cargo.toml @@ -19,9 +19,10 @@ cranelift-frontend = { workspace = true } cranelift-interpreter = { workspace = true } cranelift-native = { workspace = true } cranelift-reader = { workspace = true } -cranelift-jit = { workspace = true, features = ["selinux-fix"] } +cranelift-jit = { workspace = true, features = ["selinux-fix", "wasmtime-unwinder"] } cranelift-module = { workspace = true } cranelift-control = { workspace = true } +wasmtime-unwinder = { workspace = true, features = ["cranelift"] } file-per-thread-logger = { workspace = true } filecheck = { workspace = true } gimli = { workspace = true, features = ['std'] } diff --git a/cranelift/filetests/filetests/runtests/throw.clif b/cranelift/filetests/filetests/runtests/throw.clif new file mode 100644 index 000000000000..676768749a84 --- /dev/null +++ b/cranelift/filetests/filetests/runtests/throw.clif @@ -0,0 +1,50 @@ +test run +set preserve_frame_pointers=true +target x86_64 +target aarch64 +target riscv64 +target s390x + +function %entry() -> i64 tail { + fn0 = %main(i64) -> i64 tail + +block0: + v1 = get_frame_pointer.i64 + v2 = call fn0(v1) + return v2 +} + +; run: %entry() == 58 + +function %main(i64) -> i64 tail { + sig0 = (i64, i32, i64, i64) tail + fn0 = %throw(i64, i32, i64, i64) tail + +block0(v0: i64): + v1 = iconst.i64 42 + v2 = iconst.i64 100 + v3 = iconst.i32 1 + try_call fn0(v0, v3, v1, v2), sig0, block1(), [ tag1: block2(exn0, exn1) ] + +block1: + v4 = iconst.i64 1 + return v4 + +block2(v5: i64, v6: i64): + v7 = isub.i64 v6, v5 + return v7 +} + + +function %throw(i64, i32, i64, i64) tail { + sig0 = (i64, i64, i64, i32, i64, i64) + fn0 = %__cranelift_throw(i64, i64, i64, i32, i64, i64) + +block0(v0: i64, v1: i32, v2: i64, v3: i64): + v4 = get_frame_pointer.i64 + v5 = get_return_address.i64 + v6 = load.i64 v5 ; get caller's FP + v7 = func_addr.i64 fn0 + call_indirect sig0, v7(v0, v4, v6, v1, v2, v3) + return +} diff --git a/cranelift/filetests/src/function_runner.rs b/cranelift/filetests/src/function_runner.rs index 20d7d4c7545c..f9ab1e2bcdbd 100644 --- a/cranelift/filetests/src/function_runner.rs +++ b/cranelift/filetests/src/function_runner.rs @@ -1,9 +1,12 @@ //! Provides functionality for compiling and running CLIF IR for `run` tests. use anyhow::{Result, anyhow}; use core::mem; +use cranelift::prelude::Imm64; +use cranelift_codegen::cursor::{Cursor, FuncCursor}; use cranelift_codegen::data_value::DataValue; use cranelift_codegen::ir::{ - ExternalName, Function, InstBuilder, Signature, UserExternalName, UserFuncName, + ExternalName, Function, InstBuilder, InstructionData, LibCall, Opcode, Signature, + UserExternalName, UserFuncName, }; use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; use cranelift_codegen::{CodegenError, Context, ir, settings}; @@ -14,9 +17,10 @@ use cranelift_module::{FuncId, Linkage, Module, ModuleError}; use cranelift_native::builder_with_options; use cranelift_reader::TestFile; use pulley_interpreter::interp as pulley; +use std::cell::Cell; use std::cmp::max; -use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; use std::ptr::NonNull; use target_lexicon::Architecture; use thiserror::Error; @@ -67,7 +71,7 @@ struct DefinedFunction { /// let compiled = compiler.compile().unwrap(); /// let trampoline = compiled.get_trampoline(&func).unwrap(); /// -/// let returned = trampoline.call(&vec![DataValue::I32(2), DataValue::I32(40)]); +/// let returned = trampoline.call(&compiled, &vec![DataValue::I32(2), DataValue::I32(40)]); /// assert_eq!(vec![DataValue::I32(42)], returned); /// ``` pub struct TestFileCompiler { @@ -255,7 +259,13 @@ impl TestFileCompiler { } /// Defines the body of a function - pub fn define_function(&mut self, func: Function, ctrl_plane: &mut ControlPlane) -> Result<()> { + pub fn define_function( + &mut self, + mut func: Function, + ctrl_plane: &mut ControlPlane, + ) -> Result<()> { + Self::replace_hostcall_references(&mut func); + let defined_func = self .defined_functions .get(&func.name) @@ -271,6 +281,47 @@ impl TestFileCompiler { Ok(()) } + fn replace_hostcall_references(func: &mut Function) { + // For every `func_addr` referring to a hostcall that we + // define, replace with an `iconst` with the actual + // address. Then modify the external func references to + // harmless libcall references (that will be unused so + // ignored). + let mut funcrefs_to_remove = HashSet::new(); + let mut cursor = FuncCursor::new(func); + while let Some(_block) = cursor.next_block() { + while let Some(inst) = cursor.next_inst() { + match &cursor.func.dfg.insts[inst] { + InstructionData::FuncAddr { + opcode: Opcode::FuncAddr, + func_ref, + } => { + let ext_func = &cursor.func.dfg.ext_funcs[*func_ref]; + let hostcall_addr = match &ext_func.name { + ExternalName::TestCase(tc) if tc.raw() == b"__cranelift_throw" => { + Some(__cranelift_throw as usize) + } + _ => None, + }; + + if let Some(addr) = hostcall_addr { + funcrefs_to_remove.insert(*func_ref); + cursor.func.dfg.insts[inst] = InstructionData::UnaryImm { + opcode: Opcode::Iconst, + imm: Imm64::new(addr as i64), + }; + } + } + _ => {} + } + } + } + + for to_remove in funcrefs_to_remove { + func.dfg.ext_funcs[to_remove].name = ExternalName::LibCall(LibCall::Probestack); + } + } + /// Creates and registers a trampoline for a function if none exists. pub fn create_trampoline_for_function( &mut self, @@ -356,6 +407,13 @@ impl Drop for CompiledTestFile { } } +std::thread_local! { + /// TLS slot used to store a CompiledTestFile reference so that it + /// can be recovered when a hostcall (such as the exception-throw + /// handler) is invoked. + pub static COMPILED_TEST_FILE: Cell<*const CompiledTestFile> = Cell::new(std::ptr::null()); +} + /// A callable trampoline pub struct Trampoline<'a> { module: &'a JITModule, @@ -366,16 +424,18 @@ pub struct Trampoline<'a> { impl<'a> Trampoline<'a> { /// Call the target function of this trampoline, passing in [DataValue]s using a compiled trampoline. - pub fn call(&self, arguments: &[DataValue]) -> Vec { + pub fn call(&self, compiled: &CompiledTestFile, arguments: &[DataValue]) -> Vec { let mut values = UnboxedValues::make_arguments(arguments, &self.func_signature); let arguments_address = values.as_mut_ptr(); let function_ptr = self.module.get_finalized_function(self.func_id); let trampoline_ptr = self.module.get_finalized_function(self.trampoline_id); + COMPILED_TEST_FILE.set(compiled as *const _); unsafe { self.call_raw(trampoline_ptr, function_ptr, arguments_address); } + COMPILED_TEST_FILE.set(std::ptr::null()); values.collect_returns(&self.func_signature) } @@ -563,6 +623,71 @@ fn make_trampoline(name: UserFuncName, signature: &ir::Signature, isa: &dyn Targ func } +/// Hostcall invoked directly from a compiled function body to test +/// exception throws. +/// +/// This function does not return normally: it either uses the +/// unwinder to jump directly to a Cranelift frame further up the +/// stack, if a handler is found; or it panics, if not. +#[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "riscv64" +))] +extern "C-unwind" fn __cranelift_throw( + entry_fp: usize, + exit_fp: usize, + exit_pc: usize, + tag: u32, + payload1: usize, + payload2: usize, +) -> ! { + let compiled_test_file = unsafe { &*COMPILED_TEST_FILE.get() }; + let unwind_host = wasmtime_unwinder::UnwindHost; + let module_lookup = |pc| { + compiled_test_file + .module + .as_ref() + .unwrap() + .lookup_wasmtime_exception_data(pc) + }; + unsafe { + match wasmtime_unwinder::compute_throw_action( + &unwind_host, + module_lookup, + exit_pc, + exit_fp, + entry_fp, + tag, + ) { + wasmtime_unwinder::ThrowAction::Handler { pc, sp, fp } => { + wasmtime_unwinder::resume_to_exception_handler(pc, sp, fp, payload1, payload2); + } + wasmtime_unwinder::ThrowAction::None => { + panic!("Expected a handler to exit for throw of tag {tag} at pc {exit_pc:x}"); + } + } + } +} + +#[cfg(not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "riscv64" +)))] +extern "C-unwind" fn __cranelift_throw( + _entry_fp: usize, + _exit_fp: usize, + _exit_pc: usize, + _tag: u32, + _payload1: usize, + _payload2: usize, +) -> ! { + panic!("Throw not implemented on platforms without native backends."); +} + #[cfg(target_arch = "x86_64")] use std::arch::x86_64::__m128i; #[cfg(target_arch = "x86_64")] @@ -655,7 +780,7 @@ mod test { .unwrap(); let compiled = compiler.compile().unwrap(); let trampoline = compiled.get_trampoline(&function).unwrap(); - let returned = trampoline.call(&[]); + let returned = trampoline.call(&compiled, &[]); assert_eq!(returned, vec![DataValue::I8(-1)]) } diff --git a/cranelift/filetests/src/test_run.rs b/cranelift/filetests/src/test_run.rs index d12d1557c06c..dac83ab6d4c5 100644 --- a/cranelift/filetests/src/test_run.rs +++ b/cranelift/filetests/src/test_run.rs @@ -191,7 +191,7 @@ fn run_test( args.extend_from_slice(run_args); let trampoline = testfile.get_trampoline(func).unwrap(); - Ok(trampoline.call(&args)) + Ok(trampoline.call(&testfile, &args)) }) .map_err(|s| anyhow::anyhow!("{}", s))?; } diff --git a/cranelift/jit/Cargo.toml b/cranelift/jit/Cargo.toml index 9543428463cb..576a6313f709 100644 --- a/cranelift/jit/Cargo.toml +++ b/cranelift/jit/Cargo.toml @@ -19,6 +19,7 @@ cranelift-native = { workspace = true } cranelift-codegen = { workspace = true, features = ["std"] } cranelift-entity = { workspace = true } cranelift-control = { workspace = true } +wasmtime-unwinder = { workspace = true, optional = true, features = ["cranelift"] } anyhow = { workspace = true } region = "3.0.2" libc = { workspace = true } @@ -39,6 +40,8 @@ features = [ selinux-fix = ['memmap2'] default = [] +wasmtime-unwinder = ["dep:wasmtime-unwinder"] + [dev-dependencies] cranelift = { path = "../umbrella" } cranelift-frontend = { workspace = true } diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index 0f52b08361bb..f8c5667e2fa3 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -176,6 +176,7 @@ pub struct JITModule { declarations: ModuleDeclarations, compiled_functions: SecondaryMap>, compiled_data_objects: SecondaryMap>, + code_ranges: Vec<(usize, usize, FuncId)>, functions_to_finalize: Vec, data_objects_to_finalize: Vec, } @@ -328,6 +329,9 @@ impl JITModule { data.perform_relocations(|name| self.get_address(name)); } + self.code_ranges + .sort_unstable_by_key(|(start, _end, _)| *start); + // Now that we're done patching, prepare the memory for execution! let branch_protection = if cfg!(target_arch = "aarch64") && use_bti(&self.isa.isa_flags()) { BranchProtection::BTI @@ -358,10 +362,50 @@ impl JITModule { declarations: ModuleDeclarations::default(), compiled_functions: SecondaryMap::new(), compiled_data_objects: SecondaryMap::new(), + code_ranges: Vec::new(), functions_to_finalize: Vec::new(), data_objects_to_finalize: Vec::new(), } } + + /// Look up the Wasmtime unwind ExceptionTable and corresponding + /// base PC, if any, for a given PC that may be within one of the + /// CompiledBlobs in this module. + #[cfg(feature = "wasmtime-unwinder")] + pub fn lookup_wasmtime_exception_data<'a>( + &'a self, + pc: usize, + ) -> Option<(usize, wasmtime_unwinder::ExceptionTable<'a>)> { + // Search the sorted code-ranges for the PC. + let idx = match self + .code_ranges + .binary_search_by_key(&pc, |(start, _end, _func)| *start) + { + Ok(exact_start_match) => Some(exact_start_match), + Err(least_upper_bound) if least_upper_bound > 0 => { + let last_range_before_pc = &self.code_ranges[least_upper_bound - 1]; + if last_range_before_pc.0 <= pc && pc < last_range_before_pc.1 { + Some(least_upper_bound - 1) + } else { + None + } + } + _ => None, + }?; + + let (start, _, func) = self.code_ranges[idx]; + + // Get the ExceptionTable. The "parse" here simply reads two + // u32s for lengths and constructs borrowed slices, so it's + // cheap. + let data = self.compiled_functions[func] + .as_ref() + .unwrap() + .exception_data + .as_ref()?; + let exception_table = wasmtime_unwinder::ExceptionTable::parse(data).ok()?; + Some((start, exception_table)) + } } impl Module for JITModule { @@ -460,7 +504,32 @@ impl Module for JITModule { .collect(); self.record_function_for_perf(ptr, size, &decl.linkage_name(id)); - self.compiled_functions[id] = Some(CompiledBlob { ptr, size, relocs }); + self.compiled_functions[id] = Some(CompiledBlob { + ptr, + size, + relocs, + #[cfg(feature = "wasmtime-unwinder")] + exception_data: None, + }); + + let range_start = ptr as usize; + let range_end = range_start + size; + // These will be sorted when we finalize. + self.code_ranges.push((range_start, range_end, id)); + + #[cfg(feature = "wasmtime-unwinder")] + { + let mut exception_builder = wasmtime_unwinder::ExceptionTableBuilder::default(); + exception_builder + .add_func(0, compiled_code.buffer.call_sites()) + .map_err(|_| { + ModuleError::Compilation(cranelift_codegen::CodegenError::Unsupported( + "Invalid exception data".into(), + )) + })?; + self.compiled_functions[id].as_mut().unwrap().exception_data = + Some(exception_builder.to_vec()); + } self.functions_to_finalize.push(id); @@ -509,6 +578,8 @@ impl Module for JITModule { ptr, size, relocs: relocs.to_owned(), + #[cfg(feature = "wasmtime-unwinder")] + exception_data: None, }); self.functions_to_finalize.push(id); @@ -599,6 +670,8 @@ impl Module for JITModule { ptr, size: init.size(), relocs, + #[cfg(feature = "wasmtime-unwinder")] + exception_data: None, }); self.data_objects_to_finalize.push(id); diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs index 4313c9f7ff87..157d64e2a913 100644 --- a/cranelift/jit/src/compiled_blob.rs +++ b/cranelift/jit/src/compiled_blob.rs @@ -15,6 +15,8 @@ pub(crate) struct CompiledBlob { pub(crate) ptr: *mut u8, pub(crate) size: usize, pub(crate) relocs: Vec, + #[cfg(feature = "wasmtime-unwinder")] + pub(crate) exception_data: Option>, } unsafe impl Send for CompiledBlob {} diff --git a/cranelift/src/run.rs b/cranelift/src/run.rs index b9d0b0b9a2c8..e78b7cb638fb 100644 --- a/cranelift/src/run.rs +++ b/cranelift/src/run.rs @@ -96,7 +96,7 @@ fn run_file_contents(file_contents: String) -> Result<()> { let trampoline = compiled.get_trampoline(&func).unwrap(); command - .run(|_, args| Ok(trampoline.call(args))) + .run(|_, args| Ok(trampoline.call(&compiled, args))) .map_err(|s| anyhow::anyhow!("{}", s))?; } } diff --git a/crates/unwinder/Cargo.toml b/crates/unwinder/Cargo.toml new file mode 100644 index 000000000000..316005c3aff6 --- /dev/null +++ b/crates/unwinder/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "wasmtime-unwinder" +authors.workspace = true +version.workspace = true +description = "Wasmtime's unwind format and unwinder" +license = "Apache-2.0 WITH LLVM-exception" +documentation = "https://docs.rs/wasmtime-unwinder" +repository = "https://github.com/bytecodealliance/wasmtime" +categories = ["no-std"] +readme = "README.md" +keywords = ["unwind", "exceptions"] +edition.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +cranelift-codegen = { workspace = true, optional = true } +log = { workspace = true } +cfg-if = { workspace = true } +object = { workspace = true } +anyhow = { workspace = true } + +[features] +default = [] + +# Enable generation of unwind info from Cranelift metadata. +cranelift = ["dep:cranelift-codegen"] diff --git a/crates/unwinder/LICENSE b/crates/unwinder/LICENSE new file mode 100644 index 000000000000..f9d81955f4bc --- /dev/null +++ b/crates/unwinder/LICENSE @@ -0,0 +1,220 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/crates/unwinder/README.md b/crates/unwinder/README.md new file mode 100644 index 000000000000..f9b0c7e8f348 --- /dev/null +++ b/crates/unwinder/README.md @@ -0,0 +1,13 @@ +# `wasmtime-unwinder` + +This crate implements an unwind info format, stack walking, and +unwinding for Wasmtime. It includes logic that: + +- Can walk the Wasmstack and visit each frame; +- Can find exception handlers using an efficient format serialized + from Cranelift compilation metadata that can be mapped and used + in-place from disk; +- Provides a "throw" helper that, when called from host code that has + been invoked from Wasmcode, can find a handler; and a "resume" stub + that can be invoked to transfer control to the corresponding + handler. diff --git a/crates/wasmtime/src/runtime/vm/arch/aarch64.rs b/crates/unwinder/src/arch/aarch64.rs similarity index 77% rename from crates/wasmtime/src/runtime/vm/arch/aarch64.rs rename to crates/unwinder/src/arch/aarch64.rs index 8d2e6daefd9c..3cd1ec4dc5b6 100644 --- a/crates/wasmtime/src/runtime/vm/arch/aarch64.rs +++ b/crates/unwinder/src/arch/aarch64.rs @@ -47,9 +47,40 @@ pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { pc } +pub unsafe fn resume_to_exception_handler( + pc: usize, + sp: usize, + fp: usize, + payload1: usize, + payload2: usize, +) -> ! { + unsafe { + core::arch::asm!( + "mov x0, {}", + "mov x1, {}", + "mov sp, {}", + "mov fp, {}", + "br {}", + in(reg) payload1, + in(reg) payload2, + in(reg) sp, + in(reg) fp, + in(reg) pc, + out("x0") _, + out("x1") _, + options(nostack, nomem), + ); + + core::hint::unreachable_unchecked() + } +} + // And the current frame pointer points to the next older frame pointer. pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; +// SP of caller is FP in callee plus size of FP/return address pair. +pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = 16; + pub fn assert_fp_is_aligned(_fp: usize) { // From AAPCS64, section 6.2.3 The Frame Pointer[0]: // diff --git a/crates/unwinder/src/arch/mod.rs b/crates/unwinder/src/arch/mod.rs new file mode 100644 index 000000000000..714657a4a7fc --- /dev/null +++ b/crates/unwinder/src/arch/mod.rs @@ -0,0 +1,109 @@ +//! Architecture-specific runtime support corresponding to details of +//! Cranelift codegen or ABI support. +//! +//! This crate houses any architecture-specific tidbits required when +//! building a runtime that executes Cranelift-produced code. +//! +//! All architectures have the same interface when exposed to the rest of the +//! crate. + +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86; + use x86 as imp; + } else if #[cfg(target_arch = "aarch64")] { + mod aarch64; + use aarch64 as imp; + } else if #[cfg(target_arch = "s390x")] { + mod s390x; + use s390x as imp; + } else if #[cfg(target_arch = "riscv64")] { + mod riscv64; + use riscv64 as imp; + } +} + +// Re re-export functions from the `imp` module with one set of `pub +// use` declarations here so we can share doc-comments. + +cfg_if::cfg_if! { + if #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "riscv64" + ))] { + /// Get the current stack pointer (at the time this function is + /// executing). This may be used to check, e.g., approximate space + /// remaining on a stack, but cannot be relied upon for anything exact + /// because the stack pointer from *within this function* is read and + /// the frame is later popped. + pub use imp::get_stack_pointer; + + /// Resume execution at the given PC, SP, and FP, with the given + /// payload values, according to the tail-call ABI's exception + /// scheme. Note that this scheme does not restore any other + /// registers, so the given state is all that we need. + /// + /// # Safety + /// + /// This method requires: + /// + /// - the `sp` and `fp` to correspond to an active stack frame + /// (above the current function), in code using Cranelift's + /// `tail` calling convention. + /// + /// - The `pc` to correspond to a `try_call` handler + /// destination, as emitted in Cranelift metadata, or + /// otherwise a target that is expecting the tail-call ABI's + /// exception ABI. + /// + /// - The Rust frames between the unwind destination and this + /// frame to be unwind-safe: that is, they cannot have `Drop` + /// handlers for which safety requires that they run. + pub use imp::resume_to_exception_handler; + + /// Get the return address in the function at the next-older + /// frame from the given FP. + /// + /// # Safety + /// + /// - Requires that `fp` is a valid frame-pointer value for an + /// active stack frame (above the current function), in code + /// using Cranelift's `tail` calling convention. + pub use imp::get_next_older_pc_from_fp; + + + /// The offset of the saved old-FP value in a frame, from the + /// location pointed to by a given FP. + pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = imp::NEXT_OLDER_FP_FROM_FP_OFFSET; + + /// The offset of the next older SP value, from the value of a + /// given FP. + pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = imp::NEXT_OLDER_SP_FROM_FP_OFFSET; + + /// Assert that the given `fp` is aligned as expected by the + /// host platform's implementation of the Cranelift tail-call + /// ABI. + pub use imp::assert_fp_is_aligned; + + /// If we have the above host-specific implementations, we can + /// implement `Unwind`. + pub struct UnwindHost; + + unsafe impl crate::stackwalk::Unwind for UnwindHost { + fn next_older_fp_from_fp_offset(&self) -> usize { + NEXT_OLDER_FP_FROM_FP_OFFSET + } + fn next_older_sp_from_fp_offset(&self) -> usize { + NEXT_OLDER_SP_FROM_FP_OFFSET + } + unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize { + get_next_older_pc_from_fp(fp) + } + fn assert_fp_is_aligned(&self, fp: usize) { + assert_fp_is_aligned(fp) + } + } + } +} diff --git a/crates/unwinder/src/arch/riscv64.rs b/crates/unwinder/src/arch/riscv64.rs new file mode 100644 index 000000000000..735fa13c985e --- /dev/null +++ b/crates/unwinder/src/arch/riscv64.rs @@ -0,0 +1,57 @@ +//! Riscv64-specific definitions of architecture-specific functions in Wasmtime. + +#[inline] +#[allow(missing_docs)] +pub fn get_stack_pointer() -> usize { + let stack_pointer: usize; + unsafe { + core::arch::asm!( + "mv {}, sp", + out(reg) stack_pointer, + options(nostack,nomem), + ); + } + stack_pointer +} + +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + *(fp as *mut usize).offset(1) +} + +pub unsafe fn resume_to_exception_handler( + pc: usize, + sp: usize, + fp: usize, + payload1: usize, + payload2: usize, +) -> ! { + unsafe { + core::arch::asm!( + "mv a0, {}", + "mv a1, {}", + "mv sp, {}", + "mv fp, {}", + "jr {}", + in(reg) payload1, + in(reg) payload2, + in(reg) sp, + in(reg) fp, + in(reg) pc, + out("a0") _, + out("a1") _, + options(nostack, nomem), + ); + + core::hint::unreachable_unchecked() + } +} + +// And the current frame pointer points to the next older frame pointer. +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +// SP of caller is FP in callee plus size of FP/return address pair. +pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = 16; + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); +} diff --git a/crates/unwinder/src/arch/s390x.rs b/crates/unwinder/src/arch/s390x.rs new file mode 100644 index 000000000000..25d9ecd3d0e9 --- /dev/null +++ b/crates/unwinder/src/arch/s390x.rs @@ -0,0 +1,59 @@ +//! s390x-specific definitions of architecture-specific functions in Wasmtime. + +#[inline] +#[allow(missing_docs)] +pub fn get_stack_pointer() -> usize { + let mut sp; + unsafe { + core::arch::asm!( + "lgr {}, %r15", + out(reg) sp, + options(nostack, nomem), + ); + } + sp +} + +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + // The next older PC can be found in register %r14 at function entry, which + // was saved into slot 14 of the register save area pointed to by "FP" (the + // backchain pointer). + *(fp as *mut usize).offset(14) +} + +pub unsafe fn resume_to_exception_handler( + pc: usize, + sp: usize, + _fp: usize, + payload1: usize, + payload2: usize, +) -> ! { + unsafe { + core::arch::asm!( + "lgr %r6, {}", + "lgr %r7, {}", + "lgr %r15, {}", + "br {}", + in(reg) payload1, + in(reg) payload2, + in(reg) sp, + in(reg) pc, + out("r6") _, + out("r7") _, + options(nostack, nomem), + ); + + core::hint::unreachable_unchecked() + } +} + +// The next older "FP" (backchain pointer) was saved in the slot pointed to +// by the current "FP". +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +// SP of caller is "FP" (backchain pointer) in callee. +pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = 0; + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 8, 0, "stack should always be aligned to 8"); +} diff --git a/crates/wasmtime/src/runtime/vm/arch/x86.rs b/crates/unwinder/src/arch/x86.rs similarity index 57% rename from crates/wasmtime/src/runtime/vm/arch/x86.rs rename to crates/unwinder/src/arch/x86.rs index ef8dbb4bf7f7..cf3d6328bd53 100644 --- a/crates/wasmtime/src/runtime/vm/arch/x86.rs +++ b/crates/unwinder/src/arch/x86.rs @@ -21,9 +21,40 @@ pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { *(fp as *mut usize).offset(1) } +pub unsafe fn resume_to_exception_handler( + pc: usize, + sp: usize, + fp: usize, + payload1: usize, + payload2: usize, +) -> ! { + unsafe { + core::arch::asm!( + "mov rax, {}", + "mov rdx, {}", + "mov rsp, {}", + "mov rbp, {}", + "jmp {}", + in(reg) payload1, + in(reg) payload2, + in(reg) sp, + in(reg) fp, + in(reg) pc, + out("rax") _, + out("rdx") _, + options(nostack, nomem), + ); + + core::hint::unreachable_unchecked() + } +} + // And the current frame pointer points to the next older frame pointer. pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; +// SP of caller is FP in callee plus size of FP/return address pair. +pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = 16; + /// Frame pointers are aligned if they're aligned to twice the size of a /// pointer. pub fn assert_fp_is_aligned(fp: usize) { diff --git a/crates/unwinder/src/exception_table.rs b/crates/unwinder/src/exception_table.rs new file mode 100644 index 000000000000..ab36ce079d50 --- /dev/null +++ b/crates/unwinder/src/exception_table.rs @@ -0,0 +1,300 @@ +//! Compact representation of exception handlers associated with +//! callsites, for use when searching a Cranelift stack for a handler. +//! +//! This module implements (i) conversion from the metadata provided +//! alongside Cranelift's compilation result (as provided by +//! [`cranelift_codegen::MachBufferFinalized::call_sites`]) to its +//! format, and (ii) use of its format to find a handler efficiently. +//! +//! The format has been designed so that it can be mapped in from disk +//! and used without post-processing; this enables efficient +//! module-loading in runtimes such as Wasmtime. + +use object::{Bytes, LittleEndian, U32Bytes}; + +#[cfg(feature = "cranelift")] +use alloc::{vec, vec::Vec}; +#[cfg(feature = "cranelift")] +use cranelift_codegen::{FinalizedMachCallSite, binemit::CodeOffset}; + +/// Collector struct for exception handlers per call site. +/// +/// # Format +/// +/// We keep four different arrays (`Vec`s) that we build as we visit +/// callsites, in ascending offset (address relative to beginning of +/// code segment) order: tags, destination offsets, callsite offsets, +/// and tag/destination ranges. +/// +/// The callsite offsets and tag/destination ranges logically form a +/// sorted lookup array, allowing us to find information for any +/// single callsite. The range denotes a range of indices in the tag +/// and destination offset arrays, and those are sorted by tag per +/// callsite. Ranges are stored with the (exclusive) *end* index only; +/// the start index is implicit as the previous end, or zero if first +/// element. +/// +/// # Example +/// +/// An example of this data format: +/// +/// ```plain +/// callsites: [0x10, 0x50, 0xf0] // callsites (return addrs) at offsets 0x10, 0x50, 0xf0 +/// ranges: [2, 4, 5] // corresponding ranges for each callsite +/// tags: [1, 5, 1, -1, -1] // tags for each handler at each callsite +/// handlers: [0x40, 0x42, 0x6f, 0x71, 0xf5] // handler destinations at each callsite +/// ``` +/// +/// Expanding this out: +/// +/// ```plain +/// callsites: [0x10, 0x50, 0xf0], # PCs relative to some start of return-points. +/// ranges: [ +/// 2, # callsite 0x10 has tags/handlers indices 0..2 +/// 4, # callsite 0x50 has tags/handlers indices 2..4 +/// 5, # callsite 0xf0 has tags/handlers indices 4..5 +/// ], +/// tags: [ +/// # tags for callsite 0x10: +/// 1, +/// 5, +/// # tags for callsite 0x50: +/// 1, +/// -1, # "catch-all" +/// # tags for callsite 0xf0: +/// -1, # "catch-all" +/// ] +/// handlers: [ +/// # handlers for callsite 0x10: +/// 0x40, # relative PC to handle tag 1 (above) +/// 0x42, # relative PC to handle tag 5 +/// # handlers for callsite 0x50: +/// 0x6f, # relative PC to handle tag 1 +/// 0x71, # relative PC to handle all other tags +/// # handlers for callsite 0xf0: +/// 0xf5, # relative PC to handle all other tags +/// ] +/// ``` +#[cfg(feature = "cranelift")] +#[derive(Clone, Debug, Default)] +pub struct ExceptionTableBuilder { + pub callsites: Vec>, + pub ranges: Vec>, + pub tags: Vec>, + pub handlers: Vec>, + last_start_offset: CodeOffset, +} + +#[cfg(feature = "cranelift")] +impl ExceptionTableBuilder { + /// Add a function at a given offset from the start of the + /// compiled code section, recording information about its call + /// sites. + /// + /// Functions must be added in ascending offset order. + pub fn add_func<'a>( + &mut self, + start_offset: CodeOffset, + call_sites: impl Iterator>, + ) -> anyhow::Result<()> { + // Ensure that we see functions in offset order. + assert!(start_offset >= self.last_start_offset); + self.last_start_offset = start_offset; + + // Visit each callsite in turn, translating offsets from + // function-local to section-local. + let mut handlers = vec![]; + for call_site in call_sites { + let ret_addr = call_site.ret_addr.checked_add(start_offset).unwrap(); + handlers.extend(call_site.exception_handlers.iter().cloned()); + handlers.sort_by_key(|(tag, _dest)| *tag); + + if handlers.windows(2).any(|parts| parts[0].0 == parts[1].0) { + anyhow::bail!("Duplicate handler tag"); + } + + let start_idx = u32::try_from(self.tags.len()).unwrap(); + for (tag, dest) in handlers.drain(..) { + self.tags.push(U32Bytes::new( + LittleEndian, + tag.expand().map(|t| t.as_u32()).unwrap_or(u32::MAX), + )); + self.handlers.push(U32Bytes::new( + LittleEndian, + dest.checked_add(start_offset).unwrap(), + )); + } + let end_idx = u32::try_from(self.tags.len()).unwrap(); + + // Omit empty callsites for compactness. + if end_idx > start_idx { + self.ranges.push(U32Bytes::new(LittleEndian, end_idx)); + self.callsites.push(U32Bytes::new(LittleEndian, ret_addr)); + } + } + + Ok(()) + } + + /// Serialize the exception-handler data section, taking a closure + /// to consume slices. + pub fn serialize(&self, mut f: F) { + // Serialize the length of `callsites` / `ranges`. + let callsite_count = u32::try_from(self.callsites.len()).unwrap(); + f(&callsite_count.to_le_bytes()); + // Serialize the length of `tags` / `handlers`. + let handler_count = u32::try_from(self.handlers.len()).unwrap(); + f(&handler_count.to_le_bytes()); + + // Serialize `callsites`, `ranges`, `tags`, and `handlers` in + // that order. + f(object::bytes_of_slice(&self.callsites)); + f(object::bytes_of_slice(&self.ranges)); + f(object::bytes_of_slice(&self.tags)); + f(object::bytes_of_slice(&self.handlers)); + } + + /// Serialize the exception-handler data section to a vector of + /// bytes. + pub fn to_vec(&self) -> Vec { + let mut bytes = vec![]; + self.serialize(|slice| bytes.extend(slice.iter().cloned())); + bytes + } +} + +/// ExceptionTable deserialized from a serialized slice. +/// +/// This struct retains borrows of the various serialized parts of the +/// exception table data as produced by +/// [`ExceptionTableBuilder::serialize`]. +#[derive(Clone, Debug)] +pub struct ExceptionTable<'a> { + callsites: &'a [U32Bytes], + ranges: &'a [U32Bytes], + tags: &'a [U32Bytes], + handlers: &'a [U32Bytes], +} + +impl<'a> ExceptionTable<'a> { + /// Parse exception tables from a byte-slice as produced by + /// [`ExceptionTableBuilder::serialize`]. + pub fn parse(data: &'a [u8]) -> anyhow::Result> { + let mut data = Bytes(data); + let callsite_count = data + .read::>() + .map_err(|_| anyhow::anyhow!("Unable to read callsite count prefix"))?; + let callsite_count = usize::try_from(callsite_count.get(LittleEndian))?; + let handler_count = data + .read::>() + .map_err(|_| anyhow::anyhow!("Unable to read handler count prefix"))?; + let handler_count = usize::try_from(handler_count.get(LittleEndian))?; + let (callsites, data) = + object::slice_from_bytes::>(data.0, callsite_count) + .map_err(|_| anyhow::anyhow!("Unable to read callsites slice"))?; + let (ranges, data) = + object::slice_from_bytes::>(data, callsite_count) + .map_err(|_| anyhow::anyhow!("Unable to read ranges slice"))?; + let (tags, data) = object::slice_from_bytes::>(data, handler_count) + .map_err(|_| anyhow::anyhow!("Unable to read tags slice"))?; + let (handlers, data) = + object::slice_from_bytes::>(data, handler_count) + .map_err(|_| anyhow::anyhow!("Unable to read handlers slice"))?; + + if !data.is_empty() { + anyhow::bail!("Unexpected data at end of serialized exception table"); + } + + Ok(ExceptionTable { + callsites, + ranges, + tags, + handlers, + }) + } + + /// Look up the handler destination, if any, for a given return + /// address (as an offset into the code section) and exception + /// tag. + /// + /// Note: we use raw `u32` types for code offsets and tags here to + /// avoid dependencies on `cranelift-codegen` when this crate is + /// built without compiler backend support (runtime-only config). + pub fn lookup(&self, pc: u32, tag: u32) -> Option { + // First, look up the callsite in the sorted callsites list. + let callsite_idx = self + .callsites + .binary_search_by_key(&pc, |callsite| callsite.get(LittleEndian)) + .ok()?; + // Now get the range. + let end_idx = self.ranges[callsite_idx].get(LittleEndian); + let start_idx = if callsite_idx > 0 { + self.ranges[callsite_idx - 1].get(LittleEndian) + } else { + 0 + }; + + // Take the subslices of `tags` and `handlers` corresponding + // to this callsite. + let start_idx = usize::try_from(start_idx).unwrap(); + let end_idx = usize::try_from(end_idx).unwrap(); + let tags = &self.tags[start_idx..end_idx]; + let handlers = &self.handlers[start_idx..end_idx]; + + // Is there any handler with an exact tag match? + if let Ok(handler_idx) = tags.binary_search_by_key(&tag, |tag| tag.get(LittleEndian)) { + return Some(handlers[handler_idx].get(LittleEndian)); + } + + // If not, is there a fallback handler? Note that we serialize + // it with the tag `u32::MAX`, so it is always last in sorted + // order. + if tags.last().map(|v| v.get(LittleEndian)) == Some(u32::MAX) { + return Some(handlers.last().unwrap().get(LittleEndian)); + } + + None + } +} + +#[cfg(all(test, feature = "cranelift"))] +mod test { + use super::*; + use cranelift_codegen::entity::EntityRef; + use cranelift_codegen::ir::ExceptionTag; + + #[test] + fn serialize_exception_table() { + let callsites = [ + FinalizedMachCallSite { + ret_addr: 0x10, + exception_handlers: &[ + (Some(ExceptionTag::new(1)).into(), 0x20), + (Some(ExceptionTag::new(2)).into(), 0x30), + (None.into(), 0x40), + ], + }, + FinalizedMachCallSite { + ret_addr: 0x48, + exception_handlers: &[], + }, + FinalizedMachCallSite { + ret_addr: 0x50, + exception_handlers: &[(None.into(), 0x60)], + }, + ]; + + let mut builder = ExceptionTableBuilder::default(); + builder.add_func(0x100, callsites.into_iter()).unwrap(); + let mut bytes = vec![]; + builder.serialize(|slice| bytes.extend(slice.iter().cloned())); + + let deserialized = ExceptionTable::parse(&bytes).unwrap(); + + assert_eq!(deserialized.lookup(0x148, 1), None); + assert_eq!(deserialized.lookup(0x110, 1), Some(0x120)); + assert_eq!(deserialized.lookup(0x110, 2), Some(0x130)); + assert_eq!(deserialized.lookup(0x110, 42), Some(0x140)); + assert_eq!(deserialized.lookup(0x150, 100), Some(0x160)); + } +} diff --git a/crates/unwinder/src/lib.rs b/crates/unwinder/src/lib.rs new file mode 100644 index 000000000000..ddd1523bfd84 --- /dev/null +++ b/crates/unwinder/src/lib.rs @@ -0,0 +1,18 @@ +//! Cranelift unwinder. +#![doc = include_str!("../README.md")] +#![no_std] +#![expect(unsafe_op_in_unsafe_fn, reason = "crate isn't migrated yet")] +#![expect(clippy::allow_attributes_without_reason, reason = "crate not migrated")] + +#[cfg(feature = "cranelift")] +extern crate alloc; + +mod stackwalk; +pub use stackwalk::*; +mod arch; +#[allow(unused_imports)] // `arch` becomes empty on platforms without native-code backends. +pub use arch::*; +mod exception_table; +pub use exception_table::*; +mod throw; +pub use throw::*; diff --git a/crates/unwinder/src/stackwalk.rs b/crates/unwinder/src/stackwalk.rs new file mode 100644 index 000000000000..b2b65cc6bb8f --- /dev/null +++ b/crates/unwinder/src/stackwalk.rs @@ -0,0 +1,175 @@ +//! Stack-walking of a Wasm stack. +//! +//! A stack walk requires a first and last frame pointer (FP), and it +//! only works on code that has been compiled with frame pointers +//! enabled (`preserve_frame_pointers` Cranelift option enabled). The +//! stack walk follows the singly-linked list of saved frame pointer +//! and return address pairs on the stack that is naturally built by +//! function prologues. +//! +//! This crate makes use of the fact that Wasmtime surrounds Wasm +//! frames by trampolines both at entry and exit, and is "up the +//! stack" from the point doing the unwinding: in other words, host +//! code invokes Wasm code via an entry trampoline, that code may call +//! other Wasm code, and ultimately it calls back to host code via an +//! exit trampoline. That exit trampoline is able to provide the +//! "start FP" (FP at exit trampoline) and "end FP" (FP at entry +//! trampoline) and this stack-walker can visit all Wasm frames +//! active on the stack between those two. +//! +//! This module provides a visitor interface to frames, but is +//! agnostic to the desired use-case or consumer of the frames, and to +//! the overall runtime structure. + +use core::ops::ControlFlow; + +/// Implementation necessary to unwind the stack, used by `Backtrace`. +pub unsafe trait Unwind { + /// Returns the offset, from the current frame pointer, of where to get to + /// the previous frame pointer on the stack. + fn next_older_fp_from_fp_offset(&self) -> usize; + + /// Returns the offset, from the current frame pointer, of the + /// stack pointer of the next older frame. + fn next_older_sp_from_fp_offset(&self) -> usize; + + /// Load the return address of a frame given the frame pointer for that + /// frame. + unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize; + + /// Debug assertion that the frame pointer is aligned. + fn assert_fp_is_aligned(&self, fp: usize); +} + +/// A stack frame within a Wasm stack trace. +#[derive(Debug)] +pub struct Frame { + /// The program counter in this frame. Because every frame in the + /// stack-walk is paused at a call (as we are in host code called + /// by Wasm code below these frames), the PC is at the return + /// address, i.e., points to the instruction after the call + /// instruction. + pc: usize, + /// The frame pointer value corresponding to this frame. + fp: usize, +} + +impl Frame { + /// Get this frame's program counter. + pub fn pc(&self) -> usize { + self.pc + } + + /// Get this frame's frame pointer. + pub fn fp(&self) -> usize { + self.fp + } +} + +/// Walk through a contiguous sequence of Wasm frames starting with +/// the frame at the given PC and FP and ending at +/// `trampoline_fp`. This FP should correspond to that of a trampoline +/// that was used to enter the Wasm code. +/// +/// We require that the initial PC, FP, and `trampoline_fp` values are +/// non-null (non-zero). +pub unsafe fn visit_frames( + unwind: &dyn Unwind, + mut pc: usize, + mut fp: usize, + trampoline_fp: usize, + mut f: impl FnMut(Frame) -> ControlFlow, +) -> ControlFlow { + log::trace!("=== Tracing through contiguous sequence of Wasm frames ==="); + log::trace!("trampoline_fp = 0x{:016x}", trampoline_fp); + log::trace!(" initial pc = 0x{:016x}", pc); + log::trace!(" initial fp = 0x{:016x}", fp); + + // Safety requirements documented above. + assert_ne!(pc, 0); + assert_ne!(fp, 0); + assert_ne!(trampoline_fp, 0); + + // This loop will walk the linked list of frame pointers starting + // at `fp` and going up until `trampoline_fp`. We know that both + // `fp` and `trampoline_fp` are "trusted values" aka generated and + // maintained by Wasmtime. This means that it should be safe to + // walk the linked list of pointers and inspect Wasm frames. + // + // Note, though, that any frames outside of this range are not + // guaranteed to have valid frame pointers. For example native code + // might be using the frame pointer as a general purpose register. Thus + // we need to be careful to only walk frame pointers in this one + // contiguous linked list. + // + // To know when to stop iteration all architectures' stacks currently + // look something like this: + // + // | ... | + // | Native Frames | + // | ... | + // |-------------------| + // | ... | <-- Trampoline FP | + // | Trampoline Frame | | + // | ... | <-- Trampoline SP | + // |-------------------| Stack + // | Return Address | Grows + // | Previous FP | <-- Wasm FP Down + // | ... | | + // | Cranelift Frames | | + // | ... | V + // + // The trampoline records its own frame pointer (`trampoline_fp`), + // which is guaranteed to be above all Wasm code. To check when + // we've reached the trampoline frame, it is therefore sufficient + // to check when the next frame pointer is equal to + // `trampoline_fp`. Once that's hit then we know that the entire + // linked list has been traversed. + // + // Note that it might be possible that this loop doesn't execute + // at all. For example if the entry trampoline called Wasm code + // which `return_call`'d an exit trampoline, then `fp == + // trampoline_fp` on the entry of this function, meaning the loop + // won't actually execute anything. + while fp != trampoline_fp { + // At the start of each iteration of the loop, we know that + // `fp` is a frame pointer from Wasm code. Therefore, we know + // it is not being used as an extra general-purpose register, + // and it is safe dereference to get the PC and the next older + // frame pointer. + // + // The stack also grows down, and therefore any frame pointer + // we are dealing with should be less than the frame pointer + // on entry to Wasm code. Finally also assert that it's + // aligned correctly as an additional sanity check. + assert!(trampoline_fp > fp, "{trampoline_fp:#x} > {fp:#x}"); + unwind.assert_fp_is_aligned(fp); + + log::trace!("--- Tracing through one Wasm frame ---"); + log::trace!("pc = {:p}", pc as *const ()); + log::trace!("fp = {:p}", fp as *const ()); + + f(Frame { pc, fp })?; + + pc = unwind.get_next_older_pc_from_fp(fp); + + // We rely on this offset being zero for all supported + // architectures in + // `crates/cranelift/src/component/compiler.rs` when we set + // the Wasm exit FP. If this ever changes, we will need to + // update that code as well! + assert_eq!(unwind.next_older_fp_from_fp_offset(), 0); + + // Get the next older frame pointer from the current Wasm + // frame pointer. + let next_older_fp = *(fp as *mut usize).add(unwind.next_older_fp_from_fp_offset()); + + // Because the stack always grows down, the older FP must be greater + // than the current FP. + assert!(next_older_fp > fp, "{next_older_fp:#x} > {fp:#x}"); + fp = next_older_fp; + } + + log::trace!("=== Done tracing contiguous sequence of Wasm frames ==="); + ControlFlow::Continue(()) +} diff --git a/crates/unwinder/src/throw.rs b/crates/unwinder/src/throw.rs new file mode 100644 index 000000000000..ea1652b1d622 --- /dev/null +++ b/crates/unwinder/src/throw.rs @@ -0,0 +1,80 @@ +//! Generation of the throw-stub. +//! +//! In order to throw exceptions from within Cranelift-compiled code, +//! we provide a runtime function helper meant to be called by host +//! code that is invoked by guest code. +//! +//! The helper below must be provided a delimited range on the stack +//! corresponding to Cranelift frames above the current host code. It +//! will look for any handlers in this code, given a closure that +//! knows how to use an absolute PC to look up a module's exception +//! table and its start-of-code-segment. If a handler is found, the +//! helper below will return the SP, FP and PC that must be +//! restored. Architecture-specific helpers are provided to jump to +//! this new context with payload values. Otherwise, if no handler is +//! found, the return type indicates this, and it is the caller's +//! responsibility to invoke alternative behavior (e.g., abort the +//! program or unwind all the way to initial Cranelift-code entry). + +use crate::{ExceptionTable, Unwind}; +use core::ops::ControlFlow; + +/// Throw action to perform. +#[derive(Clone, Debug)] +pub enum ThrowAction { + /// Jump to the given handler with the given SP and FP values. + Handler { + /// Program counter of handler return point. + pc: usize, + /// Stack pointer to restore before jumping to handler. + sp: usize, + /// Frame pointer to restore before jumping to handler. + fp: usize, + }, + /// No handler found. + None, +} + +/// Implementation of stack-walking to find a handler. +/// +/// This function searches for a handler in the given range of stack +/// frames, starting from the throw stub and up to a specified entry +/// frame. +pub unsafe fn compute_throw_action<'a, F: Fn(usize) -> Option<(usize, ExceptionTable<'a>)>>( + unwind: &dyn Unwind, + module_lookup: F, + exit_pc: usize, + exit_frame: usize, + entry_frame: usize, + tag: u32, +) -> ThrowAction { + let mut last_fp = exit_frame; + match crate::stackwalk::visit_frames(unwind, exit_pc, exit_frame, entry_frame, |frame| { + if let Some((base, table)) = module_lookup(frame.pc()) { + let relative_pc = u32::try_from( + frame + .pc() + .checked_sub(base) + .expect("module lookup did not return a module base below the PC"), + ) + .expect("module larger than 4GiB"); + + if let Some(handler) = table.lookup(relative_pc, tag) { + let abs_handler_pc = base + .checked_add(usize::try_from(handler).unwrap()) + .expect("Handler address computation overflowed"); + + return ControlFlow::Break(ThrowAction::Handler { + pc: abs_handler_pc, + sp: last_fp + unwind.next_older_sp_from_fp_offset(), + fp: frame.fp(), + }); + } + } + last_fp = frame.fp(); + ControlFlow::Continue(()) + }) { + ControlFlow::Break(action) => action, + ControlFlow::Continue(()) => ThrowAction::None, + } +} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 2f5105525cd1..02188fa8fd91 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -24,6 +24,7 @@ wasmtime-jit-icache-coherence = { workspace = true, optional = true } wasmtime-cache = { workspace = true, optional = true } wasmtime-fiber = { workspace = true, optional = true } wasmtime-cranelift = { workspace = true, optional = true, features = ['pulley'] } +wasmtime-unwinder = { workspace = true, optional = true } wasmtime-winch = { workspace = true, optional = true } wasmtime-component-macro = { workspace = true, optional = true } wasmtime-component-util = { workspace = true, optional = true } @@ -94,9 +95,6 @@ mach2 = { workspace = true, optional = true } [target.'cfg(unix)'.dependencies] rustix = { workspace = true, optional = true, features = ["mm", "param"] } -[target.'cfg(target_arch = "s390x")'.dependencies] -psm = { workspace = true, optional = true } - [dev-dependencies] env_logger = { workspace = true } proptest = { workspace = true } @@ -151,7 +149,7 @@ default = [ # with the Cranelift compiler. Cranelift is the default compilation backend of # Wasmtime. If disabled then WebAssembly modules can only be created from # precompiled WebAssembly modules. -cranelift = ["dep:wasmtime-cranelift", "std"] +cranelift = ["dep:wasmtime-cranelift", "std", "wasmtime-unwinder/cranelift"] # Enables support for Winch, the WebAssembly baseline compiler. The Winch compiler # strategy in `Config` will be available. It is currently in active development @@ -256,11 +254,11 @@ runtime = [ "dep:wasmtime-slab", "dep:wasmtime-versioned-export-macros", "dep:windows-sys", - "dep:psm", "dep:rustix", "rustix/mm", "pulley-interpreter/interp", "dep:wasmtime-jit-icache-coherence", + "dep:wasmtime-unwinder", ] # Enable support for garbage collection-related things. diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 260c698acb48..c2f016fe2f0a 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -1911,7 +1911,7 @@ at https://bytecodealliance.org/security. pub(crate) fn unwinder(&self) -> &'static dyn Unwind { match &self.executor { - Executor::Interpreter(_) => &vm::UnwindPulley, + Executor::Interpreter(i) => i.unwinder(), #[cfg(has_host_compiler_backend)] Executor::Native => &vm::UnwindHost, } diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index a4bd468a5a1e..83cadee596ca 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -48,8 +48,6 @@ use wasmtime_environ::{ #[cfg(feature = "gc")] use wasmtime_environ::ModuleInternedTypeIndex; -#[cfg(has_host_compiler_backend)] -mod arch; #[cfg(feature = "component-model")] pub mod component; mod const_expr; @@ -66,7 +64,6 @@ mod store_box; mod sys; mod table; mod traphandlers; -mod unwind; mod vmcontext; #[cfg(feature = "threads")] @@ -90,8 +87,6 @@ pub(crate) use interpreter_disabled as interpreter; #[cfg(feature = "debug-builtins")] pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration; -#[cfg(has_host_compiler_backend)] -pub use crate::runtime::vm::arch::get_stack_pointer; pub use crate::runtime::vm::export::*; pub use crate::runtime::vm::gc::*; pub use crate::runtime::vm::imports::Imports; @@ -119,7 +114,6 @@ pub use crate::runtime::vm::sys::mmap::open_file_for_mmap; pub use crate::runtime::vm::sys::unwind::UnwindRegistration; pub use crate::runtime::vm::table::{Table, TableElement}; pub use crate::runtime::vm::traphandlers::*; -pub use crate::runtime::vm::unwind::*; #[cfg(feature = "component-model")] pub use crate::runtime::vm::vmcontext::VMTableDefinition; pub use crate::runtime::vm::vmcontext::{ @@ -130,6 +124,10 @@ pub use crate::runtime::vm::vmcontext::{ }; pub use send_sync_ptr::SendSyncPtr; +pub use wasmtime_unwinder::Unwind; + +#[cfg(has_host_compiler_backend)] +pub use wasmtime_unwinder::{UnwindHost, get_stack_pointer}; mod module_id; pub use module_id::CompiledModuleId; diff --git a/crates/wasmtime/src/runtime/vm/arch/mod.rs b/crates/wasmtime/src/runtime/vm/arch/mod.rs deleted file mode 100644 index 780759a49024..000000000000 --- a/crates/wasmtime/src/runtime/vm/arch/mod.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! Architecture-specific support required by Wasmtime. -//! -//! This crate houses any architecture-specific tidbits required when running -//! Wasmtime. Each architecture has its own file in the `arch` folder which is -//! referenced here. -//! -//! All architectures have the same interface when exposed to the rest of the -//! crate. - -cfg_if::cfg_if! { - if #[cfg(target_arch = "x86_64")] { - mod x86; - use x86 as imp; - } else if #[cfg(target_arch = "aarch64")] { - mod aarch64; - use aarch64 as imp; - } else if #[cfg(target_arch = "s390x")] { - mod s390x; - use s390x as imp; - } else if #[cfg(target_arch = "riscv64")] { - mod riscv64; - use riscv64 as imp; - } else { - mod unsupported; - use unsupported as imp; - } -} - -// Functions defined in this module but all the implementations delegate to each -// `imp` module. This exists to assert that each module internally provides the -// same set of functionality with the same types for all architectures. - -pub fn get_stack_pointer() -> usize { - imp::get_stack_pointer() -} - -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - imp::get_next_older_pc_from_fp(fp) -} - -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = imp::NEXT_OLDER_FP_FROM_FP_OFFSET; - -pub fn assert_fp_is_aligned(fp: usize) { - imp::assert_fp_is_aligned(fp) -} diff --git a/crates/wasmtime/src/runtime/vm/arch/riscv64.rs b/crates/wasmtime/src/runtime/vm/arch/riscv64.rs deleted file mode 100644 index 0ca7f8209af6..000000000000 --- a/crates/wasmtime/src/runtime/vm/arch/riscv64.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Riscv64-specific definitions of architecture-specific functions in Wasmtime. - -#[inline] -#[allow(missing_docs)] -pub fn get_stack_pointer() -> usize { - let stack_pointer: usize; - unsafe { - core::arch::asm!( - "mv {}, sp", - out(reg) stack_pointer, - options(nostack,nomem), - ); - } - stack_pointer -} - -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - *(fp as *mut usize).offset(1) -} - -// And the current frame pointer points to the next older frame pointer. -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); -} diff --git a/crates/wasmtime/src/runtime/vm/arch/s390x.rs b/crates/wasmtime/src/runtime/vm/arch/s390x.rs deleted file mode 100644 index 2773381243ab..000000000000 --- a/crates/wasmtime/src/runtime/vm/arch/s390x.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! s390x-specific definitions of architecture-specific functions in Wasmtime. - -#[inline] -#[allow(missing_docs)] -pub fn get_stack_pointer() -> usize { - psm::stack_pointer() as usize -} - -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - // The next older PC can be found in register %r14 at function entry, which - // was saved into slot 14 of the register save area pointed to by "FP" (the - // backchain pointer). - *(fp as *mut usize).offset(14) -} - -// The next older "FP" (backchain pointer) was saved in the slot pointed to -// by the current "FP". -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 8, 0, "stack should always be aligned to 8"); -} diff --git a/crates/wasmtime/src/runtime/vm/arch/unsupported.rs b/crates/wasmtime/src/runtime/vm/arch/unsupported.rs deleted file mode 100644 index 19aca0a3761f..000000000000 --- a/crates/wasmtime/src/runtime/vm/arch/unsupported.rs +++ /dev/null @@ -1,28 +0,0 @@ -compile_error!("Wasmtime's runtime is being compiled for an architecture that it does not support"); - -cfg_if::cfg_if! { - if #[cfg(target_arch = "riscv32")] { - compile_error!("\ -the tracking issue for riscv32 support is https://github.com/bytecodealliance/wasmtime/issues/8768 \ -"); - } else { - compile_error!("\ -if you'd like feel free to file an issue for platform support at -https://github.com/bytecodealliance/wasmtime/issues/new -"); - } -} - -pub fn get_stack_pointer() -> usize { - panic!() -} - -pub unsafe fn get_next_older_pc_from_fp(_fp: usize) -> usize { - panic!() -} - -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn assert_fp_is_aligned(_fp: usize) { - panic!() -} diff --git a/crates/wasmtime/src/runtime/vm/interpreter.rs b/crates/wasmtime/src/runtime/vm/interpreter.rs index 6ec0b188b260..b3e98ee0bd9b 100644 --- a/crates/wasmtime/src/runtime/vm/interpreter.rs +++ b/crates/wasmtime/src/runtime/vm/interpreter.rs @@ -8,6 +8,7 @@ use core::ptr::NonNull; use pulley_interpreter::interp::{DoneReason, RegType, TrapKind, Val, Vm, XRegVal}; use pulley_interpreter::{FReg, Reg, XReg}; use wasmtime_environ::{BuiltinFunctionIndex, HostCall, Trap}; +use wasmtime_unwinder::Unwind; /// Interpreter state stored within a `Store`. #[repr(transparent)] @@ -70,6 +71,11 @@ impl Interpreter { pub fn pulley(&self) -> &Vm { unsafe { self.pulley.get().as_ref() } } + + /// Get an implementation of `Unwind` used to walk the Pulley stack. + pub fn unwinder(&self) -> &'static dyn Unwind { + &UnwindPulley + } } /// Wrapper around `&mut pulley_interpreter::Vm` to enable compiling this to a @@ -80,6 +86,36 @@ pub struct InterpreterRef<'a> { _phantom: marker::PhantomData<&'a mut Vm>, } +/// An implementation of stack-walking details specifically designed +/// for unwinding Pulley's runtime stack. +pub struct UnwindPulley; + +unsafe impl Unwind for UnwindPulley { + fn next_older_fp_from_fp_offset(&self) -> usize { + 0 + } + fn next_older_sp_from_fp_offset(&self) -> usize { + if cfg!(target_pointer_width = "32") { + 8 + } else { + 16 + } + } + unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize { + // The calling convention always pushes the return pointer (aka the PC + // of the next older frame) just before this frame. + *(fp as *mut usize).offset(1) + } + fn assert_fp_is_aligned(&self, fp: usize) { + let expected = if cfg!(target_pointer_width = "32") { + 8 + } else { + 16 + }; + assert_eq!(fp % expected, 0, "stack should always be aligned"); + } +} + /// Equivalent of a native platform's `jmp_buf` (sort of). /// /// This structure ensures that all callee-save state in Pulley is saved at wasm diff --git a/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs b/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs index c884f6b37a8a..a22c54fff725 100644 --- a/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs +++ b/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs @@ -10,6 +10,7 @@ use crate::{Engine, ValRaw}; use core::marker; use core::mem; use core::ptr::NonNull; +use wasmtime_unwinder::Unwind; pub struct Interpreter { empty: Uninhabited, @@ -26,6 +27,10 @@ impl Interpreter { pub fn as_interpreter_ref(&mut self) -> InterpreterRef<'_> { match self.empty {} } + + pub fn unwinder(&self) -> &'static dyn Unwind { + match self.empty {} + } } pub struct InterpreterRef<'a> { diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index d267904e8f15..f925394528d3 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -28,7 +28,7 @@ use core::ptr::{self, NonNull}; pub use self::backtrace::Backtrace; #[cfg(feature = "gc")] -pub use self::backtrace::Frame; +pub use wasmtime_unwinder::Frame; pub use self::coredump::CoreDumpStack; pub use self::tls::tls_eager_initialize; diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index 84acb28ae2a2..97008c10d7c2 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -31,35 +31,12 @@ use crate::runtime::vm::{ #[cfg(all(feature = "gc", feature = "stack-switching"))] use crate::vm::stack_switching::{VMContRef, VMStackState}; use core::ops::ControlFlow; +use wasmtime_unwinder::Frame; /// A WebAssembly stack trace. #[derive(Debug)] pub struct Backtrace(Vec); -/// A stack frame within a Wasm stack trace. -#[derive(Debug)] -pub struct Frame { - pc: usize, - #[cfg_attr( - not(feature = "gc"), - expect(dead_code, reason = "not worth #[cfg] annotations to remove") - )] - fp: usize, -} - -impl Frame { - /// Get this frame's program counter. - pub fn pc(&self) -> usize { - self.pc - } - - /// Get this frame's frame pointer. - #[cfg(feature = "gc")] - pub fn fp(&self) -> usize { - self.fp - } -} - impl Backtrace { /// Returns an empty backtrace pub fn empty() -> Backtrace { @@ -257,7 +234,7 @@ impl Backtrace { // Handle the stack that is currently running (which may be a // continuation or the initial stack). - Self::trace_through_wasm(unwind, pc, fp, trampoline_fp, &mut f)?; + wasmtime_unwinder::visit_frames(unwind, pc, fp, trampoline_fp, &mut f)?; // Note that the rest of this function has no effect if `chain` is // `Some(VMStackChain::InitialStack(_))` (i.e., there is only one stack to @@ -318,7 +295,7 @@ impl Backtrace { debug_assert!(parent_stack_range.contains(&parent_limits.stack_limit)); }); - Self::trace_through_wasm( + wasmtime_unwinder::visit_frames( unwind, resume_pc, resume_fp, @@ -329,108 +306,6 @@ impl Backtrace { ControlFlow::Continue(()) } - /// Walk through a contiguous sequence of Wasm frames starting with the - /// frame at the given PC and FP and ending at `trampoline_sp`. - unsafe fn trace_through_wasm( - unwind: &dyn Unwind, - mut pc: usize, - mut fp: usize, - trampoline_fp: usize, - mut f: impl FnMut(Frame) -> ControlFlow<()>, - ) -> ControlFlow<()> { - log::trace!("=== Tracing through contiguous sequence of Wasm frames ==="); - log::trace!("trampoline_fp = 0x{:016x}", trampoline_fp); - log::trace!(" initial pc = 0x{:016x}", pc); - log::trace!(" initial fp = 0x{:016x}", fp); - - // We already checked for this case in the `trace_with_trap_state` - // caller. - assert_ne!(pc, 0); - assert_ne!(fp, 0); - assert_ne!(trampoline_fp, 0); - - // This loop will walk the linked list of frame pointers starting at - // `fp` and going up until `trampoline_fp`. We know that both `fp` and - // `trampoline_fp` are "trusted values" aka generated and maintained by - // Cranelift. This means that it should be safe to walk the linked list - // of pointers and inspect wasm frames. - // - // Note, though, that any frames outside of this range are not - // guaranteed to have valid frame pointers. For example native code - // might be using the frame pointer as a general purpose register. Thus - // we need to be careful to only walk frame pointers in this one - // contiguous linked list. - // - // To know when to stop iteration all architectures' stacks currently - // look something like this: - // - // | ... | - // | Native Frames | - // | ... | - // |-------------------| - // | ... | <-- Trampoline FP | - // | Trampoline Frame | | - // | ... | <-- Trampoline SP | - // |-------------------| Stack - // | Return Address | Grows - // | Previous FP | <-- Wasm FP Down - // | ... | | - // | Wasm Frames | | - // | ... | V - // - // The trampoline records its own frame pointer (`trampoline_fp`), - // which is guaranteed to be above all Wasm. To check when we've - // reached the trampoline frame, it is therefore sufficient to - // check when the next frame pointer is equal to `trampoline_fp`. Once - // that's hit then we know that the entire linked list has been - // traversed. - // - // Note that it might be possible that this loop doesn't execute at all. - // For example if the entry trampoline called wasm which `return_call`'d - // an imported function which is an exit trampoline, then - // `fp == trampoline_fp` on the entry of this function, meaning the loop - // won't actually execute anything. - while fp != trampoline_fp { - // At the start of each iteration of the loop, we know that `fp` is - // a frame pointer from Wasm code. Therefore, we know it is not - // being used as an extra general-purpose register, and it is safe - // dereference to get the PC and the next older frame pointer. - // - // The stack also grows down, and therefore any frame pointer we are - // dealing with should be less than the frame pointer on entry to - // Wasm. Finally also assert that it's aligned correctly as an - // additional sanity check. - assert!(trampoline_fp > fp, "{trampoline_fp:#x} > {fp:#x}"); - unwind.assert_fp_is_aligned(fp); - - log::trace!("--- Tracing through one Wasm frame ---"); - log::trace!("pc = {:p}", pc as *const ()); - log::trace!("fp = {:p}", fp as *const ()); - - f(Frame { pc, fp })?; - - pc = unwind.get_next_older_pc_from_fp(fp); - - // We rely on this offset being zero for all supported architectures - // in `crates/cranelift/src/component/compiler.rs` when we set the - // Wasm exit FP. If this ever changes, we will need to update that - // code as well! - assert_eq!(unwind.next_older_fp_from_fp_offset(), 0); - - // Get the next older frame pointer from the current Wasm frame - // pointer. - let next_older_fp = *(fp as *mut usize).add(unwind.next_older_fp_from_fp_offset()); - - // Because the stack always grows down, the older FP must be greater - // than the current FP. - assert!(next_older_fp > fp, "{next_older_fp:#x} > {fp:#x}"); - fp = next_older_fp; - } - - log::trace!("=== Done tracing contiguous sequence of Wasm frames ==="); - ControlFlow::Continue(()) - } - /// Iterate over the frames inside this backtrace. pub fn frames<'a>( &'a self, diff --git a/crates/wasmtime/src/runtime/vm/unwind.rs b/crates/wasmtime/src/runtime/vm/unwind.rs deleted file mode 100644 index 6deee517cb6d..000000000000 --- a/crates/wasmtime/src/runtime/vm/unwind.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! Support for low-level primitives of unwinding the stack. - -#[cfg(has_host_compiler_backend)] -use crate::runtime::vm::arch; - -/// Implementation necessary to unwind the stack, used by `Backtrace`. -pub unsafe trait Unwind { - /// Returns the offset, from the current frame pointer, of where to get to - /// the previous frame pointer on the stack. - fn next_older_fp_from_fp_offset(&self) -> usize; - - /// Load the return address of a frame given the frame pointer for that - /// frame. - unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize; - - /// Debug assertion that the frame pointer is aligned. - fn assert_fp_is_aligned(&self, fp: usize); -} - -/// A host-backed implementation of unwinding, using the native platform ABI -/// that Cranelift has. -#[cfg(has_host_compiler_backend)] -pub struct UnwindHost; - -#[cfg(has_host_compiler_backend)] -unsafe impl Unwind for UnwindHost { - fn next_older_fp_from_fp_offset(&self) -> usize { - arch::NEXT_OLDER_FP_FROM_FP_OFFSET - } - unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize { - arch::get_next_older_pc_from_fp(fp) - } - fn assert_fp_is_aligned(&self, fp: usize) { - arch::assert_fp_is_aligned(fp) - } -} - -/// An implementation specifically designed for unwinding Pulley's runtime stack -/// (which might not match the native host). -pub struct UnwindPulley; - -unsafe impl Unwind for UnwindPulley { - fn next_older_fp_from_fp_offset(&self) -> usize { - 0 - } - unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize { - // The calling convention always pushes the return pointer (aka the PC - // of the next older frame) just before this frame. - *(fp as *mut usize).offset(1) - } - fn assert_fp_is_aligned(&self, fp: usize) { - let expected = if cfg!(target_pointer_width = "32") { - 8 - } else { - 16 - }; - assert_eq!(fp % expected, 0, "stack should always be aligned"); - } -} diff --git a/fuzz/fuzz_targets/cranelift-fuzzgen.rs b/fuzz/fuzz_targets/cranelift-fuzzgen.rs index a5541ee23680..2b7c18b5f33d 100644 --- a/fuzz/fuzz_targets/cranelift-fuzzgen.rs +++ b/fuzz/fuzz_targets/cranelift-fuzzgen.rs @@ -19,7 +19,7 @@ use std::sync::atomic::Ordering; use cranelift_codegen::data_value::DataValue; use cranelift_codegen::ir::{LibCall, TrapCode}; use cranelift_codegen::isa; -use cranelift_filetests::function_runner::{TestFileCompiler, Trampoline}; +use cranelift_filetests::function_runner::{CompiledTestFile, TestFileCompiler, Trampoline}; use cranelift_fuzzgen::*; use cranelift_interpreter::environment::FuncIndex; use cranelift_interpreter::environment::FunctionStore; @@ -286,8 +286,12 @@ fn run_in_interpreter(interpreter: &mut Interpreter, args: &[DataValue]) -> RunR } } -fn run_in_host(trampoline: &Trampoline, args: &[DataValue]) -> RunResult { - let res = trampoline.call(args); +fn run_in_host( + compiled: &CompiledTestFile, + trampoline: &Trampoline, + args: &[DataValue], +) -> RunResult { + let res = trampoline.call(compiled, args); RunResult::Success(res) } @@ -413,6 +417,6 @@ fuzz_target!(|testcase: TestCase| { let compiled = compiler.compile().unwrap(); let trampoline = compiled.get_trampoline(testcase.main()).unwrap(); - run_test_inputs(&testcase, |args| run_in_host(&trampoline, args)); + run_test_inputs(&testcase, |args| run_in_host(&compiled, &trampoline, args)); } }); diff --git a/scripts/publish.rs b/scripts/publish.rs index cc76639d260f..112ae8e956fa 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -42,6 +42,9 @@ const CRATES_TO_PUBLISH: &[&str] = &[ "cranelift-object", "cranelift-interpreter", "wasmtime-jit-icache-coherence", + // Wasmtime unwinder, used by both `cranelift-jit` (optionally) and filetests, and by Wasmtime. + "wasmtime-unwinder", + // Cranelift crates that use Wasmtime unwinder. "cranelift-jit", "cranelift", // wiggle