From 071539035c887d078b1c63fd73b76294d108d9fa Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Wed, 12 Mar 2025 18:46:59 +0000 Subject: [PATCH 01/38] [pr1] base --- Cargo.toml | 2 + crates/c-api/include/wasmtime/config.h | 16 + crates/c-api/include/wasmtime/linker.h | 10 + crates/c-api/src/config.rs | 4 + crates/cli-flags/Cargo.toml | 1 + crates/cli-flags/src/lib.rs | 14 + crates/cranelift/src/func_environ.rs | 17 + crates/cranelift/src/gc/enabled.rs | 22 +- crates/cranelift/src/lib.rs | 12 +- .../src/translate/code_translator.rs | 61 +- crates/environ/src/builtin.rs | 29 + crates/environ/src/gc.rs | 9 +- crates/environ/src/lib.rs | 1 + crates/environ/src/stack_switching.rs | 78 ++ crates/environ/src/trap_encoding.rs | 16 + crates/environ/src/types.rs | 24 +- crates/environ/src/vmoffsets.rs | 127 ++++ crates/fuzzing/src/generators/config.rs | 1 + crates/misc/component-test-util/src/lib.rs | 5 +- crates/wasmtime/Cargo.toml | 8 +- crates/wasmtime/src/config.rs | 67 +- crates/wasmtime/src/engine.rs | 19 +- .../wasmtime/src/runtime/externals/global.rs | 4 + .../wasmtime/src/runtime/externals/table.rs | 5 + crates/wasmtime/src/runtime/func.rs | 243 +++--- .../src/runtime/gc/enabled/arrayref.rs | 6 + .../src/runtime/gc/enabled/structref.rs | 10 +- crates/wasmtime/src/runtime/store.rs | 146 +++- crates/wasmtime/src/runtime/type_registry.rs | 2 +- crates/wasmtime/src/runtime/types.rs | 181 ++++- crates/wasmtime/src/runtime/values.rs | 5 + crates/wasmtime/src/runtime/vm.rs | 2 + .../instance/allocator/pooling/table_pool.rs | 15 +- crates/wasmtime/src/runtime/vm/libcalls.rs | 103 +++ .../src/runtime/vm/stack_switching.rs | 705 ++++++++++++++++++ .../src/runtime/vm/stack_switching/stack.rs | 119 +++ .../runtime/vm/stack_switching/stack/dummy.rs | 75 ++ .../runtime/vm/stack_switching/stack/unix.rs | 354 +++++++++ .../vm/stack_switching/stack/unix/x86_64.rs | 86 +++ crates/wasmtime/src/runtime/vm/table.rs | 204 ++++- .../wasmtime/src/runtime/vm/traphandlers.rs | 137 ++-- .../src/runtime/vm/traphandlers/backtrace.rs | 158 +++- crates/wasmtime/src/runtime/vm/vmcontext.rs | 8 +- crates/wast-util/src/lib.rs | 7 + tests/all/main.rs | 1 + 45 files changed, 2874 insertions(+), 245 deletions(-) create mode 100644 crates/environ/src/stack_switching.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching/stack.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs create mode 100644 crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs diff --git a/Cargo.toml b/Cargo.toml index 100d63bfa58c..47218b3081db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -431,6 +431,7 @@ default = [ "gc", "gc-drc", "gc-null", + "stack-switching", "winch", "pulley", @@ -489,6 +490,7 @@ gc = ["wasmtime-cli-flags/gc", "wasmtime/gc"] gc-drc = ["gc", "wasmtime/gc-drc", "wasmtime-cli-flags/gc-drc"] gc-null = ["gc", "wasmtime/gc-null", "wasmtime-cli-flags/gc-null"] pulley = ["wasmtime-cli-flags/pulley"] +stack-switching = ["wasmtime/stack-switching", "wasmtime-cli-flags/stack-switching"] # CLI subcommands for the `wasmtime` executable. See `wasmtime $cmd --help` # for more information on each subcommand. diff --git a/crates/c-api/include/wasmtime/config.h b/crates/c-api/include/wasmtime/config.h index 4788ea9bb145..8a262859cf8c 100644 --- a/crates/c-api/include/wasmtime/config.h +++ b/crates/c-api/include/wasmtime/config.h @@ -250,6 +250,22 @@ WASMTIME_CONFIG_PROP(void, wasm_wide_arithmetic, bool) #ifdef WASMTIME_FEATURE_COMPILER +/** + * \brief Configures whether the WebAssembly function references + * proposal is enabled. + * + * This setting is `false` by default. + */ +WASMTIME_CONFIG_PROP(void, wasm_function_references, bool) + +/** + * \brief Configures whether the WebAssembly stack switching + * proposal is enabled. + * + * This setting is `false` by default. + */ +WASMTIME_CONFIG_PROP(void, wasm_stack_switching, bool) + /** * \brief Configures how JIT code will be compiled. * diff --git a/crates/c-api/include/wasmtime/linker.h b/crates/c-api/include/wasmtime/linker.h index 0a793d1e6fc9..c97d824d2efd 100644 --- a/crates/c-api/include/wasmtime/linker.h +++ b/crates/c-api/include/wasmtime/linker.h @@ -66,6 +66,16 @@ WASM_API_EXTERN void wasmtime_linker_delete(wasmtime_linker_t *linker); WASM_API_EXTERN void wasmtime_linker_allow_shadowing(wasmtime_linker_t *linker, bool allow_shadowing); +/** + * \brief Configures whether the given Linker will allow unknown exports from + * command modules. + * + * By default this setting is `false`. + */ +WASM_API_EXTERN void +wasmtime_linker_allow_unknown_exports(wasmtime_linker_t *linker, + bool allow_unknown_exports); + /** * \brief Defines a new item in this linker. * diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index 455802416cd5..25198d95af54 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -137,6 +137,10 @@ pub extern "C" fn wasmtime_config_wasm_memory64_set(c: &mut wasm_config_t, enabl } #[unsafe(no_mangle)] +pub extern "C" fn wasmtime_config_wasm_stack_switching_set(c: &mut wasm_config_t, enable: bool) { + c.config.wasm_stack_switching(enable); +} + #[cfg(any(feature = "cranelift", feature = "winch"))] pub extern "C" fn wasmtime_config_strategy_set( c: &mut wasm_config_t, diff --git a/crates/cli-flags/Cargo.toml b/crates/cli-flags/Cargo.toml index 23dc34036e85..1469c9535b15 100644 --- a/crates/cli-flags/Cargo.toml +++ b/crates/cli-flags/Cargo.toml @@ -39,3 +39,4 @@ gc-null = ["gc", "wasmtime/gc-null"] threads = ["wasmtime/threads"] memory-protection-keys = ["wasmtime/memory-protection-keys"] pulley = ["wasmtime/pulley"] +stack-switching = ["wasmtime/stack-switching"] diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs index 9079d7f882bb..1e86e1f2ca4a 100644 --- a/crates/cli-flags/src/lib.rs +++ b/crates/cli-flags/src/lib.rs @@ -331,6 +331,8 @@ wasmtime_option_group! { pub trap_on_grow_failure: Option, /// Maximum execution time of wasm code before timing out (1, 2s, 100ms, etc) pub timeout: Option, + /// Size of stacks created with cont.new instructions + pub stack_switching_stack_size: Option, /// Configures support for all WebAssembly proposals implemented. pub all_proposals: Option, /// Configure support for the bulk memory proposal. @@ -366,6 +368,8 @@ wasmtime_option_group! { pub component_model_async: Option, /// Configure support for the function-references proposal. pub function_references: Option, + /// Configure support for the stack-switching proposal. + pub stack_switching: Option, /// Configure support for the GC proposal. pub gc: Option, /// Configure support for the custom-page-sizes proposal. @@ -803,6 +807,12 @@ impl CommonOptions { config.native_unwind_info(enable); } + match_feature! { + ["stack-switching" : self.wasm.stack_switching_stack_size] + size => config.stack_switching_stack_size(size), + _ => err, + } + match_feature! { ["pooling-allocator" : self.opts.pooling_allocator.or(pooling_allocator_default)] enable => { @@ -964,6 +974,9 @@ impl CommonOptions { if let Some(enable) = self.wasm.memory64.or(all) { config.wasm_memory64(enable); } + if let Some(enable) = self.wasm.stack_switching { + config.wasm_stack_switching(enable); + } if let Some(enable) = self.wasm.custom_page_sizes.or(all) { config.wasm_custom_page_sizes(enable); } @@ -994,6 +1007,7 @@ impl CommonOptions { ("gc", gc, wasm_gc) ("gc", reference_types, wasm_reference_types) ("gc", function_references, wasm_function_references) + ("stack-switching", stack_switching, wasm_stack_switching) } Ok(()) } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 3177b0af7def..b370bfcf01b4 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -3441,3 +3441,20 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type { IndexType::I64 => I64, } } + +/// TODO(10248) This is removed in the next stack switching PR. It stops the +/// compiler from complaining about the stack switching libcalls being dead +/// code. +#[allow( + dead_code, + reason = "Dummy function to supress more dead code warnings" +)] +pub fn use_stack_switching_libcalls() { + let _ = BuiltinFunctions::delete_me_print_str; + let _ = BuiltinFunctions::delete_me_print_int; + let _ = BuiltinFunctions::delete_me_print_pointer; + + let _ = BuiltinFunctions::cont_new; + let _ = BuiltinFunctions::table_grow_cont_obj; + let _ = BuiltinFunctions::table_fill_cont_obj; +} diff --git a/crates/cranelift/src/gc/enabled.rs b/crates/cranelift/src/gc/enabled.rs index 04ae2ff9b7d8..743ea93af3e3 100644 --- a/crates/cranelift/src/gc/enabled.rs +++ b/crates/cranelift/src/gc/enabled.rs @@ -148,7 +148,12 @@ fn read_field_at_addr( .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]); builder.func.dfg.first_result(call_inst) } - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + // TODO(#10248) GC integration for stack switching + return Err(wasmtime_environ::WasmError::Unsupported( + "Stack switching feature not compatbile with GC, yet".to_string(), + )); + } }, }, }; @@ -1011,6 +1016,8 @@ pub fn translate_ref_test( | WasmHeapType::NoExtern | WasmHeapType::Func | WasmHeapType::NoFunc + | WasmHeapType::Cont + | WasmHeapType::NoCont | WasmHeapType::I31 => unreachable!("handled top, bottom, and i31 types above"), // For these abstract but non-top and non-bottom types, we check the @@ -1063,8 +1070,12 @@ pub fn translate_ref_test( func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty) } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::ConcreteCont(_) => { + // TODO(#10248) GC integration for stack switching + return Err(wasmtime_environ::WasmError::Unsupported( + "Stack switching feature not compatbile with GC, yet".to_string(), + )); + } }; builder.ins().jump(continue_block, &[result]); @@ -1403,8 +1414,9 @@ impl FuncEnvironment<'_> { WasmHeapType::Func | WasmHeapType::ConcreteFunc(_) | WasmHeapType::NoFunc => { unreachable!() } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => { + unreachable!() + } }; match (ty.nullable, might_be_i31) { diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index a98a9c04b77d..8393ae3dc124 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -61,6 +61,12 @@ pub const TRAP_HEAP_MISALIGNED: TrapCode = TrapCode::unwrap_user(Trap::HeapMisaligned as u8 + TRAP_OFFSET); pub const TRAP_TABLE_OUT_OF_BOUNDS: TrapCode = TrapCode::unwrap_user(Trap::TableOutOfBounds as u8 + TRAP_OFFSET); +pub const TRAP_UNHANDLED_TAG: TrapCode = + TrapCode::unwrap_user(Trap::UnhandledTag as u8 + TRAP_OFFSET); +pub const TRAP_CONTINUATION_ALREADY_CONSUMED: TrapCode = + TrapCode::unwrap_user(Trap::ContinuationAlreadyConsumed as u8 + TRAP_OFFSET); +pub const TRAP_DELETE_ME_DEBUG_ASSERTION: TrapCode = + TrapCode::unwrap_user(Trap::DeleteMeDebugAssertion as u8 + TRAP_OFFSET); pub const TRAP_CAST_FAILURE: TrapCode = TrapCode::unwrap_user(Trap::CastFailure as u8 + TRAP_OFFSET); @@ -202,7 +208,11 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type { match wasm_ht.top() { WasmHeapTopType::Func => pointer_type, WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => + // TODO(10248) This is added in a follow-up PR + { + unimplemented!("codegen for stack switching types not implemented, yet") + } } } diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index 8bb55974ec56..4a30e262adff 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -2876,6 +2876,56 @@ pub fn translate_operator( // representation, so we don't actually need to do anything. } + Operator::ContNew { cont_type_index: _ } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::ContBind { + argument_index: _, + result_index: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::Suspend { tag_index: _ } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::Resume { + cont_type_index: _, + resume_table: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::ResumeThrow { + cont_type_index: _, + tag_index: _, + resume_table: _, + } => { + // TODO(10248) This depends on exception handling + return Err(wasmtime_environ::WasmError::Unsupported( + "resume.throw instructions not supported, yet".to_string(), + )); + } + Operator::Switch { + cont_type_index: _, + tag_index: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::GlobalAtomicGet { .. } | Operator::GlobalAtomicSet { .. } | Operator::GlobalAtomicRmwAdd { .. } @@ -2917,17 +2967,6 @@ pub fn translate_operator( )); } - Operator::ContNew { .. } - | Operator::ContBind { .. } - | Operator::Suspend { .. } - | Operator::Resume { .. } - | Operator::ResumeThrow { .. } - | Operator::Switch { .. } => { - return Err(wasm_unsupported!( - "stack-switching operators are not yet implemented" - )); - } - Operator::I64MulWideS => { let (arg1, arg2) = state.pop2(); let arg1 = builder.ins().sextend(I128, arg1); diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index a446829af7d8..1c70dd964cd7 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -206,6 +206,32 @@ macro_rules! foreach_builtin_function { // Raises an unconditional trap where the trap information must have // been previously filled in. raise(vmctx: vmctx); + + // Creates a new continuation from a funcref. + cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer; + + // FIXME(frank-emrich) The next three builtins are used by the debug printing mechanism. + // They are not supposed to be part of the final upstreamed code. + // + // Prints a 'static str, represented as a + // pointer and a length. + delete_me_print_str(vmctx: vmctx, s: pointer, len : u64); + // Prints integer + delete_me_print_int(vmctx: vmctx, arg : u64); + // Prints pointer, formatted as hex. + delete_me_print_pointer(vmctx: vmctx, arg : pointer); + + // Returns an index for Wasm's `table.grow` instruction + // for `contobj`s. Note that the initial + // Option (i.e., the value to fill the new + // slots with) is split into two arguments: The underlying + // continuation reference and the revision count. To + // denote the continuation being `None`, `init_contref` + // may be 0. + table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: u64) -> pointer; + // `value_contref` and `value_revision` together encode + // the Option, as in previous libcall. + table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: u64, len: u64) -> bool; } }; } @@ -347,6 +373,7 @@ impl BuiltinFunctionIndex { (@get memory32_grow pointer) => (TrapSentinel::NegativeTwo); (@get table_grow_func_ref pointer) => (TrapSentinel::NegativeTwo); (@get table_grow_gc_ref pointer) => (TrapSentinel::NegativeTwo); + (@get table_grow_cont_obj pointer) => (TrapSentinel::NegativeTwo); // Atomics-related functions return a negative value indicating trap // indicate a trap. @@ -371,6 +398,8 @@ impl BuiltinFunctionIndex { (@get intern_func_ref_for_gc_heap u64) => (return None); (@get is_subtype u32) => (return None); + (@get cont_new pointer) => (TrapSentinel::Negative); + // Bool-returning functions use `false` as an indicator of a trap. (@get $name:ident bool) => (TrapSentinel::Falsy); diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs index 4fa7d8745bab..7e2215b354fc 100644 --- a/crates/environ/src/gc.rs +++ b/crates/environ/src/gc.rs @@ -40,10 +40,15 @@ pub const VM_GC_HEADER_TYPE_INDEX_OFFSET: u32 = 4; /// Get the byte size of the given Wasm type when it is stored inside the GC /// heap. pub fn byte_size_of_wasm_ty_in_gc_heap(ty: &WasmStorageType) -> u32 { + use crate::{WasmHeapType::*, WasmRefType}; match ty { WasmStorageType::I8 => 1, WasmStorageType::I16 => 2, WasmStorageType::Val(ty) => match ty { + WasmValType::Ref(WasmRefType { + nullable: _, + heap_type: ConcreteCont(_) | Cont, + }) => unimplemented!("Stack switching feature not compatbile with GC, yet"), WasmValType::I32 | WasmValType::F32 | WasmValType::Ref(_) => 4, WasmValType::I64 | WasmValType::F64 => 8, WasmValType::V128 => 16, @@ -162,7 +167,9 @@ pub trait GcTypeLayouts { WasmCompositeInnerType::Array(ty) => Some(self.array_layout(ty).into()), WasmCompositeInnerType::Struct(ty) => Some(self.struct_layout(ty).into()), WasmCompositeInnerType::Func(_) => None, - WasmCompositeInnerType::Cont(_) => None, + WasmCompositeInnerType::Cont(_) => { + unimplemented!("Stack switching feature not compatbile with GC, yet") + } } } diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 511b4a73cfec..3b42869f58b8 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -29,6 +29,7 @@ pub mod obj; mod ref_bits; mod scopevec; mod stack_map; +pub mod stack_switching; mod trap_encoding; mod tunables; mod types; diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs new file mode 100644 index 000000000000..6a781faea905 --- /dev/null +++ b/crates/environ/src/stack_switching.rs @@ -0,0 +1,78 @@ +//! This module contains basic type definitions used by the implementation of +//! the stack switching proposal. + +/// FIXME(frank-emrich) Will remove in the final upstreamed version +#[allow(dead_code, reason = "Only accessed in debug builds")] +pub const ENABLE_DEBUG_PRINTING: bool = false; + +/// FIXME(frank-emrich) Will remove in the final upstreamed version +#[macro_export] +macro_rules! debug_println { + ($( $args:expr ),+ ) => { + #[cfg(debug_assertions)] + if ENABLE_DEBUG_PRINTING { + #[cfg(feature = "std")] + println!($($args),*); + } + } +} + +/// Runtime configuration options for stack switching that can be set +/// via the command line. +/// +/// Part of wasmtime::config::Config type (which is not in scope in this crate). +#[derive(Debug, Clone)] +pub struct StackSwitchingConfig { + /// The (fixed) size of a continuation stack. + pub stack_size: usize, +} + +impl Default for StackSwitchingConfig { + fn default() -> Self { + /// Default size for continuation stacks + const DEFAULT_FIBER_SIZE: usize = 2097152; // 2MB = 512 pages of 4k + + Self { + stack_size: DEFAULT_FIBER_SIZE, + } + } +} + +/// Discriminant of variant `Absent` in +/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +pub const STACK_CHAIN_ABSENT_DISCRIMINANT: usize = 0; +/// Discriminant of variant `InitialStack` in +/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +pub const STACK_CHAIN_INITIAL_STACK_DISCRIMINANT: usize = 1; +/// Discriminant of variant `Continiation` in +/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +pub const STACK_CHAIN_CONTINUATION_DISCRIMINANT: usize = 2; + +/// Discriminant of variant `Fresh` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_FRESH_DISCRIMINANT: u32 = 0; +/// Discriminant of variant `Running` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_RUNNING_DISCRIMINANT: u32 = 1; +/// Discriminant of variant `Parent` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_PARENT_DISCRIMINANT: u32 = 2; +/// Discriminant of variant `Suspended` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_SUSPENDED_DISCRIMINANT: u32 = 3; +/// Discriminant of variant `Returned` in +/// `runtime::vm::stack_switching::VMStackState`. +pub const STACK_STATE_RETURNED_DISCRIMINANT: u32 = 4; + +/// Discriminant of variant `Return` in +/// `runtime::vm::stack_switching::ControlEffect`. +pub const CONTROL_EFFECT_RETURN_DISCRIMINANT: u32 = 0; +/// Discriminant of variant `Resume` in +/// `runtime::vm::stack_switching::ControlEffect`. +pub const CONTROL_EFFECT_RESUME_DISCRIMINANT: u32 = 1; +/// Discriminant of variant `Suspend` in +/// `runtime::vm::stack_switching::ControlEffect`. +pub const CONTROL_EFFECT_SUSPEND_DISCRIMINANT: u32 = 2; +/// Discriminant of variant `Switch` in +/// `runtime::vm::stack_switching::ControlEffect`. +pub const CONTROL_EFFECT_SWITCH_DISCRIMINANT: u32 = 3; diff --git a/crates/environ/src/trap_encoding.rs b/crates/environ/src/trap_encoding.rs index 38e5e5dd4119..b1ed2885110a 100644 --- a/crates/environ/src/trap_encoding.rs +++ b/crates/environ/src/trap_encoding.rs @@ -92,6 +92,16 @@ pub enum Trap { /// Async-lifted export failed to produce a result by calling `task.return` /// before returning `STATUS_DONE` and/or after all host tasks completed. NoAsyncResult, + + /// We are suspending to a tag for which there is no active handler. + UnhandledTag, + + /// Attempt to resume a continuation twice. + ContinuationAlreadyConsumed, + + /// FIXME(frank-emrich) Only used for stack switching debugging code, to be + /// removed from final upstreamed code. + DeleteMeDebugAssertion, // if adding a variant here be sure to update the `check!` macro below } @@ -129,6 +139,9 @@ impl Trap { CastFailure CannotEnterComponent NoAsyncResult + UnhandledTag + ContinuationAlreadyConsumed + DeleteMeDebugAssertion } None @@ -160,6 +173,9 @@ impl fmt::Display for Trap { CastFailure => "cast failure", CannotEnterComponent => "cannot enter component instance", NoAsyncResult => "async-lifted export failed to produce a result", + UnhandledTag => "unhandled tag", + ContinuationAlreadyConsumed => "continuation already consumed", + DeleteMeDebugAssertion => "triggered debug assertion", }; write!(f, "wasm trap: {desc}") } diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rs index 14829377ea43..b9e4056c6f7a 100644 --- a/crates/environ/src/types.rs +++ b/crates/environ/src/types.rs @@ -232,6 +232,14 @@ impl WasmValType { size => panic!("invalid int bits for WasmValType: {size}"), } } + + /// TODO + pub fn unwrap_ref_type(&self) -> WasmRefType { + match self { + WasmValType::Ref(ref_type) => *ref_type, + _ => panic!("Called WasmValType::unwrap_ref_type on non-reference type"), + } + } } /// WebAssembly reference type -- equivalent of `wasmparser`'s RefType @@ -801,6 +809,15 @@ impl WasmContType { pub fn new(idx: EngineOrModuleTypeIndex) -> Self { WasmContType(idx) } + + /// Returns the (module interned) index to the underlying function type. + pub fn unwrap_interned_type_index(self) -> ModuleInternedTypeIndex { + match self.0 { + EngineOrModuleTypeIndex::Engine(_) => panic!("not module interned"), + EngineOrModuleTypeIndex::Module(idx) => idx, + EngineOrModuleTypeIndex::RecGroup(_) => todo!(), + } + } } impl TypeTrace for WasmContType { @@ -2217,12 +2234,11 @@ pub trait TypeConvert { wasmparser::AbstractHeapType::Struct => WasmHeapType::Struct, wasmparser::AbstractHeapType::None => WasmHeapType::None, - wasmparser::AbstractHeapType::Exn - | wasmparser::AbstractHeapType::NoExn - | wasmparser::AbstractHeapType::Cont - | wasmparser::AbstractHeapType::NoCont => { + wasmparser::AbstractHeapType::Exn | wasmparser::AbstractHeapType::NoExn => { unimplemented!("unsupported heap type {ty:?}"); } + wasmparser::AbstractHeapType::Cont => WasmHeapType::Cont, + wasmparser::AbstractHeapType::NoCont => WasmHeapType::NoCont, }, _ => unimplemented!("unsupported heap type {ty:?}"), } diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 01c283317126..b8e03a3996e2 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -164,6 +164,12 @@ pub trait PtrSize { 4 } + /// This is the size of the largest value type (i.e. a V128). + #[inline] + fn maximum_value_size(&self) -> u8 { + self.size_of_vmglobal_definition() + } + // Offsets within `VMStoreContext` /// Return the offset of the `fuel_consumed` field of `VMStoreContext` @@ -199,6 +205,11 @@ pub trait PtrSize { self.vmstore_context_last_wasm_exit_pc() + self.size() } + /// Return the offset of the `stack_chain` field of `VMStoreContext`. + fn vmstore_context_stack_chain(&self) -> u8 { + self.vmstore_context_last_wasm_entry_fp() + self.size() + } + // Offsets within `VMMemoryDefinition` /// The offset of the `base` field. @@ -236,6 +247,122 @@ pub trait PtrSize { .unwrap() } + /// Return the size of `VMStackChain`. + fn size_of_vmstack_chain(&self) -> u8 { + 2 * self.size() + } + + // Offsets within `VMStackLimits` + + /// Return the offset of `VMStackLimits::stack_limit`. + fn vmstack_limits_stack_limit(&self) -> u8 { + 0 + } + + /// Return the offset of `VMStackLimits::last_wasm_entry_fp`. + fn vmstack_limits_last_wasm_entry_fp(&self) -> u8 { + self.size() + } + + // Offsets within `VMArray` + + /// Return the offset of `VMArray::length`. + fn vmarray_length(&self) -> u8 { + 0 + } + + /// Return the offset of `VMArray::capacity`. + fn vmarray_capacity(&self) -> u8 { + 4 + } + + /// Return the offset of `VMArray::data`. + fn vmarray_data(&self) -> u8 { + 8 + } + + /// Return the size of `VMArray`. + fn size_of_vmarray(&self) -> u8 { + 8 + self.size() + } + + // Offsets within `VMCommonStackInformation` + + /// Return the offset of `VMCommonStackInformation::limits`. + fn vmcommon_stack_information_limits(&self) -> u8 { + 0 * self.size() + } + + /// Return the offset of `VMCommonStackInformation::state`. + fn vmcommon_stack_information_state(&self) -> u8 { + 2 * self.size() + } + + /// Return the offset of `VMCommonStackInformation::handlers`. + fn vmcommon_stack_information_handlers(&self) -> u8 { + u8::try_from(align( + self.vmcommon_stack_information_state() as u32 + 4, + u32::from(self.size()), + )) + .unwrap() + } + + /// Return the offset of `VMCommonStackInformation::first_switch_handler_index`. + fn vmcommon_stack_information_first_switch_handler_index(&self) -> u8 { + self.vmcommon_stack_information_handlers() + self.size_of_vmarray() + } + + /// Return the size of `VMCommonStackInformation`. + fn size_of_vmcommon_stack_information(&self) -> u8 { + u8::try_from(align( + self.vmcommon_stack_information_first_switch_handler_index() as u32 + 4, + u32::from(self.size()), + )) + .unwrap() + } + + // Offsets within `VMContRef` + + /// Return the offset of `VMContRef::common_stack_information`. + fn vmcontref_common_stack_information(&self) -> u8 { + 0 * self.size() + } + + /// Return the offset of `VMContRef::parent_chain`. + fn vmcontref_parent_chain(&self) -> u8 { + u8::try_from(align( + (self.vmcontref_common_stack_information() + self.size_of_vmcommon_stack_information()) + as u32, + u32::from(self.size()), + )) + .unwrap() + } + + /// Return the offset of `VMContRef::last_ancestor`. + fn vmcontref_last_ancestor(&self) -> u8 { + self.vmcontref_parent_chain() + 2 * self.size() + } + + /// Return the offset of `VMContRef::revision`. + fn vmcontref_revision(&self) -> u8 { + self.vmcontref_last_ancestor() + self.size() + } + + /// Return the offset of `VMContRef::stack`. + fn vmcontref_stack(&self) -> u8 { + self.vmcontref_revision() + 8 + } + + /// Return the offset of `VMContRef::args`. + fn vmcontref_args(&self) -> u8 { + self.vmcontref_stack() + 3 * self.size() + } + + /// Return the offset of `VMContRef::values`. + fn vmcontref_values(&self) -> u8 { + self.vmcontref_args() + self.size_of_vmarray() + } + /// Return the offset to the `magic` value in this `VMContext`. #[inline] fn vmctx_magic(&self) -> u8 { diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs index 5a484a3de661..732e450e602d 100644 --- a/crates/fuzzing/src/generators/config.rs +++ b/crates/fuzzing/src/generators/config.rs @@ -144,6 +144,7 @@ impl Config { hogs_memory: _, nan_canonicalization: _, gc_types: _, + stack_switching: _, } = test.config; // Enable/disable some proposals that aren't configurable in wasm-smith diff --git a/crates/misc/component-test-util/src/lib.rs b/crates/misc/component-test-util/src/lib.rs index d300f02e7577..8b6bfeed0692 100644 --- a/crates/misc/component-test-util/src/lib.rs +++ b/crates/misc/component-test-util/src/lib.rs @@ -168,6 +168,7 @@ pub fn apply_test_config(config: &mut Config, test_config: &wasmtime_wast_util:: component_model_async, nan_canonicalization, simd, + stack_switching, hogs_memory: _, gc_types: _, @@ -192,7 +193,8 @@ pub fn apply_test_config(config: &mut Config, test_config: &wasmtime_wast_util:: // To avoid needing to enable all of them at once implicitly enable // downstream proposals once the end proposal is enabled (e.g. when enabling // gc that also enables function-references and reference-types). - let function_references = gc || function_references.unwrap_or(false); + let stack_switching = stack_switching.unwrap_or(false); + let function_references = gc || stack_switching || function_references.unwrap_or(false); let reference_types = function_references || reference_types.unwrap_or(false); let simd = relaxed_simd || simd.unwrap_or(false); @@ -210,5 +212,6 @@ pub fn apply_test_config(config: &mut Config, test_config: &wasmtime_wast_util:: .wasm_extended_const(extended_const) .wasm_wide_arithmetic(wide_arithmetic) .wasm_component_model_async(component_model_async) + .wasm_stack_switching(stack_switching) .cranelift_nan_canonicalization(nan_canonicalization); } diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 67ff4001f77b..832f29d98101 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -93,7 +93,7 @@ memfd = { workspace = true, optional = true } mach2 = { workspace = true, optional = true } [target.'cfg(unix)'.dependencies] -rustix = { workspace = true, optional = true } +rustix = { workspace = true, optional = true, features = ["mm", "param"] } [target.'cfg(target_arch = "s390x")'.dependencies] psm = { workspace = true, optional = true } @@ -143,6 +143,7 @@ default = [ 'runtime', 'component-model', 'threads', + 'stack-switching', 'std', ] @@ -308,6 +309,11 @@ threads = [ "std", ] +stack-switching = [ + "std", + "runtime" +] + # Controls whether backtraces will attempt to parse DWARF information in # WebAssembly modules and components to provide filenames and line numbers in # stack traces. diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 75c1078c127a..73059cc61bcc 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -8,7 +8,9 @@ use std::path::Path; use wasmparser::WasmFeatures; #[cfg(feature = "cache")] use wasmtime_cache::CacheConfig; -use wasmtime_environ::{ConfigTunables, TripleExt, Tunables}; +use wasmtime_environ::{ + stack_switching::StackSwitchingConfig, ConfigTunables, TripleExt, Tunables, +}; #[cfg(feature = "runtime")] use crate::memory::MemoryCreator; @@ -127,6 +129,14 @@ pub struct Config { profiling_strategy: ProfilingStrategy, tunables: ConfigTunables, + /// Runtime configuration for the stack switching feature. + /// The structure is defined in the + /// `wasmtime_environ::stack_switching` module, so that we can + /// hand out the configuration object in the interface of + /// `wasmtime_runtime::Store` trait, where the full `Config` type + /// is not in scope. + pub(crate) stack_switching_config: StackSwitchingConfig, + #[cfg(feature = "cache")] pub(crate) cache_config: CacheConfig, #[cfg(feature = "runtime")] @@ -227,6 +237,7 @@ impl Config { tunables: ConfigTunables::default(), #[cfg(any(feature = "cranelift", feature = "winch"))] compiler_config: CompilerConfig::default(), + stack_switching_config: StackSwitchingConfig::default(), target: None, #[cfg(feature = "gc")] collector: Collector::default(), @@ -777,6 +788,12 @@ impl Config { self } + /// Configures the size of the stacks created with cont.new instructions. + pub fn stack_switching_stack_size(&mut self, size: usize) -> &mut Self { + self.stack_switching_config.stack_size = size; + self + } + fn wasm_feature(&mut self, flag: WasmFeatures, enable: bool) -> &mut Self { self.enabled_features.set(flag, enable); self.disabled_features.set(flag, !enable); @@ -2024,18 +2041,37 @@ impl Config { // `threads` proposal, notably shared memory, because Rust can't // safely implement loads/stores in the face of shared memory. if self.compiler_target().is_pulley() { - return WasmFeatures::THREADS; + return WasmFeatures::THREADS | WasmFeatures::STACK_SWITCHING; } - // Other Cranelift backends are either 100% missing or complete - // at this time, so no need to further filter. - WasmFeatures::empty() + use target_lexicon::*; + match self.compiler_target() { + Triple { + architecture: Architecture::X86_64 | Architecture::X86_64h, + operating_system: + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_), + .. + } => { + // Other Cranelift backends are either 100% missing or complete + // at this time, so no need to further filter. + WasmFeatures::empty() + } + + _ => { + // On platforms other than x64 Unix-like, we don't + // support stack switching. + WasmFeatures::STACK_SWITCHING + } + } } Some(Strategy::Winch) => { let mut unsupported = WasmFeatures::GC | WasmFeatures::FUNCTION_REFERENCES | WasmFeatures::RELAXED_SIMD | WasmFeatures::TAIL_CALL + | WasmFeatures::STACK_SWITCHING | WasmFeatures::GC_TYPES; match self.compiler_target().architecture { target_lexicon::Architecture::Aarch64(_) => { @@ -2426,6 +2462,27 @@ impl Config { bail!("cannot disable the simd proposal but enable the relaxed simd proposal"); } + if features.contains(WasmFeatures::STACK_SWITCHING) { + use target_lexicon::OperatingSystem; + let model = match target.operating_system { + OperatingSystem::Windows => "update_windows_tib", + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_) => "basic", + _ => bail!("stack-switching feature not supported on this platform "), + }; + + if !self + .compiler_config + .ensure_setting_unset_or_given("stack_switch_model".into(), model.into()) + { + bail!( + "compiler option 'stack_switch_model' must be set to '{}' on this platform", + model + ); + } + } + // Apply compiler settings and flags for (k, v) in self.compiler_config.settings.iter() { compiler.set(k, v)?; diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index 532c08a98382..473282265e63 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -377,6 +377,24 @@ impl Engine { } } + // stack switch model must match the current OS + "stack_switch_model" => { + if self.features().contains(WasmFeatures::STACK_SWITCHING) { + use target_lexicon::OperatingSystem; + let expected = + match target.operating_system { + OperatingSystem::Windows => "update_windows_tib", + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_) => "basic", + _ => { return Err(String::from("stack-switching feature not supported on this platform")); } + }; + *value == FlagValue::Enum(expected) + } else { + return Ok(()) + } + } + // These settings don't affect the interface or functionality of // the module itself, so their configuration values shouldn't // matter. @@ -394,7 +412,6 @@ impl Engine { | "bb_padding_log2_minus_one" | "machine_code_cfg_info" | "tls_model" // wasmtime doesn't use tls right now - | "stack_switch_model" // wasmtime doesn't use stack switching right now | "opt_level" // opt level doesn't change semantics | "enable_alias_analysis" // alias analysis-based opts don't change semantics | "probestack_size_log2" // probestack above asserted disabled diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs index 02a9669c20f4..3bca5dd0f12b 100644 --- a/crates/wasmtime/src/runtime/externals/global.rs +++ b/crates/wasmtime/src/runtime/externals/global.rs @@ -130,6 +130,10 @@ impl Global { }) .into(), ), + HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont => { + // TODO(#10248) Required to support stack switching in the embedder API. + unimplemented!() + } HeapType::NoExtern => Ref::Extern(None), diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs index 40fd1056d6d2..19ee59340efb 100644 --- a/crates/wasmtime/src/runtime/externals/table.rs +++ b/crates/wasmtime/src/runtime/externals/table.rs @@ -188,6 +188,11 @@ impl Table { ty => unreachable!("not a top type: {ty:?}"), } } + + runtime::TableElement::ContRef(_c) => { + // TODO(#10248) Required to support stack switching in the embedder API. + unimplemented!() + } } } } diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index e75484b36cdf..868d02690e66 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1,7 +1,8 @@ use crate::prelude::*; +use crate::runtime::vm::stack_switching::VMCommonStackInformation; use crate::runtime::vm::{ ExportFunction, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMContext, - VMFuncRef, VMFunctionImport, VMOpaqueContext, + VMFuncRef, VMFunctionImport, VMOpaqueContext, VMStoreContext, }; use crate::runtime::Uninhabited; use crate::store::{AutoAssertNoGc, StoreData, StoreOpaque, Stored}; @@ -357,6 +358,7 @@ macro_rules! for_each_function_signature { } mod typed; +use crate::runtime::vm::stack_switching::VMStackChain; pub use typed::*; impl Func { @@ -1191,6 +1193,7 @@ impl Func { results.len() ); } + for (ty, arg) in ty.params().zip(params) { arg.ensure_matches_ty(opaque, &ty) .context("argument type mismatch")?; @@ -1597,103 +1600,173 @@ pub(crate) fn invoke_wasm_and_catch_traps( closure: impl FnMut(NonNull, Option>) -> bool, ) -> Result<()> { unsafe { - let exit = enter_wasm(store); + // The `enter_wasm` call below will reset the store's `stack_chain` to + // a new `InitialStack`, pointing to the stack-allocated + // `initial_stack_csi`. + let mut initial_stack_csi = VMCommonStackInformation::running_default(); + // Stores some state of the runtime just before entering Wasm. Will be + // restored upon exiting Wasm. Note that the `CallThreadState` that is + // created by the `catch_traps` call below will store a pointer to this + // stack-allocated `previous_runtime_state`. + let previous_runtime_state = EntryStoreContext::enter_wasm(store, &mut initial_stack_csi); if let Err(trap) = store.0.call_hook(CallHook::CallingWasm) { - exit_wasm(store, exit); return Err(trap); } - let result = crate::runtime::vm::catch_traps(store, closure); - exit_wasm(store, exit); + let result = crate::runtime::vm::catch_traps(store, &previous_runtime_state, closure); + core::mem::drop(previous_runtime_state); store.0.call_hook(CallHook::ReturningFromWasm)?; result.map_err(|t| crate::trap::from_runtime_box(store.0, t)) } } -/// This function is called to register state within `Store` whenever -/// WebAssembly is entered within the `Store`. -/// -/// This function sets up various limits such as: -/// -/// * The stack limit. This is what ensures that we limit the stack space -/// allocated by WebAssembly code and it's relative to the initial stack -/// pointer that called into wasm. -/// -/// This function may fail if the stack limit can't be set because an -/// interrupt already happened. -fn enter_wasm(store: &mut StoreContextMut<'_, T>) -> Option { - // If this is a recursive call, e.g. our stack limit is already set, then - // we may be able to skip this function. - // - // For synchronous stores there's nothing else to do because all wasm calls - // happen synchronously and on the same stack. This means that the previous - // stack limit will suffice for the next recursive call. - // - // For asynchronous stores then each call happens on a separate native - // stack. This means that the previous stack limit is no longer relevant - // because we're on a separate stack. - if unsafe { *store.0.vm_store_context().stack_limit.get() } != usize::MAX - && !store.0.async_support() - { - return None; - } - - // Ignore this stack pointer business on miri since we can't execute wasm - // anyway and the concept of a stack pointer on miri is a bit nebulous - // regardless. - if cfg!(miri) { - return None; - } - - // When Cranelift has support for the host then we might be running native - // compiled code meaning we need to read the actual stack pointer. If - // Cranelift can't be used though then we're guaranteed to be running pulley - // in which case this stack pointer isn't actually used as Pulley has custom - // mechanisms for stack overflow. - #[cfg(has_host_compiler_backend)] - let stack_pointer = crate::runtime::vm::get_stack_pointer(); - #[cfg(not(has_host_compiler_backend))] - let stack_pointer = { - use wasmtime_environ::TripleExt; - debug_assert!(store.engine().target().is_pulley()); - usize::MAX - }; +/// This type helps managing the state of the runtime when entering and exiting +/// Wasm. To this end, it contains a subset of the data in `VMStoreContext`.. +/// Upon entering Wasm, it updates various runtime fields and their +/// original values saved in this struct. Upon exiting Wasm, the previous values +/// are restored. +// FIXME(frank-emrich) Do the fields in here need to be (Unsafe)Cells? +pub struct EntryStoreContext { + /// If set, contains value of `stack_limit` field to restore in + /// `VMRuntimeLimits` when exiting Wasm. + pub stack_limit: Option, + /// Contains value of `last_wasm_exit_pc` field to restore in + /// `VMRuntimeLimits` when exiting Wasm. + pub last_wasm_exit_pc: usize, + /// Contains value of `last_wasm_exit_fp` field to restore in + /// `VMRuntimeLimits` when exiting Wasm. + pub last_wasm_exit_fp: usize, + /// Contains value of `last_wasm_entry_fp` field to restore in + /// `VMRuntimeLimits` when exiting Wasm. + pub last_wasm_entry_fp: usize, + /// Contains value of `stack_chain` field to restore in + /// `Store` when exiting Wasm. + pub stack_chain: VMStackChain, + + /// We need a pointer to the runtime limits, so we can update them from + /// `drop`/`exit_wasm`. + vm_store_context: *const VMStoreContext, +} - // Determine the stack pointer where, after which, any wasm code will - // immediately trap. This is checked on the entry to all wasm functions. - // - // Note that this isn't 100% precise. We are requested to give wasm - // `max_wasm_stack` bytes, but what we're actually doing is giving wasm - // probably a little less than `max_wasm_stack` because we're - // calculating the limit relative to this function's approximate stack - // pointer. Wasm will be executed on a frame beneath this one (or next - // to it). In any case it's expected to be at most a few hundred bytes - // of slop one way or another. When wasm is typically given a MB or so - // (a million bytes) the slop shouldn't matter too much. - // - // After we've got the stack limit then we store it into the `stack_limit` - // variable. - let wasm_stack_limit = stack_pointer - store.engine().config().max_wasm_stack; - let prev_stack = unsafe { - mem::replace( - &mut *store.0.vm_store_context().stack_limit.get(), - wasm_stack_limit, - ) - }; +impl EntryStoreContext { + /// This function is called to update and save state when + /// WebAssembly is entered within the `Store`. + /// + /// This updates various fields such as: + /// + /// * The stack limit. This is what ensures that we limit the stack space + /// allocated by WebAssembly code and it's relative to the initial stack + /// pointer that called into wasm. + /// + /// It also saves the different last_wasm_* values in the `VMRuntimeLimits`. + pub fn enter_wasm( + store: &mut StoreContextMut<'_, T>, + initial_stack_information: *mut VMCommonStackInformation, + ) -> Self { + let stack_limit; - Some(prev_stack) -} + // If this is a recursive call, e.g. our stack limit is already set, then + // we may be able to skip this function. + // + // For synchronous stores there's nothing else to do because all wasm calls + // happen synchronously and on the same stack. This means that the previous + // stack limit will suffice for the next recursive call. + // + // For asynchronous stores then each call happens on a separate native + // stack. This means that the previous stack limit is no longer relevant + // because we're on a separate stack. + if unsafe { *store.0.vm_store_context().stack_limit.get() } != usize::MAX + && !store.0.async_support() + { + stack_limit = None; + } + // Ignore this stack pointer business on miri since we can't execute wasm + // anyway and the concept of a stack pointer on miri is a bit nebulous + // regardless. + else if cfg!(miri) { + stack_limit = None; + } else { + // When Cranelift has support for the host then we might be running native + // compiled code meaning we need to read the actual stack pointer. If + // Cranelift can't be used though then we're guaranteed to be running pulley + // in which case this stack pointer isn't actually used as Pulley has custom + // mechanisms for stack overflow. + #[cfg(has_host_compiler_backend)] + let stack_pointer = crate::runtime::vm::get_stack_pointer(); + #[cfg(not(has_host_compiler_backend))] + let stack_pointer = { + use wasmtime_environ::TripleExt; + debug_assert!(store.engine().target().is_pulley()); + usize::MAX + }; -fn exit_wasm(store: &mut StoreContextMut<'_, T>, prev_stack: Option) { - // If we don't have a previous stack pointer to restore, then there's no - // cleanup we need to perform here. - let prev_stack = match prev_stack { - Some(stack) => stack, - None => return, - }; + // Determine the stack pointer where, after which, any wasm code will + // immediately trap. This is checked on the entry to all wasm functions. + // + // Note that this isn't 100% precise. We are requested to give wasm + // `max_wasm_stack` bytes, but what we're actually doing is giving wasm + // probably a little less than `max_wasm_stack` because we're + // calculating the limit relative to this function's approximate stack + // pointer. Wasm will be executed on a frame beneath this one (or next + // to it). In any case it's expected to be at most a few hundred bytes + // of slop one way or another. When wasm is typically given a MB or so + // (a million bytes) the slop shouldn't matter too much. + // + // After we've got the stack limit then we store it into the `stack_limit` + // variable. + let wasm_stack_limit = stack_pointer - store.engine().config().max_wasm_stack; + let prev_stack = unsafe { + mem::replace( + &mut *store.0.vm_store_context().stack_limit.get(), + wasm_stack_limit, + ) + }; + stack_limit = Some(prev_stack); + } - unsafe { - *store.0.vm_store_context().stack_limit.get() = prev_stack; + unsafe { + let last_wasm_exit_pc = *store.0.vm_store_context().last_wasm_exit_pc.get(); + let last_wasm_exit_fp = *store.0.vm_store_context().last_wasm_exit_fp.get(); + let last_wasm_entry_fp = *store.0.vm_store_context().last_wasm_entry_fp.get(); + let stack_chain = (*store.0.vm_store_context().stack_chain.get()).clone(); + + let new_stack_chain = VMStackChain::InitialStack(initial_stack_information); + *store.0.vm_store_context().stack_chain.get() = new_stack_chain; + + let vm_store_context = store.0.vm_store_context(); + + Self { + stack_limit, + last_wasm_exit_pc, + last_wasm_exit_fp, + last_wasm_entry_fp, + stack_chain, + vm_store_context, + } + } + } + + /// This function restores the values stored in this struct. We invoke this + /// function through this type's `Drop` implementation. This ensures that we + /// even restore the values if we unwind the stack (e.g., because we are + /// panicing out of a Wasm execution). + fn exit_wasm(&mut self) { + unsafe { + self.stack_limit.inspect(|limit| { + *(&*self.vm_store_context).stack_limit.get() = *limit; + }); + + *(*self.vm_store_context).last_wasm_exit_fp.get() = self.last_wasm_exit_fp; + *(*self.vm_store_context).last_wasm_exit_pc.get() = self.last_wasm_exit_pc; + *(*self.vm_store_context).last_wasm_entry_fp.get() = self.last_wasm_entry_fp; + *(*self.vm_store_context).stack_chain.get() = self.stack_chain.clone(); + } + } +} + +impl Drop for EntryStoreContext { + fn drop(&mut self) { + self.exit_wasm(); } } diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs index 14ee5b783856..ed1347e17976 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs @@ -727,6 +727,9 @@ unsafe impl WasmTy for Rooted { | HeapType::I31 | HeapType::Struct | HeapType::ConcreteStruct(_) + | HeapType::Cont + | HeapType::NoCont + | HeapType::ConcreteCont(_) | HeapType::None => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, @@ -821,6 +824,9 @@ unsafe impl WasmTy for ManuallyRooted { | HeapType::I31 | HeapType::Struct | HeapType::ConcreteStruct(_) + | HeapType::Cont + | HeapType::NoCont + | HeapType::ConcreteCont(_) | HeapType::None => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, diff --git a/crates/wasmtime/src/runtime/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/gc/enabled/structref.rs index e111c2f2e633..eb9233b00b15 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/structref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/structref.rs @@ -594,7 +594,10 @@ unsafe impl WasmTy for Rooted { | HeapType::I31 | HeapType::Array | HeapType::ConcreteArray(_) - | HeapType::None => bail!( + | HeapType::None + | HeapType::NoCont + | HeapType::Cont + | HeapType::ConcreteCont(_) => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, ), @@ -688,7 +691,10 @@ unsafe impl WasmTy for ManuallyRooted { | HeapType::I31 | HeapType::Array | HeapType::ConcreteArray(_) - | HeapType::None => bail!( + | HeapType::None + | HeapType::NoCont + | HeapType::Cont + | HeapType::ConcreteCont(_) => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, ), diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 559abf7ac410..65e9e1692bb2 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -114,6 +114,9 @@ pub use self::async_::CallHookHandler; #[cfg(feature = "async")] use self::async_::*; +use super::vm::stack_switching::stack::VMContinuationStack; +use super::vm::stack_switching::VMContRef; + /// A [`Store`] is a collection of WebAssembly instances and host-defined state. /// /// All WebAssembly instances and items will be attached to and refer to a @@ -306,6 +309,11 @@ pub struct StoreOpaque { engine: Engine, vm_store_context: VMStoreContext, + + // Contains all continuations ever allocated throughout the lifetime of this + // store. + continuations: Vec>, + instances: Vec, #[cfg(feature = "component-model")] num_component_instances: usize, @@ -524,6 +532,7 @@ impl Store { _marker: marker::PhantomPinned, engine: engine.clone(), vm_store_context: Default::default(), + continuations: Vec::new(), instances: Vec::new(), #[cfg(feature = "component-model")] num_component_instances: 0, @@ -1533,6 +1542,7 @@ impl StoreOpaque { assert!(gc_roots_list.is_empty()); self.trace_wasm_stack_roots(gc_roots_list); + self.trace_wasm_continuation_roots(gc_roots_list); self.trace_vmctx_roots(gc_roots_list); self.trace_user_roots(gc_roots_list); @@ -1540,60 +1550,106 @@ impl StoreOpaque { } #[cfg(feature = "gc")] - fn trace_wasm_stack_roots(&mut self, gc_roots_list: &mut GcRootsList) { - use crate::runtime::vm::{Backtrace, SendSyncPtr}; + fn trace_wasm_stack_frame( + &self, + gc_roots_list: &mut GcRootsList, + frame: crate::runtime::vm::Frame, + ) { + use crate::runtime::vm::SendSyncPtr; use core::ptr::NonNull; - log::trace!("Begin trace GC roots :: Wasm stack"); + let pc = frame.pc(); + debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames"); - Backtrace::trace(self, |frame| { - let pc = frame.pc(); - debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames"); + let fp = frame.fp() as *mut usize; + debug_assert!( + !fp.is_null(), + "we should always get a valid frame pointer for Wasm frames" + ); - let fp = frame.fp() as *mut usize; - debug_assert!( - !fp.is_null(), - "we should always get a valid frame pointer for Wasm frames" - ); + let module_info = self + .modules() + .lookup_module_by_pc(pc) + .expect("should have module info for Wasm frame"); - let module_info = self - .modules() - .lookup_module_by_pc(pc) - .expect("should have module info for Wasm frame"); + let stack_map = match module_info.lookup_stack_map(pc) { + Some(sm) => sm, + None => { + log::trace!("No stack map for this Wasm frame"); + return; + } + }; + log::trace!( + "We have a stack map that maps {} bytes in this Wasm frame", + stack_map.frame_size() + ); - let stack_map = match module_info.lookup_stack_map(pc) { - Some(sm) => sm, - None => { - log::trace!("No stack map for this Wasm frame"); - return core::ops::ControlFlow::Continue(()); - } - }; - log::trace!( - "We have a stack map that maps {} bytes in this Wasm frame", - stack_map.frame_size() - ); + let sp = unsafe { stack_map.sp(fp) }; + for stack_slot in unsafe { stack_map.live_gc_refs(sp) } { + let raw: u32 = unsafe { core::ptr::read(stack_slot) }; + log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}"); - let sp = unsafe { stack_map.sp(fp) }; - for stack_slot in unsafe { stack_map.live_gc_refs(sp) } { - let raw: u32 = unsafe { core::ptr::read(stack_slot) }; - log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}"); - - let gc_ref = VMGcRef::from_raw_u32(raw); - if gc_ref.is_some() { - unsafe { - gc_roots_list.add_wasm_stack_root(SendSyncPtr::new( - NonNull::new(stack_slot).unwrap(), - )); - } + let gc_ref = VMGcRef::from_raw_u32(raw); + if gc_ref.is_some() { + unsafe { + gc_roots_list + .add_wasm_stack_root(SendSyncPtr::new(NonNull::new(stack_slot).unwrap())); } } + } + } + + #[cfg(feature = "gc")] + fn trace_wasm_stack_roots(&mut self, gc_roots_list: &mut GcRootsList) { + use crate::runtime::vm::Backtrace; + log::trace!("Begin trace GC roots :: Wasm stack"); + Backtrace::trace(self, |frame| { + self.trace_wasm_stack_frame(gc_roots_list, frame); core::ops::ControlFlow::Continue(()) }); log::trace!("End trace GC roots :: Wasm stack"); } + #[cfg(feature = "gc")] + fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) { + use crate::runtime::vm::Backtrace; + log::trace!("Begin trace GC roots :: continuations"); + + for continuation in &self.continuations { + let state = continuation.common_stack_information.state; + + // FIXME(frank-emrich) In general, it is not enough to just trace + // through the stacks of continuations; we also need to look through + // their `cont.bind` arguments. However, we don't currently have + // enough RTTI information to check if any of the values in the + // buffers used by `cont.bind` are GC values. As a workaround, note + // that we currently disallow cont.bind-ing GC values altogether. + // This way, it is okay not to check them here. + + // Note that we only care about continuations that have state + // `Suspended`. + // - `Running` continuations will be handled by + // `trace_wasm_stack_roots`. + // - For `Parent` continuations, we don't know if they are the + // parent of a running continuation or a suspended one. But it + // does not matter: They will be handled when traversing the stack + // chain starting at either the running one, or the suspended + // continuations below. + // - For `Fresh` continuations, we know that there are no GC values + // on their stack, yet. + if state == crate::vm::stack_switching::VMStackState::Suspended { + Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| { + self.trace_wasm_stack_frame(gc_roots_list, frame); + core::ops::ControlFlow::Continue(()) + }); + } + } + + log::trace!("End trace GC roots :: continuations"); + } + #[cfg(feature = "gc")] fn trace_vmctx_roots(&mut self, gc_roots_list: &mut GcRootsList) { log::trace!("Begin trace GC roots :: vmctx"); @@ -1885,6 +1941,20 @@ at https://bytecodealliance.org/security. Executor::Native => &crate::runtime::vm::UnwindHost, } } + + /// Allocates a new continuation. Note that we currently don't support + /// deallocating them. Instead, all continuations remain allocated + /// throughout the store's lifetime. + pub fn allocate_continuation(&mut self) -> Result<*mut VMContRef> { + // FIXME(frank-emrich) Do we need to pin this? + let mut continuation = Box::new(VMContRef::empty()); + let stack_size = self.engine.config().stack_switching_config.stack_size; + let stack = VMContinuationStack::new(stack_size)?; + continuation.stack = stack; + let ptr = continuation.deref_mut() as *mut VMContRef; + self.continuations.push(continuation); + Ok(ptr) + } } unsafe impl crate::runtime::vm::VMStore for StoreInner { diff --git a/crates/wasmtime/src/runtime/type_registry.rs b/crates/wasmtime/src/runtime/type_registry.rs index ded58a1f9e99..58fc9c41b538 100644 --- a/crates/wasmtime/src/runtime/type_registry.rs +++ b/crates/wasmtime/src/runtime/type_registry.rs @@ -851,7 +851,7 @@ impl TypeRegistryInner { .struct_layout(s) .into(), ), - wasmtime_environ::WasmCompositeInnerType::Cont(_) => todo!(), // FIXME: #10248 stack switching support. + wasmtime_environ::WasmCompositeInnerType::Cont(_) => None, // FIXME: #10248 stack switching support. }; // Add the type to our slab. diff --git a/crates/wasmtime/src/runtime/types.rs b/crates/wasmtime/src/runtime/types.rs index 2d985acc938c..f1f87a6dadec 100644 --- a/crates/wasmtime/src/runtime/types.rs +++ b/crates/wasmtime/src/runtime/types.rs @@ -154,6 +154,12 @@ impl ValType { /// The `nullref` type, aka `(ref null none)`. pub const NULLREF: Self = ValType::Ref(RefType::NULLREF); + /// The `contref` type, aka `(ref null cont)`. + pub const CONTREF: Self = ValType::Ref(RefType::CONTREF); + + /// The `nullcontref` type, aka. `(ref null nocont)`. + pub const NULLCONTREF: Self = ValType::Ref(RefType::NULLCONTREF); + /// Returns true if `ValType` matches any of the numeric types. (e.g. `I32`, /// `I64`, `F32`, `F64`). #[inline] @@ -236,6 +242,18 @@ impl ValType { ) } + /// Is this the `contref` (aka `(ref null cont)`) type? + #[inline] + pub fn is_contref(&self) -> bool { + matches!( + self, + ValType::Ref(RefType { + is_nullable: true, + heap_type: HeapType::Cont + }) + ) + } + /// Get the underlying reference type, if this value type is a reference /// type. #[inline] @@ -437,6 +455,18 @@ impl RefType { heap_type: HeapType::None, }; + /// The `contref` type, aka `(ref null cont)`. + pub const CONTREF: Self = RefType { + is_nullable: true, + heap_type: HeapType::Cont, + }; + + /// The `nullcontref` type, aka `(ref null nocont)`. + pub const NULLCONTREF: Self = RefType { + is_nullable: true, + heap_type: HeapType::NoCont, + }; + /// Construct a new reference type. pub fn new(is_nullable: bool, heap_type: HeapType) -> RefType { RefType { @@ -697,6 +727,23 @@ pub enum HeapType { /// of `any` and `eq`) and supertypes of the `none` heap type. ConcreteStruct(StructType), + /// A reference to a continuation of a specific, concrete type. + /// + /// These are subtypes of `cont` and supertypes of `nocont`. + ConcreteCont(ContType), + + /// The `cont` heap type represents a reference to any kind of continuation. + /// + /// This is the top type for the continuation objects type hierarchy, and is + /// therefore a supertype of every continuation object. + Cont, + + /// The `nocont` heap type represents the null continuation object. + /// + /// This is the bottom type for the continuation objects type hierarchy, and + /// therefore `nocont` is a subtype of all continuation object types. + NoCont, + /// The abstract `none` heap type represents the null internal reference. /// /// This is the bottom type for the internal type hierarchy, and therefore @@ -720,6 +767,9 @@ impl Display for HeapType { HeapType::ConcreteFunc(ty) => write!(f, "(concrete func {:?})", ty.type_index()), HeapType::ConcreteArray(ty) => write!(f, "(concrete array {:?})", ty.type_index()), HeapType::ConcreteStruct(ty) => write!(f, "(concrete struct {:?})", ty.type_index()), + HeapType::ConcreteCont(ty) => write!(f, "(concrete cont {:?})", ty.type_index()), + HeapType::Cont => write!(f, "cont"), + HeapType::NoCont => write!(f, "nocont"), } } } @@ -745,6 +795,13 @@ impl From for HeapType { } } +impl From for HeapType { + #[inline] + fn from(f: ContType) -> Self { + HeapType::ConcreteCont(f) + } +} + impl HeapType { /// Is this the abstract `extern` heap type? pub fn is_extern(&self) -> bool { @@ -776,6 +833,11 @@ impl HeapType { matches!(self, HeapType::None) } + /// Is this the abstract `cont` heap type? + pub fn is_cont(&self) -> bool { + matches!(self, HeapType::Cont) + } + /// Is this an abstract type? /// /// Types that are not abstract are concrete, user-defined types. @@ -790,7 +852,10 @@ impl HeapType { pub fn is_concrete(&self) -> bool { matches!( self, - HeapType::ConcreteFunc(_) | HeapType::ConcreteArray(_) | HeapType::ConcreteStruct(_) + HeapType::ConcreteFunc(_) + | HeapType::ConcreteArray(_) + | HeapType::ConcreteStruct(_) + | HeapType::ConcreteCont(_) ) } @@ -836,6 +901,21 @@ impl HeapType { self.as_concrete_array().unwrap() } + /// Is this a concrete, user-defined continuation type? + pub fn is_concrete_cont(&self) -> bool { + matches!(self, HeapType::ConcreteCont(_)) + } + + /// Get the underlying concrete, user-defined continuation type, if any. + /// + /// Returns `None` if this is not a concrete continuation type. + pub fn as_concrete_cont(&self) -> Option<&ContType> { + match self { + HeapType::ConcreteCont(f) => Some(f), + _ => None, + } + } + /// Is this a concrete, user-defined struct type? pub fn is_concrete_struct(&self) -> bool { matches!(self, HeapType::ConcreteStruct(_)) @@ -851,6 +931,12 @@ impl HeapType { } } + /// Get the underlying concrete, user-defined type, panicking if this is not + /// a concrete continuation type. + pub fn unwrap_concrete_cont(&self) -> &ContType { + self.as_concrete_cont().unwrap() + } + /// Get the underlying concrete, user-defined type, panicking if this is not /// a concrete struct type. pub fn unwrap_concrete_struct(&self) -> &StructType { @@ -876,6 +962,8 @@ impl HeapType { | HeapType::Struct | HeapType::ConcreteStruct(_) | HeapType::None => HeapType::Any, + + HeapType::Cont | HeapType::ConcreteCont(_) | HeapType::NoCont => HeapType::Cont, } } @@ -883,7 +971,7 @@ impl HeapType { #[inline] pub fn is_top(&self) -> bool { match self { - HeapType::Any | HeapType::Extern | HeapType::Func => true, + HeapType::Any | HeapType::Extern | HeapType::Func | HeapType::Cont => true, _ => false, } } @@ -907,6 +995,8 @@ impl HeapType { | HeapType::Struct | HeapType::ConcreteStruct(_) | HeapType::None => HeapType::None, + + HeapType::Cont | HeapType::ConcreteCont(_) | HeapType::NoCont => HeapType::NoCont, } } @@ -914,7 +1004,7 @@ impl HeapType { #[inline] pub fn is_bottom(&self) -> bool { match self { - HeapType::None | HeapType::NoExtern | HeapType::NoFunc => true, + HeapType::None | HeapType::NoExtern | HeapType::NoFunc | HeapType::NoCont => true, _ => false, } } @@ -952,6 +1042,18 @@ impl HeapType { (HeapType::Func, HeapType::Func) => true, (HeapType::Func, _) => false, + (HeapType::Cont, HeapType::Cont) => true, + (HeapType::Cont, _) => false, + + (HeapType::NoCont, HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont) => { + true + } + (HeapType::NoCont, _) => false, + + (HeapType::ConcreteCont(_), HeapType::Cont) => true, + (HeapType::ConcreteCont(a), HeapType::ConcreteCont(b)) => a.matches(b), + (HeapType::ConcreteCont(_), _) => false, + ( HeapType::None, HeapType::None @@ -1035,10 +1137,13 @@ impl HeapType { | HeapType::I31 | HeapType::Array | HeapType::Struct + | HeapType::Cont + | HeapType::NoCont | HeapType::None => true, HeapType::ConcreteFunc(ty) => ty.comes_from_same_engine(engine), HeapType::ConcreteArray(ty) => ty.comes_from_same_engine(engine), HeapType::ConcreteStruct(ty) => ty.comes_from_same_engine(engine), + HeapType::ConcreteCont(ty) => ty.comes_from_same_engine(engine), } } @@ -1063,6 +1168,11 @@ impl HeapType { HeapType::ConcreteStruct(a) => { WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::Engine(a.type_index())) } + HeapType::Cont => WasmHeapType::Cont, + HeapType::NoCont => WasmHeapType::NoCont, + HeapType::ConcreteCont(c) => { + WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Engine(c.type_index())) + } } } @@ -1093,16 +1203,22 @@ impl HeapType { | WasmHeapType::ConcreteArray(EngineOrModuleTypeIndex::Module(_)) | WasmHeapType::ConcreteArray(EngineOrModuleTypeIndex::RecGroup(_)) | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::Module(_)) - | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::RecGroup(_)) => { + | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::RecGroup(_)) + | WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Module(_)) + | WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::RecGroup(_)) => { panic!("HeapType::from_wasm_type on non-canonicalized-for-runtime-usage heap type") } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::Cont => HeapType::Cont, + WasmHeapType::NoCont => HeapType::NoCont, + WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Engine(idx)) => { + HeapType::ConcreteCont(ContType::from_shared_type_index(engine, *idx)) + } } } pub(crate) fn as_registered_type(&self) -> Option<&RegisteredType> { match self { + HeapType::ConcreteCont(c) => Some(&c.registered_type), HeapType::ConcreteFunc(f) => Some(&f.registered_type), HeapType::ConcreteArray(a) => Some(&a.registered_type), HeapType::ConcreteStruct(a) => Some(&a.registered_type), @@ -1116,6 +1232,8 @@ impl HeapType { | HeapType::I31 | HeapType::Array | HeapType::Struct + | HeapType::Cont + | HeapType::NoCont | HeapType::None => None, } } @@ -1125,6 +1243,7 @@ impl HeapType { match self.top() { Self::Any | Self::Extern => true, Self::Func => false, + Self::Cont => false, ty => unreachable!("not a top type: {ty:?}"), } } @@ -2413,6 +2532,56 @@ impl FuncType { } } +// Continuation types +/// A WebAssembly continuation descriptor. +#[derive(Debug, Clone, Hash)] +pub struct ContType { + registered_type: RegisteredType, +} + +impl ContType { + /// Get the engine that this function type is associated with. + pub fn engine(&self) -> &Engine { + self.registered_type.engine() + } + + pub(crate) fn comes_from_same_engine(&self, engine: &Engine) -> bool { + Engine::same(self.registered_type.engine(), engine) + } + + pub(crate) fn type_index(&self) -> VMSharedTypeIndex { + self.registered_type.index() + } + + /// Does this continuation type match the other continuation type? + /// + /// That is, is this continuation type a subtype of the other continuation type? + /// + /// # Panics + /// + /// Panics if either type is associated with a different engine from the + /// other. + pub fn matches(&self, other: &ContType) -> bool { + assert!(self.comes_from_same_engine(other.engine())); + + // Avoid matching on structure for subtyping checks when we have + // precisely the same type. + // TODO(dhil): Implement subtype check later. + self.type_index() == other.type_index() + } + + pub(crate) fn from_shared_type_index(engine: &Engine, index: VMSharedTypeIndex) -> ContType { + let ty = RegisteredType::root(engine, index).expect( + "VMSharedTypeIndex is not registered in the Engine! Wrong \ + engine? Didn't root the index somewhere?", + ); + assert!(ty.is_cont()); + Self { + registered_type: ty, + } + } +} + // Global Types /// A WebAssembly global descriptor. diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs index 3d606d795900..0207d35f9094 100644 --- a/crates/wasmtime/src/runtime/values.rs +++ b/crates/wasmtime/src/runtime/values.rs @@ -280,6 +280,11 @@ impl Val { HeapType::NoFunc => Ref::Func(None), + HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont => { + // TODO(#10248): Required to support stack switching in the embedder API. + unimplemented!() + } + HeapType::Extern => ExternRef::_from_raw(store, raw.get_externref()).into(), HeapType::NoExtern => Ref::Extern(None), diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 189f93dc86a5..eb751fb45675 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -42,6 +42,7 @@ mod vmcontext; #[cfg(feature = "threads")] mod parking_spot; +pub mod stack_switching; // Note that `debug_builtins` here is disabled with a feature or a lack of a // native compilation backend because it's only here to assist in debugging @@ -95,6 +96,7 @@ pub use crate::runtime::vm::vmcontext::{ VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw, }; + pub use send_sync_ptr::SendSyncPtr; mod module_id; diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs index 1c26e59b15ac..12c78116e79c 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs @@ -8,7 +8,6 @@ use crate::runtime::vm::{ SendSyncPtr, Table, }; use crate::{prelude::*, vm::HostAlignedByteCount}; -use std::mem; use std::ptr::NonNull; use wasmtime_environ::{Module, Tunables}; @@ -31,7 +30,7 @@ impl TablePool { /// Create a new `TablePool`. pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result { let table_size = HostAlignedByteCount::new_rounded_up( - mem::size_of::<*mut u8>() + crate::runtime::vm::table::MAX_TABLE_ELEM_SIZE .checked_mul(config.limits.table_elements) .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, )?; @@ -133,13 +132,15 @@ impl TablePool { match (|| { let base = self.get(allocation_index); + let element_size = crate::vm::table::wasm_to_table_type(ty.ref_type).element_size(); + unsafe { - commit_pages(base, self.table_elements * mem::size_of::<*mut u8>())?; + commit_pages(base, self.table_elements * element_size)?; } let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut( base.cast(), - self.table_elements * mem::size_of::<*mut u8>(), + self.table_elements * element_size, )) .unwrap(); unsafe { @@ -198,8 +199,10 @@ impl TablePool { // doesn't overflow? The only check that exists is for the boundary // condition that table.size() * mem::size_of::<*mut u8>() is less than // a host page smaller than usize::MAX. - let size = HostAlignedByteCount::new_rounded_up(table.size() * mem::size_of::<*mut u8>()) - .expect("table entry size doesn't overflow"); + let size = HostAlignedByteCount::new_rounded_up( + table.size() * table.element_type().element_size(), + ) + .expect("table entry size doesn't overflow"); // `memset` the first `keep_resident` bytes. let size_to_memset = size.min(self.keep_resident); diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 0447c4c29b58..e489465579b2 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -54,6 +54,8 @@ //! } //! ``` +use super::stack_switching::VMContObj; +use super::stack_switching::VMContRef; use crate::prelude::*; use crate::runtime::vm::table::{Table, TableElementType}; use crate::runtime::vm::vmcontext::VMFuncRef; @@ -219,6 +221,7 @@ unsafe fn table_grow_func_ref( let element = match instance.table_element_type(table_index) { TableElementType::Func => NonNull::new(init_value.cast::()).into(), TableElementType::GcRef => unreachable!(), + TableElementType::Cont => unreachable!(), }; let result = instance @@ -248,6 +251,40 @@ unsafe fn table_grow_gc_ref( .clone_gc_ref(&r) }) .into(), + TableElementType::Cont => unreachable!(), + }; + + let result = instance + .table_grow(store, table_index, delta, element)? + .map(AllocationSize); + Ok(result) +} + +unsafe fn table_grow_cont_obj( + store: &mut dyn VMStore, + instance: &mut Instance, + table_index: u32, + delta: u64, + // The following two values together form the intitial Option. + // A None value is indicated by the pointer being null. + init_value_contref: *mut u8, + init_value_revision: u64, +) -> Result> { + use core::ptr::NonNull; + let init_value = if init_value_contref.is_null() { + None + } else { + // SAFETY: We just checked that the pointer is non-null + let contref = NonNull::new_unchecked(init_value_contref as *mut VMContRef); + let contobj = VMContObj::new(contref, init_value_revision); + Some(contobj) + }; + + let table_index = TableIndex::from_u32(table_index); + + let element = match instance.table_element_type(table_index) { + TableElementType::Cont => init_value.into(), + _ => panic!("Wrong table growing function"), }; let result = instance @@ -274,6 +311,7 @@ unsafe fn table_fill_func_ref( Ok(()) } TableElementType::GcRef => unreachable!(), + TableElementType::Cont => unreachable!(), } } @@ -297,6 +335,38 @@ unsafe fn table_fill_gc_ref( table.fill(Some(gc_store), dst, gc_ref.into(), len)?; Ok(()) } + + TableElementType::Cont => unreachable!(), + } +} + +unsafe fn table_fill_cont_obj( + store: &mut dyn VMStore, + instance: &mut Instance, + table_index: u32, + dst: u64, + value_contref: *mut u8, + value_revision: u64, + len: u64, +) -> Result<()> { + use core::ptr::NonNull; + let table_index = TableIndex::from_u32(table_index); + let table = &mut *instance.get_table(table_index); + match table.element_type() { + TableElementType::Cont => { + let contobj = if value_contref.is_null() { + None + } else { + // SAFETY: We just checked that the pointer is non-null + let contref = NonNull::new_unchecked(value_contref as *mut VMContRef); + let contobj = VMContObj::new(contref, value_revision); + Some(contobj) + }; + + table.fill(store.optional_gc_store_mut()?, dst, contobj.into(), len)?; + Ok(()) + } + _ => panic!("Wrong table filling function"), } } @@ -1359,3 +1429,36 @@ pub mod relocs { } } } + +// Builtins for continuations. These are thin wrappers around the +// respective definitions in stack_switching.rs. +fn cont_new( + store: &mut dyn VMStore, + instance: &mut Instance, + func: *mut u8, + param_count: u32, + result_count: u32, +) -> Result, TrapReason> { + let ans = + crate::vm::stack_switching::cont_new(store, instance, func, param_count, result_count)?; + Ok(Some(AllocationSize(ans.cast::() as usize))) +} + +fn delete_me_print_str(_store: &mut dyn VMStore, _instance: &mut Instance, s: *const u8, len: u64) { + let len = + usize::try_from(len).map_err(|_error| TrapReason::User(anyhow::anyhow!("len too large!"))); + let str = unsafe { core::slice::from_raw_parts(s, len.unwrap()) }; + let _s = core::str::from_utf8(str).unwrap(); + #[cfg(feature = "std")] + print!("{_s}") +} + +fn delete_me_print_int(_store: &mut dyn VMStore, _instance: &mut Instance, _arg: u64) { + #[cfg(feature = "std")] + print!("{_arg}") +} + +fn delete_me_print_pointer(_store: &mut dyn VMStore, _instance: &mut Instance, _arg: *const u8) { + #[cfg(feature = "std")] + print!("{_arg:p}") +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs new file mode 100644 index 000000000000..59203e817f4a --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -0,0 +1,705 @@ +//! This module contains the runtime components of the implementation of the +//! stack switching proposal. + +use core::{marker::PhantomPinned, ptr::NonNull}; + +use stack::VMContinuationStack; +#[allow(unused)] +use wasmtime_environ::{debug_println, stack_switching::ENABLE_DEBUG_PRINTING}; + +use crate::vm::{Instance, TrapReason, VMFuncRef, VMStore}; +use crate::ValRaw; + +pub mod stack; + +/// A continuation object is a handle to a continuation reference +/// (i.e. an actual stack). A continuation object only be consumed +/// once. The linearity is checked dynamically in the generated code +/// by comparing the revision witness embedded in the pointer to the +/// actual revision counter on the continuation reference. +/// +/// In the optimized implementation, the continuation logically +/// represented by a VMContObj not only encompasses the pointed-to +/// VMContRef, but also all of its parents: +/// +/// ```text +/// +/// +----------------+ +/// +-->| VMContRef | +/// | +----------------+ +/// | ^ +/// | | parent +/// | | +/// | +----------------+ +/// | | VMContRef | +/// | +----------------+ +/// | ^ +/// | | parent +/// last ancestor | | +/// | +----------------+ +/// +---| VMContRef | <-- VMContObj +/// +----------------+ +/// ``` +/// +/// For performance reasons, the VMContRef at the bottom of this chain +/// (i.e., the one pointed to by the VMContObj) has a pointer to the +/// other end of the chain (i.e., its last ancestor). +// FIXME(frank-emrich) Does this actually need to be 16-byte aligned any +// more? Now that we use I128 on the Cranelift side (see +// [wasmtime_cranelift::stack_switching::fatpointer::pointer_type]), it +// should be fine to use the natural alignment of the type. +#[repr(C, align(16))] +#[derive(Debug, Clone, Copy)] +pub struct VMContObj { + pub revision: u64, + pub contref: NonNull, +} + +impl VMContObj { + pub fn new(contref: NonNull, revision: u64) -> Self { + Self { contref, revision } + } +} + +unsafe impl Send for VMContObj {} +unsafe impl Sync for VMContObj {} + +#[test] +fn null_pointer_optimization() { + // The Rust spec does not technically guarantee that the null pointer + // optimization applies to a struct containing a `NonNull`. + assert_eq!( + core::mem::size_of::>(), + core::mem::size_of::() + ); +} + +/// This type is used to save (and subsequently restore) a subset of the data in +/// `VMRuntimeLimits`. See documentation of `VMStackChain` for the exact uses. +#[repr(C)] +#[derive(Debug, Default, Clone)] +pub struct VMStackLimits { + /// Saved version of `stack_limit` field of `VMRuntimeLimits` + pub stack_limit: usize, + /// Saved version of `last_wasm_entry_fp` field of `VMRuntimeLimits` + pub last_wasm_entry_fp: usize, +} + +#[test] +fn check_vm_stack_limits_offsets() { + use core::mem::offset_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMStackLimits, stack_limit), + usize::from(offsets.ptr.vmstack_limits_stack_limit()) + ); + assert_eq!( + offset_of!(VMStackLimits, last_wasm_entry_fp), + usize::from(offsets.ptr.vmstack_limits_last_wasm_entry_fp()) + ); +} + +/// This type represents "common" information that we need to save both for the +/// initial stack and each continuation. +#[repr(C)] +#[derive(Debug, Clone)] +pub struct VMCommonStackInformation { + /// Saves subset of `VMRuntimeLimits` for this stack. See documentation of + /// `VMStackChain` for the exact uses. + pub limits: VMStackLimits, + /// For the initial stack, this field must only have one of the following values: + /// - Running + /// - Parent + pub state: VMStackState, + + /// Only in use when state is `Parent`. Otherwise, the list must be empty. + /// + /// Represents the handlers that this stack installed when resume-ing a + /// continuation. + /// + /// Note that for any resume instruction, we can re-order the handler + /// clauses without changing behavior such that all the suspend handlers + /// come first, followed by all the switch handler (while maintaining the + /// original ordering within the two groups). + /// Thus, we assume that the given resume instruction has the following + /// shape: + /// + /// (resume $ct + /// (on $tag_0 $block_0) ... (on $tag_{n-1} $block_{n-1}) + /// (on $tag_n switch) ... (on $tag_m switch) + /// ) + /// + /// On resume, the handler list is then filled with m + 1 (i.e., one per + /// handler clause) entries such that the i-th entry, using 0-based + /// indexing, is the identifier of $tag_i (represented as *mut + /// VMTagDefinition). + /// Further, `first_switch_handler_index` (see below) is set to n (i.e., the + /// 0-based index of the first switch handler). + /// + /// Note that the actual data buffer (i.e., the one `handler.data` points + /// to) is always allocated on the stack that this `CommonStackInformation` + /// struct describes. + pub handlers: VMHandlerList, + + /// Only used when state is `Parent`. See documentation of `handlers` above. + pub first_switch_handler_index: u32, +} + +impl VMCommonStackInformation { + /// Default value with state set to `Running` + pub fn running_default() -> Self { + Self { + limits: VMStackLimits::default(), + state: VMStackState::Running, + handlers: VMHandlerList::empty(), + first_switch_handler_index: 0, + } + } +} + +#[test] +fn check_vm_common_stack_information_offsets() { + use core::mem::offset_of; + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmcommon_stack_information()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, limits), + usize::from(offsets.ptr.vmcommon_stack_information_limits()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, state), + usize::from(offsets.ptr.vmcommon_stack_information_state()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, handlers), + usize::from(offsets.ptr.vmcommon_stack_information_handlers()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, first_switch_handler_index), + usize::from( + offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() + ) + ); +} + +impl VMStackLimits { + /// Default value, but uses the given value for `stack_limit`. + pub fn with_stack_limit(stack_limit: usize) -> Self { + Self { + stack_limit, + ..Default::default() + } + } +} + +#[repr(C)] +#[derive(Debug, Clone)] +/// Reference to a stack-allocated buffer ("array"), storing data of some type +/// `T`. +pub struct VMArray { + /// Number of currently occupied slots. + pub length: u32, + /// Number of slots in the data buffer. Note that this is *not* the size of + /// the buffer in bytes! + pub capacity: u32, + /// The actual data buffer + pub data: *mut T, +} + +impl VMArray { + /// Creates empty `Array` + pub fn empty() -> Self { + Self { + length: 0, + capacity: 0, + data: core::ptr::null_mut(), + } + } + + /// Makes `Array` empty. + pub fn clear(&mut self) { + *self = Self::empty(); + } +} + +#[test] +fn check_vm_array_offsets() { + use core::mem::offset_of; + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + // Note that the type parameter has no influence on the size and offsets. + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::>(), + usize::from(offsets.ptr.size_of_vmarray()) + ); + assert_eq!( + offset_of!(VMArray<()>, length), + usize::from(offsets.ptr.vmarray_length()) + ); + assert_eq!( + offset_of!(VMArray<()>, capacity), + usize::from(offsets.ptr.vmarray_capacity()) + ); + assert_eq!( + offset_of!(VMArray<()>, data), + usize::from(offsets.ptr.vmarray_data()) + ); +} + +/// Type used for passing payloads to and from continuations. The actual type +/// argument should be wasmtime::runtime::vm::vmcontext::ValRaw, but we don't +/// have access to that here. +pub type VMPayloads = VMArray; + +/// Type for a list of handlers, represented by the handled tag. Thus, the +/// stored data is actually `*mut VMTagDefinition`, but we don't havr access to +/// that here. +pub type VMHandlerList = VMArray<*mut u8>; + +/// The main type representing a continuation. +#[repr(C)] +pub struct VMContRef { + /// The `CommonStackInformation` of this continuation's stack. + pub common_stack_information: VMCommonStackInformation, + + /// The parent of this continuation, which may be another continuation, the + /// initial stack, or absent (in case of a suspended continuation). + pub parent_chain: VMStackChain, + + /// Only used if `common_stack_information.state` is `Suspended` or `Fresh`. In + /// that case, this points to the end of the stack chain (i.e., the + /// continuation in the parent chain whose own `parent_chain` field is + /// `VMStackChain::Absent`). + /// Note that this may be a pointer to iself (if the state is `Fresh`, this is always the case). + pub last_ancestor: *mut VMContRef, + + /// Revision counter. + pub revision: u64, + + /// The underlying stack. + pub stack: VMContinuationStack, + + /// Used to store only + /// 1. The arguments to the function passed to cont.new + /// 2. The return values of that function + /// + /// Note that the actual data buffer (i.e., the one `args.data` points + /// to) is always allocated on this continuation's stack. + pub args: VMPayloads, + + /// Once a continuation has been suspended (using suspend or switch), + /// this buffer is used to pass payloads to and from the continuation. + /// More concretely, it is used to + /// - Pass payloads from a suspend instruction to the corresponding handler. + /// - Pass payloads to a continuation using cont.bind or resume + /// - Pass payloads to the continuation being switched to when using switch. + /// + /// Note that the actual data buffer (i.e., the one `values.data` points + /// to) is always allocated on this continuation's stack. + pub values: VMPayloads, + + /// Tell the compiler that this structure has potential self-references + /// through the `last_ancestor` pointer. + _marker: core::marker::PhantomPinned, +} + +impl VMContRef { + pub fn fiber_stack(&self) -> &VMContinuationStack { + &self.stack + } + + pub fn detach_stack(&mut self) -> VMContinuationStack { + core::mem::replace(&mut self.stack, VMContinuationStack::unallocated()) + } + + /// This is effectively a `Default` implementation, without calling it + /// so. Used to create `VMContRef`s when initializing pooling allocator. + #[allow(clippy::cast_possible_truncation)] + pub fn empty() -> Self { + let limits = VMStackLimits::with_stack_limit(Default::default()); + let state = VMStackState::Fresh; + let handlers = VMHandlerList::empty(); + let common_stack_information = VMCommonStackInformation { + limits, + state, + handlers, + first_switch_handler_index: 0, + }; + let parent_chain = VMStackChain::Absent; + let last_ancestor = core::ptr::null_mut(); + let stack = VMContinuationStack::unallocated(); + let args = VMPayloads::empty(); + let values = VMPayloads::empty(); + let revision = 0; + let _marker = PhantomPinned; + + Self { + common_stack_information, + parent_chain, + last_ancestor, + stack, + args, + values, + revision, + _marker, + } + } +} + +impl Drop for VMContRef { + fn drop(&mut self) { + // Note that continuation references do not own their parents, and we + // don't drop them here. + + // We would like to enforce the invariant that any continuation that + // was created for a cont.new (rather than, say, just living in a + // pool and never being touched), either ran to completion or was + // cancelled. But failing to do so should yield a custom error, + // instead of panicking here. + } +} + +// These are required so the WasmFX pooling allocator can store a Vec of +// `VMContRef`s. +unsafe impl Send for VMContRef {} +unsafe impl Sync for VMContRef {} + +#[test] +fn check_vm_contref_offsets() { + use core::mem::offset_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMContRef, common_stack_information), + usize::from(offsets.ptr.vmcontref_common_stack_information()) + ); + assert_eq!( + offset_of!(VMContRef, parent_chain), + usize::from(offsets.ptr.vmcontref_parent_chain()) + ); + assert_eq!( + offset_of!(VMContRef, last_ancestor), + usize::from(offsets.ptr.vmcontref_last_ancestor()) + ); + // Some 32-bit platforms need this to be 8-byte aligned, some don't. + // So we need to make sure it always is, without padding. + assert_eq!(u8::vmcontref_revision(&4) % 8, 0); + assert_eq!(u8::vmcontref_revision(&8) % 8, 0); + assert_eq!( + offset_of!(VMContRef, revision), + usize::from(offsets.ptr.vmcontref_revision()) + ); + assert_eq!( + offset_of!(VMContRef, stack), + usize::from(offsets.ptr.vmcontref_stack()) + ); + assert_eq!( + offset_of!(VMContRef, args), + usize::from(offsets.ptr.vmcontref_args()) + ); + assert_eq!( + offset_of!(VMContRef, values), + usize::from(offsets.ptr.vmcontref_values()) + ); +} + +/// Implements `cont.new` instructions (i.e., creation of continuations). +#[inline(always)] +pub fn cont_new( + store: &mut dyn VMStore, + instance: &mut Instance, + func: *mut u8, + param_count: u32, + result_count: u32, +) -> Result<*mut VMContRef, TrapReason> { + let caller_vmctx = instance.vmctx(); + + let stack_size = store.engine().config().stack_switching_config.stack_size; + + let contref = store.allocate_continuation()?; + let contref = unsafe { contref.as_mut().unwrap() }; + + let tsp = contref.stack.top().unwrap(); + contref.parent_chain = VMStackChain::Absent; + // The continuation is fresh, which is a special case of being suspended. + // Thus we need to set the correct end of the continuation chain: itself. + contref.last_ancestor = contref; + + // The initialization function will allocate the actual args/return value buffer and + // update this object (if needed). + let contref_args_ptr = &mut contref.args as *mut _ as *mut VMArray; + + contref.stack.initialize( + func.cast::(), + caller_vmctx.as_ptr(), + contref_args_ptr, + param_count, + result_count, + ); + + // Now that the initial stack pointer was set by the initialization + // function, use it to determine stack limit. + let stack_pointer = contref.stack.control_context_stack_pointer(); + // Same caveat regarding stack_limit here as descibed in + // `wasmtime::runtime::func::EntryStoreContext::enter_wasm`. + let wasm_stack_limit = core::cmp::max( + stack_pointer - store.engine().config().max_wasm_stack, + tsp as usize - stack_size, + ); + let limits = VMStackLimits::with_stack_limit(wasm_stack_limit); + let csi = &mut contref.common_stack_information; + csi.state = VMStackState::Fresh; + csi.limits = limits; + + debug_println!("Created contref @ {:p}", contref); + Ok(contref) +} + +/// This type represents a linked lists ("chain") of stacks, where the a +/// node's successor denotes its parent. +/// A additionally, a `CommonStackInformation` object is associated with +/// each stack in the list. +/// Here, a "stack" is one of the following: +/// - A continuation (i.e., created with cont.new). +/// - The initial stack. This is the stack that we were on when entering +/// Wasm (i.e., when executing +/// `crate::runtime::func::invoke_wasm_and_catch_traps`). +/// This stack never has a parent. +/// In terms of the memory allocation that this stack resides on, it will +/// usually be the main stack, but doesn't have to: If we are running +/// inside a continuation while executing a host call, which in turn +/// re-renters Wasm, the initial stack is actually the stack of that +/// continuation. +/// +/// Note that the linked list character of `VMStackChain` arises from the fact +/// that `VMStackChain::Continuation` variants have a pointer to a +/// `VMContRef`, which in turn has a `parent_chain` value of type +/// `VMStackChain`. This is how the stack chain reflects the parent-child +/// relationships between continuations/stacks. This also shows how the +/// initial stack (mentioned above) cannot have a parent. +/// +/// There are generally two uses of `VMStackChain`: +/// +/// 1. The `stack_chain` field in the `StoreOpaque` contains such a +/// chain of stacks, where the head of the list denotes the stack that is +/// currently executing (either a continuation or the initial stack). Note +/// that in this case, the linked list must contain 0 or more `Continuation` +/// elements, followed by a final `InitialStack` element. In particular, +/// this list always ends with `InitialStack` and never contains an `Absent` +/// variant. +/// +/// 2. When a continuation is suspended, its chain of parents eventually +/// ends with an `Absent` variant in its `parent_chain` field. Note that a +/// suspended continuation never appears in the stack chain in the +/// VMContext! +/// +/// +/// As mentioned before, each stack in a `VMStackChain` has a corresponding +/// `CommonStackInformation` object. For continuations, this is stored in +/// the `common_stack_information` field of the corresponding `VMContRef`. +/// For the initial stack, the `InitialStack` variant contains a pointer to +/// a `CommonStackInformation`. The latter will be allocated allocated on +/// the stack frame that executed by `invoke_wasm_and_catch_traps`. +/// +/// The following invariants hold for these `VMStackLimits` objects, +/// and the data in `VMRuntimeLimits`. +/// +/// Currently executing stack: For the currently executing stack (i.e., the +/// stack that is at the head of the store's `stack_chain` list), the +/// associated `VMStackLimits` object contains stale/undefined data. Instead, +/// the live data describing the limits for the currently executing stack is +/// always maintained in `VMRuntimeLimits`. Note that as a general rule +/// independently from any execution of continuations, the `last_wasm_exit*` +/// fields in the `VMRuntimeLimits` contain undefined values while executing +/// wasm. +/// +/// Parents of currently executing stack: For stacks that appear in the tail +/// of the store's `stack_chain` list (i.e., stacks that are not currently +/// executing themselves, but are an ancestor of the currently executing +/// stack), we have the following: All the fields in the stack's +/// `VMStackLimits` are valid, describing the stack's stack limit, and +/// pointers where executing for that stack entered and exited WASM. +/// +/// Suspended continuations: For suspended continuations (including their +/// ancestors), we have the following. Note that the initial stack can never +/// be in this state. The `stack_limit` and `last_enter_wasm_sp` fields of +/// the corresponding `VMStackLimits` object contain valid data, while the +/// `last_exit_wasm_*` fields contain arbitrary values. There is only one +/// exception to this: Note that a continuation that has been created with +/// cont.new, but never been resumed so far, is considered "suspended". +/// However, its `last_enter_wasm_sp` field contains undefined data. This is +/// justified, because when resume-ing a continuation for the first time, a +/// native-to-wasm trampoline is called, which sets up the +/// `last_wasm_entry_sp` in the `VMRuntimeLimits` with the correct value, +/// thus restoring the necessary invariant. +#[derive(Debug, Clone, PartialEq)] +#[repr(usize, C)] +pub enum VMStackChain { + /// For suspended continuations, denotes the end of their chain of + /// ancestors. + Absent = wasmtime_environ::stack_switching::STACK_CHAIN_ABSENT_DISCRIMINANT, + /// Represents the initial stack (i.e., where we entered Wasm from the + /// host by executing + /// `crate::runtime::func::invoke_wasm_and_catch_traps`). Therefore, it + /// does not have a parent. The `CommonStackInformation` that this + /// variant points to is stored in the stack frame of + /// `invoke_wasm_and_catch_traps`. + InitialStack(*mut VMCommonStackInformation) = + wasmtime_environ::stack_switching::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT, + /// Represents a continuation's stack. + Continuation(*mut VMContRef) = + wasmtime_environ::stack_switching::STACK_CHAIN_CONTINUATION_DISCRIMINANT, +} + +impl VMStackChain { + /// Indicates if `self` is a `InitialStack` variant. + pub fn is_initial_stack(&self) -> bool { + matches!(self, VMStackChain::InitialStack(_)) + } + + /// Returns an iterator over the continuations in this chain. + /// We don't implement `IntoIterator` because our iterator is unsafe, so at + /// least this gives us some way of indicating this, even though the actual + /// unsafety lies in the `next` function. + /// + /// # Safety + /// + /// This function is not unsafe per see, but it returns an object + /// whose usage is unsafe. + pub unsafe fn into_continuation_iter(self) -> ContinuationIterator { + ContinuationIterator(self) + } + + /// Returns an iterator over the stack limits in this chain. + /// We don't implement `IntoIterator` because our iterator is unsafe, so at + /// least this gives us some way of indicating this, even though the actual + /// unsafety lies in the `next` function. + /// + /// # Safety + /// + /// This function is not unsafe per see, but it returns an object + /// whose usage is unsafe. + pub unsafe fn into_stack_limits_iter(self) -> StackLimitsIterator { + StackLimitsIterator(self) + } +} + +#[test] +fn check_vm_stack_chain_offsets() { + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmstack_chain()) + ); +} + +/// Iterator for Continuations in a stack chain. +pub struct ContinuationIterator(VMStackChain); + +/// Iterator for VMStackLimits in a stack chain. +pub struct StackLimitsIterator(VMStackChain); + +impl Iterator for ContinuationIterator { + type Item = *mut VMContRef; + + fn next(&mut self) -> Option { + match self.0 { + VMStackChain::Absent | VMStackChain::InitialStack(_) => None, + VMStackChain::Continuation(ptr) => { + let continuation = unsafe { ptr.as_mut().unwrap() }; + self.0 = continuation.parent_chain.clone(); + Some(ptr) + } + } + } +} + +impl Iterator for StackLimitsIterator { + type Item = *mut VMStackLimits; + + fn next(&mut self) -> Option { + match self.0 { + VMStackChain::Absent => None, + VMStackChain::InitialStack(csi) => { + let stack_limits = unsafe { &mut (*csi).limits } as *mut VMStackLimits; + self.0 = VMStackChain::Absent; + Some(stack_limits) + } + VMStackChain::Continuation(ptr) => { + let continuation = unsafe { ptr.as_mut().unwrap() }; + let stack_limits = + (&mut continuation.common_stack_information.limits) as *mut VMStackLimits; + self.0 = continuation.parent_chain.clone(); + Some(stack_limits) + } + } + } +} + +/// Encodes the life cycle of a `VMContRef`. +#[derive(Debug, Clone, Copy, PartialEq)] +#[repr(u32)] +pub enum VMStackState { + /// The `VMContRef` has been created, but neither `resume` or `switch` has ever been + /// called on it. During this stage, we may add arguments using `cont.bind`. + Fresh = wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT, + /// The continuation is running, meaning that it is the one currently + /// executing code. + Running = wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT, + /// The continuation is suspended because it executed a resume instruction + /// that has not finished yet. In other words, it became the parent of + /// another continuation (which may itself be `Running`, a `Parent`, or + /// `Suspended`). + Parent = wasmtime_environ::stack_switching::STACK_STATE_PARENT_DISCRIMINANT, + /// The continuation was suspended by a `suspend` or `switch` instruction. + Suspended = wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT, + /// The function originally passed to `cont.new` has returned normally. + /// Note that there is no guarantee that a VMContRef will ever + /// reach this status, as it may stay suspended until being dropped. + Returned = wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT, +} + +/// Universal control effect. This structure encodes return signal, resume +/// signal, suspension signal, and the handler to suspend to in a single variant +/// type. This instance is used at runtime. There is a codegen counterpart in +/// `cranelift/src/stack-switching/control_effect.rs`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +#[allow(dead_code)] +pub enum ControlEffect { + /// Used to signal that a continuation has returned and control switches + /// back to the parent. + Return = wasmtime_environ::stack_switching::CONTROL_EFFECT_RETURN_DISCRIMINANT, + /// Used to signal to a continuation that it is being resumed. + Resume = wasmtime_environ::stack_switching::CONTROL_EFFECT_RESUME_DISCRIMINANT, + /// Used to signal that a continuation has invoked a `suspend` instruction. + Suspend { + /// The index of the handler to be used in the parent continuation to + /// switch back to. + handler_index: u32, + } = wasmtime_environ::stack_switching::CONTROL_EFFECT_SUSPEND_DISCRIMINANT, + /// Used to signal that a continuation has invoked a `suspend` instruction. + Switch = wasmtime_environ::stack_switching::CONTROL_EFFECT_SWITCH_DISCRIMINANT, +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs new file mode 100644 index 000000000000..12637585df42 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs @@ -0,0 +1,119 @@ +//! This module contains a modified version of the `wasmtime_fiber` crate, +//! specialized for executing stack switching continuations. + +#![allow(missing_docs)] + +use anyhow::Result; +use core::ops::Range; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; + +cfg_if::cfg_if! { + if #[cfg(all(feature = "stack-switching",unix, target_arch = "x86_64"))] { + pub mod unix; + use unix as imp; + } else { + pub mod dummy; + use dummy as imp; + } +} + +/// Represents an execution stack to use for a fiber. +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack(imp::VMContinuationStack); + +impl VMContinuationStack { + /// Creates a new fiber stack of the given size. + pub fn new(size: usize) -> Result { + Ok(Self(imp::VMContinuationStack::new(size)?)) + } + + /// Returns a stack of size 0. + pub fn unallocated() -> Self { + Self(imp::VMContinuationStack::unallocated()) + } + + /// Is this stack unallocated/of size 0? + pub fn is_unallocated(&self) -> bool { + imp::VMContinuationStack::is_unallocated(&self.0) + } + + /// Creates a new fiber stack with the given pointer to the bottom of the + /// stack plus the byte length of the stack. + /// + /// The `bottom` pointer should be addressable for `len` bytes. The page + /// beneath `bottom` should be unmapped as a guard page. + /// + /// # Safety + /// + /// This is unsafe because there is no validation of the given pointer. + /// + /// The caller must properly allocate the stack space with a guard page and + /// make the pages accessible for correct behavior. + pub unsafe fn from_raw_parts(bottom: *mut u8, guard_size: usize, len: usize) -> Result { + Ok(Self(imp::VMContinuationStack::from_raw_parts( + bottom, guard_size, len, + )?)) + } + + /// Is this a manually-managed stack created from raw parts? If so, it is up + /// to whoever created it to manage the stack's memory allocation. + pub fn is_from_raw_parts(&self) -> bool { + self.0.is_from_raw_parts() + } + + /// Gets the top of the stack. + /// + /// Returns `None` if the platform does not support getting the top of the + /// stack. + pub fn top(&self) -> Option<*mut u8> { + self.0.top() + } + + /// Returns the range of where this stack resides in memory if the platform + /// supports it. + pub fn range(&self) -> Option> { + self.0.range() + } + + /// Returns the instruction pointer stored in the Fiber's ControlContext. + pub fn control_context_instruction_pointer(&self) -> usize { + self.0.control_context_instruction_pointer() + } + + /// Returns the frame pointer stored in the Fiber's ControlContext. + pub fn control_context_frame_pointer(&self) -> usize { + self.0.control_context_frame_pointer() + } + + /// Returns the stack pointer stored in the Fiber's ControlContext. + pub fn control_context_stack_pointer(&self) -> usize { + self.0.control_context_stack_pointer() + } + + /// Initializes this stack, such that it will execute the function denoted + /// by `func_ref`. `parameter_count` and `return_value_count` must be the + /// corresponding number of parameters and return values of `func_ref`. + /// `args` must point to the `args` field of the `VMContRef` owning this pointer. + /// + /// It will be updated by this function to correctly describe + /// the buffer used by this function for its arguments and return values. + pub fn initialize( + &self, + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + parameter_count: u32, + return_value_count: u32, + ) { + self.0.initialize( + func_ref, + caller_vmctx, + args, + parameter_count, + return_value_count, + ) + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs new file mode 100644 index 000000000000..e2042e352c19 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs @@ -0,0 +1,75 @@ +use anyhow::Result; +use core::ops::Range; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; + +#[allow(dead_code)] +#[derive(Debug, PartialEq, Eq)] +pub enum Allocator { + Mmap, + Custom, +} + +/// Making sure that this has the same size as the non-dummy version, to +/// make some tests happy. +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack { + _top: *mut u8, + _len: usize, + _allocator: Allocator, +} + +impl VMContinuationStack { + pub fn new(_size: usize) -> Result { + anyhow::bail!("Stack switching disabled or not implemented on this platform") + } + + pub fn unallocated() -> Self { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn is_unallocated(&self) -> bool { + panic!("Stack switching disabled or not implemented on this platform") + } + + #[allow(clippy::missing_safety_doc)] + pub unsafe fn from_raw_parts(_base: *mut u8, _guard_size: usize, _len: usize) -> Result { + anyhow::bail!("Stack switching disabled or not implemented on this platform") + } + + pub fn is_from_raw_parts(&self) -> bool { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn top(&self) -> Option<*mut u8> { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn range(&self) -> Option> { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_instruction_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_frame_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_stack_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn initialize( + &self, + _func_ref: *const VMFuncRef, + _caller_vmctx: *mut VMContext, + _args: *mut VMArray, + _parameter_count: u32, + _return_value_count: u32, + ) { + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs new file mode 100644 index 000000000000..e50e1f2c7fbe --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -0,0 +1,354 @@ +//! The stack layout is expected to look like so: +//! +//! +//! ```text +//! 0xB000 +-----------------------+ <- top of stack (TOS) +//! | saved RIP | +//! 0xAff8 +-----------------------+ +//! | saved RBP | +//! 0xAff0 +-----------------------+ +//! | saved RSP | +//! 0xAfe8 +-----------------------+ <- beginning of "control context", +//! | args_capacity | +//! 0xAfe0 +-----------------------+ +//! | args buffer, size: | +//! | (16 * args_capacity) | +//! 0xAfc0 +-----------------------+ <- below: beginning of usable stack space +//! | | (16-byte aligned) +//! | | +//! ~ ... ~ <- actual native stack space to use +//! | | +//! 0x1000 +-----------------------+ +//! | guard page | <- (not currently enabled) +//! 0x0000 +-----------------------+ +//! ``` +//! +//! The "control context" indicates how to resume a computation. The layout is +//! determined by Cranelift's stack_switch instruction, which reads and writes +//! these fields. The fields are used as follows, where we distinguish two +//! cases: +//! +//! 1. +//! If the continuation is currently active (i.e., running directly, or ancestor +//! of the running continuation), it stores the PC, RSP, and RBP of the *parent* +//! of the running continuation. +//! +//! 2. +//! If the picture shows a suspended computation, the fields store the PC, RSP, +//! and RBP at the time of the suspension. +//! +//! Note that this design ensures that external tools can construct backtraces +//! in the presence of stack switching by using frame pointers only: The +//! wasmtime_continuation_start trampoline uses the address of the RBP field in the +//! control context (0xAff0 above) as its frame pointer. This means that when +//! passing the wasmtime_continuation_start frame while doing frame pointer walking, +//! the parent of that frame is the last frame in the parent of this +//! continuation. +//! +//! Wasmtime's own mechanism for constructing backtraces also relies on frame +//! pointer chains. However, it understands continuations and does not rely on +//! the trickery outlined here to go from the frames in one continuation to the +//! parent. +//! +//! The args buffer is used as follows: It is used by the array calling +//! trampoline to read and store the arguments and return values of the function +//! running inside the continuation. If this function has m parameters and n +//! return values, then args_capacity is defined as max(m, n) and the size of +//! the args buffer is args_capacity * 16 bytes. The start address (0xAfc0 in +//! the example above, thus assuming args_capacity = 2) is saved as the `data` +//! field of the VMContRef's `args` object. + +#![allow(unused_macros)] + +use core::ptr::NonNull; +use std::io; +use std::ops::Range; +use std::ptr; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, VMOpaqueContext, ValRaw}; + +#[derive(Debug, PartialEq, Eq)] +pub enum Allocator { + Mmap, + Custom, +} + +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack { + // The top of the stack; for stacks allocated by the fiber implementation itself, + // the base address of the allocation will be `top.sub(len.unwrap())` + top: *mut u8, + // The length of the stack + len: usize, + // allocation strategy + allocator: Allocator, +} + +impl VMContinuationStack { + pub fn new(size: usize) -> io::Result { + // Round up our stack size request to the nearest multiple of the + // page size. + let page_size = rustix::param::page_size(); + let size = if size == 0 { + page_size + } else { + (size + (page_size - 1)) & (!(page_size - 1)) + }; + + unsafe { + // Add in one page for a guard page and then ask for some memory. + let mmap_len = size + page_size; + let mmap = rustix::mm::mmap_anonymous( + ptr::null_mut(), + mmap_len, + rustix::mm::ProtFlags::empty(), + rustix::mm::MapFlags::PRIVATE, + )?; + + rustix::mm::mprotect( + mmap.cast::().add(page_size).cast(), + size, + rustix::mm::MprotectFlags::READ | rustix::mm::MprotectFlags::WRITE, + )?; + + Ok(Self { + top: mmap.cast::().add(mmap_len), + len: mmap_len, + allocator: Allocator::Mmap, + }) + } + } + + pub fn unallocated() -> Self { + Self { + top: std::ptr::null_mut(), + len: 0, + allocator: Allocator::Custom, + } + } + + pub fn is_unallocated(&self) -> bool { + debug_assert_eq!(self.len == 0, self.top == std::ptr::null_mut()); + self.len == 0 + } + + #[allow(clippy::missing_safety_doc)] + pub unsafe fn from_raw_parts( + base: *mut u8, + _guard_size: usize, + len: usize, + ) -> io::Result { + Ok(Self { + top: base.add(len), + len, + allocator: Allocator::Custom, + }) + } + + pub fn is_from_raw_parts(&self) -> bool { + self.allocator == Allocator::Custom + } + + pub fn top(&self) -> Option<*mut u8> { + Some(self.top) + } + + pub fn range(&self) -> Option> { + let base = unsafe { self.top.sub(self.len) as usize }; + Some(base..base + self.len) + } + + pub fn control_context_instruction_pointer(&self) -> usize { + // See picture at top of this file: + // RIP is stored 8 bytes below top of stack. + unsafe { + let ptr = self.top.sub(8) as *mut usize; + *ptr + } + } + + pub fn control_context_frame_pointer(&self) -> usize { + // See picture at top of this file: + // RBP is stored 16 bytes below top of stack. + unsafe { + let ptr = self.top.sub(16) as *mut usize; + *ptr + } + } + + pub fn control_context_stack_pointer(&self) -> usize { + // See picture at top of this file: + // RSP is stored 24 bytes below top of stack. + unsafe { + let ptr = self.top.sub(24) as *mut usize; + *ptr + } + } + + /// This function installs the launchpad for the computation to run on the + /// fiber, such that executing a `stack_switch` instruction on the stack + /// actually runs the desired computation. + /// + /// Concretely, switching to the stack prepared by this function + /// causes that we enter `wasmtime_continuation_start`, which then in turn + /// calls `fiber_start` with the following arguments: + /// TOS, func_ref, caller_vmctx, args_ptr, args_capacity + /// + /// Note that at this point we also allocate the args buffer + /// (see picture at the top of this file). + /// We define `args_capacity` as the max of parameter and return value count. + /// Then the size s of the actual buffer size is calculated as follows: + /// s = size_of(ValRaw) * `args_capacity`, + /// + /// Note that this value is used below, and we may have s = 0. + /// + /// The layout of the VMContinuationStack near the top of stack (TOS) + /// *after* running this function is as follows: + /// + /// + /// Offset from | + /// TOS | Contents + /// ---------------|------------------------------------------------------- + /// -0x08 | address of wasmtime_continuation_start function (future PC) + /// -0x10 | TOS - 0x10 (future RBP) + /// -0x18 | TOS - 0x40 - s (future RSP) + /// -0x20 | args_capacity + /// + /// + /// The data stored behind the args buffer is as follows: + /// + /// Offset from | + /// TOS | Contents + /// ---------------|------------------------------------------------------- + /// -0x28 - s | func_ref + /// -0x30 - s | caller_vmctx + /// -0x38 - s | args (of type *mut ArrayRef) + /// -0x40 - s | return_value_count + pub fn initialize( + &self, + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + parameter_count: u32, + return_value_count: u32, + ) { + let tos = self.top; + + unsafe { + let store = |tos_neg_offset, value| { + let target = tos.sub(tos_neg_offset) as *mut usize; + target.write(value) + }; + + let args = &mut *args; + let args_capacity = std::cmp::max(parameter_count, return_value_count); + // The args object must currently be empty. + debug_assert_eq!(args.capacity, 0); + debug_assert_eq!(args.length, 0); + + // The size of the args buffer + let s = args_capacity as usize * std::mem::size_of::(); + + // The actual pointer to the buffer + let args_data_ptr = if args_capacity == 0 { + 0 + } else { + tos as usize - 0x20 - s + }; + + args.capacity = args_capacity; + args.data = args_data_ptr as *mut ValRaw; + + let to_store = [ + // Data near top of stack: + (0x08, wasmtime_continuation_start as usize), + (0x10, tos.sub(0x10) as usize), + (0x18, tos.sub(0x40 + s) as usize), + (0x20, args_capacity as usize), + // Data after the args buffer: + (0x28 + s, func_ref as usize), + (0x30 + s, caller_vmctx as usize), + (0x38 + s, args as *mut VMArray as usize), + (0x40 + s, return_value_count as usize), + ]; + + for (offset, data) in to_store { + store(offset, data); + } + } + } +} + +impl Drop for VMContinuationStack { + fn drop(&mut self) { + unsafe { + match self.allocator { + Allocator::Mmap => { + let ret = rustix::mm::munmap(self.top.sub(self.len) as _, self.len); + debug_assert!(ret.is_ok()); + } + Allocator::Custom => {} // It's the creator's responsibility to reclaim the memory. + } + } + } +} + +unsafe extern "C" { + #[allow(dead_code)] // only used in inline assembly for some platforms + fn wasmtime_continuation_start(); +} + +/// This function is responsible for actually running a wasm function inside a +/// continuation. It is only ever called from `wasmtime_continuation_start`. Hence, it +/// must never return. +unsafe extern "C" fn fiber_start( + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + return_value_count: u32, +) { + unsafe { + let func_ref = func_ref.as_ref().expect("Non-null function reference"); + let caller_vmxtx = VMOpaqueContext::from_vmcontext(NonNull::new_unchecked(caller_vmctx)); + let args = &mut *args; + let params_and_returns: NonNull<[ValRaw]> = if args.capacity == 0 { + NonNull::from(&[]) + } else { + std::slice::from_raw_parts_mut(args.data, args.capacity as usize).into() + }; + + // NOTE(frank-emrich) The usage of the `caller_vmctx` is probably not + // 100% correct here. Currently, we determine the "caller" vmctx when + // initilizing the fiber stack/continuation (i.e. as part of + // `cont.new`). However, we may subsequenly `resume` the continuation + // from a different Wasm instance. The way to fix this would be to make + // the currently active `VMContext` an additional parameter of + // `wasmtime_continuation_switch` and pipe it through to this point. However, + // since the caller vmctx is only really used to access stuff in the + // underlying `Store`, it's fine to be slightly sloppy about the exact + // value we set. + func_ref.array_call(None, caller_vmxtx, params_and_returns); // TODO(dhil): we are ignoring the boolean return value + // here... we probably shouldn't. + + // The array call trampoline should have just written + // `return_value_count` values to the `args` buffer. Let's reflect that + // in its length field, to make various bounds checks happy. + args.length = return_value_count; + + // Note that after this function returns, wasmtime_continuation_start + // will switch back to the parent stack. + } +} + +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + } else { + // Note that this shoul be unreachable: In stack.rs, we currently select + // the module defined in the current file only if we are on unix AND + // x86_64. + compile_error!("the stack switching feature is not supported on this CPU architecture"); + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs new file mode 100644 index 000000000000..61ffd66f68e6 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs @@ -0,0 +1,86 @@ +// A WORD OF CAUTION +// +// This entire file basically needs to be kept in sync with itself. It's not +// really possible to modify just one bit of this file without understanding +// all the other bits. Documentation tries to reference various bits here and +// there but try to make sure to read over everything before tweaking things! + +use wasmtime_asm_macros::asm_func; + +// This is a pretty special function that has no real signature. Its use is to +// be the "base" function of all fibers. This entrypoint is used in +// `wasmtime_continuation_init` to bootstrap the execution of a new fiber. +// +// We also use this function as a persistent frame on the stack to emit dwarf +// information to unwind into the caller. This allows us to unwind from the +// fiber's stack back to the initial stack that the fiber was called from. We use +// special dwarf directives here to do so since this is a pretty nonstandard +// function. +// +// If you're curious a decent introduction to CFI things and unwinding is at +// https://www.imperialviolet.org/2017/01/18/cfi.html +// +// Note that this function is never called directly. It is only ever entered +// when a `stack_switch` instruction loads its address when switching to a stack +// prepared by `FiberStack::initialize`. +// +// Executing `stack_switch` on a stack prepared by `FiberStack::initialize` as +// described in the comment on `FiberStack::initialize` leads to the following +// values in various registers when execution of wasmtime_continuation_start begins: +// +// RSP: TOS - 0x40 - (16 * `args_capacity`) +// RBP: TOS - 0x10 +asm_func!( + "wasmtime_continuation_start", + " + // TODO(frank-emrich): Restore DWARF information for this function. In + // the meantime, debugging is possible using frame pointer walking. + + + // + // Note that the next 4 instructions amount to calling fiber_start + // with the following arguments: + // 1. func_ref + // 2. caller_vmctx + // 3. args (of type *mut ArrayRef) + // 4. return_value_count + // + + pop rcx // return_value_count + pop rdx // args + pop rsi // caller_vmctx + pop rdi // func_ref + // Note that RBP already contains the right frame pointer to build a + // frame pointer chain including the parent continuation: + // The current value of RBP is where we store the parent RBP in the + // control context! + call {fiber_start} + + // Return to the parent continuation. + // RBP is callee-saved (no matter if it's used as a frame pointe or + // not), so its value is still TOS - 0x10. + // Use that fact to obtain saved parent RBP, RSP, and RIP from control + // context near TOS. + mov rsi, 0x08[rbp] // putting new RIP in temp register + mov rsp, -0x08[rbp] + mov rbp, [rbp] + + // The stack_switch instruction uses register RDI for the payload. + // Here, the payload indicates that we are returning (value 0). + // See the test case below to keep this in sync with + // ControlEffect::return_() + mov rdi, 0 + + jmp rsi + ", + fiber_start = sym super::fiber_start, +); + +#[test] +fn test_return_payload() { + // The following assumption is baked into `wasmtime_continuation_start`. + assert_eq!( + wasmtime_environ::stack_switching::CONTROL_EFFECT_RETURN_DISCRIMINANT, + 0 + ); +} diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index dc52dd32fead..c0e1515f008e 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -5,6 +5,7 @@ #![cfg_attr(feature = "gc", allow(irrefutable_let_patterns))] use crate::prelude::*; +use crate::runtime::vm::stack_switching::VMContObj; use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition}; use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VMStore}; use core::alloc::Layout; @@ -34,12 +35,16 @@ pub enum TableElement { /// (which has access to the info needed for lazy initialization) /// will replace it when fetched. UninitFunc, + + /// A `contref` + ContRef(Option), } #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum TableElementType { Func, GcRef, + Cont, } impl TableElementType { @@ -47,9 +52,19 @@ impl TableElementType { match (val, self) { (TableElement::FuncRef(_), TableElementType::Func) => true, (TableElement::GcRef(_), TableElementType::GcRef) => true, + (TableElement::ContRef(_), TableElementType::Cont) => true, _ => false, } } + + /// Returns the size required to actually store an element of this particular type + pub fn element_size(&self) -> usize { + match self { + TableElementType::Func => core::mem::size_of::(), + TableElementType::GcRef => core::mem::size_of::>(), + TableElementType::Cont => core::mem::size_of::(), + } + } } // The usage of `*mut VMFuncRef` is safe w.r.t. thread safety, this just relies @@ -74,6 +89,7 @@ impl TableElement { Self::FuncRef(e) => e, Self::UninitFunc => panic!("Uninitialized table element value outside of table slot"), Self::GcRef(_) => panic!("GC reference is not a function reference"), + Self::ContRef(_) => panic!("Continuation reference is not a function reference"), } } @@ -105,6 +121,18 @@ impl From for TableElement { } } +impl From> for TableElement { + fn from(c: Option) -> TableElement { + TableElement::ContRef(c) + } +} + +impl From for TableElement { + fn from(c: VMContObj) -> TableElement { + TableElement::ContRef(Some(c)) + } +} + #[derive(Copy, Clone)] #[repr(transparent)] struct TaggedFuncRef(*mut VMFuncRef); @@ -140,10 +168,40 @@ impl TaggedFuncRef { } pub type FuncTableElem = Option>; +pub type ContTableElem = Option; + +/// The maximum of the sizes of any of the table element types +#[allow(dead_code, reason = "Only used if pooling allocator is enabled")] +pub const MAX_TABLE_ELEM_SIZE: usize = { + let sizes = [ + core::mem::size_of::(), + core::mem::size_of::(), + core::mem::size_of::>(), + ]; + + // This is equivalent to `|data| {data.iter().reduce(std::cmp::max).unwrap()}`, + // but as a `const` function, so we can use it to define a constant. + const fn slice_max(data: &[usize]) -> usize { + match data { + [] => 0, + [head, tail @ ..] => { + let tail_max = slice_max(tail); + if *head >= tail_max { + *head + } else { + tail_max + } + } + } + } + + slice_max(&sizes) +}; pub enum StaticTable { Func(StaticFuncTable), GcRef(StaticGcRefTable), + Cont(StaticContTable), } impl From for StaticTable { @@ -158,6 +216,12 @@ impl From for StaticTable { } } +impl From for StaticTable { + fn from(value: StaticContTable) -> Self { + Self::Cont(value) + } +} + pub struct StaticFuncTable { /// Where data for this table is stored. The length of this list is the /// maximum size of the table. @@ -176,9 +240,18 @@ pub struct StaticGcRefTable { size: usize, } +pub struct StaticContTable { + /// Where data for this table is stored. The length of this list is the + /// maximum size of the table. + data: SendSyncPtr<[ContTableElem]>, + /// The current size of the table. + size: usize, +} + pub enum DynamicTable { Func(DynamicFuncTable), GcRef(DynamicGcRefTable), + Cont(DynamicContTable), } impl From for DynamicTable { @@ -193,6 +266,12 @@ impl From for DynamicTable { } } +impl From for DynamicTable { + fn from(value: DynamicContTable) -> Self { + Self::Cont(value) + } +} + pub struct DynamicFuncTable { /// Dynamically managed storage space for this table. The length of this /// vector is the current size of the table. @@ -211,6 +290,14 @@ pub struct DynamicGcRefTable { maximum: Option, } +pub struct DynamicContTable { + /// Dynamically managed storage space for this table. The length of this + /// vector is the current size of the table. + elements: Vec, + /// Maximum size that `elements` can grow to. + maximum: Option, +} + /// Represents an instance's table. pub enum Table { /// A "static" table where storage space is managed externally, currently @@ -241,6 +328,13 @@ impl From for Table { } } +impl From for Table { + fn from(value: StaticContTable) -> Self { + let t: StaticTable = value.into(); + t.into() + } +} + impl From for Table { fn from(value: DynamicTable) -> Self { Self::Dynamic(value) @@ -261,11 +355,18 @@ impl From for Table { } } -fn wasm_to_table_type(ty: WasmRefType) -> TableElementType { +impl From for Table { + fn from(value: DynamicContTable) -> Self { + let t: DynamicTable = value.into(); + t.into() + } +} + +pub(crate) fn wasm_to_table_type(ty: WasmRefType) -> TableElementType { match ty.heap_type.top() { WasmHeapTopType::Func => TableElementType::Func, WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => TableElementType::Cont, } } @@ -326,6 +427,10 @@ impl Table { elements: unsafe { alloc_dynamic_table_elements(minimum)? }, maximum, })), + TableElementType::Cont => Ok(Self::from(DynamicContTable { + elements: vec![None; minimum], + maximum: maximum, + })), } } @@ -385,6 +490,26 @@ impl Table { )); Ok(Self::from(StaticGcRefTable { data, size })) } + TableElementType::Cont => { + let len = { + let data = data.as_non_null().as_ref(); + let (before, data, after) = data.align_to::(); + assert!(before.is_empty()); + assert!(after.is_empty()); + data.len() + }; + ensure!( + usize::try_from(ty.limits.min).unwrap() <= len, + "initial table size of {} exceeds the pooling allocator's \ + configured maximum table size of {len} elements", + ty.limits.min, + ); + let data = SendSyncPtr::new(NonNull::slice_from_raw_parts( + data.as_non_null().cast::(), + cmp::min(len, max), + )); + Ok(Self::from(StaticContTable { data, size })) + } } } @@ -441,6 +566,9 @@ impl Table { Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => { TableElementType::GcRef } + Table::Static(StaticTable::Cont(_)) | Table::Dynamic(DynamicTable::Cont(_)) => { + TableElementType::Cont + } } } @@ -455,10 +583,12 @@ impl Table { match self { Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size, Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size, + Table::Static(StaticTable::Cont(StaticContTable { size, .. })) => *size, Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(), Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { elements.len() } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => elements.len(), } } @@ -470,10 +600,12 @@ impl Table { /// when it is being constrained by an instance allocator. pub fn maximum(&self) -> Option { match self { + Table::Static(StaticTable::Cont(StaticContTable { data, .. })) => Some(data.len()), Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()), Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()), Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum, Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum, + Table::Dynamic(DynamicTable::Cont(DynamicContTable { maximum, .. })) => *maximum, } } @@ -572,6 +704,10 @@ impl Table { let (funcrefs, _lazy_init) = self.funcrefs_mut(); funcrefs[start..end].fill(TaggedFuncRef::UNINIT); } + TableElement::ContRef(c) => { + let contrefs = self.contrefs_mut(); + contrefs[start..end].fill(c); + } } Ok(()) @@ -652,6 +788,12 @@ impl Table { } *size = new_size; } + Table::Static(StaticTable::Cont(StaticContTable { data, size })) => { + unsafe { + debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none())); + } + *size = new_size; + } // These calls to `resize` could move the base address of // `elements`. If this table's limits declare it to be fixed-size, @@ -666,6 +808,9 @@ impl Table { Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { elements.resize_with(usize::try_from(new_size).unwrap(), || None); } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => { + elements.resize(usize::try_from(new_size).unwrap(), None); + } } self.fill( @@ -698,6 +843,11 @@ impl Table { let r = r.as_ref().map(|r| gc_store.unwrap().clone_gc_ref(r)); TableElement::GcRef(r) }), + TableElementType::Cont => self + .contrefs() + .get(index) + .copied() + .map(|e| TableElement::ContRef(e)), } } @@ -725,6 +875,9 @@ impl Table { TableElement::GcRef(e) => { *self.gc_refs_mut().get_mut(index).ok_or(())? = e; } + TableElement::ContRef(c) => { + *self.contrefs_mut().get_mut(index).ok_or(())? = c; + } } Ok(()) } @@ -792,6 +945,10 @@ impl Table { current_elements: *size, } } + Table::Static(StaticTable::Cont(StaticContTable { data, size })) => VMTableDefinition { + base: data.cast().into(), + current_elements: *size, + }, Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => { VMTableDefinition { base: NonNull::<[FuncTableElem]>::from(&mut elements[..]) @@ -808,6 +965,14 @@ impl Table { current_elements: elements.len(), } } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => { + VMTableDefinition { + base: NonNull::<[Option]>::from(&mut elements[..]) + .cast() + .into(), + current_elements: elements.len().try_into().unwrap(), + } + } } } @@ -876,6 +1041,32 @@ impl Table { } } + fn contrefs(&self) -> &[Option] { + assert_eq!(self.element_type(), TableElementType::Cont); + match self { + Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe { + slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) + }, + Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe { + slice::from_raw_parts(data.as_ptr().cast(), usize::try_from(*size).unwrap()) + }, + _ => unreachable!(), + } + } + + fn contrefs_mut(&mut self) -> &mut [Option] { + assert_eq!(self.element_type(), TableElementType::Cont); + match self { + Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe { + slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) + }, + Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe { + slice::from_raw_parts_mut(data.as_ptr().cast(), usize::try_from(*size).unwrap()) + }, + _ => unreachable!(), + } + } + /// Get this table's GC references as a slice. /// /// Panics if this is not a table of GC references. @@ -924,6 +1115,11 @@ impl Table { ); } } + TableElementType::Cont => { + // `contref` are `Copy`, so just do a mempcy + dst_table.contrefs_mut()[dst_range] + .copy_from_slice(&src_table.contrefs()[src_range]); + } } } @@ -972,6 +1168,10 @@ impl Table { } } } + TableElementType::Cont => { + // `contref` are `Copy`, so just do a memmove + self.contrefs_mut().copy_within(src_range, dst_range.start); + } } } } diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 62da746f404c..019a56c5dab7 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -20,12 +20,15 @@ use crate::runtime::module::lookup_code; use crate::runtime::store::{ExecutorRef, StoreOpaque}; use crate::runtime::vm::sys::traphandlers; use crate::runtime::vm::{Instance, InterpreterRef, VMContext, VMStoreContext}; -use crate::{StoreContextMut, WasmBacktrace}; +use crate::{EntryStoreContext, StoreContextMut, WasmBacktrace}; use core::cell::Cell; use core::ops::Range; use core::ptr::{self, NonNull}; pub use self::backtrace::Backtrace; +#[cfg(feature = "gc")] +pub use self::backtrace::Frame; + pub use self::coredump::CoreDumpStack; pub use self::tls::tls_eager_initialize; #[cfg(feature = "async")] @@ -356,46 +359,51 @@ impl From for TrapReason { /// longjmp'd over and none of its destructors on the stack may be run. pub unsafe fn catch_traps( store: &mut StoreContextMut<'_, T>, + old_state: &EntryStoreContext, mut closure: F, ) -> Result<(), Box> where F: FnMut(NonNull, Option>) -> bool, { let caller = store.0.default_caller(); - let result = CallThreadState::new(store.0, caller).with(|cx| match store.0.executor() { - // In interpreted mode directly invoke the host closure since we won't - // be using host-based `setjmp`/`longjmp` as that's not going to save - // the context we want. - ExecutorRef::Interpreter(r) => { - cx.jmp_buf - .set(CallThreadState::JMP_BUF_INTERPRETER_SENTINEL); - closure(caller, Some(r)) - } + let result = + CallThreadState::new(store.0, caller, old_state).with(|cx| match store.0.executor() { + // In interpreted mode directly invoke the host closure since we won't + // be using host-based `setjmp`/`longjmp` as that's not going to save + // the context we want. + ExecutorRef::Interpreter(r) => { + cx.jmp_buf + .set(CallThreadState::JMP_BUF_INTERPRETER_SENTINEL); + closure(caller, Some(r)) + } - // In native mode, however, defer to C to do the `setjmp` since Rust - // doesn't understand `setjmp`. - // - // Note that here we pass a function pointer to C to catch longjmp - // within, here it's `call_closure`, and that passes `None` for the - // interpreter since this branch is only ever taken if the interpreter - // isn't present. - #[cfg(has_host_compiler_backend)] - ExecutorRef::Native => traphandlers::wasmtime_setjmp( - cx.jmp_buf.as_ptr(), - { - extern "C" fn call_closure(payload: *mut u8, caller: NonNull) -> bool - where - F: FnMut(NonNull, Option>) -> bool, + // In native mode, however, defer to C to do the `setjmp` since Rust + // doesn't understand `setjmp`. + // + // Note that here we pass a function pointer to C to catch longjmp + // within, here it's `call_closure`, and that passes `None` for the + // interpreter since this branch is only ever taken if the interpreter + // isn't present. + #[cfg(has_host_compiler_backend)] + ExecutorRef::Native => traphandlers::wasmtime_setjmp( + cx.jmp_buf.as_ptr(), { - unsafe { (*(payload as *mut F))(caller, None) } - } - - call_closure:: - }, - &mut closure as *mut F as *mut u8, - caller, - ), - }); + extern "C" fn call_closure( + payload: *mut u8, + caller: NonNull, + ) -> bool + where + F: FnMut(NonNull, Option>) -> bool, + { + unsafe { (*(payload as *mut F))(caller, None) } + } + + call_closure:: + }, + &mut closure as *mut F as *mut u8, + caller, + ), + }); return match result { Ok(x) => Ok(x), @@ -413,7 +421,9 @@ where // usage of its accessor methods. mod call_thread_state { use super::*; + use crate::runtime::vm::stack_switching::VMStackChain; use crate::runtime::vm::Unwind; + use crate::EntryStoreContext; /// Temporary state stored on the stack which is registered in the `tls` module /// below for calls into wasm. @@ -433,17 +443,16 @@ mod call_thread_state { #[cfg(all(has_native_signals, unix))] pub(crate) async_guard_range: Range<*mut u8>, - // The values of `VMStoreContext::last_wasm_{exit_{pc,fp},entry_sp}` for - // the *previous* `CallThreadState` for this same store/limits. Our - // *current* last wasm PC/FP/SP are saved in `self.vm_store_context`. We - // save a copy of the old registers here because the `VMStoreContext` - // typically doesn't change across nested calls into Wasm (i.e. they are - // typically calls back into the same store and `self.vm_store_context - // == self.prev.vm_store_context`) and we must to maintain the list of - // contiguous-Wasm-frames stack regions for backtracing purposes. - old_last_wasm_exit_fp: Cell, - old_last_wasm_exit_pc: Cell, - old_last_wasm_entry_fp: Cell, + // The state of the runtime for the *previous* `CallThreadState` for + // this same store. Our *current* state is saved in `self.limits`, + // `self.stack_chain`, etc. We need access to the old values of these + // fields because the `VMStoreContext` typically doesn't change across + // nested calls into Wasm (i.e. they are typically calls back into the + // same store and `self.vm_store_context == self.prev.vm_store_context`) and we must to + // maintain the list of contiguous-Wasm-frames stack regions for + // backtracing purposes. + // FIXME(frank-emrich) Does this need to be an (Unsafe)Cell? + old_state: *const EntryStoreContext, } impl Drop for CallThreadState { @@ -451,13 +460,6 @@ mod call_thread_state { // Unwind information should not be present as it should have // already been processed. debug_assert!(self.unwind.replace(None).is_none()); - - unsafe { - let cx = self.vm_store_context.as_ref(); - *cx.last_wasm_exit_fp.get() = self.old_last_wasm_exit_fp.get(); - *cx.last_wasm_exit_pc.get() = self.old_last_wasm_exit_pc.get(); - *cx.last_wasm_entry_fp.get() = self.old_last_wasm_entry_fp.get(); - } } } @@ -465,7 +467,11 @@ mod call_thread_state { pub const JMP_BUF_INTERPRETER_SENTINEL: *mut u8 = 1 as *mut u8; #[inline] - pub(super) fn new(store: &mut StoreOpaque, caller: NonNull) -> CallThreadState { + pub(super) fn new( + store: &mut StoreOpaque, + caller: NonNull, + old_state: *const EntryStoreContext, + ) -> CallThreadState { let vm_store_context = unsafe { Instance::from_vmctx(caller, |i| i.vm_store_context()) .read() @@ -490,31 +496,28 @@ mod call_thread_state { #[cfg(all(has_native_signals, unix))] async_guard_range: store.async_guard_range(), prev: Cell::new(ptr::null()), - old_last_wasm_exit_fp: Cell::new(unsafe { - *vm_store_context.as_ref().last_wasm_exit_fp.get() - }), - old_last_wasm_exit_pc: Cell::new(unsafe { - *vm_store_context.as_ref().last_wasm_exit_pc.get() - }), - old_last_wasm_entry_fp: Cell::new(unsafe { - *vm_store_context.as_ref().last_wasm_entry_fp.get() - }), + old_state, } } /// Get the saved FP upon exit from Wasm for the previous `CallThreadState`. - pub fn old_last_wasm_exit_fp(&self) -> usize { - self.old_last_wasm_exit_fp.get() + pub unsafe fn old_last_wasm_exit_fp(&self) -> usize { + (&*self.old_state).last_wasm_exit_fp } /// Get the saved PC upon exit from Wasm for the previous `CallThreadState`. - pub fn old_last_wasm_exit_pc(&self) -> usize { - self.old_last_wasm_exit_pc.get() + pub unsafe fn old_last_wasm_exit_pc(&self) -> usize { + (&*self.old_state).last_wasm_exit_pc } /// Get the saved FP upon entry into Wasm for the previous `CallThreadState`. - pub fn old_last_wasm_entry_fp(&self) -> usize { - self.old_last_wasm_entry_fp.get() + pub unsafe fn old_last_wasm_entry_fp(&self) -> usize { + (&*self.old_state).last_wasm_entry_fp + } + + /// Get the saved `VMStackChain` for the previous `CallThreadState`. + pub unsafe fn old_stack_chain(&self) -> VMStackChain { + (&*self.old_state).stack_chain.clone() } /// Get the previous `CallThreadState`. diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index 77fbfe58b534..c147097eff3b 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -23,10 +23,12 @@ use crate::prelude::*; use crate::runtime::store::StoreOpaque; +use crate::runtime::vm::stack_switching::VMStackChain; use crate::runtime::vm::{ traphandlers::{tls, CallThreadState}, Unwind, VMStoreContext, }; +use crate::vm::stack_switching::{VMContRef, VMStackState}; use core::ops::ControlFlow; /// A WebAssembly stack trace. @@ -107,6 +109,38 @@ impl Backtrace { }); } + // Walk the stack of the given continuation, which must be suspended, and + // all of its parent continuations (if any). + #[allow(dead_code, reason = "Only used by GC code at the moment")] + pub fn trace_suspended_continuation( + store: &StoreOpaque, + continuation: &VMContRef, + f: impl FnMut(Frame) -> ControlFlow<()>, + ) { + assert_eq!( + continuation.common_stack_information.state, + VMStackState::Suspended + ); + + let unwind = store.unwinder(); + + let pc = continuation.stack.control_context_instruction_pointer(); + let fp = continuation.stack.control_context_frame_pointer(); + let trampoline_fp = continuation + .common_stack_information + .limits + .last_wasm_entry_fp; + + unsafe { + // FIXME(frank-emrich) Casting from *const to *mut pointer is + // terrible, but we won't actually modify any of the continuations + // here. + let stack_chain = + VMStackChain::Continuation(continuation as *const VMContRef as *mut VMContRef); + Self::trace_through_continuations(unwind, stack_chain, pc, fp, trampoline_fp, f); + } + } + /// Walk the current Wasm stack, calling `f` for each frame we walk. /// /// If Wasm hit a trap, and we calling this from the trap handler, then the @@ -141,7 +175,17 @@ impl Backtrace { } }; + let stack_chain = (*(*vm_store_context).stack_chain.get()).clone(); + + // The first value in `activations` is for the most recently running + // wasm. We thus provide the stack chain of `first_wasm_state` to + // traverse the potential continuation stacks. For the subsequent + // activations, we unconditionally use `None` as the corresponding stack + // chain. This is justified because only the most recent execution of + // wasm may execute off the initial stack (see comments in + // `wasmtime::invoke_wasm_and_catch_traps` for details). let activations = core::iter::once(( + stack_chain, last_wasm_exit_pc, last_wasm_exit_fp, *(*vm_store_context).last_wasm_entry_fp.get(), @@ -149,25 +193,31 @@ impl Backtrace { .chain( state .iter() + .flat_map(|state| state.iter()) .filter(|state| core::ptr::eq(vm_store_context, state.vm_store_context.as_ptr())) .map(|state| { ( + state.old_stack_chain(), state.old_last_wasm_exit_pc(), state.old_last_wasm_exit_fp(), state.old_last_wasm_entry_fp(), ) }), ) - .take_while(|&(pc, fp, sp)| { - if pc == 0 { - debug_assert_eq!(fp, 0); - debug_assert_eq!(sp, 0); + .take_while(|(chain, pc, fp, sp)| { + if *pc == 0 { + debug_assert_eq!(*fp, 0); + debug_assert_eq!(*sp, 0); + } else { + debug_assert_ne!(chain.clone(), VMStackChain::Absent) } - pc != 0 + *pc != 0 }); - for (pc, fp, sp) in activations { - if let ControlFlow::Break(()) = Self::trace_through_wasm(unwind, pc, fp, sp, &mut f) { + for (chain, pc, fp, sp) in activations { + if let ControlFlow::Break(()) = + Self::trace_through_continuations(unwind, chain, pc, fp, sp, &mut f) + { log::trace!("====== Done Capturing Backtrace (closure break) ======"); return; } @@ -176,6 +226,100 @@ impl Backtrace { log::trace!("====== Done Capturing Backtrace (reached end of activations) ======"); } + /// Traces through a sequence of stacks, creating a backtrace for each one, + /// beginning at the given `pc` and `fp`. + /// + /// If `chain` is `InitialStack`, we are tracing through the initial stack, + /// and this function behaves like `trace_through_wasm`. + /// Otherwise, we can interpret `chain` as a linked list of stacks, which + /// ends with the initial stack. We then trace through each of these stacks + /// individually, up to (and including) the initial stack. + unsafe fn trace_through_continuations( + unwind: &dyn Unwind, + chain: VMStackChain, + pc: usize, + fp: usize, + trampoline_fp: usize, + mut f: impl FnMut(Frame) -> ControlFlow<()>, + ) -> ControlFlow<()> { + use crate::runtime::vm::stack_switching::{VMContRef, VMStackLimits}; + + // Handle the stack that is currently running (which may be a + // continuation or the initial stack). + Self::trace_through_wasm(unwind, pc, fp, trampoline_fp, &mut f)?; + + // Note that the rest of this function has no effect if `chain` is + // `Some(VMStackChain::InitialStack(_))` (i.e., there is only one stack to + // trace through: the initial stack) + + assert_ne!(chain, VMStackChain::Absent); + let stack_limits_vec: Vec<*mut VMStackLimits> = + chain.clone().into_stack_limits_iter().collect(); + let continuations_vec: Vec<*mut VMContRef> = + chain.clone().into_continuation_iter().collect(); + + // The VMStackLimits of the currently running stack (whether that's a + // continuation or the initial stack) contains undefined data, the + // information about that stack is saved in the Store's + // `VMRuntimeLimits` and handled at the top of this function + // already. That's why we ignore `stack_limits_vec[0]`. + // + // Note that a continuation stack's control context stores + // information about how to resume execution *in its parent*. Thus, + // we combine the information from continuations_vec[i] with + // stack_limits_vec[i + 1] below to get information about a + // particular stack. + // + // There must be exactly one more `VMStackLimits` object than there + // are continuations, due to the initial stack having one, too. + assert_eq!(stack_limits_vec.len(), continuations_vec.len() + 1); + + for i in 0..continuations_vec.len() { + let (continuation, parent_continuation, parent_limits) = unsafe { + // The continuation whose control context we want to + // access, to get information about how to continue + // execution in its parent. + let continuation = &*continuations_vec[i]; + + // The stack limits describing the parent of `continuation`. + let parent_limits = &*stack_limits_vec[i + 1]; + + // The parent of `continuation`, if the parent is itself a + // continuation. Otherwise, if `continuation` is the last + // continuation (i.e., its parent is the initial stack), this is + // None. + let parent_continuation = if i + 1 < continuations_vec.len() { + Some(&*continuations_vec[i + 1]) + } else { + None + }; + (continuation, parent_continuation, parent_limits) + }; + let fiber_stack = continuation.fiber_stack(); + let resume_pc = fiber_stack.control_context_instruction_pointer(); + let resume_fp = fiber_stack.control_context_frame_pointer(); + + // If the parent is indeed a continuation, we know the + // boundaries of its stack and can perform some extra debugging + // checks. + let parent_stack_range = parent_continuation.and_then(|p| p.fiber_stack().range()); + parent_stack_range.inspect(|parent_stack_range| { + debug_assert!(parent_stack_range.contains(&resume_fp)); + debug_assert!(parent_stack_range.contains(&parent_limits.last_wasm_entry_fp)); + debug_assert!(parent_stack_range.contains(&parent_limits.stack_limit)); + }); + + Self::trace_through_wasm( + unwind, + resume_pc, + resume_fp, + parent_limits.last_wasm_entry_fp, + &mut f, + )? + } + ControlFlow::Continue(()) + } + /// Walk through a contiguous sequence of Wasm frames starting with the /// frame at the given PC and FP and ending at `trampoline_sp`. unsafe fn trace_through_wasm( diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 9af40e983f5f..83f27fd9142b 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -7,6 +7,7 @@ pub use self::vm_host_func_context::VMArrayCallHostFuncContext; use crate::prelude::*; use crate::runtime::vm::{GcStore, InterpreterRef, VMGcRef, VmPtr, VmSafe}; use crate::store::StoreOpaque; +use crate::vm::stack_switching::VMStackChain; use core::cell::UnsafeCell; use core::ffi::c_void; use core::fmt; @@ -500,7 +501,7 @@ impl VMGlobalDefinition { global.init_gc_ref(store.gc_store_mut()?, r.as_ref()) } WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(), - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => *global.as_func_ref_mut() = raw.get_funcref().cast(), // TODO(#10248): temporary hack. }, } Ok(global) @@ -1051,6 +1052,10 @@ pub struct VMStoreContext { /// Used to find the end of a contiguous sequence of Wasm frames when /// walking the stack. pub last_wasm_entry_fp: UnsafeCell, + + /// Stack information used by stack switching instructions. See documentation + /// on `VMStackChain` for details. + pub stack_chain: UnsafeCell, } // The `VMStoreContext` type is a pod-type with no destructor, and we don't @@ -1072,6 +1077,7 @@ impl Default for VMStoreContext { last_wasm_exit_fp: UnsafeCell::new(0), last_wasm_exit_pc: UnsafeCell::new(0), last_wasm_entry_fp: UnsafeCell::new(0), + stack_chain: UnsafeCell::new(VMStackChain::Absent), } } } diff --git a/crates/wast-util/src/lib.rs b/crates/wast-util/src/lib.rs index 7fac6a30cb9a..70d5aa7310d1 100644 --- a/crates/wast-util/src/lib.rs +++ b/crates/wast-util/src/lib.rs @@ -187,6 +187,7 @@ macro_rules! foreach_config_option { component_model_async simd gc_types + stack_switching } }; } @@ -291,6 +292,7 @@ impl Compiler { || config.gc() || config.relaxed_simd() || config.gc_types() + || config.stack_switching() { return true; } @@ -303,6 +305,11 @@ impl Compiler { if config.threads() { return true; } + + // Stack switching is not supported by Pulley. + if config.stack_switching() { + return true; + } } } diff --git a/tests/all/main.rs b/tests/all/main.rs index 1922c1393a8d..d786421de239 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -42,6 +42,7 @@ mod stack_overflow; mod store; mod structs; mod table; +#[cfg(all(feature = "stack-switching", unix, target_arch = "x86_64"))] mod tags; mod threads; mod traps; From 175d1898c7966b7f9832cdbc2cd2c824389b4d44 Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Wed, 12 Mar 2025 20:20:34 +0000 Subject: [PATCH 02/38] prtest:full From f8162ae5a6654109512ee38f564fc5e4a0d6aeb5 Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Mon, 21 Apr 2025 22:20:41 +0100 Subject: [PATCH 03/38] make sure to use ControlFlow result in trace_suspended_continuation --- .../src/runtime/vm/traphandlers/backtrace.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index c147097eff3b..23068ca19f7b 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -117,6 +117,8 @@ impl Backtrace { continuation: &VMContRef, f: impl FnMut(Frame) -> ControlFlow<()>, ) { + log::trace!("====== Capturing Backtrace (suspended continuation) ======"); + assert_eq!( continuation.common_stack_information.state, VMStackState::Suspended @@ -137,8 +139,16 @@ impl Backtrace { // here. let stack_chain = VMStackChain::Continuation(continuation as *const VMContRef as *mut VMContRef); - Self::trace_through_continuations(unwind, stack_chain, pc, fp, trampoline_fp, f); + + if let ControlFlow::Break(()) = + Self::trace_through_continuations(unwind, stack_chain, pc, fp, trampoline_fp, f) + { + log::trace!("====== Done Capturing Backtrace (closure break) ======"); + return; + } } + + log::trace!("====== Done Capturing Backtrace (reached end of stack chain) ======"); } /// Walk the current Wasm stack, calling `f` for each frame we walk. From 47da5a4d648a208b74c5fb6bd7ab1fe2b25c8bb4 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 19 May 2025 19:27:18 -0500 Subject: [PATCH 04/38] stack-switching: cleanup: remove stray c-api changes These are remnants of unrelated wasmfx wasmtime experiments, possibly suitable for later submission against upstream. --- crates/c-api/include/wasmtime/config.h | 8 -------- crates/c-api/include/wasmtime/linker.h | 10 ---------- 2 files changed, 18 deletions(-) diff --git a/crates/c-api/include/wasmtime/config.h b/crates/c-api/include/wasmtime/config.h index b96470e4eee7..482f3c1b7ce1 100644 --- a/crates/c-api/include/wasmtime/config.h +++ b/crates/c-api/include/wasmtime/config.h @@ -250,14 +250,6 @@ WASMTIME_CONFIG_PROP(void, wasm_wide_arithmetic, bool) #ifdef WASMTIME_FEATURE_COMPILER -/** - * \brief Configures whether the WebAssembly function references - * proposal is enabled. - * - * This setting is `false` by default. - */ -WASMTIME_CONFIG_PROP(void, wasm_function_references, bool) - /** * \brief Configures whether the WebAssembly stack switching * proposal is enabled. diff --git a/crates/c-api/include/wasmtime/linker.h b/crates/c-api/include/wasmtime/linker.h index c97d824d2efd..0a793d1e6fc9 100644 --- a/crates/c-api/include/wasmtime/linker.h +++ b/crates/c-api/include/wasmtime/linker.h @@ -66,16 +66,6 @@ WASM_API_EXTERN void wasmtime_linker_delete(wasmtime_linker_t *linker); WASM_API_EXTERN void wasmtime_linker_allow_shadowing(wasmtime_linker_t *linker, bool allow_shadowing); -/** - * \brief Configures whether the given Linker will allow unknown exports from - * command modules. - * - * By default this setting is `false`. - */ -WASM_API_EXTERN void -wasmtime_linker_allow_unknown_exports(wasmtime_linker_t *linker, - bool allow_unknown_exports); - /** * \brief Defines a new item in this linker. * From 0c6c58acadba7b3e81ec967c5f129a110c8080ac Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 11:11:29 -0500 Subject: [PATCH 05/38] stack-switching: reuse async_stack_size --- crates/cli-flags/src/lib.rs | 25 ++++++++++++----- crates/environ/src/stack_switching.rs | 21 --------------- crates/wasmtime/src/config.rs | 27 ++++--------------- crates/wasmtime/src/runtime/store.rs | 2 +- .../src/runtime/vm/stack_switching.rs | 2 +- 5 files changed, 25 insertions(+), 52 deletions(-) diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs index 16305436eab9..71f615e38cfd 100644 --- a/crates/cli-flags/src/lib.rs +++ b/crates/cli-flags/src/lib.rs @@ -331,8 +331,6 @@ wasmtime_option_group! { pub trap_on_grow_failure: Option, /// Maximum execution time of wasm code before timing out (1, 2s, 100ms, etc) pub timeout: Option, - /// Size of stacks created with cont.new instructions - pub stack_switching_stack_size: Option, /// Configures support for all WebAssembly proposals implemented. pub all_proposals: Option, /// Configure support for the bulk memory proposal. @@ -819,10 +817,21 @@ impl CommonOptions { config.native_unwind_info(enable); } - match_feature! { - ["stack-switching" : self.wasm.stack_switching_stack_size] - size => config.stack_switching_stack_size(size), - _ => err, + // async_stack_size enabled by either async or stack-switching, so + // cannot directly use match_feature! + #[cfg(any(feature = "async", feature = "stack-switching"))] + { + if let Some(size) = self.wasm.async_stack_size { + config.async_stack_size(size); + } + } + #[cfg(not(any(feature = "async", feature = "stack-switching")))] + { + if let Some(size) = self.wasm.async_stack_size { + anyhow::bail!(concat!( + "support for async/stack-switching disabled at compile time" + )); + } } match_feature! { @@ -930,6 +939,8 @@ impl CommonOptions { ); } + #[cfg(any(feature = "async", feature = "stack-switching"))] + match_feature! { ["async" : self.wasm.async_stack_size] size => config.async_stack_size(size), @@ -947,7 +958,7 @@ impl CommonOptions { // If `-Wasync-stack-size` isn't passed then automatically adjust it // to the wasm stack size provided here too. That prevents the need // to pass both when one can generally be inferred from the other. - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] if self.wasm.async_stack_size.is_none() { const DEFAULT_HOST_STACK: usize = 512 << 10; config.async_stack_size(max + DEFAULT_HOST_STACK); diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs index 6a781faea905..ee8fa5011130 100644 --- a/crates/environ/src/stack_switching.rs +++ b/crates/environ/src/stack_switching.rs @@ -17,27 +17,6 @@ macro_rules! debug_println { } } -/// Runtime configuration options for stack switching that can be set -/// via the command line. -/// -/// Part of wasmtime::config::Config type (which is not in scope in this crate). -#[derive(Debug, Clone)] -pub struct StackSwitchingConfig { - /// The (fixed) size of a continuation stack. - pub stack_size: usize, -} - -impl Default for StackSwitchingConfig { - fn default() -> Self { - /// Default size for continuation stacks - const DEFAULT_FIBER_SIZE: usize = 2097152; // 2MB = 512 pages of 4k - - Self { - stack_size: DEFAULT_FIBER_SIZE, - } - } -} - /// Discriminant of variant `Absent` in /// `wasmtime::runtime::vm::stack_switching::VMStackChain`. pub const STACK_CHAIN_ABSENT_DISCRIMINANT: usize = 0; diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 7291d484e810..72916f67191c 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -8,9 +8,7 @@ use std::path::Path; use wasmparser::WasmFeatures; #[cfg(feature = "cache")] use wasmtime_cache::CacheConfig; -use wasmtime_environ::{ - stack_switching::StackSwitchingConfig, ConfigTunables, TripleExt, Tunables, -}; +use wasmtime_environ::{ConfigTunables, TripleExt, Tunables}; #[cfg(feature = "runtime")] use crate::memory::MemoryCreator; @@ -129,14 +127,6 @@ pub struct Config { profiling_strategy: ProfilingStrategy, tunables: ConfigTunables, - /// Runtime configuration for the stack switching feature. - /// The structure is defined in the - /// `wasmtime_environ::stack_switching` module, so that we can - /// hand out the configuration object in the interface of - /// `wasmtime_runtime::Store` trait, where the full `Config` type - /// is not in scope. - pub(crate) stack_switching_config: StackSwitchingConfig, - #[cfg(feature = "cache")] pub(crate) cache_config: CacheConfig, #[cfg(feature = "runtime")] @@ -157,7 +147,7 @@ pub struct Config { pub(crate) wasm_backtrace: bool, pub(crate) wasm_backtrace_details_env_used: bool, pub(crate) native_unwind_info: Option, - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] pub(crate) async_stack_size: usize, #[cfg(feature = "async")] pub(crate) async_stack_zeroing: bool, @@ -237,7 +227,6 @@ impl Config { tunables: ConfigTunables::default(), #[cfg(any(feature = "cranelift", feature = "winch"))] compiler_config: CompilerConfig::default(), - stack_switching_config: StackSwitchingConfig::default(), target: None, #[cfg(feature = "gc")] collector: Collector::default(), @@ -263,7 +252,7 @@ impl Config { native_unwind_info: None, enabled_features: WasmFeatures::empty(), disabled_features: WasmFeatures::empty(), - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] async_stack_size: 2 << 20, #[cfg(feature = "async")] async_stack_zeroing: false, @@ -747,7 +736,7 @@ impl Config { /// /// The `Engine::new` method will fail if the value for this option is /// smaller than the [`Config::max_wasm_stack`] option. - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] pub fn async_stack_size(&mut self, size: usize) -> &mut Self { self.async_stack_size = size; self @@ -787,12 +776,6 @@ impl Config { self } - /// Configures the size of the stacks created with cont.new instructions. - pub fn stack_switching_stack_size(&mut self, size: usize) -> &mut Self { - self.stack_switching_config.stack_size = size; - self - } - fn wasm_feature(&mut self, flag: WasmFeatures, enable: bool) -> &mut Self { self.enabled_features.set(flag, enable); self.disabled_features.set(flag, !enable); @@ -2242,7 +2225,7 @@ impl Config { panic!("should have returned an error by now") } - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] if self.async_support && self.max_wasm_stack > self.async_stack_size { bail!("max_wasm_stack size cannot exceed the async_stack_size"); } diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index b80a491575ba..cb0522e2f100 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -2039,7 +2039,7 @@ at https://bytecodealliance.org/security. pub fn allocate_continuation(&mut self) -> Result<*mut VMContRef> { // FIXME(frank-emrich) Do we need to pin this? let mut continuation = Box::new(VMContRef::empty()); - let stack_size = self.engine.config().stack_switching_config.stack_size; + let stack_size = self.engine.config().async_stack_size; let stack = VMContinuationStack::new(stack_size)?; continuation.stack = stack; let ptr = continuation.deref_mut() as *mut VMContRef; diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 59203e817f4a..6e00b330567d 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -432,7 +432,7 @@ pub fn cont_new( ) -> Result<*mut VMContRef, TrapReason> { let caller_vmctx = instance.vmctx(); - let stack_size = store.engine().config().stack_switching_config.stack_size; + let stack_size = store.engine().config().async_stack_size; let contref = store.allocate_continuation()?; let contref = unsafe { contref.as_mut().unwrap() }; From cb0df54e680a25c69359115a5705eca341525197 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 11:16:15 -0500 Subject: [PATCH 06/38] stack-switching: delete delete_me debugging --- crates/cranelift/src/func_environ.rs | 4 ---- crates/cranelift/src/lib.rs | 2 -- crates/environ/src/builtin.rs | 11 ----------- crates/wasmtime/src/runtime/vm/libcalls.rs | 19 ------------------- 4 files changed, 36 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 01c22c3ddc4d..cda65bb6da38 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -3524,10 +3524,6 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type { reason = "Dummy function to supress more dead code warnings" )] pub fn use_stack_switching_libcalls() { - let _ = BuiltinFunctions::delete_me_print_str; - let _ = BuiltinFunctions::delete_me_print_int; - let _ = BuiltinFunctions::delete_me_print_pointer; - let _ = BuiltinFunctions::cont_new; let _ = BuiltinFunctions::table_grow_cont_obj; let _ = BuiltinFunctions::table_fill_cont_obj; diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 60ab8ebe72bd..adf8812a804f 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -65,8 +65,6 @@ pub const TRAP_UNHANDLED_TAG: TrapCode = TrapCode::unwrap_user(Trap::UnhandledTag as u8 + TRAP_OFFSET); pub const TRAP_CONTINUATION_ALREADY_CONSUMED: TrapCode = TrapCode::unwrap_user(Trap::ContinuationAlreadyConsumed as u8 + TRAP_OFFSET); -pub const TRAP_DELETE_ME_DEBUG_ASSERTION: TrapCode = - TrapCode::unwrap_user(Trap::DeleteMeDebugAssertion as u8 + TRAP_OFFSET); pub const TRAP_CAST_FAILURE: TrapCode = TrapCode::unwrap_user(Trap::CastFailure as u8 + TRAP_OFFSET); diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index 6eaa816fea50..2d2e7929b104 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -216,17 +216,6 @@ macro_rules! foreach_builtin_function { // Creates a new continuation from a funcref. cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer; - // FIXME(frank-emrich) The next three builtins are used by the debug printing mechanism. - // They are not supposed to be part of the final upstreamed code. - // - // Prints a 'static str, represented as a - // pointer and a length. - delete_me_print_str(vmctx: vmctx, s: pointer, len : u64); - // Prints integer - delete_me_print_int(vmctx: vmctx, arg : u64); - // Prints pointer, formatted as hex. - delete_me_print_pointer(vmctx: vmctx, arg : pointer); - // Returns an index for Wasm's `table.grow` instruction // for `contobj`s. Note that the initial // Option (i.e., the value to fill the new diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 381b646f6fcf..3c38944ef9fb 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -1441,22 +1441,3 @@ fn cont_new( crate::vm::stack_switching::cont_new(store, instance, func, param_count, result_count)?; Ok(Some(AllocationSize(ans.cast::() as usize))) } - -fn delete_me_print_str(_store: &mut dyn VMStore, _instance: &mut Instance, s: *const u8, len: u64) { - let len = - usize::try_from(len).map_err(|_error| TrapReason::User(anyhow::anyhow!("len too large!"))); - let str = unsafe { core::slice::from_raw_parts(s, len.unwrap()) }; - let _s = core::str::from_utf8(str).unwrap(); - #[cfg(feature = "std")] - print!("{_s}") -} - -fn delete_me_print_int(_store: &mut dyn VMStore, _instance: &mut Instance, _arg: u64) { - #[cfg(feature = "std")] - print!("{_arg}") -} - -fn delete_me_print_pointer(_store: &mut dyn VMStore, _instance: &mut Instance, _arg: *const u8) { - #[cfg(feature = "std")] - print!("{_arg:p}") -} From 8f0ba05cddc5c24df061fefac8067f0882628795 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 11:31:52 -0500 Subject: [PATCH 07/38] stack-switching: address feedback in environ::types --- crates/environ/src/types.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rs index 15ebdd9a6fd4..aeb6ab0f6027 100644 --- a/crates/environ/src/types.rs +++ b/crates/environ/src/types.rs @@ -233,7 +233,9 @@ impl WasmValType { } } - /// TODO + /// Returns the contained reference type. + /// + /// Panics if the value type is not a vmgcref pub fn unwrap_ref_type(&self) -> WasmRefType { match self { WasmValType::Ref(ref_type) => *ref_type, @@ -811,7 +813,7 @@ impl WasmContType { } /// Returns the (module interned) index to the underlying function type. - pub fn unwrap_interned_type_index(self) -> ModuleInternedTypeIndex { + pub fn unwrap_module_type_index(self) -> ModuleInternedTypeIndex { match self.0 { EngineOrModuleTypeIndex::Engine(_) => panic!("not module interned"), EngineOrModuleTypeIndex::Module(idx) => idx, From f93903afa766d5c534fca7d77d068d8e0dfad85c Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 11:35:17 -0500 Subject: [PATCH 08/38] stack-switching: remove unused code from vmoffsets --- crates/environ/src/vmoffsets.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index d4790bc4d099..700eab3bf6d4 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -162,12 +162,6 @@ pub trait PtrSize { 4 } - /// This is the size of the largest value type (i.e. a V128). - #[inline] - fn maximum_value_size(&self) -> u8 { - self.size_of_vmglobal_definition() - } - // Offsets within `VMStoreContext` /// Return the offset of the `fuel_consumed` field of `VMStoreContext` From 4237cdc25740341237a4ca12606f0a0a7ea1d29e Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 13:38:38 -0500 Subject: [PATCH 09/38] stack-switching: drop dependency on std --- crates/wasmtime/Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 6c07910f678b..c35d167dc966 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -311,8 +311,7 @@ threads = [ ] stack-switching = [ - "std", - "runtime" + "runtime", ] # Controls whether backtraces will attempt to parse DWARF information in From 85f593ffea2872fa9bfe2b530e5ff5acc1ff4877 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 13:42:59 -0500 Subject: [PATCH 10/38] stack-switching: add compilation checks to ci matrix --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b59832c60c2f..f585e1455716 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -371,6 +371,9 @@ jobs: -p wasmtime --no-default-features --features threads -p wasmtime --no-default-features --features runtime,threads -p wasmtime --no-default-features --features cranelift,threads + -p wasmtime --no-default-features --features stack-switching + -p wasmtime --no-default-features --features cranelift,stack-switching + -p wasmtime --no-default-features --features runtime,stack-switching -p wasmtime --features incremental-cache -p wasmtime --features profile-pulley -p wasmtime --all-features From f3a1cab679d47631d0049b4ae5016359671d022e Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 14:05:19 -0500 Subject: [PATCH 11/38] stack-switching: remove debug_println cruft --- crates/environ/src/stack_switching.rs | 16 ---------------- .../wasmtime/src/runtime/vm/stack_switching.rs | 4 +--- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs index ee8fa5011130..4bef7f5c435e 100644 --- a/crates/environ/src/stack_switching.rs +++ b/crates/environ/src/stack_switching.rs @@ -1,22 +1,6 @@ //! This module contains basic type definitions used by the implementation of //! the stack switching proposal. -/// FIXME(frank-emrich) Will remove in the final upstreamed version -#[allow(dead_code, reason = "Only accessed in debug builds")] -pub const ENABLE_DEBUG_PRINTING: bool = false; - -/// FIXME(frank-emrich) Will remove in the final upstreamed version -#[macro_export] -macro_rules! debug_println { - ($( $args:expr ),+ ) => { - #[cfg(debug_assertions)] - if ENABLE_DEBUG_PRINTING { - #[cfg(feature = "std")] - println!($($args),*); - } - } -} - /// Discriminant of variant `Absent` in /// `wasmtime::runtime::vm::stack_switching::VMStackChain`. pub const STACK_CHAIN_ABSENT_DISCRIMINANT: usize = 0; diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 6e00b330567d..a6647bafbf2e 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -4,8 +4,6 @@ use core::{marker::PhantomPinned, ptr::NonNull}; use stack::VMContinuationStack; -#[allow(unused)] -use wasmtime_environ::{debug_println, stack_switching::ENABLE_DEBUG_PRINTING}; use crate::vm::{Instance, TrapReason, VMFuncRef, VMStore}; use crate::ValRaw; @@ -469,7 +467,7 @@ pub fn cont_new( csi.state = VMStackState::Fresh; csi.limits = limits; - debug_println!("Created contref @ {:p}", contref); + log::trace!("Created contref @ {:p}", contref); Ok(contref) } From 257f958cc006e7eb02f67a982adb6b66b20306ae Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 14:05:41 -0500 Subject: [PATCH 12/38] stack-switching: export environ consts consistently --- crates/environ/src/lib.rs | 3 ++- .../src/runtime/vm/stack_switching.rs | 25 +++++++++---------- .../src/runtime/vm/stack_switching/stack.rs | 2 +- .../vm/stack_switching/stack/unix/x86_64.rs | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 449a4fb44288..43501dd940d7 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -33,7 +33,7 @@ pub mod obj; mod ref_bits; mod scopevec; mod stack_map; -pub mod stack_switching; +mod stack_switching; mod trap_encoding; mod tunables; mod types; @@ -52,6 +52,7 @@ pub use crate::module_types::*; pub use crate::ref_bits::*; pub use crate::scopevec::ScopeVec; pub use crate::stack_map::*; +pub use crate::stack_switching::*; pub use crate::trap_encoding::*; pub use crate::tunables::*; pub use crate::types::*; diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index a6647bafbf2e..49c7038244ed 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -553,7 +553,7 @@ pub fn cont_new( pub enum VMStackChain { /// For suspended continuations, denotes the end of their chain of /// ancestors. - Absent = wasmtime_environ::stack_switching::STACK_CHAIN_ABSENT_DISCRIMINANT, + Absent = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT, /// Represents the initial stack (i.e., where we entered Wasm from the /// host by executing /// `crate::runtime::func::invoke_wasm_and_catch_traps`). Therefore, it @@ -561,10 +561,9 @@ pub enum VMStackChain { /// variant points to is stored in the stack frame of /// `invoke_wasm_and_catch_traps`. InitialStack(*mut VMCommonStackInformation) = - wasmtime_environ::stack_switching::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT, + wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT, /// Represents a continuation's stack. - Continuation(*mut VMContRef) = - wasmtime_environ::stack_switching::STACK_CHAIN_CONTINUATION_DISCRIMINANT, + Continuation(*mut VMContRef) = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT, } impl VMStackChain { @@ -662,21 +661,21 @@ impl Iterator for StackLimitsIterator { pub enum VMStackState { /// The `VMContRef` has been created, but neither `resume` or `switch` has ever been /// called on it. During this stage, we may add arguments using `cont.bind`. - Fresh = wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT, + Fresh = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT, /// The continuation is running, meaning that it is the one currently /// executing code. - Running = wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT, + Running = wasmtime_environ::STACK_STATE_RUNNING_DISCRIMINANT, /// The continuation is suspended because it executed a resume instruction /// that has not finished yet. In other words, it became the parent of /// another continuation (which may itself be `Running`, a `Parent`, or /// `Suspended`). - Parent = wasmtime_environ::stack_switching::STACK_STATE_PARENT_DISCRIMINANT, + Parent = wasmtime_environ::STACK_STATE_PARENT_DISCRIMINANT, /// The continuation was suspended by a `suspend` or `switch` instruction. - Suspended = wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT, + Suspended = wasmtime_environ::STACK_STATE_SUSPENDED_DISCRIMINANT, /// The function originally passed to `cont.new` has returned normally. /// Note that there is no guarantee that a VMContRef will ever /// reach this status, as it may stay suspended until being dropped. - Returned = wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT, + Returned = wasmtime_environ::STACK_STATE_RETURNED_DISCRIMINANT, } /// Universal control effect. This structure encodes return signal, resume @@ -689,15 +688,15 @@ pub enum VMStackState { pub enum ControlEffect { /// Used to signal that a continuation has returned and control switches /// back to the parent. - Return = wasmtime_environ::stack_switching::CONTROL_EFFECT_RETURN_DISCRIMINANT, + Return = wasmtime_environ::CONTROL_EFFECT_RETURN_DISCRIMINANT, /// Used to signal to a continuation that it is being resumed. - Resume = wasmtime_environ::stack_switching::CONTROL_EFFECT_RESUME_DISCRIMINANT, + Resume = wasmtime_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT, /// Used to signal that a continuation has invoked a `suspend` instruction. Suspend { /// The index of the handler to be used in the parent continuation to /// switch back to. handler_index: u32, - } = wasmtime_environ::stack_switching::CONTROL_EFFECT_SUSPEND_DISCRIMINANT, + } = wasmtime_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT, /// Used to signal that a continuation has invoked a `suspend` instruction. - Switch = wasmtime_environ::stack_switching::CONTROL_EFFECT_SWITCH_DISCRIMINANT, + Switch = wasmtime_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT, } diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs index 12637585df42..fe78a81a237a 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs @@ -10,7 +10,7 @@ use crate::runtime::vm::stack_switching::VMArray; use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; cfg_if::cfg_if! { - if #[cfg(all(feature = "stack-switching",unix, target_arch = "x86_64"))] { + if #[cfg(all(feature = "stack-switching", unix, target_arch = "x86_64"))] { pub mod unix; use unix as imp; } else { diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs index 61ffd66f68e6..cda649c01dce 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs @@ -80,7 +80,7 @@ asm_func!( fn test_return_payload() { // The following assumption is baked into `wasmtime_continuation_start`. assert_eq!( - wasmtime_environ::stack_switching::CONTROL_EFFECT_RETURN_DISCRIMINANT, + wasmtime_environ::CONTROL_EFFECT_RETURN_DISCRIMINANT, 0 ); } From ba91bd055ba9d6ef0dc045009411b808d8de1b48 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 20 May 2025 15:00:27 -0500 Subject: [PATCH 13/38] stack-switching: export vm pub items consistently --- crates/environ/src/stack_switching.rs | 24 +++++++++---------- crates/wasmtime/src/runtime/func.rs | 4 ++-- crates/wasmtime/src/runtime/store.rs | 10 ++++---- crates/wasmtime/src/runtime/vm.rs | 3 ++- .../src/runtime/vm/stack_switching.rs | 7 +++--- .../src/runtime/vm/stack_switching/stack.rs | 4 ++-- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs index 4bef7f5c435e..6c6be122449c 100644 --- a/crates/environ/src/stack_switching.rs +++ b/crates/environ/src/stack_switching.rs @@ -2,40 +2,40 @@ //! the stack switching proposal. /// Discriminant of variant `Absent` in -/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +/// `wasmtime::runtime::vm::VMStackChain`. pub const STACK_CHAIN_ABSENT_DISCRIMINANT: usize = 0; /// Discriminant of variant `InitialStack` in -/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +/// `wasmtime::runtime::vm::VMStackChain`. pub const STACK_CHAIN_INITIAL_STACK_DISCRIMINANT: usize = 1; /// Discriminant of variant `Continiation` in -/// `wasmtime::runtime::vm::stack_switching::VMStackChain`. +/// `wasmtime::runtime::vm::VMStackChain`. pub const STACK_CHAIN_CONTINUATION_DISCRIMINANT: usize = 2; /// Discriminant of variant `Fresh` in -/// `runtime::vm::stack_switching::VMStackState`. +/// `runtime::vm::VMStackState`. pub const STACK_STATE_FRESH_DISCRIMINANT: u32 = 0; /// Discriminant of variant `Running` in -/// `runtime::vm::stack_switching::VMStackState`. +/// `runtime::vm::VMStackState`. pub const STACK_STATE_RUNNING_DISCRIMINANT: u32 = 1; /// Discriminant of variant `Parent` in -/// `runtime::vm::stack_switching::VMStackState`. +/// `runtime::vm::VMStackState`. pub const STACK_STATE_PARENT_DISCRIMINANT: u32 = 2; /// Discriminant of variant `Suspended` in -/// `runtime::vm::stack_switching::VMStackState`. +/// `runtime::vm::VMStackState`. pub const STACK_STATE_SUSPENDED_DISCRIMINANT: u32 = 3; /// Discriminant of variant `Returned` in -/// `runtime::vm::stack_switching::VMStackState`. +/// `runtime::vm::VMStackState`. pub const STACK_STATE_RETURNED_DISCRIMINANT: u32 = 4; /// Discriminant of variant `Return` in -/// `runtime::vm::stack_switching::ControlEffect`. +/// `runtime::vm::ControlEffect`. pub const CONTROL_EFFECT_RETURN_DISCRIMINANT: u32 = 0; /// Discriminant of variant `Resume` in -/// `runtime::vm::stack_switching::ControlEffect`. +/// `runtime::vm::ControlEffect`. pub const CONTROL_EFFECT_RESUME_DISCRIMINANT: u32 = 1; /// Discriminant of variant `Suspend` in -/// `runtime::vm::stack_switching::ControlEffect`. +/// `runtime::vm::ControlEffect`. pub const CONTROL_EFFECT_SUSPEND_DISCRIMINANT: u32 = 2; /// Discriminant of variant `Switch` in -/// `runtime::vm::stack_switching::ControlEffect`. +/// `runtime::vm::ControlEffect`. pub const CONTROL_EFFECT_SWITCH_DISCRIMINANT: u32 = 3; diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index 307749e2ae58..30e10e704043 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1,5 +1,5 @@ use crate::prelude::*; -use crate::runtime::vm::stack_switching::VMCommonStackInformation; +use crate::runtime::vm::VMCommonStackInformation; use crate::runtime::vm::{ ExportFunction, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport, VMOpaqueContext, VMStoreContext, @@ -358,7 +358,7 @@ macro_rules! for_each_function_signature { } mod typed; -use crate::runtime::vm::stack_switching::VMStackChain; +use crate::runtime::vm::VMStackChain; pub use typed::*; impl Func { diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index cb0522e2f100..219e15807320 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -86,7 +86,8 @@ use crate::runtime::vm::GcRootsList; use crate::runtime::vm::{ ExportGlobal, GcStore, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, Interpreter, InterpreterRef, ModuleRuntimeInfo, OnDemandInstanceAllocator, SignalHandler, - StoreBox, StorePtr, Unwind, VMContext, VMFuncRef, VMGcRef, VMStoreContext, + StoreBox, StorePtr, Unwind, VMContRef, VMContext, VMContinuationStack, VMFuncRef, VMGcRef, + VMStoreContext, }; use crate::trampoline::VMHostGlobalContext; use crate::RootSet; @@ -116,9 +117,6 @@ use self::async_::*; #[cfg(feature = "gc")] mod gc; -use super::vm::stack_switching::stack::VMContinuationStack; -use super::vm::stack_switching::VMContRef; - /// A [`Store`] is a collection of WebAssembly instances and host-defined state. /// /// All WebAssembly instances and items will be attached to and refer to a @@ -1700,7 +1698,7 @@ impl StoreOpaque { #[cfg(feature = "gc")] fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) { - use crate::runtime::vm::Backtrace; + use crate::{runtime::vm::Backtrace, vm::VMStackState}; log::trace!("Begin trace GC roots :: continuations"); for continuation in &self.continuations { @@ -1725,7 +1723,7 @@ impl StoreOpaque { // continuations below. // - For `Fresh` continuations, we know that there are no GC values // on their stack, yet. - if state == crate::vm::stack_switching::VMStackState::Suspended { + if state == VMStackState::Suspended { Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| { self.trace_wasm_stack_frame(gc_roots_list, frame); core::ops::ControlFlow::Continue(()) diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 5d0d64a7986c..282bd9f8eebb 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -33,6 +33,7 @@ mod memory; mod mmap_vec; mod provenance; mod send_sync_ptr; +mod stack_switching; mod store_box; mod sys; mod table; @@ -42,7 +43,6 @@ mod vmcontext; #[cfg(feature = "threads")] mod parking_spot; -pub mod stack_switching; // Note that `debug_builtins` here is disabled with a feature or a lack of a // native compilation backend because it's only here to assist in debugging @@ -83,6 +83,7 @@ pub use crate::runtime::vm::memory::{ }; pub use crate::runtime::vm::mmap_vec::MmapVec; pub use crate::runtime::vm::provenance::*; +pub use crate::runtime::vm::stack_switching::*; pub use crate::runtime::vm::store_box::*; #[cfg(feature = "std")] pub use crate::runtime::vm::sys::mmap::open_file_for_mmap; diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 49c7038244ed..5f1d84b4bc77 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -1,14 +1,13 @@ //! This module contains the runtime components of the implementation of the //! stack switching proposal. -use core::{marker::PhantomPinned, ptr::NonNull}; - -use stack::VMContinuationStack; +mod stack; use crate::vm::{Instance, TrapReason, VMFuncRef, VMStore}; use crate::ValRaw; +use core::{marker::PhantomPinned, ptr::NonNull}; -pub mod stack; +pub use stack::*; /// A continuation object is a handle to a continuation reference /// (i.e. an actual stack). A continuation object only be consumed diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs index fe78a81a237a..44e3947abbc8 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs @@ -11,10 +11,10 @@ use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; cfg_if::cfg_if! { if #[cfg(all(feature = "stack-switching", unix, target_arch = "x86_64"))] { - pub mod unix; + mod unix; use unix as imp; } else { - pub mod dummy; + mod dummy; use dummy as imp; } } From 7fbb3fa7be7a4631d67b0dd7dc55a48957931e87 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 22 May 2025 12:26:17 -0500 Subject: [PATCH 14/38] table_pool: reduced capacity for large elements VMContRef elements which takes up two words and we don't want to double the size of all tables in order to support storing these. This change changes the table to target storing the requested max number of elements if they are "nominally" sized with (potentially) reduced capacity for non-nominally sized types when encountered. Continuations are the only type of element which may result in fewer table slots being available than requested. --- .../runtime/vm/instance/allocator/pooling.rs | 6 +- .../instance/allocator/pooling/table_pool.rs | 102 +++++++++++++----- crates/wasmtime/src/runtime/vm/table.rs | 6 +- 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs index 647dfbaa1711..ee45e972ec45 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs @@ -130,7 +130,11 @@ pub struct InstanceLimits { /// Maximum number of tables per instance. pub max_tables_per_module: u32, - /// Maximum number of table elements per table. + /// Maximum number of word-size elements per table. + /// + /// Note that tables for element types such as continuations + /// that use more than one word of storage may store fewer + /// elements. pub table_elements: usize, /// Maximum number of linear memories per instance. diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs index 12c78116e79c..d89e2ead1894 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs @@ -23,14 +23,14 @@ pub struct TablePool { max_total_tables: usize, tables_per_instance: usize, keep_resident: HostAlignedByteCount, - table_elements: usize, + nominal_table_elements: usize, } impl TablePool { /// Create a new `TablePool`. pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result { let table_size = HostAlignedByteCount::new_rounded_up( - crate::runtime::vm::table::MAX_TABLE_ELEM_SIZE + crate::runtime::vm::table::NOMINAL_MAX_TABLE_ELEM_SIZE .checked_mul(config.limits.table_elements) .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, )?; @@ -45,14 +45,17 @@ impl TablePool { let mapping = Mmap::accessible_reserved(allocation_size, allocation_size) .context("failed to create table pool mapping")?; + let keep_resident = HostAlignedByteCount::new_rounded_up(config.table_keep_resident)?; + assert!(keep_resident <= table_size); + Ok(Self { index_allocator: SimpleIndexAllocator::new(config.limits.total_tables), mapping, table_size, max_total_tables, tables_per_instance, - keep_resident: HostAlignedByteCount::new_rounded_up(config.table_keep_resident)?, - table_elements: usize::try_from(config.limits.table_elements).unwrap(), + keep_resident, + nominal_table_elements: usize::try_from(config.limits.table_elements).unwrap(), }) } @@ -77,12 +80,12 @@ impl TablePool { } for (i, table) in module.tables.iter().skip(module.num_imported_tables) { - if table.limits.min > u64::try_from(self.table_elements)? { + if table.limits.min > u64::try_from(self.nominal_table_elements)? { bail!( "table index {} has a minimum element size of {} which exceeds the limit of {}", i.as_u32(), table.limits.min, - self.table_elements, + self.nominal_table_elements, ); } } @@ -114,6 +117,20 @@ impl TablePool { } } + /// Returns the number of bytes occupied by table entry data + /// + /// This is typically just the `nominal_table_elements` multiplied by + /// the size of the table's element type, but may be less in the case + /// of types such as VMContRef for which less capacity will be avialable + /// (maintaining a consistent table size in the pool). + fn data_size(&self, table_type: crate::vm::table::TableElementType) -> usize { + let element_size = table_type.element_size(); + let elements = self + .nominal_table_elements + .min(self.table_size.byte_count() / element_size); + elements * element_size + } + /// Allocate a single table for the given instance allocation request. pub fn allocate( &self, @@ -131,18 +148,13 @@ impl TablePool { match (|| { let base = self.get(allocation_index); - - let element_size = crate::vm::table::wasm_to_table_type(ty.ref_type).element_size(); - + let data_size = self.data_size(crate::vm::table::wasm_to_table_type(ty.ref_type)); unsafe { - commit_pages(base, self.table_elements * element_size)?; + commit_pages(base, data_size)?; } - let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut( - base.cast(), - self.table_elements * element_size, - )) - .unwrap(); + let ptr = + NonNull::new(std::ptr::slice_from_raw_parts_mut(base.cast(), data_size)).unwrap(); unsafe { Table::new_static( ty, @@ -194,15 +206,8 @@ impl TablePool { ) { assert!(table.is_static()); let base = self.get(allocation_index); - - // XXX Should we check that table.size() * mem::size_of::<*mut u8>() - // doesn't overflow? The only check that exists is for the boundary - // condition that table.size() * mem::size_of::<*mut u8>() is less than - // a host page smaller than usize::MAX. - let size = HostAlignedByteCount::new_rounded_up( - table.size() * table.element_type().element_size(), - ) - .expect("table entry size doesn't overflow"); + let size = HostAlignedByteCount::new_rounded_up(self.data_size(table.element_type())) + .expect("table entry size doesn't overflow"); // `memset` the first `keep_resident` bytes. let size_to_memset = size.min(self.keep_resident); @@ -240,7 +245,7 @@ mod tests { assert_eq!(pool.table_size, host_page_size); assert_eq!(pool.max_total_tables, 7); - assert_eq!(pool.table_elements, 100); + assert_eq!(pool.nominal_table_elements, 100); let base = pool.mapping.as_ptr() as usize; @@ -255,4 +260,51 @@ mod tests { Ok(()) } + + #[test] + fn test_table_pool_continuations_capacity() -> Result<()> { + let mkpool = |table_elements: usize| -> Result { + TablePool::new(&PoolingInstanceAllocatorConfig { + limits: InstanceLimits { + table_elements, + total_tables: 7, + max_memory_size: 0, + max_memories_per_module: 0, + ..Default::default() + }, + ..Default::default() + }) + }; + + let host_page_size = HostAlignedByteCount::host_page_size(); + let words_per_page = host_page_size.byte_count() / size_of::<*const u8>(); + let pool_big = mkpool(words_per_page - 1)?; + let pool_small = mkpool(5)?; + + assert_eq!(pool_small.table_size, host_page_size); + assert_eq!(pool_big.table_size, host_page_size); + + // table should store nominal_table_elements of data for func in both cases + let func_table_type = crate::vm::table::TableElementType::Func; + assert_eq!( + pool_small.data_size(func_table_type), + pool_small.nominal_table_elements * func_table_type.element_size() + ); + assert_eq!( + pool_big.data_size(func_table_type), + pool_big.nominal_table_elements * func_table_type.element_size() + ); + + // In the "big" case, continuations should fill page size (capacity limited). + // In the "small" case, continuations should fill only part of the page, capping + // at the requested table size for nominal elements. + let cont_table_type = crate::vm::table::TableElementType::Cont; + assert_eq!( + pool_small.data_size(cont_table_type), + pool_small.nominal_table_elements * cont_table_type.element_size() + ); + assert_eq!(pool_big.data_size(cont_table_type), host_page_size); + + Ok(()) + } } diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index 31cac5f58394..ce8c471f36be 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -172,10 +172,10 @@ pub type ContTableElem = Option; /// The maximum of the sizes of any of the table element types #[allow(dead_code, reason = "Only used if pooling allocator is enabled")] -pub const MAX_TABLE_ELEM_SIZE: usize = { +pub const NOMINAL_MAX_TABLE_ELEM_SIZE: usize = { + // ContTableElem intentionally excluded for "nominal" calculation. let sizes = [ core::mem::size_of::(), - core::mem::size_of::(), core::mem::size_of::>(), ]; @@ -429,7 +429,7 @@ impl Table { })), TableElementType::Cont => Ok(Self::from(DynamicContTable { elements: vec![None; minimum], - maximum: maximum, + maximum, })), } } From e823b74cb84e27b016047849e45bcf1e8825c446 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 27 May 2025 19:00:28 +0000 Subject: [PATCH 15/38] stack-switching: extend conditional compilation A fair bit of the definitions for stack switching are still enabled, but this patch takes things a bit further to avoid compilation problems; notably, cont_new is now not compiled in unless the feature is enabled. --- crates/cli-flags/src/lib.rs | 2 +- crates/environ/Cargo.toml | 1 + crates/environ/src/builtin.rs | 1 + crates/wasmtime/Cargo.toml | 1 + crates/wasmtime/src/runtime/store.rs | 1 + crates/wasmtime/src/runtime/vm/libcalls.rs | 1 + crates/wasmtime/src/runtime/vm/stack_switching.rs | 1 + 7 files changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs index 16d64271fa70..9fbe01d5281c 100644 --- a/crates/cli-flags/src/lib.rs +++ b/crates/cli-flags/src/lib.rs @@ -830,7 +830,7 @@ impl CommonOptions { } #[cfg(not(any(feature = "async", feature = "stack-switching")))] { - if let Some(size) = self.wasm.async_stack_size { + if let Some(_size) = self.wasm.async_stack_size { anyhow::bail!(concat!( "support for async/stack-switching disabled at compile time" )); diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index e85aef15b347..ffdf9570d20c 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -69,6 +69,7 @@ compile = [ "dep:wasm-encoder", "dep:wasmprinter", ] +stack-switching = [] threads = ['std'] wmemcheck = ['std'] std = [ diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index fb4ea4f10eb9..ae020c79d85c 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -228,6 +228,7 @@ macro_rules! foreach_builtin_function { raise(vmctx: vmctx); // Creates a new continuation from a funcref. + #[cfg(feature = "stack-switching")] cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer; // Returns an index for Wasm's `table.grow` instruction diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index c35d167dc966..e155decb7003 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -312,6 +312,7 @@ threads = [ stack-switching = [ "runtime", + "wasmtime-environ/stack-switching", ] # Controls whether backtraces will attempt to parse DWARF information in diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 5e2b19eac4f9..7bf75d770235 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -2038,6 +2038,7 @@ at https://bytecodealliance.org/security. /// Allocates a new continuation. Note that we currently don't support /// deallocating them. Instead, all continuations remain allocated /// throughout the store's lifetime. + #[cfg(feature = "stack-switching")] pub fn allocate_continuation(&mut self) -> Result<*mut VMContRef> { // FIXME(frank-emrich) Do we need to pin this? let mut continuation = Box::new(VMContRef::empty()); diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index b395ea790f4a..e3885f0d013a 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -1548,6 +1548,7 @@ fn raise(_store: &mut dyn VMStore, _instance: &mut Instance) { // Builtins for continuations. These are thin wrappers around the // respective definitions in stack_switching.rs. +#[cfg(feature = "stack-switching")] fn cont_new( store: &mut dyn VMStore, instance: &mut Instance, diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 5f1d84b4bc77..1085a14c48dd 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -419,6 +419,7 @@ fn check_vm_contref_offsets() { } /// Implements `cont.new` instructions (i.e., creation of continuations). +#[cfg(feature = "stack-switching")] #[inline(always)] pub fn cont_new( store: &mut dyn VMStore, From b63e0720e3c8e115c53237bc08d9e17b95173001 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 27 May 2025 19:58:51 +0000 Subject: [PATCH 16/38] stack-switching: formatting fixes --- crates/wasmtime/src/runtime/vm/stack_switching.rs | 2 +- .../wasmtime/src/runtime/vm/stack_switching/stack/unix.rs | 6 ++++-- .../src/runtime/vm/stack_switching/stack/unix/x86_64.rs | 5 +---- crates/wasmtime/src/runtime/vm/table.rs | 6 +----- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 1085a14c48dd..b3b6165e1ea0 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -3,8 +3,8 @@ mod stack; -use crate::vm::{Instance, TrapReason, VMFuncRef, VMStore}; use crate::ValRaw; +use crate::vm::{Instance, TrapReason, VMFuncRef, VMStore}; use core::{marker::PhantomPinned, ptr::NonNull}; pub use stack::*; diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs index e50e1f2c7fbe..930431ac6c02 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -329,8 +329,10 @@ unsafe extern "C" fn fiber_start( // since the caller vmctx is only really used to access stuff in the // underlying `Store`, it's fine to be slightly sloppy about the exact // value we set. - func_ref.array_call(None, caller_vmxtx, params_and_returns); // TODO(dhil): we are ignoring the boolean return value - // here... we probably shouldn't. + // + // TODO(dhil): we are ignoring the boolean return value + // here... we probably shouldn't. + func_ref.array_call(None, caller_vmxtx, params_and_returns); // The array call trampoline should have just written // `return_value_count` values to the `args` buffer. Let's reflect that diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs index cda649c01dce..3a2292dd8e3a 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs @@ -79,8 +79,5 @@ asm_func!( #[test] fn test_return_payload() { // The following assumption is baked into `wasmtime_continuation_start`. - assert_eq!( - wasmtime_environ::CONTROL_EFFECT_RETURN_DISCRIMINANT, - 0 - ); + assert_eq!(wasmtime_environ::CONTROL_EFFECT_RETURN_DISCRIMINANT, 0); } diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index 42e3b6ea5cf8..7a6aa37ac910 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -186,11 +186,7 @@ pub const NOMINAL_MAX_TABLE_ELEM_SIZE: usize = { [] => 0, [head, tail @ ..] => { let tail_max = slice_max(tail); - if *head >= tail_max { - *head - } else { - tail_max - } + if *head >= tail_max { *head } else { tail_max } } } } From f69a569063cc07250c2273d1365280c3bd9752e4 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 27 May 2025 20:18:30 +0000 Subject: [PATCH 17/38] stack-switching: address new clippy checks In addition, to get clippy to fully pass, plumbed in additional config to make winch paths happy; there's no impl for winch yet but plumbing through the feature is required to make paths incorporating macros at various layers satisfied (and it is expected we'll use the features in the future). --- crates/cranelift/Cargo.toml | 1 + crates/wasmtime/Cargo.toml | 2 ++ crates/wasmtime/src/config.rs | 2 +- crates/wasmtime/src/runtime/vm/table.rs | 8 ++++---- crates/winch/Cargo.toml | 1 + winch/codegen/Cargo.toml | 1 + 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index c4ef5cbd2d14..8321f55ff087 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -45,4 +45,5 @@ wmemcheck = ["wasmtime-environ/wmemcheck"] gc = ["wasmtime-environ/gc"] gc-drc = ["gc", "wasmtime-environ/gc-drc"] gc-null = ["gc", "wasmtime-environ/gc-null"] +stack-switching = [] threads = ["wasmtime-environ/threads"] diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index e155decb7003..c49fc4aa8dcb 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -313,6 +313,8 @@ threads = [ stack-switching = [ "runtime", "wasmtime-environ/stack-switching", + "wasmtime-cranelift?/stack-switching", + "wasmtime-winch?/stack-switching", ] # Controls whether backtraces will attempt to parse DWARF information in diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 2a378e77dc5d..6e59ac7da865 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -2474,7 +2474,7 @@ impl Config { if !self .compiler_config - .ensure_setting_unset_or_given("stack_switch_model".into(), model.into()) + .ensure_setting_unset_or_given("stack_switch_model", model) { bail!( "compiler option 'stack_switch_model' must be set to '{}' on this platform", diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index 7a6aa37ac910..9cfa186701d7 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -812,7 +812,7 @@ impl Table { elements.resize_with(new_size, || None); } Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => { - elements.resize(usize::try_from(new_size).unwrap(), None); + elements.resize(new_size, None); } } @@ -977,7 +977,7 @@ impl Table { base: NonNull::<[Option]>::from(&mut elements[..]) .cast() .into(), - current_elements: elements.len().try_into().unwrap(), + current_elements: elements.len(), } } } @@ -1051,7 +1051,7 @@ impl Table { slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) }, Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe { - slice::from_raw_parts(data.as_ptr().cast(), usize::try_from(*size).unwrap()) + slice::from_raw_parts(data.as_ptr().cast(), *size) }, _ => unreachable!(), } @@ -1064,7 +1064,7 @@ impl Table { slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) }, Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe { - slice::from_raw_parts_mut(data.as_ptr().cast(), usize::try_from(*size).unwrap()) + slice::from_raw_parts_mut(data.as_ptr().cast(), *size) }, _ => unreachable!(), } diff --git a/crates/winch/Cargo.toml b/crates/winch/Cargo.toml index 6be5ed09bbf8..b966107b829b 100644 --- a/crates/winch/Cargo.toml +++ b/crates/winch/Cargo.toml @@ -31,5 +31,6 @@ all-arch = ["winch-codegen/all-arch"] gc = ['winch-codegen/gc'] gc-drc = ['winch-codegen/gc-drc'] gc-null = ['winch-codegen/gc-null'] +stack-switching = ['winch-codegen/stack-switching'] threads = ['winch-codegen/threads'] wmemcheck = ['winch-codegen/wmemcheck'] diff --git a/winch/codegen/Cargo.toml b/winch/codegen/Cargo.toml index e60429d66af0..d8b87aca2d9f 100644 --- a/winch/codegen/Cargo.toml +++ b/winch/codegen/Cargo.toml @@ -38,5 +38,6 @@ all-arch = [ gc = ['wasmtime-environ/gc'] gc-drc = ['wasmtime-environ/gc-drc'] gc-null = ['wasmtime-environ/gc-null'] +stack-switching = ['wasmtime-environ/stack-switching'] threads = ['wasmtime-environ/threads'] wmemcheck = ['wasmtime-environ/wmemcheck'] From e3e8d6a08c7c65e9924dadf199f1e11f880eb246 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 27 May 2025 21:43:47 +0000 Subject: [PATCH 18/38] stack-switching: more conditional compilation fixes --- crates/cranelift/src/func_environ.rs | 1 + crates/wasmtime/src/runtime/store.rs | 5 ++--- crates/wasmtime/src/runtime/vm/stack_switching.rs | 12 +++++------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 9e4e133ab21d..dc1ab3e0d668 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -3813,6 +3813,7 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type { /// TODO(10248) This is removed in the next stack switching PR. It stops the /// compiler from complaining about the stack switching libcalls being dead /// code. +#[cfg(feature = "stack-switching")] #[allow( dead_code, reason = "Dummy function to supress more dead code warnings" diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 7bf75d770235..01338febfdb0 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -87,8 +87,7 @@ use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::{ ExportGlobal, GcStore, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, Interpreter, InterpreterRef, ModuleRuntimeInfo, OnDemandInstanceAllocator, SignalHandler, - StoreBox, StorePtr, Unwind, VMContRef, VMContext, VMContinuationStack, VMFuncRef, VMGcRef, - VMStoreContext, + StoreBox, StorePtr, Unwind, VMContRef, VMContext, VMFuncRef, VMGcRef, VMStoreContext, }; use crate::trampoline::VMHostGlobalContext; use crate::{Engine, Module, Trap, Val, ValRaw, module::ModuleRegistry}; @@ -2043,7 +2042,7 @@ at https://bytecodealliance.org/security. // FIXME(frank-emrich) Do we need to pin this? let mut continuation = Box::new(VMContRef::empty()); let stack_size = self.engine.config().async_stack_size; - let stack = VMContinuationStack::new(stack_size)?; + let stack = crate::vm::VMContinuationStack::new(stack_size)?; continuation.stack = stack; let ptr = continuation.deref_mut() as *mut VMContRef; self.continuations.push(continuation); diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index b3b6165e1ea0..b600e82521f5 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -3,8 +3,6 @@ mod stack; -use crate::ValRaw; -use crate::vm::{Instance, TrapReason, VMFuncRef, VMStore}; use core::{marker::PhantomPinned, ptr::NonNull}; pub use stack::*; @@ -422,12 +420,12 @@ fn check_vm_contref_offsets() { #[cfg(feature = "stack-switching")] #[inline(always)] pub fn cont_new( - store: &mut dyn VMStore, - instance: &mut Instance, + store: &mut dyn crate::vm::VMStore, + instance: &mut crate::vm::Instance, func: *mut u8, param_count: u32, result_count: u32, -) -> Result<*mut VMContRef, TrapReason> { +) -> Result<*mut VMContRef, crate::vm::TrapReason> { let caller_vmctx = instance.vmctx(); let stack_size = store.engine().config().async_stack_size; @@ -443,10 +441,10 @@ pub fn cont_new( // The initialization function will allocate the actual args/return value buffer and // update this object (if needed). - let contref_args_ptr = &mut contref.args as *mut _ as *mut VMArray; + let contref_args_ptr = &mut contref.args as *mut _ as *mut VMArray; contref.stack.initialize( - func.cast::(), + func.cast::(), caller_vmctx.as_ptr(), contref_args_ptr, param_count, From 12b2d3f7bf05eb880bd2f3acaee2f19d043b22d0 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 27 May 2025 21:54:21 +0000 Subject: [PATCH 19/38] stack-switching: additional conditional compile on table builtins for continuations --- crates/environ/src/builtin.rs | 3 +++ crates/wasmtime/src/runtime/vm/libcalls.rs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index ae020c79d85c..5285f29ece8c 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -238,9 +238,12 @@ macro_rules! foreach_builtin_function { // continuation reference and the revision count. To // denote the continuation being `None`, `init_contref` // may be 0. + #[cfg(feature = "stack-switching")] table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: u64) -> pointer; + // `value_contref` and `value_revision` together encode // the Option, as in previous libcall. + #[cfg(feature = "stack-switching")] table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: u64, len: u64) -> bool; } }; diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index e3885f0d013a..b9731f89e145 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -273,6 +273,7 @@ unsafe fn table_grow_gc_ref( Ok(result) } +#[cfg(feature = "stack-switching")] unsafe fn table_grow_cont_obj( store: &mut dyn VMStore, instance: &mut Instance, @@ -353,6 +354,7 @@ unsafe fn table_fill_gc_ref( } } +#[cfg(feature = "stack-switching")] unsafe fn table_fill_cont_obj( store: &mut dyn VMStore, instance: &mut Instance, From fc60266211a0297aa6ca157b911741acda33d32c Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 28 May 2025 16:02:33 +0000 Subject: [PATCH 20/38] stack-switching: additional conditional compile fixes --- crates/wasmtime/src/runtime/vm/libcalls.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index b9731f89e145..f00197842b86 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -54,8 +54,8 @@ //! } //! ``` -use super::stack_switching::VMContObj; -use super::stack_switching::VMContRef; +#[cfg(feature = "stack-switching")] +use super::stack_switching::{VMContObj, VMContRef}; use crate::prelude::*; #[cfg(feature = "gc")] use crate::runtime::vm::VMGcRef; From 7e7766298d43361c60bf01e7fc4884b0185377f3 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 28 May 2025 16:49:51 +0000 Subject: [PATCH 21/38] stack-switching: additional conditional compile in store --- crates/wasmtime/src/runtime/store.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 01338febfdb0..259731972e9a 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -83,11 +83,13 @@ use crate::module::RegisteredModuleId; use crate::prelude::*; #[cfg(feature = "gc")] use crate::runtime::vm::GcRootsList; +#[cfg(feature = "stack-switching")] +use crate::runtime::vm::VMContRef; use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::{ ExportGlobal, GcStore, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, Interpreter, InterpreterRef, ModuleRuntimeInfo, OnDemandInstanceAllocator, SignalHandler, - StoreBox, StorePtr, Unwind, VMContRef, VMContext, VMFuncRef, VMGcRef, VMStoreContext, + StoreBox, StorePtr, Unwind, VMContext, VMFuncRef, VMGcRef, VMStoreContext, }; use crate::trampoline::VMHostGlobalContext; use crate::{Engine, Module, Trap, Val, ValRaw, module::ModuleRegistry}; @@ -324,6 +326,7 @@ pub struct StoreOpaque { // Contains all continuations ever allocated throughout the lifetime of this // store. + #[cfg(feature = "stack-switching")] continuations: Vec>, instances: Vec, @@ -547,6 +550,7 @@ impl Store { _marker: marker::PhantomPinned, engine: engine.clone(), vm_store_context: Default::default(), + #[cfg(feature = "stack-switching")] continuations: Vec::new(), instances: Vec::new(), #[cfg(feature = "component-model")] @@ -1629,6 +1633,7 @@ impl StoreOpaque { assert!(gc_roots_list.is_empty()); self.trace_wasm_stack_roots(gc_roots_list); + #[cfg(feature = "stack-switching")] self.trace_wasm_continuation_roots(gc_roots_list); self.trace_vmctx_roots(gc_roots_list); self.trace_user_roots(gc_roots_list); @@ -1699,7 +1704,7 @@ impl StoreOpaque { log::trace!("End trace GC roots :: Wasm stack"); } - #[cfg(feature = "gc")] + #[cfg(all(feature = "gc", feature = "stack-switching"))] fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) { use crate::{runtime::vm::Backtrace, vm::VMStackState}; log::trace!("Begin trace GC roots :: continuations"); From 17645fa211da8b29d75c8806ddb7472d1ae38eb4 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 28 May 2025 18:31:56 +0000 Subject: [PATCH 22/38] stack-switching: remove overly strict assertion --- .../src/runtime/vm/instance/allocator/pooling/table_pool.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs index c35847805861..6045a9aefc3c 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs @@ -46,7 +46,6 @@ impl TablePool { .context("failed to create table pool mapping")?; let keep_resident = HostAlignedByteCount::new_rounded_up(config.table_keep_resident)?; - assert!(keep_resident <= table_size); Ok(Self { index_allocator: SimpleIndexAllocator::new(config.limits.total_tables), From da37984c754e30a290811aead3fefbb4f23478c1 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 28 May 2025 18:45:57 +0000 Subject: [PATCH 23/38] stack-switching: remove errantly dropped no_mangle in config c-api --- crates/c-api/src/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index f38b03488588..3113ab1627b9 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -145,6 +145,7 @@ pub extern "C" fn wasmtime_config_wasm_stack_switching_set(c: &mut wasm_config_t c.config.wasm_stack_switching(enable); } +#[unsafe(no_mangle)] #[cfg(any(feature = "cranelift", feature = "winch"))] pub extern "C" fn wasmtime_config_strategy_set( c: &mut wasm_config_t, From 5db7836326489e7e291c4afa96c620a8df5b9f75 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 30 May 2025 14:28:58 -0500 Subject: [PATCH 24/38] stack-switching: VMContObj::from_raw_parts --- crates/wasmtime/src/runtime/vm/libcalls.rs | 23 +++---------------- .../src/runtime/vm/stack_switching.rs | 12 ++++++++++ 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index f00197842b86..17ee85a86272 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -55,7 +55,7 @@ //! ``` #[cfg(feature = "stack-switching")] -use super::stack_switching::{VMContObj, VMContRef}; +use super::stack_switching::VMContObj; use crate::prelude::*; #[cfg(feature = "gc")] use crate::runtime::vm::VMGcRef; @@ -284,15 +284,7 @@ unsafe fn table_grow_cont_obj( init_value_contref: *mut u8, init_value_revision: u64, ) -> Result> { - use core::ptr::NonNull; - let init_value = if init_value_contref.is_null() { - None - } else { - // SAFETY: We just checked that the pointer is non-null - let contref = NonNull::new_unchecked(init_value_contref as *mut VMContRef); - let contobj = VMContObj::new(contref, init_value_revision); - Some(contobj) - }; + let init_value = VMContObj::from_raw_parts(init_value_contref, init_value_revision); let table_index = TableIndex::from_u32(table_index); @@ -364,20 +356,11 @@ unsafe fn table_fill_cont_obj( value_revision: u64, len: u64, ) -> Result<()> { - use core::ptr::NonNull; let table_index = TableIndex::from_u32(table_index); let table = &mut *instance.get_table(table_index); match table.element_type() { TableElementType::Cont => { - let contobj = if value_contref.is_null() { - None - } else { - // SAFETY: We just checked that the pointer is non-null - let contref = NonNull::new_unchecked(value_contref as *mut VMContRef); - let contobj = VMContObj::new(contref, value_revision); - Some(contobj) - }; - + let contobj = VMContObj::from_raw_parts(value_contref, value_revision); table.fill(store.optional_gc_store_mut(), dst, contobj.into(), len)?; Ok(()) } diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index b600e82521f5..6b5699ccf571 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -54,6 +54,18 @@ impl VMContObj { pub fn new(contref: NonNull, revision: u64) -> Self { Self { contref, revision } } + + /// Construction a VMContinuationObject from a pointer and revision + /// + /// The `contref` pointer may be null in which case None will be returned. + /// + /// # Safety + /// + /// Behavior will be undefined if a pointer to data that is not a + /// VMContRef is provided. + pub unsafe fn from_raw_parts(contref: *mut u8, revision: u64) -> Option { + NonNull::new(contref.cast::()).map(|contref| Self::new(contref, revision)) + } } unsafe impl Send for VMContObj {} From 7b72aa3dfe718cc0a3cc4bfd5296a0d285bde479 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 30 May 2025 14:38:36 -0500 Subject: [PATCH 25/38] stack-switching: remove duplicate async_stack_size feature check --- crates/cli-flags/src/lib.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs index 9fbe01d5281c..5a407ef8e56f 100644 --- a/crates/cli-flags/src/lib.rs +++ b/crates/cli-flags/src/lib.rs @@ -942,13 +942,6 @@ impl CommonOptions { ); } - #[cfg(any(feature = "async", feature = "stack-switching"))] - - match_feature! { - ["async" : self.wasm.async_stack_size] - size => config.async_stack_size(size), - _ => err, - } match_feature! { ["async" : self.wasm.async_stack_zeroing] enable => config.async_stack_zeroing(enable), From 22bf039e801f0e92048a2ce508dc27289fbb27ae Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 30 May 2025 14:59:42 -0500 Subject: [PATCH 26/38] stack-switching: VMArray -> VMHostArray --- .../wasmtime/src/runtime/vm/stack_switching.rs | 18 +++++++++--------- .../src/runtime/vm/stack_switching/stack.rs | 4 ++-- .../runtime/vm/stack_switching/stack/dummy.rs | 4 ++-- .../runtime/vm/stack_switching/stack/unix.rs | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 6b5699ccf571..f1c6249b00ab 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -215,7 +215,7 @@ impl VMStackLimits { #[derive(Debug, Clone)] /// Reference to a stack-allocated buffer ("array"), storing data of some type /// `T`. -pub struct VMArray { +pub struct VMHostArray { /// Number of currently occupied slots. pub length: u32, /// Number of slots in the data buffer. Note that this is *not* the size of @@ -225,7 +225,7 @@ pub struct VMArray { pub data: *mut T, } -impl VMArray { +impl VMHostArray { /// Creates empty `Array` pub fn empty() -> Self { Self { @@ -252,19 +252,19 @@ fn check_vm_array_offsets() { let module = Module::new(); let offsets = VMOffsets::new(HostPtr, &module); assert_eq!( - size_of::>(), + size_of::>(), usize::from(offsets.ptr.size_of_vmarray()) ); assert_eq!( - offset_of!(VMArray<()>, length), + offset_of!(VMHostArray<()>, length), usize::from(offsets.ptr.vmarray_length()) ); assert_eq!( - offset_of!(VMArray<()>, capacity), + offset_of!(VMHostArray<()>, capacity), usize::from(offsets.ptr.vmarray_capacity()) ); assert_eq!( - offset_of!(VMArray<()>, data), + offset_of!(VMHostArray<()>, data), usize::from(offsets.ptr.vmarray_data()) ); } @@ -272,12 +272,12 @@ fn check_vm_array_offsets() { /// Type used for passing payloads to and from continuations. The actual type /// argument should be wasmtime::runtime::vm::vmcontext::ValRaw, but we don't /// have access to that here. -pub type VMPayloads = VMArray; +pub type VMPayloads = VMHostArray; /// Type for a list of handlers, represented by the handled tag. Thus, the /// stored data is actually `*mut VMTagDefinition`, but we don't havr access to /// that here. -pub type VMHandlerList = VMArray<*mut u8>; +pub type VMHandlerList = VMHostArray<*mut u8>; /// The main type representing a continuation. #[repr(C)] @@ -453,7 +453,7 @@ pub fn cont_new( // The initialization function will allocate the actual args/return value buffer and // update this object (if needed). - let contref_args_ptr = &mut contref.args as *mut _ as *mut VMArray; + let contref_args_ptr = &mut contref.args as *mut _ as *mut VMHostArray; contref.stack.initialize( func.cast::(), diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs index 44e3947abbc8..cb658c03b143 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs @@ -6,7 +6,7 @@ use anyhow::Result; use core::ops::Range; -use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::stack_switching::VMHostArray; use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; cfg_if::cfg_if! { @@ -104,7 +104,7 @@ impl VMContinuationStack { &self, func_ref: *const VMFuncRef, caller_vmctx: *mut VMContext, - args: *mut VMArray, + args: *mut VMHostArray, parameter_count: u32, return_value_count: u32, ) { diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs index e2042e352c19..86de598185ea 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs @@ -1,7 +1,7 @@ use anyhow::Result; use core::ops::Range; -use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::stack_switching::VMHostArray; use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; #[allow(dead_code)] @@ -67,7 +67,7 @@ impl VMContinuationStack { &self, _func_ref: *const VMFuncRef, _caller_vmctx: *mut VMContext, - _args: *mut VMArray, + _args: *mut VMHostArray, _parameter_count: u32, _return_value_count: u32, ) { diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs index 930431ac6c02..78bd8d7a3df1 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -65,7 +65,7 @@ use std::io; use std::ops::Range; use std::ptr; -use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::stack_switching::VMHostArray; use crate::runtime::vm::{VMContext, VMFuncRef, VMOpaqueContext, ValRaw}; #[derive(Debug, PartialEq, Eq)] @@ -230,7 +230,7 @@ impl VMContinuationStack { &self, func_ref: *const VMFuncRef, caller_vmctx: *mut VMContext, - args: *mut VMArray, + args: *mut VMHostArray, parameter_count: u32, return_value_count: u32, ) { @@ -270,7 +270,7 @@ impl VMContinuationStack { // Data after the args buffer: (0x28 + s, func_ref as usize), (0x30 + s, caller_vmctx as usize), - (0x38 + s, args as *mut VMArray as usize), + (0x38 + s, args as *mut VMHostArray as usize), (0x40 + s, return_value_count as usize), ]; @@ -306,7 +306,7 @@ unsafe extern "C" { unsafe extern "C" fn fiber_start( func_ref: *const VMFuncRef, caller_vmctx: *mut VMContext, - args: *mut VMArray, + args: *mut VMHostArray, return_value_count: u32, ) { unsafe { From 2f994c32a765030fbc18c9ee6527383f9827d4f8 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 30 May 2025 16:17:43 -0500 Subject: [PATCH 27/38] stack-switching: remove unnecessary clippy exception --- crates/wasmtime/src/runtime/vm/stack_switching.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index f1c6249b00ab..e692a8826ffe 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -337,7 +337,6 @@ impl VMContRef { /// This is effectively a `Default` implementation, without calling it /// so. Used to create `VMContRef`s when initializing pooling allocator. - #[allow(clippy::cast_possible_truncation)] pub fn empty() -> Self { let limits = VMStackLimits::with_stack_limit(Default::default()); let state = VMStackState::Fresh; From 6a60fe64684704435cc26c3f1527a7a0273aabb0 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 30 May 2025 16:23:21 -0500 Subject: [PATCH 28/38] stack-switching: fix docs referenced VMRuntimeLimits --- crates/wasmtime/src/runtime/func.rs | 4 ++-- .../wasmtime/src/runtime/vm/stack_switching.rs | 16 ++++++++-------- .../src/runtime/vm/traphandlers/backtrace.rs | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index 1bad88bf0509..6e6c5903df9c 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1642,7 +1642,7 @@ pub(crate) fn invoke_wasm_and_catch_traps( /// are restored. pub(crate) struct EntryStoreContext { /// If set, contains value of `stack_limit` field to restore in - /// `VMRuntimeLimits` when exiting Wasm. + /// `VMStoreContext` when exiting Wasm. pub stack_limit: Option, /// Contains value of `last_wasm_exit_pc` field to restore in /// `VMStoreContext` when exiting Wasm. @@ -1672,7 +1672,7 @@ impl EntryStoreContext { /// allocated by WebAssembly code and it's relative to the initial stack /// pointer that called into wasm. /// - /// It also saves the different last_wasm_* values in the `VMRuntimeLimits`. + /// It also saves the different last_wasm_* values in the `VMStoreContext`. pub fn enter_wasm( store: &mut StoreContextMut<'_, T>, initial_stack_information: *mut VMCommonStackInformation, diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index e692a8826ffe..544fb88a0bd2 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -82,13 +82,13 @@ fn null_pointer_optimization() { } /// This type is used to save (and subsequently restore) a subset of the data in -/// `VMRuntimeLimits`. See documentation of `VMStackChain` for the exact uses. +/// `VMStoreContext`. See documentation of `VMStackChain` for the exact uses. #[repr(C)] #[derive(Debug, Default, Clone)] pub struct VMStackLimits { - /// Saved version of `stack_limit` field of `VMRuntimeLimits` + /// Saved version of `stack_limit` field of `VMStoreContext` pub stack_limit: usize, - /// Saved version of `last_wasm_entry_fp` field of `VMRuntimeLimits` + /// Saved version of `last_wasm_entry_fp` field of `VMStoreContext` pub last_wasm_entry_fp: usize, } @@ -114,7 +114,7 @@ fn check_vm_stack_limits_offsets() { #[repr(C)] #[derive(Debug, Clone)] pub struct VMCommonStackInformation { - /// Saves subset of `VMRuntimeLimits` for this stack. See documentation of + /// Saves subset of `VMStoreContext` for this stack. See documentation of /// `VMStackChain` for the exact uses. pub limits: VMStackLimits, /// For the initial stack, this field must only have one of the following values: @@ -527,15 +527,15 @@ pub fn cont_new( /// the stack frame that executed by `invoke_wasm_and_catch_traps`. /// /// The following invariants hold for these `VMStackLimits` objects, -/// and the data in `VMRuntimeLimits`. +/// and the data in `VMStoreContext`. /// /// Currently executing stack: For the currently executing stack (i.e., the /// stack that is at the head of the store's `stack_chain` list), the /// associated `VMStackLimits` object contains stale/undefined data. Instead, /// the live data describing the limits for the currently executing stack is -/// always maintained in `VMRuntimeLimits`. Note that as a general rule +/// always maintained in `VMStoreContext`. Note that as a general rule /// independently from any execution of continuations, the `last_wasm_exit*` -/// fields in the `VMRuntimeLimits` contain undefined values while executing +/// fields in the `VMStoreContext` contain undefined values while executing /// wasm. /// /// Parents of currently executing stack: For stacks that appear in the tail @@ -555,7 +555,7 @@ pub fn cont_new( /// However, its `last_enter_wasm_sp` field contains undefined data. This is /// justified, because when resume-ing a continuation for the first time, a /// native-to-wasm trampoline is called, which sets up the -/// `last_wasm_entry_sp` in the `VMRuntimeLimits` with the correct value, +/// `last_wasm_entry_sp` in the `VMStoreContext` with the correct value, /// thus restoring the necessary invariant. #[derive(Debug, Clone, PartialEq)] #[repr(usize, C)] diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index cde668013540..8fdf1425a678 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -271,7 +271,7 @@ impl Backtrace { // The VMStackLimits of the currently running stack (whether that's a // continuation or the initial stack) contains undefined data, the // information about that stack is saved in the Store's - // `VMRuntimeLimits` and handled at the top of this function + // `VMStoreContext` and handled at the top of this function // already. That's why we ignore `stack_limits_vec[0]`. // // Note that a continuation stack's control context stores From 4ebb04b2c28057b503f516071c437d3920a5f570 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 30 May 2025 16:37:48 -0500 Subject: [PATCH 29/38] stack-switching: fix doc typo --- crates/wasmtime/src/runtime/vm/stack_switching.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 544fb88a0bd2..409879159a9f 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -482,7 +482,7 @@ pub fn cont_new( /// This type represents a linked lists ("chain") of stacks, where the a /// node's successor denotes its parent. -/// A additionally, a `CommonStackInformation` object is associated with +/// Additionally, a `CommonStackInformation` object is associated with /// each stack in the list. /// Here, a "stack" is one of the following: /// - A continuation (i.e., created with cont.new). From 1daeb0f05423cea331c75e80c6608b70ad8c41d9 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 30 May 2025 17:13:50 -0500 Subject: [PATCH 30/38] stack-switching: follow recommendations for type casts --- .../runtime/vm/stack_switching/stack/unix.rs | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs index 78bd8d7a3df1..f4e5db61588a 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -156,7 +156,7 @@ impl VMContinuationStack { } pub fn range(&self) -> Option> { - let base = unsafe { self.top.sub(self.len) as usize }; + let base = unsafe { self.top.sub(self.len).addr() }; Some(base..base + self.len) } @@ -164,7 +164,7 @@ impl VMContinuationStack { // See picture at top of this file: // RIP is stored 8 bytes below top of stack. unsafe { - let ptr = self.top.sub(8) as *mut usize; + let ptr = self.top.sub(8).cast::(); *ptr } } @@ -173,7 +173,7 @@ impl VMContinuationStack { // See picture at top of this file: // RBP is stored 16 bytes below top of stack. unsafe { - let ptr = self.top.sub(16) as *mut usize; + let ptr = self.top.sub(16).cast::(); *ptr } } @@ -182,7 +182,7 @@ impl VMContinuationStack { // See picture at top of this file: // RSP is stored 24 bytes below top of stack. unsafe { - let ptr = self.top.sub(24) as *mut usize; + let ptr = self.top.sub(24).cast::(); *ptr } } @@ -238,40 +238,41 @@ impl VMContinuationStack { unsafe { let store = |tos_neg_offset, value| { - let target = tos.sub(tos_neg_offset) as *mut usize; + let target = tos.sub(tos_neg_offset).cast::(); target.write(value) }; - let args = &mut *args; + let args_ref = &mut *args; let args_capacity = std::cmp::max(parameter_count, return_value_count); // The args object must currently be empty. - debug_assert_eq!(args.capacity, 0); - debug_assert_eq!(args.length, 0); + debug_assert_eq!(args_ref.capacity, 0); + debug_assert_eq!(args_ref.length, 0); - // The size of the args buffer - let s = args_capacity as usize * std::mem::size_of::(); - - // The actual pointer to the buffer + let args_data_size = + usize::try_from(args_capacity).unwrap() * std::mem::size_of::(); let args_data_ptr = if args_capacity == 0 { - 0 + ptr::null_mut() } else { - tos as usize - 0x20 - s + tos.sub(0x20 + args_data_size) }; - args.capacity = args_capacity; - args.data = args_data_ptr as *mut ValRaw; + args_ref.capacity = args_capacity; + args_ref.data = args_data_ptr.cast::(); let to_store = [ // Data near top of stack: (0x08, wasmtime_continuation_start as usize), - (0x10, tos.sub(0x10) as usize), - (0x18, tos.sub(0x40 + s) as usize), - (0x20, args_capacity as usize), + (0x10, tos.sub(0x10).addr()), + (0x18, tos.sub(0x40 + args_data_size).addr()), + (0x20, usize::try_from(args_capacity).unwrap()), // Data after the args buffer: - (0x28 + s, func_ref as usize), - (0x30 + s, caller_vmctx as usize), - (0x38 + s, args as *mut VMHostArray as usize), - (0x40 + s, return_value_count as usize), + (0x28 + args_data_size, func_ref.addr()), + (0x30 + args_data_size, caller_vmctx.addr()), + (0x38 + args_data_size, args.addr()), + ( + 0x40 + args_data_size, + usize::try_from(return_value_count).unwrap(), + ), ]; for (offset, data) in to_store { @@ -316,7 +317,8 @@ unsafe extern "C" fn fiber_start( let params_and_returns: NonNull<[ValRaw]> = if args.capacity == 0 { NonNull::from(&[]) } else { - std::slice::from_raw_parts_mut(args.data, args.capacity as usize).into() + std::slice::from_raw_parts_mut(args.data, usize::try_from(args.capacity).unwrap()) + .into() }; // NOTE(frank-emrich) The usage of the `caller_vmctx` is probably not From e3a3d4cfb9a077480a225d396e6a9aeeea4f16c8 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 30 May 2025 17:14:53 -0500 Subject: [PATCH 31/38] stack-switching: use usize::next_multiple_of --- crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs index f4e5db61588a..6f9c4fa6057a 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -94,7 +94,7 @@ impl VMContinuationStack { let size = if size == 0 { page_size } else { - (size + (page_size - 1)) & (!(page_size - 1)) + size.next_multiple_of(page_size) }; unsafe { From 9bab3fd0a688663765b4fa708534122a00b4149a Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 3 Jun 2025 19:54:03 +0000 Subject: [PATCH 32/38] stack-switching: update outdated comment --- crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs index 6f9c4fa6057a..0a3aaadc9cc0 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -302,8 +302,7 @@ unsafe extern "C" { } /// This function is responsible for actually running a wasm function inside a -/// continuation. It is only ever called from `wasmtime_continuation_start`. Hence, it -/// must never return. +/// continuation. It is only ever called from `wasmtime_continuation_start`. unsafe extern "C" fn fiber_start( func_ref: *const VMFuncRef, caller_vmctx: *mut VMContext, From 3f3e366aad2e475640244a1ac03a3296a2001d00 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 3 Jun 2025 19:55:51 +0000 Subject: [PATCH 33/38] stack-switching: use feature gate instead of allow(dead_code) --- crates/wasmtime/src/runtime/vm/table.rs | 2 +- crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index 9cfa186701d7..eda6d04fc875 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -171,7 +171,7 @@ pub type FuncTableElem = Option>; pub type ContTableElem = Option; /// The maximum of the sizes of any of the table element types -#[allow(dead_code, reason = "Only used if pooling allocator is enabled")] +#[cfg(feature = "pooling-allocator")] pub const NOMINAL_MAX_TABLE_ELEM_SIZE: usize = { // ContTableElem intentionally excluded for "nominal" calculation. let sizes = [ diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index 8fdf1425a678..65d5d3c1c625 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -111,7 +111,7 @@ impl Backtrace { // Walk the stack of the given continuation, which must be suspended, and // all of its parent continuations (if any). - #[allow(dead_code, reason = "Only used by GC code at the moment")] + #[cfg(feature = "gc")] pub fn trace_suspended_continuation( store: &StoreOpaque, continuation: &VMContRef, From f6dd45f58c9d8030e6ba0b52fca4f97ff4b1e037 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 3 Jun 2025 20:11:06 +0000 Subject: [PATCH 34/38] stack-switching: rework backtrace using chunks/zip --- .../src/runtime/vm/traphandlers/backtrace.rs | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index 65d5d3c1c625..e1524b49f057 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -284,27 +284,25 @@ impl Backtrace { // are continuations, due to the initial stack having one, too. assert_eq!(stack_limits_vec.len(), continuations_vec.len() + 1); - for i in 0..continuations_vec.len() { - let (continuation, parent_continuation, parent_limits) = unsafe { - // The continuation whose control context we want to - // access, to get information about how to continue - // execution in its parent. - let continuation = &*continuations_vec[i]; - - // The stack limits describing the parent of `continuation`. - let parent_limits = &*stack_limits_vec[i + 1]; - - // The parent of `continuation`, if the parent is itself a - // continuation. Otherwise, if `continuation` is the last - // continuation (i.e., its parent is the initial stack), this is - // None. - let parent_continuation = if i + 1 < continuations_vec.len() { - Some(&*continuations_vec[i + 1]) - } else { - None - }; - (continuation, parent_continuation, parent_limits) - }; + for (conts, &parent_limits) in continuations_vec + .chunks(2) + .zip(stack_limits_vec.iter().skip(1)) + { + // The continuation whose control context we want to + // access, to get information about how to continue + // execution in its parent. + let continuation = conts[0]; + let continuation = unsafe { &*continuation }; + + // The stack limits describing the parent of `continuation`. + let parent_limits = unsafe { &*parent_limits }; + + // The parent of `continuation`, if the parent is itself a + // continuation. Otherwise, if `continuation` is the last + // continuation (i.e., its parent is the initial stack), this is + // None. + let parent_continuation = conts.get(1).map(|&p| unsafe { &*p }); + let fiber_stack = continuation.fiber_stack(); let resume_pc = fiber_stack.control_context_instruction_pointer(); let resume_fp = fiber_stack.control_context_frame_pointer(); From d93132ece43e5489c1d191124dce5853f7a44a9f Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 4 Jun 2025 16:24:00 +0000 Subject: [PATCH 35/38] stack-switching: move tests to footer module This is a bit more consistent with the prevailing style in tree and (subjectively) makes finding the tests as a reader more straightforward. Tests left unchanged sans some import cleanup. --- .../src/runtime/vm/stack_switching.rs | 274 +++++++++--------- 1 file changed, 131 insertions(+), 143 deletions(-) diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 409879159a9f..39edf71ce6d5 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -71,16 +71,6 @@ impl VMContObj { unsafe impl Send for VMContObj {} unsafe impl Sync for VMContObj {} -#[test] -fn null_pointer_optimization() { - // The Rust spec does not technically guarantee that the null pointer - // optimization applies to a struct containing a `NonNull`. - assert_eq!( - core::mem::size_of::>(), - core::mem::size_of::() - ); -} - /// This type is used to save (and subsequently restore) a subset of the data in /// `VMStoreContext`. See documentation of `VMStackChain` for the exact uses. #[repr(C)] @@ -92,23 +82,6 @@ pub struct VMStackLimits { pub last_wasm_entry_fp: usize, } -#[test] -fn check_vm_stack_limits_offsets() { - use core::mem::offset_of; - use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; - - let module = Module::new(); - let offsets = VMOffsets::new(HostPtr, &module); - assert_eq!( - offset_of!(VMStackLimits, stack_limit), - usize::from(offsets.ptr.vmstack_limits_stack_limit()) - ); - assert_eq!( - offset_of!(VMStackLimits, last_wasm_entry_fp), - usize::from(offsets.ptr.vmstack_limits_last_wasm_entry_fp()) - ); -} - /// This type represents "common" information that we need to save both for the /// initial stack and each continuation. #[repr(C)] @@ -167,40 +140,6 @@ impl VMCommonStackInformation { } } -#[test] -fn check_vm_common_stack_information_offsets() { - use core::mem::offset_of; - use std::mem::size_of; - use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; - - let module = Module::new(); - let offsets = VMOffsets::new(HostPtr, &module); - assert_eq!( - size_of::(), - usize::from(offsets.ptr.size_of_vmcommon_stack_information()) - ); - assert_eq!( - offset_of!(VMCommonStackInformation, limits), - usize::from(offsets.ptr.vmcommon_stack_information_limits()) - ); - assert_eq!( - offset_of!(VMCommonStackInformation, state), - usize::from(offsets.ptr.vmcommon_stack_information_state()) - ); - assert_eq!( - offset_of!(VMCommonStackInformation, handlers), - usize::from(offsets.ptr.vmcommon_stack_information_handlers()) - ); - assert_eq!( - offset_of!(VMCommonStackInformation, first_switch_handler_index), - usize::from( - offsets - .ptr - .vmcommon_stack_information_first_switch_handler_index() - ) - ); -} - impl VMStackLimits { /// Default value, but uses the given value for `stack_limit`. pub fn with_stack_limit(stack_limit: usize) -> Self { @@ -241,34 +180,6 @@ impl VMHostArray { } } -#[test] -fn check_vm_array_offsets() { - use core::mem::offset_of; - use std::mem::size_of; - use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; - - // Note that the type parameter has no influence on the size and offsets. - - let module = Module::new(); - let offsets = VMOffsets::new(HostPtr, &module); - assert_eq!( - size_of::>(), - usize::from(offsets.ptr.size_of_vmarray()) - ); - assert_eq!( - offset_of!(VMHostArray<()>, length), - usize::from(offsets.ptr.vmarray_length()) - ); - assert_eq!( - offset_of!(VMHostArray<()>, capacity), - usize::from(offsets.ptr.vmarray_capacity()) - ); - assert_eq!( - offset_of!(VMHostArray<()>, data), - usize::from(offsets.ptr.vmarray_data()) - ); -} - /// Type used for passing payloads to and from continuations. The actual type /// argument should be wasmtime::runtime::vm::vmcontext::ValRaw, but we don't /// have access to that here. @@ -386,47 +297,6 @@ impl Drop for VMContRef { unsafe impl Send for VMContRef {} unsafe impl Sync for VMContRef {} -#[test] -fn check_vm_contref_offsets() { - use core::mem::offset_of; - use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; - - let module = Module::new(); - let offsets = VMOffsets::new(HostPtr, &module); - assert_eq!( - offset_of!(VMContRef, common_stack_information), - usize::from(offsets.ptr.vmcontref_common_stack_information()) - ); - assert_eq!( - offset_of!(VMContRef, parent_chain), - usize::from(offsets.ptr.vmcontref_parent_chain()) - ); - assert_eq!( - offset_of!(VMContRef, last_ancestor), - usize::from(offsets.ptr.vmcontref_last_ancestor()) - ); - // Some 32-bit platforms need this to be 8-byte aligned, some don't. - // So we need to make sure it always is, without padding. - assert_eq!(u8::vmcontref_revision(&4) % 8, 0); - assert_eq!(u8::vmcontref_revision(&8) % 8, 0); - assert_eq!( - offset_of!(VMContRef, revision), - usize::from(offsets.ptr.vmcontref_revision()) - ); - assert_eq!( - offset_of!(VMContRef, stack), - usize::from(offsets.ptr.vmcontref_stack()) - ); - assert_eq!( - offset_of!(VMContRef, args), - usize::from(offsets.ptr.vmcontref_args()) - ); - assert_eq!( - offset_of!(VMContRef, values), - usize::from(offsets.ptr.vmcontref_values()) - ); -} - /// Implements `cont.new` instructions (i.e., creation of continuations). #[cfg(feature = "stack-switching")] #[inline(always)] @@ -608,19 +478,6 @@ impl VMStackChain { } } -#[test] -fn check_vm_stack_chain_offsets() { - use std::mem::size_of; - use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; - - let module = Module::new(); - let offsets = VMOffsets::new(HostPtr, &module); - assert_eq!( - size_of::(), - usize::from(offsets.ptr.size_of_vmstack_chain()) - ); -} - /// Iterator for Continuations in a stack chain. pub struct ContinuationIterator(VMStackChain); @@ -709,3 +566,134 @@ pub enum ControlEffect { /// Used to signal that a continuation has invoked a `suspend` instruction. Switch = wasmtime_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT, } + +#[cfg(test)] +mod tests { + use core::mem::{offset_of, size_of}; + + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + use super::*; + + #[test] + fn null_pointer_optimization() { + // The Rust spec does not technically guarantee that the null pointer + // optimization applies to a struct containing a `NonNull`. + assert_eq!(size_of::>(), size_of::()); + } + + #[test] + fn check_vm_stack_limits_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMStackLimits, stack_limit), + usize::from(offsets.ptr.vmstack_limits_stack_limit()) + ); + assert_eq!( + offset_of!(VMStackLimits, last_wasm_entry_fp), + usize::from(offsets.ptr.vmstack_limits_last_wasm_entry_fp()) + ); + } + + #[test] + fn check_vm_common_stack_information_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmcommon_stack_information()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, limits), + usize::from(offsets.ptr.vmcommon_stack_information_limits()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, state), + usize::from(offsets.ptr.vmcommon_stack_information_state()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, handlers), + usize::from(offsets.ptr.vmcommon_stack_information_handlers()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, first_switch_handler_index), + usize::from( + offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() + ) + ); + } + + #[test] + fn check_vm_array_offsets() { + // Note that the type parameter has no influence on the size and offsets. + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::>(), + usize::from(offsets.ptr.size_of_vmarray()) + ); + assert_eq!( + offset_of!(VMHostArray<()>, length), + usize::from(offsets.ptr.vmarray_length()) + ); + assert_eq!( + offset_of!(VMHostArray<()>, capacity), + usize::from(offsets.ptr.vmarray_capacity()) + ); + assert_eq!( + offset_of!(VMHostArray<()>, data), + usize::from(offsets.ptr.vmarray_data()) + ); + } + + #[test] + fn check_vm_contref_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMContRef, common_stack_information), + usize::from(offsets.ptr.vmcontref_common_stack_information()) + ); + assert_eq!( + offset_of!(VMContRef, parent_chain), + usize::from(offsets.ptr.vmcontref_parent_chain()) + ); + assert_eq!( + offset_of!(VMContRef, last_ancestor), + usize::from(offsets.ptr.vmcontref_last_ancestor()) + ); + // Some 32-bit platforms need this to be 8-byte aligned, some don't. + // So we need to make sure it always is, without padding. + assert_eq!(u8::vmcontref_revision(&4) % 8, 0); + assert_eq!(u8::vmcontref_revision(&8) % 8, 0); + assert_eq!( + offset_of!(VMContRef, revision), + usize::from(offsets.ptr.vmcontref_revision()) + ); + assert_eq!( + offset_of!(VMContRef, stack), + usize::from(offsets.ptr.vmcontref_stack()) + ); + assert_eq!( + offset_of!(VMContRef, args), + usize::from(offsets.ptr.vmcontref_args()) + ); + assert_eq!( + offset_of!(VMContRef, values), + usize::from(offsets.ptr.vmcontref_values()) + ); + } + + #[test] + fn check_vm_stack_chain_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmstack_chain()) + ); + } +} From ce4e0889c4b69215da9533daf8bd68fe7145c3f8 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 4 Jun 2025 16:25:52 +0000 Subject: [PATCH 36/38] stack-swictchding: verify stack_chain offsets at runtime --- crates/wasmtime/src/runtime/vm/vmcontext.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index bbc17400ff98..0a0f07a90b7d 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -1190,6 +1190,10 @@ mod test_vmstore_context { offset_of!(VMStoreContext, last_wasm_entry_fp), usize::from(offsets.ptr.vmstore_context_last_wasm_entry_fp()) ); + assert_eq!( + offset_of!(VMStoreContext, stack_chain), + usize::from(offsets.ptr.vmstore_context_stack_chain()) + ) } } From d6a3ae31df01015ab6744acf04a82be7a6e2c3b6 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 4 Jun 2025 16:42:56 +0000 Subject: [PATCH 37/38] fixup! stack-switching: use feature gate instead of allow(dead_code) --- crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index e1524b49f057..84acb28ae2a2 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -28,6 +28,7 @@ use crate::runtime::vm::{ Unwind, VMStoreContext, traphandlers::{CallThreadState, tls}, }; +#[cfg(all(feature = "gc", feature = "stack-switching"))] use crate::vm::stack_switching::{VMContRef, VMStackState}; use core::ops::ControlFlow; @@ -111,7 +112,7 @@ impl Backtrace { // Walk the stack of the given continuation, which must be suspended, and // all of its parent continuations (if any). - #[cfg(feature = "gc")] + #[cfg(all(feature = "gc", feature = "stack-switching"))] pub fn trace_suspended_continuation( store: &StoreOpaque, continuation: &VMContRef, From 60c9cdb707fc674857fa549a8459f8c602be0c82 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 4 Jun 2025 19:13:35 +0000 Subject: [PATCH 38/38] stack-switching: document continuation roots tracing using match arms --- crates/wasmtime/src/runtime/store.rs | 35 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 6c6ac9214cf1..260c698acb48 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -1604,23 +1604,24 @@ impl StoreOpaque { // buffers used by `cont.bind` are GC values. As a workaround, note // that we currently disallow cont.bind-ing GC values altogether. // This way, it is okay not to check them here. - - // Note that we only care about continuations that have state - // `Suspended`. - // - `Running` continuations will be handled by - // `trace_wasm_stack_roots`. - // - For `Parent` continuations, we don't know if they are the - // parent of a running continuation or a suspended one. But it - // does not matter: They will be handled when traversing the stack - // chain starting at either the running one, or the suspended - // continuations below. - // - For `Fresh` continuations, we know that there are no GC values - // on their stack, yet. - if state == VMStackState::Suspended { - Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| { - self.trace_wasm_stack_frame(gc_roots_list, frame); - core::ops::ControlFlow::Continue(()) - }); + match state { + VMStackState::Suspended => { + Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| { + self.trace_wasm_stack_frame(gc_roots_list, frame); + core::ops::ControlFlow::Continue(()) + }); + } + VMStackState::Running => { + // Handled by `trace_wasm_stack_roots`. + } + VMStackState::Parent => { + // We don't know whether our child is suspended or running, but in + // either case things should be hanlded correctly when traversing + // further along in the chain, nothing required at this point. + } + VMStackState::Fresh | VMStackState::Returned => { + // Fresh/Returned continuations have no gc values on their stack. + } } }