diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c1d9b52fe03..16ef0d61c2eb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -371,6 +371,9 @@ jobs: -p wasmtime --no-default-features --features threads -p wasmtime --no-default-features --features runtime,threads -p wasmtime --no-default-features --features cranelift,threads + -p wasmtime --no-default-features --features stack-switching + -p wasmtime --no-default-features --features cranelift,stack-switching + -p wasmtime --no-default-features --features runtime,stack-switching -p wasmtime --features incremental-cache -p wasmtime --features profile-pulley -p wasmtime --all-features diff --git a/Cargo.toml b/Cargo.toml index fd01b1e565b9..20ff33d7ab9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -432,6 +432,7 @@ default = [ "gc", "gc-drc", "gc-null", + "stack-switching", "winch", "pulley", @@ -490,6 +491,7 @@ gc = ["wasmtime-cli-flags/gc", "wasmtime/gc"] gc-drc = ["gc", "wasmtime/gc-drc", "wasmtime-cli-flags/gc-drc"] gc-null = ["gc", "wasmtime/gc-null", "wasmtime-cli-flags/gc-null"] pulley = ["wasmtime-cli-flags/pulley"] +stack-switching = ["wasmtime/stack-switching", "wasmtime-cli-flags/stack-switching"] # CLI subcommands for the `wasmtime` executable. See `wasmtime $cmd --help` # for more information on each subcommand. diff --git a/crates/c-api/include/wasmtime/config.h b/crates/c-api/include/wasmtime/config.h index f67cde471c32..482f3c1b7ce1 100644 --- a/crates/c-api/include/wasmtime/config.h +++ b/crates/c-api/include/wasmtime/config.h @@ -250,6 +250,14 @@ WASMTIME_CONFIG_PROP(void, wasm_wide_arithmetic, bool) #ifdef WASMTIME_FEATURE_COMPILER +/** + * \brief Configures whether the WebAssembly stack switching + * proposal is enabled. + * + * This setting is `false` by default. + */ +WASMTIME_CONFIG_PROP(void, wasm_stack_switching, bool) + /** * \brief Configures how JIT code will be compiled. * diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rs index 4ee308464151..3113ab1627b9 100644 --- a/crates/c-api/src/config.rs +++ b/crates/c-api/src/config.rs @@ -140,6 +140,11 @@ pub extern "C" fn wasmtime_config_wasm_memory64_set(c: &mut wasm_config_t, enabl c.config.wasm_memory64(enable); } +#[unsafe(no_mangle)] +pub extern "C" fn wasmtime_config_wasm_stack_switching_set(c: &mut wasm_config_t, enable: bool) { + c.config.wasm_stack_switching(enable); +} + #[unsafe(no_mangle)] #[cfg(any(feature = "cranelift", feature = "winch"))] pub extern "C" fn wasmtime_config_strategy_set( diff --git a/crates/cli-flags/Cargo.toml b/crates/cli-flags/Cargo.toml index 23dc34036e85..1469c9535b15 100644 --- a/crates/cli-flags/Cargo.toml +++ b/crates/cli-flags/Cargo.toml @@ -39,3 +39,4 @@ gc-null = ["gc", "wasmtime/gc-null"] threads = ["wasmtime/threads"] memory-protection-keys = ["wasmtime/memory-protection-keys"] pulley = ["wasmtime/pulley"] +stack-switching = ["wasmtime/stack-switching"] diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs index 1ad80292a656..9fbe01d5281c 100644 --- a/crates/cli-flags/src/lib.rs +++ b/crates/cli-flags/src/lib.rs @@ -377,6 +377,8 @@ wasmtime_option_group! { pub component_model_error_context: Option, /// Configure support for the function-references proposal. pub function_references: Option, + /// Configure support for the stack-switching proposal. + pub stack_switching: Option, /// Configure support for the GC proposal. pub gc: Option, /// Configure support for the custom-page-sizes proposal. @@ -818,6 +820,23 @@ impl CommonOptions { config.native_unwind_info(enable); } + // async_stack_size enabled by either async or stack-switching, so + // cannot directly use match_feature! + #[cfg(any(feature = "async", feature = "stack-switching"))] + { + if let Some(size) = self.wasm.async_stack_size { + config.async_stack_size(size); + } + } + #[cfg(not(any(feature = "async", feature = "stack-switching")))] + { + if let Some(_size) = self.wasm.async_stack_size { + anyhow::bail!(concat!( + "support for async/stack-switching disabled at compile time" + )); + } + } + match_feature! { ["pooling-allocator" : self.opts.pooling_allocator.or(pooling_allocator_default)] enable => { @@ -923,6 +942,8 @@ impl CommonOptions { ); } + #[cfg(any(feature = "async", feature = "stack-switching"))] + match_feature! { ["async" : self.wasm.async_stack_size] size => config.async_stack_size(size), @@ -940,7 +961,7 @@ impl CommonOptions { // If `-Wasync-stack-size` isn't passed then automatically adjust it // to the wasm stack size provided here too. That prevents the need // to pass both when one can generally be inferred from the other. - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] if self.wasm.async_stack_size.is_none() { const DEFAULT_HOST_STACK: usize = 512 << 10; config.async_stack_size(max + DEFAULT_HOST_STACK); @@ -983,6 +1004,9 @@ impl CommonOptions { if let Some(enable) = self.wasm.memory64.or(all) { config.wasm_memory64(enable); } + if let Some(enable) = self.wasm.stack_switching { + config.wasm_stack_switching(enable); + } if let Some(enable) = self.wasm.custom_page_sizes.or(all) { config.wasm_custom_page_sizes(enable); } @@ -1023,6 +1047,7 @@ impl CommonOptions { ("gc", gc, wasm_gc) ("gc", reference_types, wasm_reference_types) ("gc", function_references, wasm_function_references) + ("stack-switching", stack_switching, wasm_stack_switching) } Ok(()) } diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index c4ef5cbd2d14..8321f55ff087 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -45,4 +45,5 @@ wmemcheck = ["wasmtime-environ/wmemcheck"] gc = ["wasmtime-environ/gc"] gc-drc = ["gc", "wasmtime-environ/gc-drc"] gc-null = ["gc", "wasmtime-environ/gc-null"] +stack-switching = [] threads = ["wasmtime-environ/threads"] diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 3e1920c5f1f4..dc1ab3e0d668 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -3809,3 +3809,17 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type { IndexType::I64 => I64, } } + +/// TODO(10248) This is removed in the next stack switching PR. It stops the +/// compiler from complaining about the stack switching libcalls being dead +/// code. +#[cfg(feature = "stack-switching")] +#[allow( + dead_code, + reason = "Dummy function to supress more dead code warnings" +)] +pub fn use_stack_switching_libcalls() { + let _ = BuiltinFunctions::cont_new; + let _ = BuiltinFunctions::table_grow_cont_obj; + let _ = BuiltinFunctions::table_fill_cont_obj; +} diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs index a3ef83309796..eadcacbf90e8 100644 --- a/crates/cranelift/src/func_environ/gc/enabled.rs +++ b/crates/cranelift/src/func_environ/gc/enabled.rs @@ -153,7 +153,12 @@ fn read_field_at_addr( .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]); builder.func.dfg.first_result(call_inst) } - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + // TODO(#10248) GC integration for stack switching + return Err(wasmtime_environ::WasmError::Unsupported( + "Stack switching feature not compatbile with GC, yet".to_string(), + )); + } }, }, }; @@ -1032,6 +1037,8 @@ pub fn translate_ref_test( | WasmHeapType::NoExtern | WasmHeapType::Func | WasmHeapType::NoFunc + | WasmHeapType::Cont + | WasmHeapType::NoCont | WasmHeapType::I31 => unreachable!("handled top, bottom, and i31 types above"), // For these abstract but non-top and non-bottom types, we check the @@ -1086,8 +1093,12 @@ pub fn translate_ref_test( func_env.is_subtype(builder, actual_shared_ty, expected_shared_ty) } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::ConcreteCont(_) => { + // TODO(#10248) GC integration for stack switching + return Err(wasmtime_environ::WasmError::Unsupported( + "Stack switching feature not compatbile with GC, yet".to_string(), + )); + } }; builder.ins().jump(continue_block, &[result.into()]); @@ -1409,8 +1420,9 @@ impl FuncEnvironment<'_> { WasmHeapType::Func | WasmHeapType::ConcreteFunc(_) | WasmHeapType::NoFunc => { unreachable!() } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => { + unreachable!() + } }; match (ty.nullable, might_be_i31) { diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index d05344f88171..8ad33365f094 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -61,6 +61,10 @@ pub const TRAP_HEAP_MISALIGNED: TrapCode = TrapCode::unwrap_user(Trap::HeapMisaligned as u8 + TRAP_OFFSET); pub const TRAP_TABLE_OUT_OF_BOUNDS: TrapCode = TrapCode::unwrap_user(Trap::TableOutOfBounds as u8 + TRAP_OFFSET); +pub const TRAP_UNHANDLED_TAG: TrapCode = + TrapCode::unwrap_user(Trap::UnhandledTag as u8 + TRAP_OFFSET); +pub const TRAP_CONTINUATION_ALREADY_CONSUMED: TrapCode = + TrapCode::unwrap_user(Trap::ContinuationAlreadyConsumed as u8 + TRAP_OFFSET); pub const TRAP_CAST_FAILURE: TrapCode = TrapCode::unwrap_user(Trap::CastFailure as u8 + TRAP_OFFSET); @@ -202,7 +206,11 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type { match wasm_ht.top() { WasmHeapTopType::Func => pointer_type, WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => + // TODO(10248) This is added in a follow-up PR + { + unimplemented!("codegen for stack switching types not implemented, yet") + } } } diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index d106113ad7f9..01bdd307b053 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -2898,6 +2898,56 @@ pub fn translate_operator( // representation, so we don't actually need to do anything. } + Operator::ContNew { cont_type_index: _ } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::ContBind { + argument_index: _, + result_index: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::Suspend { tag_index: _ } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::Resume { + cont_type_index: _, + resume_table: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::ResumeThrow { + cont_type_index: _, + tag_index: _, + resume_table: _, + } => { + // TODO(10248) This depends on exception handling + return Err(wasmtime_environ::WasmError::Unsupported( + "resume.throw instructions not supported, yet".to_string(), + )); + } + Operator::Switch { + cont_type_index: _, + tag_index: _, + } => { + // TODO(10248) This is added in a follow-up PR + return Err(wasmtime_environ::WasmError::Unsupported( + "codegen for stack switching instructions not implemented, yet".to_string(), + )); + } + Operator::GlobalAtomicGet { .. } | Operator::GlobalAtomicSet { .. } | Operator::GlobalAtomicRmwAdd { .. } @@ -2939,17 +2989,6 @@ pub fn translate_operator( )); } - Operator::ContNew { .. } - | Operator::ContBind { .. } - | Operator::Suspend { .. } - | Operator::Resume { .. } - | Operator::ResumeThrow { .. } - | Operator::Switch { .. } => { - return Err(wasm_unsupported!( - "stack-switching operators are not yet implemented" - )); - } - Operator::I64MulWideS => { let (arg1, arg2) = state.pop2(); let arg1 = builder.ins().sextend(I128, arg1); diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index e85aef15b347..ffdf9570d20c 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -69,6 +69,7 @@ compile = [ "dep:wasm-encoder", "dep:wasmprinter", ] +stack-switching = [] threads = ['std'] wmemcheck = ['std'] std = [ diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index 583c7560fedd..5285f29ece8c 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -226,6 +226,25 @@ macro_rules! foreach_builtin_function { // Raises an unconditional trap where the trap information must have // been previously filled in. raise(vmctx: vmctx); + + // Creates a new continuation from a funcref. + #[cfg(feature = "stack-switching")] + cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer; + + // Returns an index for Wasm's `table.grow` instruction + // for `contobj`s. Note that the initial + // Option (i.e., the value to fill the new + // slots with) is split into two arguments: The underlying + // continuation reference and the revision count. To + // denote the continuation being `None`, `init_contref` + // may be 0. + #[cfg(feature = "stack-switching")] + table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: u64) -> pointer; + + // `value_contref` and `value_revision` together encode + // the Option, as in previous libcall. + #[cfg(feature = "stack-switching")] + table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: u64, len: u64) -> bool; } }; } @@ -367,6 +386,7 @@ impl BuiltinFunctionIndex { (@get memory32_grow pointer) => (TrapSentinel::NegativeTwo); (@get table_grow_func_ref pointer) => (TrapSentinel::NegativeTwo); (@get table_grow_gc_ref pointer) => (TrapSentinel::NegativeTwo); + (@get table_grow_cont_obj pointer) => (TrapSentinel::NegativeTwo); // Atomics-related functions return a negative value indicating trap // indicate a trap. @@ -406,6 +426,8 @@ impl BuiltinFunctionIndex { (@get fma_f32x4 f32x4) => (return None); (@get fma_f64x2 f64x2) => (return None); + (@get cont_new pointer) => (TrapSentinel::Negative); + // Bool-returning functions use `false` as an indicator of a trap. (@get $name:ident bool) => (TrapSentinel::Falsy); diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs index 05918524f421..556071b24dcf 100644 --- a/crates/environ/src/gc.rs +++ b/crates/environ/src/gc.rs @@ -40,10 +40,15 @@ pub const VM_GC_HEADER_TYPE_INDEX_OFFSET: u32 = 4; /// Get the byte size of the given Wasm type when it is stored inside the GC /// heap. pub fn byte_size_of_wasm_ty_in_gc_heap(ty: &WasmStorageType) -> u32 { + use crate::{WasmHeapType::*, WasmRefType}; match ty { WasmStorageType::I8 => 1, WasmStorageType::I16 => 2, WasmStorageType::Val(ty) => match ty { + WasmValType::Ref(WasmRefType { + nullable: _, + heap_type: ConcreteCont(_) | Cont, + }) => unimplemented!("Stack switching feature not compatbile with GC, yet"), WasmValType::I32 | WasmValType::F32 | WasmValType::Ref(_) => 4, WasmValType::I64 | WasmValType::F64 => 8, WasmValType::V128 => 16, @@ -178,7 +183,9 @@ pub trait GcTypeLayouts { WasmCompositeInnerType::Array(ty) => Some(self.array_layout(ty).into()), WasmCompositeInnerType::Struct(ty) => Some(self.struct_layout(ty).into()), WasmCompositeInnerType::Func(_) => None, - WasmCompositeInnerType::Cont(_) => None, + WasmCompositeInnerType::Cont(_) => { + unimplemented!("Stack switching feature not compatbile with GC, yet") + } } } diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 66149fcedbbf..43501dd940d7 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -33,6 +33,7 @@ pub mod obj; mod ref_bits; mod scopevec; mod stack_map; +mod stack_switching; mod trap_encoding; mod tunables; mod types; @@ -51,6 +52,7 @@ pub use crate::module_types::*; pub use crate::ref_bits::*; pub use crate::scopevec::ScopeVec; pub use crate::stack_map::*; +pub use crate::stack_switching::*; pub use crate::trap_encoding::*; pub use crate::tunables::*; pub use crate::types::*; diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs new file mode 100644 index 000000000000..6c6be122449c --- /dev/null +++ b/crates/environ/src/stack_switching.rs @@ -0,0 +1,41 @@ +//! This module contains basic type definitions used by the implementation of +//! the stack switching proposal. + +/// Discriminant of variant `Absent` in +/// `wasmtime::runtime::vm::VMStackChain`. +pub const STACK_CHAIN_ABSENT_DISCRIMINANT: usize = 0; +/// Discriminant of variant `InitialStack` in +/// `wasmtime::runtime::vm::VMStackChain`. +pub const STACK_CHAIN_INITIAL_STACK_DISCRIMINANT: usize = 1; +/// Discriminant of variant `Continiation` in +/// `wasmtime::runtime::vm::VMStackChain`. +pub const STACK_CHAIN_CONTINUATION_DISCRIMINANT: usize = 2; + +/// Discriminant of variant `Fresh` in +/// `runtime::vm::VMStackState`. +pub const STACK_STATE_FRESH_DISCRIMINANT: u32 = 0; +/// Discriminant of variant `Running` in +/// `runtime::vm::VMStackState`. +pub const STACK_STATE_RUNNING_DISCRIMINANT: u32 = 1; +/// Discriminant of variant `Parent` in +/// `runtime::vm::VMStackState`. +pub const STACK_STATE_PARENT_DISCRIMINANT: u32 = 2; +/// Discriminant of variant `Suspended` in +/// `runtime::vm::VMStackState`. +pub const STACK_STATE_SUSPENDED_DISCRIMINANT: u32 = 3; +/// Discriminant of variant `Returned` in +/// `runtime::vm::VMStackState`. +pub const STACK_STATE_RETURNED_DISCRIMINANT: u32 = 4; + +/// Discriminant of variant `Return` in +/// `runtime::vm::ControlEffect`. +pub const CONTROL_EFFECT_RETURN_DISCRIMINANT: u32 = 0; +/// Discriminant of variant `Resume` in +/// `runtime::vm::ControlEffect`. +pub const CONTROL_EFFECT_RESUME_DISCRIMINANT: u32 = 1; +/// Discriminant of variant `Suspend` in +/// `runtime::vm::ControlEffect`. +pub const CONTROL_EFFECT_SUSPEND_DISCRIMINANT: u32 = 2; +/// Discriminant of variant `Switch` in +/// `runtime::vm::ControlEffect`. +pub const CONTROL_EFFECT_SWITCH_DISCRIMINANT: u32 = 3; diff --git a/crates/environ/src/trap_encoding.rs b/crates/environ/src/trap_encoding.rs index e415fd5d77c6..3c8fb354d8ae 100644 --- a/crates/environ/src/trap_encoding.rs +++ b/crates/environ/src/trap_encoding.rs @@ -93,6 +93,16 @@ pub enum Trap { /// before returning `STATUS_DONE` and/or after all host tasks completed. NoAsyncResult, + /// We are suspending to a tag for which there is no active handler. + UnhandledTag, + + /// Attempt to resume a continuation twice. + ContinuationAlreadyConsumed, + + /// FIXME(frank-emrich) Only used for stack switching debugging code, to be + /// removed from final upstreamed code. + DeleteMeDebugAssertion, + /// A Pulley opcode was executed at runtime when the opcode was disabled at /// compile time. DisabledOpcode, @@ -133,6 +143,9 @@ impl Trap { CastFailure CannotEnterComponent NoAsyncResult + UnhandledTag + ContinuationAlreadyConsumed + DeleteMeDebugAssertion DisabledOpcode } @@ -165,6 +178,9 @@ impl fmt::Display for Trap { CastFailure => "cast failure", CannotEnterComponent => "cannot enter component instance", NoAsyncResult => "async-lifted export failed to produce a result", + UnhandledTag => "unhandled tag", + ContinuationAlreadyConsumed => "continuation already consumed", + DeleteMeDebugAssertion => "triggered debug assertion", DisabledOpcode => "pulley opcode disabled at compile time was executed", }; write!(f, "wasm trap: {desc}") diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rs index c47da9fb8c0f..bfe669e17a17 100644 --- a/crates/environ/src/types.rs +++ b/crates/environ/src/types.rs @@ -232,6 +232,16 @@ impl WasmValType { size => panic!("invalid int bits for WasmValType: {size}"), } } + + /// Returns the contained reference type. + /// + /// Panics if the value type is not a vmgcref + pub fn unwrap_ref_type(&self) -> WasmRefType { + match self { + WasmValType::Ref(ref_type) => *ref_type, + _ => panic!("Called WasmValType::unwrap_ref_type on non-reference type"), + } + } } /// WebAssembly reference type -- equivalent of `wasmparser`'s RefType @@ -801,6 +811,15 @@ impl WasmContType { pub fn new(idx: EngineOrModuleTypeIndex) -> Self { WasmContType(idx) } + + /// Returns the (module interned) index to the underlying function type. + pub fn unwrap_module_type_index(self) -> ModuleInternedTypeIndex { + match self.0 { + EngineOrModuleTypeIndex::Engine(_) => panic!("not module interned"), + EngineOrModuleTypeIndex::Module(idx) => idx, + EngineOrModuleTypeIndex::RecGroup(_) => todo!(), + } + } } impl TypeTrace for WasmContType { @@ -2239,11 +2258,9 @@ pub trait TypeConvert { wasmparser::AbstractHeapType::Array => WasmHeapType::Array, wasmparser::AbstractHeapType::Struct => WasmHeapType::Struct, wasmparser::AbstractHeapType::None => WasmHeapType::None, - - wasmparser::AbstractHeapType::Exn - | wasmparser::AbstractHeapType::NoExn - | wasmparser::AbstractHeapType::Cont - | wasmparser::AbstractHeapType::NoCont => { + wasmparser::AbstractHeapType::Cont => WasmHeapType::Cont, + wasmparser::AbstractHeapType::NoCont => WasmHeapType::NoCont, + wasmparser::AbstractHeapType::Exn | wasmparser::AbstractHeapType::NoExn => { return Err(wasm_unsupported!("unsupported heap type {ty:?}")); } }, diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 4278ac2e0f43..700eab3bf6d4 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -217,6 +217,11 @@ pub trait PtrSize { self.vmstore_context_last_wasm_exit_pc() + self.size() } + /// Return the offset of the `stack_chain` field of `VMStoreContext`. + fn vmstore_context_stack_chain(&self) -> u8 { + self.vmstore_context_last_wasm_entry_fp() + self.size() + } + // Offsets within `VMMemoryDefinition` /// The offset of the `base` field. @@ -254,6 +259,122 @@ pub trait PtrSize { .unwrap() } + /// Return the size of `VMStackChain`. + fn size_of_vmstack_chain(&self) -> u8 { + 2 * self.size() + } + + // Offsets within `VMStackLimits` + + /// Return the offset of `VMStackLimits::stack_limit`. + fn vmstack_limits_stack_limit(&self) -> u8 { + 0 + } + + /// Return the offset of `VMStackLimits::last_wasm_entry_fp`. + fn vmstack_limits_last_wasm_entry_fp(&self) -> u8 { + self.size() + } + + // Offsets within `VMArray` + + /// Return the offset of `VMArray::length`. + fn vmarray_length(&self) -> u8 { + 0 + } + + /// Return the offset of `VMArray::capacity`. + fn vmarray_capacity(&self) -> u8 { + 4 + } + + /// Return the offset of `VMArray::data`. + fn vmarray_data(&self) -> u8 { + 8 + } + + /// Return the size of `VMArray`. + fn size_of_vmarray(&self) -> u8 { + 8 + self.size() + } + + // Offsets within `VMCommonStackInformation` + + /// Return the offset of `VMCommonStackInformation::limits`. + fn vmcommon_stack_information_limits(&self) -> u8 { + 0 * self.size() + } + + /// Return the offset of `VMCommonStackInformation::state`. + fn vmcommon_stack_information_state(&self) -> u8 { + 2 * self.size() + } + + /// Return the offset of `VMCommonStackInformation::handlers`. + fn vmcommon_stack_information_handlers(&self) -> u8 { + u8::try_from(align( + self.vmcommon_stack_information_state() as u32 + 4, + u32::from(self.size()), + )) + .unwrap() + } + + /// Return the offset of `VMCommonStackInformation::first_switch_handler_index`. + fn vmcommon_stack_information_first_switch_handler_index(&self) -> u8 { + self.vmcommon_stack_information_handlers() + self.size_of_vmarray() + } + + /// Return the size of `VMCommonStackInformation`. + fn size_of_vmcommon_stack_information(&self) -> u8 { + u8::try_from(align( + self.vmcommon_stack_information_first_switch_handler_index() as u32 + 4, + u32::from(self.size()), + )) + .unwrap() + } + + // Offsets within `VMContRef` + + /// Return the offset of `VMContRef::common_stack_information`. + fn vmcontref_common_stack_information(&self) -> u8 { + 0 * self.size() + } + + /// Return the offset of `VMContRef::parent_chain`. + fn vmcontref_parent_chain(&self) -> u8 { + u8::try_from(align( + (self.vmcontref_common_stack_information() + self.size_of_vmcommon_stack_information()) + as u32, + u32::from(self.size()), + )) + .unwrap() + } + + /// Return the offset of `VMContRef::last_ancestor`. + fn vmcontref_last_ancestor(&self) -> u8 { + self.vmcontref_parent_chain() + 2 * self.size() + } + + /// Return the offset of `VMContRef::revision`. + fn vmcontref_revision(&self) -> u8 { + self.vmcontref_last_ancestor() + self.size() + } + + /// Return the offset of `VMContRef::stack`. + fn vmcontref_stack(&self) -> u8 { + self.vmcontref_revision() + 8 + } + + /// Return the offset of `VMContRef::args`. + fn vmcontref_args(&self) -> u8 { + self.vmcontref_stack() + 3 * self.size() + } + + /// Return the offset of `VMContRef::values`. + fn vmcontref_values(&self) -> u8 { + self.vmcontref_args() + self.size_of_vmarray() + } + /// Return the offset to the `magic` value in this `VMContext`. #[inline] fn vmctx_magic(&self) -> u8 { diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs index debfc9146fc1..72b004561179 100644 --- a/crates/fuzzing/src/generators/config.rs +++ b/crates/fuzzing/src/generators/config.rs @@ -150,6 +150,7 @@ impl Config { hogs_memory: _, nan_canonicalization: _, gc_types: _, + stack_switching: _, } = test.config; // Enable/disable some proposals that aren't configurable in wasm-smith diff --git a/crates/test-util/src/wasmtime_wast.rs b/crates/test-util/src/wasmtime_wast.rs index da1223b8b3d1..db5d9bc770d8 100644 --- a/crates/test-util/src/wasmtime_wast.rs +++ b/crates/test-util/src/wasmtime_wast.rs @@ -45,6 +45,7 @@ pub fn apply_test_config(config: &mut Config, test_config: &wast::TestConfig) { simd, exceptions, legacy_exceptions, + stack_switching, hogs_memory: _, gc_types: _, @@ -67,8 +68,8 @@ pub fn apply_test_config(config: &mut Config, test_config: &wast::TestConfig) { let component_model_error_context = component_model_error_context.unwrap_or(false); let nan_canonicalization = nan_canonicalization.unwrap_or(false); let relaxed_simd = relaxed_simd.unwrap_or(false); - let exceptions = exceptions.unwrap_or(false); let legacy_exceptions = legacy_exceptions.unwrap_or(false); + let stack_switching = stack_switching.unwrap_or(false); // Some proposals in wasm depend on previous proposals. For example the gc // proposal depends on function-references which depends on reference-types. @@ -79,6 +80,8 @@ pub fn apply_test_config(config: &mut Config, test_config: &wast::TestConfig) { let reference_types = function_references || reference_types.unwrap_or(false); let simd = relaxed_simd || simd.unwrap_or(false); + let exceptions = stack_switching || exceptions.unwrap_or(false); + config .wasm_multi_memory(multi_memory) .wasm_threads(threads) @@ -98,6 +101,7 @@ pub fn apply_test_config(config: &mut Config, test_config: &wast::TestConfig) { .wasm_component_model_async_stackful(component_model_async_stackful) .wasm_component_model_error_context(component_model_error_context) .wasm_exceptions(exceptions) + .wasm_stack_switching(stack_switching) .cranelift_nan_canonicalization(nan_canonicalization); #[expect(deprecated, reason = "forwarding legacy-exceptions")] config.wasm_legacy_exceptions(legacy_exceptions); diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs index f7668be4ceeb..7477552e5db8 100644 --- a/crates/test-util/src/wast.rs +++ b/crates/test-util/src/wast.rs @@ -248,6 +248,7 @@ macro_rules! foreach_config_option { gc_types exceptions legacy_exceptions + stack_switching } }; } @@ -349,6 +350,8 @@ impl Compiler { || config.relaxed_simd() || config.gc_types() || config.exceptions() + || config.legacy_exceptions() + || config.stack_switching() || config.legacy_exceptions(); if cfg!(target_arch = "x86_64") { @@ -365,7 +368,9 @@ impl Compiler { false } - Compiler::CraneliftPulley => config.threads() || config.legacy_exceptions(), + Compiler::CraneliftPulley => { + config.threads() || config.legacy_exceptions() || config.stack_switching() + } } } diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index e7ef39593f33..c49fc4aa8dcb 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -93,7 +93,7 @@ memfd = { workspace = true, optional = true } mach2 = { workspace = true, optional = true } [target.'cfg(unix)'.dependencies] -rustix = { workspace = true, optional = true } +rustix = { workspace = true, optional = true, features = ["mm", "param"] } [target.'cfg(target_arch = "s390x")'.dependencies] psm = { workspace = true, optional = true } @@ -144,6 +144,7 @@ default = [ 'runtime', 'component-model', 'threads', + 'stack-switching', 'std', ] @@ -309,6 +310,13 @@ threads = [ "std", ] +stack-switching = [ + "runtime", + "wasmtime-environ/stack-switching", + "wasmtime-cranelift?/stack-switching", + "wasmtime-winch?/stack-switching", +] + # Controls whether backtraces will attempt to parse DWARF information in # WebAssembly modules and components to provide filenames and line numbers in # stack traces. diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 69237266b69f..6e59ac7da865 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -147,7 +147,7 @@ pub struct Config { pub(crate) wasm_backtrace: bool, pub(crate) wasm_backtrace_details_env_used: bool, pub(crate) native_unwind_info: Option, - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] pub(crate) async_stack_size: usize, #[cfg(feature = "async")] pub(crate) async_stack_zeroing: bool, @@ -252,7 +252,7 @@ impl Config { native_unwind_info: None, enabled_features: WasmFeatures::empty(), disabled_features: WasmFeatures::empty(), - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] async_stack_size: 2 << 20, #[cfg(feature = "async")] async_stack_zeroing: false, @@ -736,7 +736,7 @@ impl Config { /// /// The `Engine::new` method will fail if the value for this option is /// smaller than the [`Config::max_wasm_stack`] option. - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] pub fn async_stack_size(&mut self, size: usize) -> &mut Self { self.async_stack_size = size; self @@ -2040,10 +2040,31 @@ impl Config { // Pulley at this time fundamentally doesn't support the // `threads` proposal, notably shared memory, because Rust can't // safely implement loads/stores in the face of shared memory. + // Stack switching is not implemented, either. if self.compiler_target().is_pulley() { unsupported |= WasmFeatures::THREADS; + unsupported |= WasmFeatures::STACK_SWITCHING; } + use target_lexicon::*; + match self.compiler_target() { + Triple { + architecture: Architecture::X86_64 | Architecture::X86_64h, + operating_system: + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_), + .. + } => { + // Stack switching supported on (non-Pulley) Cranelift. + } + + _ => { + // On platforms other than x64 Unix-like, we don't + // support stack switching. + unsupported |= WasmFeatures::STACK_SWITCHING; + } + } unsupported } Some(Strategy::Winch) => { @@ -2053,7 +2074,8 @@ impl Config { | WasmFeatures::TAIL_CALL | WasmFeatures::GC_TYPES | WasmFeatures::EXCEPTIONS - | WasmFeatures::LEGACY_EXCEPTIONS; + | WasmFeatures::LEGACY_EXCEPTIONS + | WasmFeatures::STACK_SWITCHING; match self.compiler_target().architecture { target_lexicon::Architecture::Aarch64(_) => { unsupported |= WasmFeatures::SIMD; @@ -2164,7 +2186,7 @@ impl Config { panic!("should have returned an error by now") } - #[cfg(feature = "async")] + #[cfg(any(feature = "async", feature = "stack-switching"))] if self.async_support && self.max_wasm_stack > self.async_stack_size { bail!("max_wasm_stack size cannot exceed the async_stack_size"); } @@ -2440,6 +2462,27 @@ impl Config { bail!("cannot disable the simd proposal but enable the relaxed simd proposal"); } + if features.contains(WasmFeatures::STACK_SWITCHING) { + use target_lexicon::OperatingSystem; + let model = match target.operating_system { + OperatingSystem::Windows => "update_windows_tib", + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_) => "basic", + _ => bail!("stack-switching feature not supported on this platform "), + }; + + if !self + .compiler_config + .ensure_setting_unset_or_given("stack_switch_model", model) + { + bail!( + "compiler option 'stack_switch_model' must be set to '{}' on this platform", + model + ); + } + } + // Apply compiler settings and flags for (k, v) in self.compiler_config.settings.iter() { compiler.set(k, v)?; diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index e5b8514c1588..14a2f8569ac4 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -387,6 +387,24 @@ impl Engine { } } + // stack switch model must match the current OS + "stack_switch_model" => { + if self.features().contains(WasmFeatures::STACK_SWITCHING) { + use target_lexicon::OperatingSystem; + let expected = + match target.operating_system { + OperatingSystem::Windows => "update_windows_tib", + OperatingSystem::Linux + | OperatingSystem::MacOSX(_) + | OperatingSystem::Darwin(_) => "basic", + _ => { return Err(String::from("stack-switching feature not supported on this platform")); } + }; + *value == FlagValue::Enum(expected) + } else { + return Ok(()) + } + } + // These settings don't affect the interface or functionality of // the module itself, so their configuration values shouldn't // matter. @@ -405,7 +423,6 @@ impl Engine { | "log2_min_function_alignment" | "machine_code_cfg_info" | "tls_model" // wasmtime doesn't use tls right now - | "stack_switch_model" // wasmtime doesn't use stack switching right now | "opt_level" // opt level doesn't change semantics | "enable_alias_analysis" // alias analysis-based opts don't change semantics | "probestack_size_log2" // probestack above asserted disabled diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs index 0d9c7df4ffac..ef3527c99735 100644 --- a/crates/wasmtime/src/runtime/externals/global.rs +++ b/crates/wasmtime/src/runtime/externals/global.rs @@ -126,6 +126,11 @@ impl Global { ExternRef::from_cloned_gc_ref(&mut store, r) })), + HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont => { + // TODO(#10248) Required to support stack switching in the embedder API. + unimplemented!() + } + HeapType::NoExtern => Ref::Extern(None), HeapType::Any diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs index 293cf476adbe..803e914f6a1d 100644 --- a/crates/wasmtime/src/runtime/externals/table.rs +++ b/crates/wasmtime/src/runtime/externals/table.rs @@ -185,6 +185,11 @@ impl Table { ty => unreachable!("not a top type: {ty:?}"), } } + + runtime::TableElement::ContRef(_c) => { + // TODO(#10248) Required to support stack switching in the embedder API. + unimplemented!() + } } } } diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index 0d59031c042f..1bad88bf0509 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1,8 +1,9 @@ use crate::prelude::*; use crate::runtime::Uninhabited; use crate::runtime::vm::{ - ExportFunction, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMContext, - VMFuncRef, VMFunctionImport, VMOpaqueContext, VMStoreContext, + ExportFunction, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, + VMCommonStackInformation, VMContext, VMFuncRef, VMFunctionImport, VMOpaqueContext, + VMStoreContext, }; use crate::store::{AutoAssertNoGc, StoreData, StoreOpaque, Stored}; use crate::type_registry::RegisteredType; @@ -357,6 +358,7 @@ macro_rules! for_each_function_signature { } mod typed; +use crate::runtime::vm::VMStackChain; pub use typed::*; impl Func { @@ -1203,6 +1205,7 @@ impl Func { results.len() ); } + for (ty, arg) in ty.params().zip(params) { arg.ensure_matches_ty(opaque, &ty) .context("argument type mismatch")?; @@ -1611,7 +1614,15 @@ pub(crate) fn invoke_wasm_and_catch_traps( closure: impl FnMut(NonNull, Option>) -> bool, ) -> Result<()> { unsafe { - let previous_runtime_state = EntryStoreContext::enter_wasm(store); + // The `enter_wasm` call below will reset the store context's + // `stack_chain` to a new `InitialStack`, pointing to the + // stack-allocated `initial_stack_csi`. + let mut initial_stack_csi = VMCommonStackInformation::running_default(); + // Stores some state of the runtime just before entering Wasm. Will be + // restored upon exiting Wasm. Note that the `CallThreadState` that is + // created by the `catch_traps` call below will store a pointer to this + // stack-allocated `previous_runtime_state`. + let previous_runtime_state = EntryStoreContext::enter_wasm(store, &mut initial_stack_csi); if let Err(trap) = store.0.call_hook(CallHook::CallingWasm) { // `previous_runtime_state` implicitly dropped here @@ -1642,6 +1653,9 @@ pub(crate) struct EntryStoreContext { /// Contains value of `last_wasm_entry_fp` field to restore in /// `VMStoreContext` when exiting Wasm. pub last_wasm_entry_fp: usize, + /// Contains value of `stack_chain` field to restore in + /// `VMStoreContext` when exiting Wasm. + pub stack_chain: VMStackChain, /// We need a pointer to the runtime limits, so we can update them from /// `drop`/`exit_wasm`. @@ -1659,7 +1673,10 @@ impl EntryStoreContext { /// pointer that called into wasm. /// /// It also saves the different last_wasm_* values in the `VMRuntimeLimits`. - pub fn enter_wasm(store: &mut StoreContextMut<'_, T>) -> Self { + pub fn enter_wasm( + store: &mut StoreContextMut<'_, T>, + initial_stack_information: *mut VMCommonStackInformation, + ) -> Self { let stack_limit; // If this is a recursive call, e.g. our stack limit is already set, then @@ -1728,6 +1745,11 @@ impl EntryStoreContext { let last_wasm_exit_fp = *store.0.vm_store_context().last_wasm_exit_fp.get(); let last_wasm_entry_fp = *store.0.vm_store_context().last_wasm_entry_fp.get(); + let stack_chain = (*store.0.vm_store_context().stack_chain.get()).clone(); + + let new_stack_chain = VMStackChain::InitialStack(initial_stack_information); + *store.0.vm_store_context().stack_chain.get() = new_stack_chain; + let vm_store_context = store.0.vm_store_context(); Self { @@ -1735,6 +1757,7 @@ impl EntryStoreContext { last_wasm_exit_pc, last_wasm_exit_fp, last_wasm_entry_fp, + stack_chain, vm_store_context, } } @@ -1754,6 +1777,7 @@ impl EntryStoreContext { *(*self.vm_store_context).last_wasm_exit_fp.get() = self.last_wasm_exit_fp; *(*self.vm_store_context).last_wasm_exit_pc.get() = self.last_wasm_exit_pc; *(*self.vm_store_context).last_wasm_entry_fp.get() = self.last_wasm_entry_fp; + *(*self.vm_store_context).stack_chain.get() = self.stack_chain.clone(); } } } diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs index 2be0d122b912..4ed1c6c313a8 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs @@ -892,6 +892,9 @@ unsafe impl WasmTy for Rooted { | HeapType::I31 | HeapType::Struct | HeapType::ConcreteStruct(_) + | HeapType::Cont + | HeapType::NoCont + | HeapType::ConcreteCont(_) | HeapType::None => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, @@ -986,6 +989,9 @@ unsafe impl WasmTy for ManuallyRooted { | HeapType::I31 | HeapType::Struct | HeapType::ConcreteStruct(_) + | HeapType::Cont + | HeapType::NoCont + | HeapType::ConcreteCont(_) | HeapType::None => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, diff --git a/crates/wasmtime/src/runtime/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/gc/enabled/structref.rs index 74ee0b99dac1..0d65424a4d18 100644 --- a/crates/wasmtime/src/runtime/gc/enabled/structref.rs +++ b/crates/wasmtime/src/runtime/gc/enabled/structref.rs @@ -687,7 +687,10 @@ unsafe impl WasmTy for Rooted { | HeapType::I31 | HeapType::Array | HeapType::ConcreteArray(_) - | HeapType::None => bail!( + | HeapType::None + | HeapType::NoCont + | HeapType::Cont + | HeapType::ConcreteCont(_) => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, ), @@ -781,7 +784,10 @@ unsafe impl WasmTy for ManuallyRooted { | HeapType::I31 | HeapType::Array | HeapType::ConcreteArray(_) - | HeapType::None => bail!( + | HeapType::None + | HeapType::NoCont + | HeapType::Cont + | HeapType::ConcreteCont(_) => bail!( "type mismatch: expected `(ref {ty})`, got `(ref {})`", self._ty(store)?, ), diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 71b79a2ee4d7..259731972e9a 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -83,6 +83,8 @@ use crate::module::RegisteredModuleId; use crate::prelude::*; #[cfg(feature = "gc")] use crate::runtime::vm::GcRootsList; +#[cfg(feature = "stack-switching")] +use crate::runtime::vm::VMContRef; use crate::runtime::vm::mpk::ProtectionKey; use crate::runtime::vm::{ ExportGlobal, GcStore, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, @@ -321,6 +323,12 @@ pub struct StoreOpaque { engine: Engine, vm_store_context: VMStoreContext, + + // Contains all continuations ever allocated throughout the lifetime of this + // store. + #[cfg(feature = "stack-switching")] + continuations: Vec>, + instances: Vec, #[cfg(feature = "component-model")] num_component_instances: usize, @@ -542,6 +550,8 @@ impl Store { _marker: marker::PhantomPinned, engine: engine.clone(), vm_store_context: Default::default(), + #[cfg(feature = "stack-switching")] + continuations: Vec::new(), instances: Vec::new(), #[cfg(feature = "component-model")] num_component_instances: 0, @@ -1623,6 +1633,8 @@ impl StoreOpaque { assert!(gc_roots_list.is_empty()); self.trace_wasm_stack_roots(gc_roots_list); + #[cfg(feature = "stack-switching")] + self.trace_wasm_continuation_roots(gc_roots_list); self.trace_vmctx_roots(gc_roots_list); self.trace_user_roots(gc_roots_list); @@ -1630,60 +1642,106 @@ impl StoreOpaque { } #[cfg(feature = "gc")] - fn trace_wasm_stack_roots(&mut self, gc_roots_list: &mut GcRootsList) { - use crate::runtime::vm::{Backtrace, SendSyncPtr}; + fn trace_wasm_stack_frame( + &self, + gc_roots_list: &mut GcRootsList, + frame: crate::runtime::vm::Frame, + ) { + use crate::runtime::vm::SendSyncPtr; use core::ptr::NonNull; - log::trace!("Begin trace GC roots :: Wasm stack"); + let pc = frame.pc(); + debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames"); - Backtrace::trace(self, |frame| { - let pc = frame.pc(); - debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames"); + let fp = frame.fp() as *mut usize; + debug_assert!( + !fp.is_null(), + "we should always get a valid frame pointer for Wasm frames" + ); - let fp = frame.fp() as *mut usize; - debug_assert!( - !fp.is_null(), - "we should always get a valid frame pointer for Wasm frames" - ); + let module_info = self + .modules() + .lookup_module_by_pc(pc) + .expect("should have module info for Wasm frame"); - let module_info = self - .modules() - .lookup_module_by_pc(pc) - .expect("should have module info for Wasm frame"); + let stack_map = match module_info.lookup_stack_map(pc) { + Some(sm) => sm, + None => { + log::trace!("No stack map for this Wasm frame"); + return; + } + }; + log::trace!( + "We have a stack map that maps {} bytes in this Wasm frame", + stack_map.frame_size() + ); - let stack_map = match module_info.lookup_stack_map(pc) { - Some(sm) => sm, - None => { - log::trace!("No stack map for this Wasm frame"); - return core::ops::ControlFlow::Continue(()); - } - }; - log::trace!( - "We have a stack map that maps {} bytes in this Wasm frame", - stack_map.frame_size() - ); + let sp = unsafe { stack_map.sp(fp) }; + for stack_slot in unsafe { stack_map.live_gc_refs(sp) } { + let raw: u32 = unsafe { core::ptr::read(stack_slot) }; + log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}"); - let sp = unsafe { stack_map.sp(fp) }; - for stack_slot in unsafe { stack_map.live_gc_refs(sp) } { - let raw: u32 = unsafe { core::ptr::read(stack_slot) }; - log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}"); - - let gc_ref = VMGcRef::from_raw_u32(raw); - if gc_ref.is_some() { - unsafe { - gc_roots_list.add_wasm_stack_root(SendSyncPtr::new( - NonNull::new(stack_slot).unwrap(), - )); - } + let gc_ref = VMGcRef::from_raw_u32(raw); + if gc_ref.is_some() { + unsafe { + gc_roots_list + .add_wasm_stack_root(SendSyncPtr::new(NonNull::new(stack_slot).unwrap())); } } + } + } + + #[cfg(feature = "gc")] + fn trace_wasm_stack_roots(&mut self, gc_roots_list: &mut GcRootsList) { + use crate::runtime::vm::Backtrace; + log::trace!("Begin trace GC roots :: Wasm stack"); + Backtrace::trace(self, |frame| { + self.trace_wasm_stack_frame(gc_roots_list, frame); core::ops::ControlFlow::Continue(()) }); log::trace!("End trace GC roots :: Wasm stack"); } + #[cfg(all(feature = "gc", feature = "stack-switching"))] + fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) { + use crate::{runtime::vm::Backtrace, vm::VMStackState}; + log::trace!("Begin trace GC roots :: continuations"); + + for continuation in &self.continuations { + let state = continuation.common_stack_information.state; + + // FIXME(frank-emrich) In general, it is not enough to just trace + // through the stacks of continuations; we also need to look through + // their `cont.bind` arguments. However, we don't currently have + // enough RTTI information to check if any of the values in the + // buffers used by `cont.bind` are GC values. As a workaround, note + // that we currently disallow cont.bind-ing GC values altogether. + // This way, it is okay not to check them here. + + // Note that we only care about continuations that have state + // `Suspended`. + // - `Running` continuations will be handled by + // `trace_wasm_stack_roots`. + // - For `Parent` continuations, we don't know if they are the + // parent of a running continuation or a suspended one. But it + // does not matter: They will be handled when traversing the stack + // chain starting at either the running one, or the suspended + // continuations below. + // - For `Fresh` continuations, we know that there are no GC values + // on their stack, yet. + if state == VMStackState::Suspended { + Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| { + self.trace_wasm_stack_frame(gc_roots_list, frame); + core::ops::ControlFlow::Continue(()) + }); + } + } + + log::trace!("End trace GC roots :: continuations"); + } + #[cfg(feature = "gc")] fn trace_vmctx_roots(&mut self, gc_roots_list: &mut GcRootsList) { log::trace!("Begin trace GC roots :: vmctx"); @@ -1980,6 +2038,21 @@ at https://bytecodealliance.org/security. Executor::Native => &crate::runtime::vm::UnwindHost, } } + + /// Allocates a new continuation. Note that we currently don't support + /// deallocating them. Instead, all continuations remain allocated + /// throughout the store's lifetime. + #[cfg(feature = "stack-switching")] + pub fn allocate_continuation(&mut self) -> Result<*mut VMContRef> { + // FIXME(frank-emrich) Do we need to pin this? + let mut continuation = Box::new(VMContRef::empty()); + let stack_size = self.engine.config().async_stack_size; + let stack = crate::vm::VMContinuationStack::new(stack_size)?; + continuation.stack = stack; + let ptr = continuation.deref_mut() as *mut VMContRef; + self.continuations.push(continuation); + Ok(ptr) + } } unsafe impl crate::runtime::vm::VMStore for StoreInner { diff --git a/crates/wasmtime/src/runtime/type_registry.rs b/crates/wasmtime/src/runtime/type_registry.rs index 1d12837b2381..cf0921df02c3 100644 --- a/crates/wasmtime/src/runtime/type_registry.rs +++ b/crates/wasmtime/src/runtime/type_registry.rs @@ -934,7 +934,7 @@ impl TypeRegistryInner { .struct_layout(s) .into(), ), - wasmtime_environ::WasmCompositeInnerType::Cont(_) => todo!(), // FIXME: #10248 stack switching support. + wasmtime_environ::WasmCompositeInnerType::Cont(_) => None, // FIXME: #10248 stack switching support. }; // Add the type to our slab. diff --git a/crates/wasmtime/src/runtime/types.rs b/crates/wasmtime/src/runtime/types.rs index 98c9605232bb..b6853468cd6b 100644 --- a/crates/wasmtime/src/runtime/types.rs +++ b/crates/wasmtime/src/runtime/types.rs @@ -158,6 +158,12 @@ impl ValType { /// The `nullref` type, aka `(ref null none)`. pub const NULLREF: Self = ValType::Ref(RefType::NULLREF); + /// The `contref` type, aka `(ref null cont)`. + pub const CONTREF: Self = ValType::Ref(RefType::CONTREF); + + /// The `nullcontref` type, aka. `(ref null nocont)`. + pub const NULLCONTREF: Self = ValType::Ref(RefType::NULLCONTREF); + /// Returns true if `ValType` matches any of the numeric types. (e.g. `I32`, /// `I64`, `F32`, `F64`). #[inline] @@ -240,6 +246,18 @@ impl ValType { ) } + /// Is this the `contref` (aka `(ref null cont)`) type? + #[inline] + pub fn is_contref(&self) -> bool { + matches!( + self, + ValType::Ref(RefType { + is_nullable: true, + heap_type: HeapType::Cont + }) + ) + } + /// Get the underlying reference type, if this value type is a reference /// type. #[inline] @@ -458,6 +476,18 @@ impl RefType { heap_type: HeapType::None, }; + /// The `contref` type, aka `(ref null cont)`. + pub const CONTREF: Self = RefType { + is_nullable: true, + heap_type: HeapType::Cont, + }; + + /// The `nullcontref` type, aka `(ref null nocont)`. + pub const NULLCONTREF: Self = RefType { + is_nullable: true, + heap_type: HeapType::NoCont, + }; + /// Construct a new reference type. pub fn new(is_nullable: bool, heap_type: HeapType) -> RefType { RefType { @@ -718,6 +748,23 @@ pub enum HeapType { /// of `any` and `eq`) and supertypes of the `none` heap type. ConcreteStruct(StructType), + /// A reference to a continuation of a specific, concrete type. + /// + /// These are subtypes of `cont` and supertypes of `nocont`. + ConcreteCont(ContType), + + /// The `cont` heap type represents a reference to any kind of continuation. + /// + /// This is the top type for the continuation objects type hierarchy, and is + /// therefore a supertype of every continuation object. + Cont, + + /// The `nocont` heap type represents the null continuation object. + /// + /// This is the bottom type for the continuation objects type hierarchy, and + /// therefore `nocont` is a subtype of all continuation object types. + NoCont, + /// The abstract `none` heap type represents the null internal reference. /// /// This is the bottom type for the internal type hierarchy, and therefore @@ -741,6 +788,9 @@ impl Display for HeapType { HeapType::ConcreteFunc(ty) => write!(f, "(concrete func {:?})", ty.type_index()), HeapType::ConcreteArray(ty) => write!(f, "(concrete array {:?})", ty.type_index()), HeapType::ConcreteStruct(ty) => write!(f, "(concrete struct {:?})", ty.type_index()), + HeapType::ConcreteCont(ty) => write!(f, "(concrete cont {:?})", ty.type_index()), + HeapType::Cont => write!(f, "cont"), + HeapType::NoCont => write!(f, "nocont"), } } } @@ -766,6 +816,13 @@ impl From for HeapType { } } +impl From for HeapType { + #[inline] + fn from(f: ContType) -> Self { + HeapType::ConcreteCont(f) + } +} + impl HeapType { /// Is this the abstract `extern` heap type? pub fn is_extern(&self) -> bool { @@ -797,6 +854,11 @@ impl HeapType { matches!(self, HeapType::None) } + /// Is this the abstract `cont` heap type? + pub fn is_cont(&self) -> bool { + matches!(self, HeapType::Cont) + } + /// Is this an abstract type? /// /// Types that are not abstract are concrete, user-defined types. @@ -811,7 +873,10 @@ impl HeapType { pub fn is_concrete(&self) -> bool { matches!( self, - HeapType::ConcreteFunc(_) | HeapType::ConcreteArray(_) | HeapType::ConcreteStruct(_) + HeapType::ConcreteFunc(_) + | HeapType::ConcreteArray(_) + | HeapType::ConcreteStruct(_) + | HeapType::ConcreteCont(_) ) } @@ -857,6 +922,21 @@ impl HeapType { self.as_concrete_array().unwrap() } + /// Is this a concrete, user-defined continuation type? + pub fn is_concrete_cont(&self) -> bool { + matches!(self, HeapType::ConcreteCont(_)) + } + + /// Get the underlying concrete, user-defined continuation type, if any. + /// + /// Returns `None` if this is not a concrete continuation type. + pub fn as_concrete_cont(&self) -> Option<&ContType> { + match self { + HeapType::ConcreteCont(f) => Some(f), + _ => None, + } + } + /// Is this a concrete, user-defined struct type? pub fn is_concrete_struct(&self) -> bool { matches!(self, HeapType::ConcreteStruct(_)) @@ -872,6 +952,12 @@ impl HeapType { } } + /// Get the underlying concrete, user-defined type, panicking if this is not + /// a concrete continuation type. + pub fn unwrap_concrete_cont(&self) -> &ContType { + self.as_concrete_cont().unwrap() + } + /// Get the underlying concrete, user-defined type, panicking if this is not /// a concrete struct type. pub fn unwrap_concrete_struct(&self) -> &StructType { @@ -897,6 +983,8 @@ impl HeapType { | HeapType::Struct | HeapType::ConcreteStruct(_) | HeapType::None => HeapType::Any, + + HeapType::Cont | HeapType::ConcreteCont(_) | HeapType::NoCont => HeapType::Cont, } } @@ -904,7 +992,7 @@ impl HeapType { #[inline] pub fn is_top(&self) -> bool { match self { - HeapType::Any | HeapType::Extern | HeapType::Func => true, + HeapType::Any | HeapType::Extern | HeapType::Func | HeapType::Cont => true, _ => false, } } @@ -928,6 +1016,8 @@ impl HeapType { | HeapType::Struct | HeapType::ConcreteStruct(_) | HeapType::None => HeapType::None, + + HeapType::Cont | HeapType::ConcreteCont(_) | HeapType::NoCont => HeapType::NoCont, } } @@ -935,7 +1025,7 @@ impl HeapType { #[inline] pub fn is_bottom(&self) -> bool { match self { - HeapType::None | HeapType::NoExtern | HeapType::NoFunc => true, + HeapType::None | HeapType::NoExtern | HeapType::NoFunc | HeapType::NoCont => true, _ => false, } } @@ -973,6 +1063,18 @@ impl HeapType { (HeapType::Func, HeapType::Func) => true, (HeapType::Func, _) => false, + (HeapType::Cont, HeapType::Cont) => true, + (HeapType::Cont, _) => false, + + (HeapType::NoCont, HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont) => { + true + } + (HeapType::NoCont, _) => false, + + (HeapType::ConcreteCont(_), HeapType::Cont) => true, + (HeapType::ConcreteCont(a), HeapType::ConcreteCont(b)) => a.matches(b), + (HeapType::ConcreteCont(_), _) => false, + ( HeapType::None, HeapType::None @@ -1056,10 +1158,13 @@ impl HeapType { | HeapType::I31 | HeapType::Array | HeapType::Struct + | HeapType::Cont + | HeapType::NoCont | HeapType::None => true, HeapType::ConcreteFunc(ty) => ty.comes_from_same_engine(engine), HeapType::ConcreteArray(ty) => ty.comes_from_same_engine(engine), HeapType::ConcreteStruct(ty) => ty.comes_from_same_engine(engine), + HeapType::ConcreteCont(ty) => ty.comes_from_same_engine(engine), } } @@ -1084,6 +1189,11 @@ impl HeapType { HeapType::ConcreteStruct(a) => { WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::Engine(a.type_index())) } + HeapType::Cont => WasmHeapType::Cont, + HeapType::NoCont => WasmHeapType::NoCont, + HeapType::ConcreteCont(c) => { + WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Engine(c.type_index())) + } } } @@ -1114,16 +1224,22 @@ impl HeapType { | WasmHeapType::ConcreteArray(EngineOrModuleTypeIndex::Module(_)) | WasmHeapType::ConcreteArray(EngineOrModuleTypeIndex::RecGroup(_)) | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::Module(_)) - | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::RecGroup(_)) => { + | WasmHeapType::ConcreteStruct(EngineOrModuleTypeIndex::RecGroup(_)) + | WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Module(_)) + | WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::RecGroup(_)) => { panic!("HeapType::from_wasm_type on non-canonicalized-for-runtime-usage heap type") } - - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapType::Cont => HeapType::Cont, + WasmHeapType::NoCont => HeapType::NoCont, + WasmHeapType::ConcreteCont(EngineOrModuleTypeIndex::Engine(idx)) => { + HeapType::ConcreteCont(ContType::from_shared_type_index(engine, *idx)) + } } } pub(crate) fn as_registered_type(&self) -> Option<&RegisteredType> { match self { + HeapType::ConcreteCont(c) => Some(&c.registered_type), HeapType::ConcreteFunc(f) => Some(&f.registered_type), HeapType::ConcreteArray(a) => Some(&a.registered_type), HeapType::ConcreteStruct(a) => Some(&a.registered_type), @@ -1137,6 +1253,8 @@ impl HeapType { | HeapType::I31 | HeapType::Array | HeapType::Struct + | HeapType::Cont + | HeapType::NoCont | HeapType::None => None, } } @@ -1146,6 +1264,7 @@ impl HeapType { match self.top() { Self::Any | Self::Extern => true, Self::Func => false, + Self::Cont => false, ty => unreachable!("not a top type: {ty:?}"), } } @@ -2450,6 +2569,53 @@ impl FuncType { } } +// Continuation types +/// A WebAssembly continuation descriptor. +#[derive(Debug, Clone, Hash)] +pub struct ContType { + registered_type: RegisteredType, +} + +impl ContType { + /// Get the engine that this function type is associated with. + pub fn engine(&self) -> &Engine { + self.registered_type.engine() + } + + pub(crate) fn comes_from_same_engine(&self, engine: &Engine) -> bool { + Engine::same(self.registered_type.engine(), engine) + } + + pub(crate) fn type_index(&self) -> VMSharedTypeIndex { + self.registered_type.index() + } + + /// Does this continuation type match the other continuation type? + /// + /// That is, is this continuation type a subtype of the other continuation type? + /// + /// # Panics + /// + /// Panics if either type is associated with a different engine from the + /// other. + pub fn matches(&self, other: &ContType) -> bool { + assert!(self.comes_from_same_engine(other.engine())); + + // Avoid matching on structure for subtyping checks when we have + // precisely the same type. + // TODO(dhil): Implement subtype check later. + self.type_index() == other.type_index() + } + + pub(crate) fn from_shared_type_index(engine: &Engine, index: VMSharedTypeIndex) -> ContType { + let ty = RegisteredType::root(engine, index); + assert!(ty.is_cont()); + Self { + registered_type: ty, + } + } +} + // Global Types /// A WebAssembly global descriptor. diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs index 2ec62236851b..8e85608c8f86 100644 --- a/crates/wasmtime/src/runtime/values.rs +++ b/crates/wasmtime/src/runtime/values.rs @@ -280,6 +280,11 @@ impl Val { HeapType::NoFunc => Ref::Func(None), + HeapType::NoCont | HeapType::ConcreteCont(_) | HeapType::Cont => { + // TODO(#10248): Required to support stack switching in the embedder API. + unimplemented!() + } + HeapType::Extern => ExternRef::_from_raw(store, raw.get_externref()).into(), HeapType::NoExtern => Ref::Extern(None), diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index ab5d5ae1bef7..2228b4601191 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -61,6 +61,7 @@ mod memory; mod mmap_vec; mod provenance; mod send_sync_ptr; +mod stack_switching; mod store_box; mod sys; mod table; @@ -110,6 +111,7 @@ pub use crate::runtime::vm::memory::{ }; pub use crate::runtime::vm::mmap_vec::MmapVec; pub use crate::runtime::vm::provenance::*; +pub use crate::runtime::vm::stack_switching::*; pub use crate::runtime::vm::store_box::*; #[cfg(feature = "std")] pub use crate::runtime::vm::sys::mmap::open_file_for_mmap; @@ -125,6 +127,7 @@ pub use crate::runtime::vm::vmcontext::{ VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMStoreContext, VMTable, VMTagImport, VMWasmCallFunction, ValRaw, }; + pub use send_sync_ptr::SendSyncPtr; mod module_id; diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs index a8ff4dd71b57..ce3a0586601e 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs @@ -130,7 +130,11 @@ pub struct InstanceLimits { /// Maximum number of tables per instance. pub max_tables_per_module: u32, - /// Maximum number of table elements per table. + /// Maximum number of word-size elements per table. + /// + /// Note that tables for element types such as continuations + /// that use more than one word of storage may store fewer + /// elements. pub table_elements: usize, /// Maximum number of linear memories per instance. diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs index 910292a696d6..6045a9aefc3c 100644 --- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs +++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/table_pool.rs @@ -8,7 +8,6 @@ use crate::runtime::vm::{ mmap::AlignedLength, }; use crate::{prelude::*, vm::HostAlignedByteCount}; -use std::mem; use std::ptr::NonNull; use wasmtime_environ::{Module, Tunables}; @@ -24,14 +23,14 @@ pub struct TablePool { max_total_tables: usize, tables_per_instance: usize, keep_resident: HostAlignedByteCount, - table_elements: usize, + nominal_table_elements: usize, } impl TablePool { /// Create a new `TablePool`. pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result { let table_size = HostAlignedByteCount::new_rounded_up( - mem::size_of::<*mut u8>() + crate::runtime::vm::table::NOMINAL_MAX_TABLE_ELEM_SIZE .checked_mul(config.limits.table_elements) .ok_or_else(|| anyhow!("table size exceeds addressable memory"))?, )?; @@ -46,14 +45,16 @@ impl TablePool { let mapping = Mmap::accessible_reserved(allocation_size, allocation_size) .context("failed to create table pool mapping")?; + let keep_resident = HostAlignedByteCount::new_rounded_up(config.table_keep_resident)?; + Ok(Self { index_allocator: SimpleIndexAllocator::new(config.limits.total_tables), mapping, table_size, max_total_tables, tables_per_instance, - keep_resident: HostAlignedByteCount::new_rounded_up(config.table_keep_resident)?, - table_elements: config.limits.table_elements, + keep_resident, + nominal_table_elements: config.limits.table_elements, }) } @@ -78,12 +79,12 @@ impl TablePool { } for (i, table) in module.tables.iter().skip(module.num_imported_tables) { - if table.limits.min > u64::try_from(self.table_elements)? { + if table.limits.min > u64::try_from(self.nominal_table_elements)? { bail!( "table index {} has a minimum element size of {} which exceeds the limit of {}", i.as_u32(), table.limits.min, - self.table_elements, + self.nominal_table_elements, ); } } @@ -115,6 +116,20 @@ impl TablePool { } } + /// Returns the number of bytes occupied by table entry data + /// + /// This is typically just the `nominal_table_elements` multiplied by + /// the size of the table's element type, but may be less in the case + /// of types such as VMContRef for which less capacity will be avialable + /// (maintaining a consistent table size in the pool). + fn data_size(&self, table_type: crate::vm::table::TableElementType) -> usize { + let element_size = table_type.element_size(); + let elements = self + .nominal_table_elements + .min(self.table_size.byte_count() / element_size); + elements * element_size + } + /// Allocate a single table for the given instance allocation request. pub fn allocate( &self, @@ -132,16 +147,13 @@ impl TablePool { match (|| { let base = self.get(allocation_index); - + let data_size = self.data_size(crate::vm::table::wasm_to_table_type(ty.ref_type)); unsafe { - commit_pages(base, self.table_elements * mem::size_of::<*mut u8>())?; + commit_pages(base, data_size)?; } - let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut( - base.cast(), - self.table_elements * mem::size_of::<*mut u8>(), - )) - .unwrap(); + let ptr = + NonNull::new(std::ptr::slice_from_raw_parts_mut(base.cast(), data_size)).unwrap(); unsafe { Table::new_static( ty, @@ -193,12 +205,7 @@ impl TablePool { ) { assert!(table.is_static()); let base = self.get(allocation_index); - - // XXX Should we check that table.size() * mem::size_of::<*mut u8>() - // doesn't overflow? The only check that exists is for the boundary - // condition that table.size() * mem::size_of::<*mut u8>() is less than - // a host page smaller than usize::MAX. - let size = HostAlignedByteCount::new_rounded_up(table.size() * mem::size_of::<*mut u8>()) + let size = HostAlignedByteCount::new_rounded_up(self.data_size(table.element_type())) .expect("table entry size doesn't overflow"); // `memset` the first `keep_resident` bytes. @@ -237,7 +244,7 @@ mod tests { assert_eq!(pool.table_size, host_page_size); assert_eq!(pool.max_total_tables, 7); - assert_eq!(pool.table_elements, 100); + assert_eq!(pool.nominal_table_elements, 100); let base = pool.mapping.as_ptr() as usize; @@ -252,4 +259,51 @@ mod tests { Ok(()) } + + #[test] + fn test_table_pool_continuations_capacity() -> Result<()> { + let mkpool = |table_elements: usize| -> Result { + TablePool::new(&PoolingInstanceAllocatorConfig { + limits: InstanceLimits { + table_elements, + total_tables: 7, + max_memory_size: 0, + max_memories_per_module: 0, + ..Default::default() + }, + ..Default::default() + }) + }; + + let host_page_size = HostAlignedByteCount::host_page_size(); + let words_per_page = host_page_size.byte_count() / size_of::<*const u8>(); + let pool_big = mkpool(words_per_page - 1)?; + let pool_small = mkpool(5)?; + + assert_eq!(pool_small.table_size, host_page_size); + assert_eq!(pool_big.table_size, host_page_size); + + // table should store nominal_table_elements of data for func in both cases + let func_table_type = crate::vm::table::TableElementType::Func; + assert_eq!( + pool_small.data_size(func_table_type), + pool_small.nominal_table_elements * func_table_type.element_size() + ); + assert_eq!( + pool_big.data_size(func_table_type), + pool_big.nominal_table_elements * func_table_type.element_size() + ); + + // In the "big" case, continuations should fill page size (capacity limited). + // In the "small" case, continuations should fill only part of the page, capping + // at the requested table size for nominal elements. + let cont_table_type = crate::vm::table::TableElementType::Cont; + assert_eq!( + pool_small.data_size(cont_table_type), + pool_small.nominal_table_elements * cont_table_type.element_size() + ); + assert_eq!(pool_big.data_size(cont_table_type), host_page_size); + + Ok(()) + } } diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 6f9d71e15620..f00197842b86 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -54,6 +54,8 @@ //! } //! ``` +#[cfg(feature = "stack-switching")] +use super::stack_switching::{VMContObj, VMContRef}; use crate::prelude::*; #[cfg(feature = "gc")] use crate::runtime::vm::VMGcRef; @@ -232,6 +234,7 @@ unsafe fn table_grow_func_ref( let element = match instance.table_element_type(table_index) { TableElementType::Func => NonNull::new(init_value.cast::()).into(), TableElementType::GcRef => unreachable!(), + TableElementType::Cont => unreachable!(), }; let result = instance @@ -261,6 +264,41 @@ unsafe fn table_grow_gc_ref( .clone_gc_ref(&r) }) .into(), + TableElementType::Cont => unreachable!(), + }; + + let result = instance + .table_grow(store, table_index, delta, element)? + .map(AllocationSize); + Ok(result) +} + +#[cfg(feature = "stack-switching")] +unsafe fn table_grow_cont_obj( + store: &mut dyn VMStore, + instance: &mut Instance, + table_index: u32, + delta: u64, + // The following two values together form the intitial Option. + // A None value is indicated by the pointer being null. + init_value_contref: *mut u8, + init_value_revision: u64, +) -> Result> { + use core::ptr::NonNull; + let init_value = if init_value_contref.is_null() { + None + } else { + // SAFETY: We just checked that the pointer is non-null + let contref = NonNull::new_unchecked(init_value_contref as *mut VMContRef); + let contobj = VMContObj::new(contref, init_value_revision); + Some(contobj) + }; + + let table_index = TableIndex::from_u32(table_index); + + let element = match instance.table_element_type(table_index) { + TableElementType::Cont => init_value.into(), + _ => panic!("Wrong table growing function"), }; let result = instance @@ -287,6 +325,7 @@ unsafe fn table_fill_func_ref( Ok(()) } TableElementType::GcRef => unreachable!(), + TableElementType::Cont => unreachable!(), } } @@ -310,6 +349,39 @@ unsafe fn table_fill_gc_ref( table.fill(Some(gc_store), dst, gc_ref.into(), len)?; Ok(()) } + + TableElementType::Cont => unreachable!(), + } +} + +#[cfg(feature = "stack-switching")] +unsafe fn table_fill_cont_obj( + store: &mut dyn VMStore, + instance: &mut Instance, + table_index: u32, + dst: u64, + value_contref: *mut u8, + value_revision: u64, + len: u64, +) -> Result<()> { + use core::ptr::NonNull; + let table_index = TableIndex::from_u32(table_index); + let table = &mut *instance.get_table(table_index); + match table.element_type() { + TableElementType::Cont => { + let contobj = if value_contref.is_null() { + None + } else { + // SAFETY: We just checked that the pointer is non-null + let contref = NonNull::new_unchecked(value_contref as *mut VMContRef); + let contobj = VMContObj::new(contref, value_revision); + Some(contobj) + }; + + table.fill(store.optional_gc_store_mut(), dst, contobj.into(), len)?; + Ok(()) + } + _ => panic!("Wrong table filling function"), } } @@ -1475,3 +1547,18 @@ fn raise(_store: &mut dyn VMStore, _instance: &mut Instance) { #[cfg(not(has_host_compiler_backend))] unreachable!() } + +// Builtins for continuations. These are thin wrappers around the +// respective definitions in stack_switching.rs. +#[cfg(feature = "stack-switching")] +fn cont_new( + store: &mut dyn VMStore, + instance: &mut Instance, + func: *mut u8, + param_count: u32, + result_count: u32, +) -> Result, TrapReason> { + let ans = + crate::vm::stack_switching::cont_new(store, instance, func, param_count, result_count)?; + Ok(Some(AllocationSize(ans.cast::() as usize))) +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs new file mode 100644 index 000000000000..b600e82521f5 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -0,0 +1,700 @@ +//! This module contains the runtime components of the implementation of the +//! stack switching proposal. + +mod stack; + +use core::{marker::PhantomPinned, ptr::NonNull}; + +pub use stack::*; + +/// A continuation object is a handle to a continuation reference +/// (i.e. an actual stack). A continuation object only be consumed +/// once. The linearity is checked dynamically in the generated code +/// by comparing the revision witness embedded in the pointer to the +/// actual revision counter on the continuation reference. +/// +/// In the optimized implementation, the continuation logically +/// represented by a VMContObj not only encompasses the pointed-to +/// VMContRef, but also all of its parents: +/// +/// ```text +/// +/// +----------------+ +/// +-->| VMContRef | +/// | +----------------+ +/// | ^ +/// | | parent +/// | | +/// | +----------------+ +/// | | VMContRef | +/// | +----------------+ +/// | ^ +/// | | parent +/// last ancestor | | +/// | +----------------+ +/// +---| VMContRef | <-- VMContObj +/// +----------------+ +/// ``` +/// +/// For performance reasons, the VMContRef at the bottom of this chain +/// (i.e., the one pointed to by the VMContObj) has a pointer to the +/// other end of the chain (i.e., its last ancestor). +// FIXME(frank-emrich) Does this actually need to be 16-byte aligned any +// more? Now that we use I128 on the Cranelift side (see +// [wasmtime_cranelift::stack_switching::fatpointer::pointer_type]), it +// should be fine to use the natural alignment of the type. +#[repr(C, align(16))] +#[derive(Debug, Clone, Copy)] +pub struct VMContObj { + pub revision: u64, + pub contref: NonNull, +} + +impl VMContObj { + pub fn new(contref: NonNull, revision: u64) -> Self { + Self { contref, revision } + } +} + +unsafe impl Send for VMContObj {} +unsafe impl Sync for VMContObj {} + +#[test] +fn null_pointer_optimization() { + // The Rust spec does not technically guarantee that the null pointer + // optimization applies to a struct containing a `NonNull`. + assert_eq!( + core::mem::size_of::>(), + core::mem::size_of::() + ); +} + +/// This type is used to save (and subsequently restore) a subset of the data in +/// `VMRuntimeLimits`. See documentation of `VMStackChain` for the exact uses. +#[repr(C)] +#[derive(Debug, Default, Clone)] +pub struct VMStackLimits { + /// Saved version of `stack_limit` field of `VMRuntimeLimits` + pub stack_limit: usize, + /// Saved version of `last_wasm_entry_fp` field of `VMRuntimeLimits` + pub last_wasm_entry_fp: usize, +} + +#[test] +fn check_vm_stack_limits_offsets() { + use core::mem::offset_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMStackLimits, stack_limit), + usize::from(offsets.ptr.vmstack_limits_stack_limit()) + ); + assert_eq!( + offset_of!(VMStackLimits, last_wasm_entry_fp), + usize::from(offsets.ptr.vmstack_limits_last_wasm_entry_fp()) + ); +} + +/// This type represents "common" information that we need to save both for the +/// initial stack and each continuation. +#[repr(C)] +#[derive(Debug, Clone)] +pub struct VMCommonStackInformation { + /// Saves subset of `VMRuntimeLimits` for this stack. See documentation of + /// `VMStackChain` for the exact uses. + pub limits: VMStackLimits, + /// For the initial stack, this field must only have one of the following values: + /// - Running + /// - Parent + pub state: VMStackState, + + /// Only in use when state is `Parent`. Otherwise, the list must be empty. + /// + /// Represents the handlers that this stack installed when resume-ing a + /// continuation. + /// + /// Note that for any resume instruction, we can re-order the handler + /// clauses without changing behavior such that all the suspend handlers + /// come first, followed by all the switch handler (while maintaining the + /// original ordering within the two groups). + /// Thus, we assume that the given resume instruction has the following + /// shape: + /// + /// (resume $ct + /// (on $tag_0 $block_0) ... (on $tag_{n-1} $block_{n-1}) + /// (on $tag_n switch) ... (on $tag_m switch) + /// ) + /// + /// On resume, the handler list is then filled with m + 1 (i.e., one per + /// handler clause) entries such that the i-th entry, using 0-based + /// indexing, is the identifier of $tag_i (represented as *mut + /// VMTagDefinition). + /// Further, `first_switch_handler_index` (see below) is set to n (i.e., the + /// 0-based index of the first switch handler). + /// + /// Note that the actual data buffer (i.e., the one `handler.data` points + /// to) is always allocated on the stack that this `CommonStackInformation` + /// struct describes. + pub handlers: VMHandlerList, + + /// Only used when state is `Parent`. See documentation of `handlers` above. + pub first_switch_handler_index: u32, +} + +impl VMCommonStackInformation { + /// Default value with state set to `Running` + pub fn running_default() -> Self { + Self { + limits: VMStackLimits::default(), + state: VMStackState::Running, + handlers: VMHandlerList::empty(), + first_switch_handler_index: 0, + } + } +} + +#[test] +fn check_vm_common_stack_information_offsets() { + use core::mem::offset_of; + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmcommon_stack_information()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, limits), + usize::from(offsets.ptr.vmcommon_stack_information_limits()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, state), + usize::from(offsets.ptr.vmcommon_stack_information_state()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, handlers), + usize::from(offsets.ptr.vmcommon_stack_information_handlers()) + ); + assert_eq!( + offset_of!(VMCommonStackInformation, first_switch_handler_index), + usize::from( + offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() + ) + ); +} + +impl VMStackLimits { + /// Default value, but uses the given value for `stack_limit`. + pub fn with_stack_limit(stack_limit: usize) -> Self { + Self { + stack_limit, + ..Default::default() + } + } +} + +#[repr(C)] +#[derive(Debug, Clone)] +/// Reference to a stack-allocated buffer ("array"), storing data of some type +/// `T`. +pub struct VMArray { + /// Number of currently occupied slots. + pub length: u32, + /// Number of slots in the data buffer. Note that this is *not* the size of + /// the buffer in bytes! + pub capacity: u32, + /// The actual data buffer + pub data: *mut T, +} + +impl VMArray { + /// Creates empty `Array` + pub fn empty() -> Self { + Self { + length: 0, + capacity: 0, + data: core::ptr::null_mut(), + } + } + + /// Makes `Array` empty. + pub fn clear(&mut self) { + *self = Self::empty(); + } +} + +#[test] +fn check_vm_array_offsets() { + use core::mem::offset_of; + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + // Note that the type parameter has no influence on the size and offsets. + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::>(), + usize::from(offsets.ptr.size_of_vmarray()) + ); + assert_eq!( + offset_of!(VMArray<()>, length), + usize::from(offsets.ptr.vmarray_length()) + ); + assert_eq!( + offset_of!(VMArray<()>, capacity), + usize::from(offsets.ptr.vmarray_capacity()) + ); + assert_eq!( + offset_of!(VMArray<()>, data), + usize::from(offsets.ptr.vmarray_data()) + ); +} + +/// Type used for passing payloads to and from continuations. The actual type +/// argument should be wasmtime::runtime::vm::vmcontext::ValRaw, but we don't +/// have access to that here. +pub type VMPayloads = VMArray; + +/// Type for a list of handlers, represented by the handled tag. Thus, the +/// stored data is actually `*mut VMTagDefinition`, but we don't havr access to +/// that here. +pub type VMHandlerList = VMArray<*mut u8>; + +/// The main type representing a continuation. +#[repr(C)] +pub struct VMContRef { + /// The `CommonStackInformation` of this continuation's stack. + pub common_stack_information: VMCommonStackInformation, + + /// The parent of this continuation, which may be another continuation, the + /// initial stack, or absent (in case of a suspended continuation). + pub parent_chain: VMStackChain, + + /// Only used if `common_stack_information.state` is `Suspended` or `Fresh`. In + /// that case, this points to the end of the stack chain (i.e., the + /// continuation in the parent chain whose own `parent_chain` field is + /// `VMStackChain::Absent`). + /// Note that this may be a pointer to iself (if the state is `Fresh`, this is always the case). + pub last_ancestor: *mut VMContRef, + + /// Revision counter. + pub revision: u64, + + /// The underlying stack. + pub stack: VMContinuationStack, + + /// Used to store only + /// 1. The arguments to the function passed to cont.new + /// 2. The return values of that function + /// + /// Note that the actual data buffer (i.e., the one `args.data` points + /// to) is always allocated on this continuation's stack. + pub args: VMPayloads, + + /// Once a continuation has been suspended (using suspend or switch), + /// this buffer is used to pass payloads to and from the continuation. + /// More concretely, it is used to + /// - Pass payloads from a suspend instruction to the corresponding handler. + /// - Pass payloads to a continuation using cont.bind or resume + /// - Pass payloads to the continuation being switched to when using switch. + /// + /// Note that the actual data buffer (i.e., the one `values.data` points + /// to) is always allocated on this continuation's stack. + pub values: VMPayloads, + + /// Tell the compiler that this structure has potential self-references + /// through the `last_ancestor` pointer. + _marker: core::marker::PhantomPinned, +} + +impl VMContRef { + pub fn fiber_stack(&self) -> &VMContinuationStack { + &self.stack + } + + pub fn detach_stack(&mut self) -> VMContinuationStack { + core::mem::replace(&mut self.stack, VMContinuationStack::unallocated()) + } + + /// This is effectively a `Default` implementation, without calling it + /// so. Used to create `VMContRef`s when initializing pooling allocator. + #[allow(clippy::cast_possible_truncation)] + pub fn empty() -> Self { + let limits = VMStackLimits::with_stack_limit(Default::default()); + let state = VMStackState::Fresh; + let handlers = VMHandlerList::empty(); + let common_stack_information = VMCommonStackInformation { + limits, + state, + handlers, + first_switch_handler_index: 0, + }; + let parent_chain = VMStackChain::Absent; + let last_ancestor = core::ptr::null_mut(); + let stack = VMContinuationStack::unallocated(); + let args = VMPayloads::empty(); + let values = VMPayloads::empty(); + let revision = 0; + let _marker = PhantomPinned; + + Self { + common_stack_information, + parent_chain, + last_ancestor, + stack, + args, + values, + revision, + _marker, + } + } +} + +impl Drop for VMContRef { + fn drop(&mut self) { + // Note that continuation references do not own their parents, and we + // don't drop them here. + + // We would like to enforce the invariant that any continuation that + // was created for a cont.new (rather than, say, just living in a + // pool and never being touched), either ran to completion or was + // cancelled. But failing to do so should yield a custom error, + // instead of panicking here. + } +} + +// These are required so the WasmFX pooling allocator can store a Vec of +// `VMContRef`s. +unsafe impl Send for VMContRef {} +unsafe impl Sync for VMContRef {} + +#[test] +fn check_vm_contref_offsets() { + use core::mem::offset_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMContRef, common_stack_information), + usize::from(offsets.ptr.vmcontref_common_stack_information()) + ); + assert_eq!( + offset_of!(VMContRef, parent_chain), + usize::from(offsets.ptr.vmcontref_parent_chain()) + ); + assert_eq!( + offset_of!(VMContRef, last_ancestor), + usize::from(offsets.ptr.vmcontref_last_ancestor()) + ); + // Some 32-bit platforms need this to be 8-byte aligned, some don't. + // So we need to make sure it always is, without padding. + assert_eq!(u8::vmcontref_revision(&4) % 8, 0); + assert_eq!(u8::vmcontref_revision(&8) % 8, 0); + assert_eq!( + offset_of!(VMContRef, revision), + usize::from(offsets.ptr.vmcontref_revision()) + ); + assert_eq!( + offset_of!(VMContRef, stack), + usize::from(offsets.ptr.vmcontref_stack()) + ); + assert_eq!( + offset_of!(VMContRef, args), + usize::from(offsets.ptr.vmcontref_args()) + ); + assert_eq!( + offset_of!(VMContRef, values), + usize::from(offsets.ptr.vmcontref_values()) + ); +} + +/// Implements `cont.new` instructions (i.e., creation of continuations). +#[cfg(feature = "stack-switching")] +#[inline(always)] +pub fn cont_new( + store: &mut dyn crate::vm::VMStore, + instance: &mut crate::vm::Instance, + func: *mut u8, + param_count: u32, + result_count: u32, +) -> Result<*mut VMContRef, crate::vm::TrapReason> { + let caller_vmctx = instance.vmctx(); + + let stack_size = store.engine().config().async_stack_size; + + let contref = store.allocate_continuation()?; + let contref = unsafe { contref.as_mut().unwrap() }; + + let tsp = contref.stack.top().unwrap(); + contref.parent_chain = VMStackChain::Absent; + // The continuation is fresh, which is a special case of being suspended. + // Thus we need to set the correct end of the continuation chain: itself. + contref.last_ancestor = contref; + + // The initialization function will allocate the actual args/return value buffer and + // update this object (if needed). + let contref_args_ptr = &mut contref.args as *mut _ as *mut VMArray; + + contref.stack.initialize( + func.cast::(), + caller_vmctx.as_ptr(), + contref_args_ptr, + param_count, + result_count, + ); + + // Now that the initial stack pointer was set by the initialization + // function, use it to determine stack limit. + let stack_pointer = contref.stack.control_context_stack_pointer(); + // Same caveat regarding stack_limit here as descibed in + // `wasmtime::runtime::func::EntryStoreContext::enter_wasm`. + let wasm_stack_limit = core::cmp::max( + stack_pointer - store.engine().config().max_wasm_stack, + tsp as usize - stack_size, + ); + let limits = VMStackLimits::with_stack_limit(wasm_stack_limit); + let csi = &mut contref.common_stack_information; + csi.state = VMStackState::Fresh; + csi.limits = limits; + + log::trace!("Created contref @ {:p}", contref); + Ok(contref) +} + +/// This type represents a linked lists ("chain") of stacks, where the a +/// node's successor denotes its parent. +/// A additionally, a `CommonStackInformation` object is associated with +/// each stack in the list. +/// Here, a "stack" is one of the following: +/// - A continuation (i.e., created with cont.new). +/// - The initial stack. This is the stack that we were on when entering +/// Wasm (i.e., when executing +/// `crate::runtime::func::invoke_wasm_and_catch_traps`). +/// This stack never has a parent. +/// In terms of the memory allocation that this stack resides on, it will +/// usually be the main stack, but doesn't have to: If we are running +/// inside a continuation while executing a host call, which in turn +/// re-renters Wasm, the initial stack is actually the stack of that +/// continuation. +/// +/// Note that the linked list character of `VMStackChain` arises from the fact +/// that `VMStackChain::Continuation` variants have a pointer to a +/// `VMContRef`, which in turn has a `parent_chain` value of type +/// `VMStackChain`. This is how the stack chain reflects the parent-child +/// relationships between continuations/stacks. This also shows how the +/// initial stack (mentioned above) cannot have a parent. +/// +/// There are generally two uses of `VMStackChain`: +/// +/// 1. The `stack_chain` field in the `StoreOpaque` contains such a +/// chain of stacks, where the head of the list denotes the stack that is +/// currently executing (either a continuation or the initial stack). Note +/// that in this case, the linked list must contain 0 or more `Continuation` +/// elements, followed by a final `InitialStack` element. In particular, +/// this list always ends with `InitialStack` and never contains an `Absent` +/// variant. +/// +/// 2. When a continuation is suspended, its chain of parents eventually +/// ends with an `Absent` variant in its `parent_chain` field. Note that a +/// suspended continuation never appears in the stack chain in the +/// VMContext! +/// +/// +/// As mentioned before, each stack in a `VMStackChain` has a corresponding +/// `CommonStackInformation` object. For continuations, this is stored in +/// the `common_stack_information` field of the corresponding `VMContRef`. +/// For the initial stack, the `InitialStack` variant contains a pointer to +/// a `CommonStackInformation`. The latter will be allocated allocated on +/// the stack frame that executed by `invoke_wasm_and_catch_traps`. +/// +/// The following invariants hold for these `VMStackLimits` objects, +/// and the data in `VMRuntimeLimits`. +/// +/// Currently executing stack: For the currently executing stack (i.e., the +/// stack that is at the head of the store's `stack_chain` list), the +/// associated `VMStackLimits` object contains stale/undefined data. Instead, +/// the live data describing the limits for the currently executing stack is +/// always maintained in `VMRuntimeLimits`. Note that as a general rule +/// independently from any execution of continuations, the `last_wasm_exit*` +/// fields in the `VMRuntimeLimits` contain undefined values while executing +/// wasm. +/// +/// Parents of currently executing stack: For stacks that appear in the tail +/// of the store's `stack_chain` list (i.e., stacks that are not currently +/// executing themselves, but are an ancestor of the currently executing +/// stack), we have the following: All the fields in the stack's +/// `VMStackLimits` are valid, describing the stack's stack limit, and +/// pointers where executing for that stack entered and exited WASM. +/// +/// Suspended continuations: For suspended continuations (including their +/// ancestors), we have the following. Note that the initial stack can never +/// be in this state. The `stack_limit` and `last_enter_wasm_sp` fields of +/// the corresponding `VMStackLimits` object contain valid data, while the +/// `last_exit_wasm_*` fields contain arbitrary values. There is only one +/// exception to this: Note that a continuation that has been created with +/// cont.new, but never been resumed so far, is considered "suspended". +/// However, its `last_enter_wasm_sp` field contains undefined data. This is +/// justified, because when resume-ing a continuation for the first time, a +/// native-to-wasm trampoline is called, which sets up the +/// `last_wasm_entry_sp` in the `VMRuntimeLimits` with the correct value, +/// thus restoring the necessary invariant. +#[derive(Debug, Clone, PartialEq)] +#[repr(usize, C)] +pub enum VMStackChain { + /// For suspended continuations, denotes the end of their chain of + /// ancestors. + Absent = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT, + /// Represents the initial stack (i.e., where we entered Wasm from the + /// host by executing + /// `crate::runtime::func::invoke_wasm_and_catch_traps`). Therefore, it + /// does not have a parent. The `CommonStackInformation` that this + /// variant points to is stored in the stack frame of + /// `invoke_wasm_and_catch_traps`. + InitialStack(*mut VMCommonStackInformation) = + wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT, + /// Represents a continuation's stack. + Continuation(*mut VMContRef) = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT, +} + +impl VMStackChain { + /// Indicates if `self` is a `InitialStack` variant. + pub fn is_initial_stack(&self) -> bool { + matches!(self, VMStackChain::InitialStack(_)) + } + + /// Returns an iterator over the continuations in this chain. + /// We don't implement `IntoIterator` because our iterator is unsafe, so at + /// least this gives us some way of indicating this, even though the actual + /// unsafety lies in the `next` function. + /// + /// # Safety + /// + /// This function is not unsafe per see, but it returns an object + /// whose usage is unsafe. + pub unsafe fn into_continuation_iter(self) -> ContinuationIterator { + ContinuationIterator(self) + } + + /// Returns an iterator over the stack limits in this chain. + /// We don't implement `IntoIterator` because our iterator is unsafe, so at + /// least this gives us some way of indicating this, even though the actual + /// unsafety lies in the `next` function. + /// + /// # Safety + /// + /// This function is not unsafe per see, but it returns an object + /// whose usage is unsafe. + pub unsafe fn into_stack_limits_iter(self) -> StackLimitsIterator { + StackLimitsIterator(self) + } +} + +#[test] +fn check_vm_stack_chain_offsets() { + use std::mem::size_of; + use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets}; + + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmstack_chain()) + ); +} + +/// Iterator for Continuations in a stack chain. +pub struct ContinuationIterator(VMStackChain); + +/// Iterator for VMStackLimits in a stack chain. +pub struct StackLimitsIterator(VMStackChain); + +impl Iterator for ContinuationIterator { + type Item = *mut VMContRef; + + fn next(&mut self) -> Option { + match self.0 { + VMStackChain::Absent | VMStackChain::InitialStack(_) => None, + VMStackChain::Continuation(ptr) => { + let continuation = unsafe { ptr.as_mut().unwrap() }; + self.0 = continuation.parent_chain.clone(); + Some(ptr) + } + } + } +} + +impl Iterator for StackLimitsIterator { + type Item = *mut VMStackLimits; + + fn next(&mut self) -> Option { + match self.0 { + VMStackChain::Absent => None, + VMStackChain::InitialStack(csi) => { + let stack_limits = unsafe { &mut (*csi).limits } as *mut VMStackLimits; + self.0 = VMStackChain::Absent; + Some(stack_limits) + } + VMStackChain::Continuation(ptr) => { + let continuation = unsafe { ptr.as_mut().unwrap() }; + let stack_limits = + (&mut continuation.common_stack_information.limits) as *mut VMStackLimits; + self.0 = continuation.parent_chain.clone(); + Some(stack_limits) + } + } + } +} + +/// Encodes the life cycle of a `VMContRef`. +#[derive(Debug, Clone, Copy, PartialEq)] +#[repr(u32)] +pub enum VMStackState { + /// The `VMContRef` has been created, but neither `resume` or `switch` has ever been + /// called on it. During this stage, we may add arguments using `cont.bind`. + Fresh = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT, + /// The continuation is running, meaning that it is the one currently + /// executing code. + Running = wasmtime_environ::STACK_STATE_RUNNING_DISCRIMINANT, + /// The continuation is suspended because it executed a resume instruction + /// that has not finished yet. In other words, it became the parent of + /// another continuation (which may itself be `Running`, a `Parent`, or + /// `Suspended`). + Parent = wasmtime_environ::STACK_STATE_PARENT_DISCRIMINANT, + /// The continuation was suspended by a `suspend` or `switch` instruction. + Suspended = wasmtime_environ::STACK_STATE_SUSPENDED_DISCRIMINANT, + /// The function originally passed to `cont.new` has returned normally. + /// Note that there is no guarantee that a VMContRef will ever + /// reach this status, as it may stay suspended until being dropped. + Returned = wasmtime_environ::STACK_STATE_RETURNED_DISCRIMINANT, +} + +/// Universal control effect. This structure encodes return signal, resume +/// signal, suspension signal, and the handler to suspend to in a single variant +/// type. This instance is used at runtime. There is a codegen counterpart in +/// `cranelift/src/stack-switching/control_effect.rs`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +#[allow(dead_code)] +pub enum ControlEffect { + /// Used to signal that a continuation has returned and control switches + /// back to the parent. + Return = wasmtime_environ::CONTROL_EFFECT_RETURN_DISCRIMINANT, + /// Used to signal to a continuation that it is being resumed. + Resume = wasmtime_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT, + /// Used to signal that a continuation has invoked a `suspend` instruction. + Suspend { + /// The index of the handler to be used in the parent continuation to + /// switch back to. + handler_index: u32, + } = wasmtime_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT, + /// Used to signal that a continuation has invoked a `suspend` instruction. + Switch = wasmtime_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT, +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs new file mode 100644 index 000000000000..44e3947abbc8 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs @@ -0,0 +1,119 @@ +//! This module contains a modified version of the `wasmtime_fiber` crate, +//! specialized for executing stack switching continuations. + +#![allow(missing_docs)] + +use anyhow::Result; +use core::ops::Range; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; + +cfg_if::cfg_if! { + if #[cfg(all(feature = "stack-switching", unix, target_arch = "x86_64"))] { + mod unix; + use unix as imp; + } else { + mod dummy; + use dummy as imp; + } +} + +/// Represents an execution stack to use for a fiber. +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack(imp::VMContinuationStack); + +impl VMContinuationStack { + /// Creates a new fiber stack of the given size. + pub fn new(size: usize) -> Result { + Ok(Self(imp::VMContinuationStack::new(size)?)) + } + + /// Returns a stack of size 0. + pub fn unallocated() -> Self { + Self(imp::VMContinuationStack::unallocated()) + } + + /// Is this stack unallocated/of size 0? + pub fn is_unallocated(&self) -> bool { + imp::VMContinuationStack::is_unallocated(&self.0) + } + + /// Creates a new fiber stack with the given pointer to the bottom of the + /// stack plus the byte length of the stack. + /// + /// The `bottom` pointer should be addressable for `len` bytes. The page + /// beneath `bottom` should be unmapped as a guard page. + /// + /// # Safety + /// + /// This is unsafe because there is no validation of the given pointer. + /// + /// The caller must properly allocate the stack space with a guard page and + /// make the pages accessible for correct behavior. + pub unsafe fn from_raw_parts(bottom: *mut u8, guard_size: usize, len: usize) -> Result { + Ok(Self(imp::VMContinuationStack::from_raw_parts( + bottom, guard_size, len, + )?)) + } + + /// Is this a manually-managed stack created from raw parts? If so, it is up + /// to whoever created it to manage the stack's memory allocation. + pub fn is_from_raw_parts(&self) -> bool { + self.0.is_from_raw_parts() + } + + /// Gets the top of the stack. + /// + /// Returns `None` if the platform does not support getting the top of the + /// stack. + pub fn top(&self) -> Option<*mut u8> { + self.0.top() + } + + /// Returns the range of where this stack resides in memory if the platform + /// supports it. + pub fn range(&self) -> Option> { + self.0.range() + } + + /// Returns the instruction pointer stored in the Fiber's ControlContext. + pub fn control_context_instruction_pointer(&self) -> usize { + self.0.control_context_instruction_pointer() + } + + /// Returns the frame pointer stored in the Fiber's ControlContext. + pub fn control_context_frame_pointer(&self) -> usize { + self.0.control_context_frame_pointer() + } + + /// Returns the stack pointer stored in the Fiber's ControlContext. + pub fn control_context_stack_pointer(&self) -> usize { + self.0.control_context_stack_pointer() + } + + /// Initializes this stack, such that it will execute the function denoted + /// by `func_ref`. `parameter_count` and `return_value_count` must be the + /// corresponding number of parameters and return values of `func_ref`. + /// `args` must point to the `args` field of the `VMContRef` owning this pointer. + /// + /// It will be updated by this function to correctly describe + /// the buffer used by this function for its arguments and return values. + pub fn initialize( + &self, + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + parameter_count: u32, + return_value_count: u32, + ) { + self.0.initialize( + func_ref, + caller_vmctx, + args, + parameter_count, + return_value_count, + ) + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs new file mode 100644 index 000000000000..e2042e352c19 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs @@ -0,0 +1,75 @@ +use anyhow::Result; +use core::ops::Range; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw}; + +#[allow(dead_code)] +#[derive(Debug, PartialEq, Eq)] +pub enum Allocator { + Mmap, + Custom, +} + +/// Making sure that this has the same size as the non-dummy version, to +/// make some tests happy. +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack { + _top: *mut u8, + _len: usize, + _allocator: Allocator, +} + +impl VMContinuationStack { + pub fn new(_size: usize) -> Result { + anyhow::bail!("Stack switching disabled or not implemented on this platform") + } + + pub fn unallocated() -> Self { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn is_unallocated(&self) -> bool { + panic!("Stack switching disabled or not implemented on this platform") + } + + #[allow(clippy::missing_safety_doc)] + pub unsafe fn from_raw_parts(_base: *mut u8, _guard_size: usize, _len: usize) -> Result { + anyhow::bail!("Stack switching disabled or not implemented on this platform") + } + + pub fn is_from_raw_parts(&self) -> bool { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn top(&self) -> Option<*mut u8> { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn range(&self) -> Option> { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_instruction_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_frame_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn control_context_stack_pointer(&self) -> usize { + panic!("Stack switching disabled or not implemented on this platform") + } + + pub fn initialize( + &self, + _func_ref: *const VMFuncRef, + _caller_vmctx: *mut VMContext, + _args: *mut VMArray, + _parameter_count: u32, + _return_value_count: u32, + ) { + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs new file mode 100644 index 000000000000..930431ac6c02 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs @@ -0,0 +1,356 @@ +//! The stack layout is expected to look like so: +//! +//! +//! ```text +//! 0xB000 +-----------------------+ <- top of stack (TOS) +//! | saved RIP | +//! 0xAff8 +-----------------------+ +//! | saved RBP | +//! 0xAff0 +-----------------------+ +//! | saved RSP | +//! 0xAfe8 +-----------------------+ <- beginning of "control context", +//! | args_capacity | +//! 0xAfe0 +-----------------------+ +//! | args buffer, size: | +//! | (16 * args_capacity) | +//! 0xAfc0 +-----------------------+ <- below: beginning of usable stack space +//! | | (16-byte aligned) +//! | | +//! ~ ... ~ <- actual native stack space to use +//! | | +//! 0x1000 +-----------------------+ +//! | guard page | <- (not currently enabled) +//! 0x0000 +-----------------------+ +//! ``` +//! +//! The "control context" indicates how to resume a computation. The layout is +//! determined by Cranelift's stack_switch instruction, which reads and writes +//! these fields. The fields are used as follows, where we distinguish two +//! cases: +//! +//! 1. +//! If the continuation is currently active (i.e., running directly, or ancestor +//! of the running continuation), it stores the PC, RSP, and RBP of the *parent* +//! of the running continuation. +//! +//! 2. +//! If the picture shows a suspended computation, the fields store the PC, RSP, +//! and RBP at the time of the suspension. +//! +//! Note that this design ensures that external tools can construct backtraces +//! in the presence of stack switching by using frame pointers only: The +//! wasmtime_continuation_start trampoline uses the address of the RBP field in the +//! control context (0xAff0 above) as its frame pointer. This means that when +//! passing the wasmtime_continuation_start frame while doing frame pointer walking, +//! the parent of that frame is the last frame in the parent of this +//! continuation. +//! +//! Wasmtime's own mechanism for constructing backtraces also relies on frame +//! pointer chains. However, it understands continuations and does not rely on +//! the trickery outlined here to go from the frames in one continuation to the +//! parent. +//! +//! The args buffer is used as follows: It is used by the array calling +//! trampoline to read and store the arguments and return values of the function +//! running inside the continuation. If this function has m parameters and n +//! return values, then args_capacity is defined as max(m, n) and the size of +//! the args buffer is args_capacity * 16 bytes. The start address (0xAfc0 in +//! the example above, thus assuming args_capacity = 2) is saved as the `data` +//! field of the VMContRef's `args` object. + +#![allow(unused_macros)] + +use core::ptr::NonNull; +use std::io; +use std::ops::Range; +use std::ptr; + +use crate::runtime::vm::stack_switching::VMArray; +use crate::runtime::vm::{VMContext, VMFuncRef, VMOpaqueContext, ValRaw}; + +#[derive(Debug, PartialEq, Eq)] +pub enum Allocator { + Mmap, + Custom, +} + +#[derive(Debug)] +#[repr(C)] +pub struct VMContinuationStack { + // The top of the stack; for stacks allocated by the fiber implementation itself, + // the base address of the allocation will be `top.sub(len.unwrap())` + top: *mut u8, + // The length of the stack + len: usize, + // allocation strategy + allocator: Allocator, +} + +impl VMContinuationStack { + pub fn new(size: usize) -> io::Result { + // Round up our stack size request to the nearest multiple of the + // page size. + let page_size = rustix::param::page_size(); + let size = if size == 0 { + page_size + } else { + (size + (page_size - 1)) & (!(page_size - 1)) + }; + + unsafe { + // Add in one page for a guard page and then ask for some memory. + let mmap_len = size + page_size; + let mmap = rustix::mm::mmap_anonymous( + ptr::null_mut(), + mmap_len, + rustix::mm::ProtFlags::empty(), + rustix::mm::MapFlags::PRIVATE, + )?; + + rustix::mm::mprotect( + mmap.cast::().add(page_size).cast(), + size, + rustix::mm::MprotectFlags::READ | rustix::mm::MprotectFlags::WRITE, + )?; + + Ok(Self { + top: mmap.cast::().add(mmap_len), + len: mmap_len, + allocator: Allocator::Mmap, + }) + } + } + + pub fn unallocated() -> Self { + Self { + top: std::ptr::null_mut(), + len: 0, + allocator: Allocator::Custom, + } + } + + pub fn is_unallocated(&self) -> bool { + debug_assert_eq!(self.len == 0, self.top == std::ptr::null_mut()); + self.len == 0 + } + + #[allow(clippy::missing_safety_doc)] + pub unsafe fn from_raw_parts( + base: *mut u8, + _guard_size: usize, + len: usize, + ) -> io::Result { + Ok(Self { + top: base.add(len), + len, + allocator: Allocator::Custom, + }) + } + + pub fn is_from_raw_parts(&self) -> bool { + self.allocator == Allocator::Custom + } + + pub fn top(&self) -> Option<*mut u8> { + Some(self.top) + } + + pub fn range(&self) -> Option> { + let base = unsafe { self.top.sub(self.len) as usize }; + Some(base..base + self.len) + } + + pub fn control_context_instruction_pointer(&self) -> usize { + // See picture at top of this file: + // RIP is stored 8 bytes below top of stack. + unsafe { + let ptr = self.top.sub(8) as *mut usize; + *ptr + } + } + + pub fn control_context_frame_pointer(&self) -> usize { + // See picture at top of this file: + // RBP is stored 16 bytes below top of stack. + unsafe { + let ptr = self.top.sub(16) as *mut usize; + *ptr + } + } + + pub fn control_context_stack_pointer(&self) -> usize { + // See picture at top of this file: + // RSP is stored 24 bytes below top of stack. + unsafe { + let ptr = self.top.sub(24) as *mut usize; + *ptr + } + } + + /// This function installs the launchpad for the computation to run on the + /// fiber, such that executing a `stack_switch` instruction on the stack + /// actually runs the desired computation. + /// + /// Concretely, switching to the stack prepared by this function + /// causes that we enter `wasmtime_continuation_start`, which then in turn + /// calls `fiber_start` with the following arguments: + /// TOS, func_ref, caller_vmctx, args_ptr, args_capacity + /// + /// Note that at this point we also allocate the args buffer + /// (see picture at the top of this file). + /// We define `args_capacity` as the max of parameter and return value count. + /// Then the size s of the actual buffer size is calculated as follows: + /// s = size_of(ValRaw) * `args_capacity`, + /// + /// Note that this value is used below, and we may have s = 0. + /// + /// The layout of the VMContinuationStack near the top of stack (TOS) + /// *after* running this function is as follows: + /// + /// + /// Offset from | + /// TOS | Contents + /// ---------------|------------------------------------------------------- + /// -0x08 | address of wasmtime_continuation_start function (future PC) + /// -0x10 | TOS - 0x10 (future RBP) + /// -0x18 | TOS - 0x40 - s (future RSP) + /// -0x20 | args_capacity + /// + /// + /// The data stored behind the args buffer is as follows: + /// + /// Offset from | + /// TOS | Contents + /// ---------------|------------------------------------------------------- + /// -0x28 - s | func_ref + /// -0x30 - s | caller_vmctx + /// -0x38 - s | args (of type *mut ArrayRef) + /// -0x40 - s | return_value_count + pub fn initialize( + &self, + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + parameter_count: u32, + return_value_count: u32, + ) { + let tos = self.top; + + unsafe { + let store = |tos_neg_offset, value| { + let target = tos.sub(tos_neg_offset) as *mut usize; + target.write(value) + }; + + let args = &mut *args; + let args_capacity = std::cmp::max(parameter_count, return_value_count); + // The args object must currently be empty. + debug_assert_eq!(args.capacity, 0); + debug_assert_eq!(args.length, 0); + + // The size of the args buffer + let s = args_capacity as usize * std::mem::size_of::(); + + // The actual pointer to the buffer + let args_data_ptr = if args_capacity == 0 { + 0 + } else { + tos as usize - 0x20 - s + }; + + args.capacity = args_capacity; + args.data = args_data_ptr as *mut ValRaw; + + let to_store = [ + // Data near top of stack: + (0x08, wasmtime_continuation_start as usize), + (0x10, tos.sub(0x10) as usize), + (0x18, tos.sub(0x40 + s) as usize), + (0x20, args_capacity as usize), + // Data after the args buffer: + (0x28 + s, func_ref as usize), + (0x30 + s, caller_vmctx as usize), + (0x38 + s, args as *mut VMArray as usize), + (0x40 + s, return_value_count as usize), + ]; + + for (offset, data) in to_store { + store(offset, data); + } + } + } +} + +impl Drop for VMContinuationStack { + fn drop(&mut self) { + unsafe { + match self.allocator { + Allocator::Mmap => { + let ret = rustix::mm::munmap(self.top.sub(self.len) as _, self.len); + debug_assert!(ret.is_ok()); + } + Allocator::Custom => {} // It's the creator's responsibility to reclaim the memory. + } + } + } +} + +unsafe extern "C" { + #[allow(dead_code)] // only used in inline assembly for some platforms + fn wasmtime_continuation_start(); +} + +/// This function is responsible for actually running a wasm function inside a +/// continuation. It is only ever called from `wasmtime_continuation_start`. Hence, it +/// must never return. +unsafe extern "C" fn fiber_start( + func_ref: *const VMFuncRef, + caller_vmctx: *mut VMContext, + args: *mut VMArray, + return_value_count: u32, +) { + unsafe { + let func_ref = func_ref.as_ref().expect("Non-null function reference"); + let caller_vmxtx = VMOpaqueContext::from_vmcontext(NonNull::new_unchecked(caller_vmctx)); + let args = &mut *args; + let params_and_returns: NonNull<[ValRaw]> = if args.capacity == 0 { + NonNull::from(&[]) + } else { + std::slice::from_raw_parts_mut(args.data, args.capacity as usize).into() + }; + + // NOTE(frank-emrich) The usage of the `caller_vmctx` is probably not + // 100% correct here. Currently, we determine the "caller" vmctx when + // initilizing the fiber stack/continuation (i.e. as part of + // `cont.new`). However, we may subsequenly `resume` the continuation + // from a different Wasm instance. The way to fix this would be to make + // the currently active `VMContext` an additional parameter of + // `wasmtime_continuation_switch` and pipe it through to this point. However, + // since the caller vmctx is only really used to access stuff in the + // underlying `Store`, it's fine to be slightly sloppy about the exact + // value we set. + // + // TODO(dhil): we are ignoring the boolean return value + // here... we probably shouldn't. + func_ref.array_call(None, caller_vmxtx, params_and_returns); + + // The array call trampoline should have just written + // `return_value_count` values to the `args` buffer. Let's reflect that + // in its length field, to make various bounds checks happy. + args.length = return_value_count; + + // Note that after this function returns, wasmtime_continuation_start + // will switch back to the parent stack. + } +} + +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + } else { + // Note that this shoul be unreachable: In stack.rs, we currently select + // the module defined in the current file only if we are on unix AND + // x86_64. + compile_error!("the stack switching feature is not supported on this CPU architecture"); + } +} diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs new file mode 100644 index 000000000000..3a2292dd8e3a --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs @@ -0,0 +1,83 @@ +// A WORD OF CAUTION +// +// This entire file basically needs to be kept in sync with itself. It's not +// really possible to modify just one bit of this file without understanding +// all the other bits. Documentation tries to reference various bits here and +// there but try to make sure to read over everything before tweaking things! + +use wasmtime_asm_macros::asm_func; + +// This is a pretty special function that has no real signature. Its use is to +// be the "base" function of all fibers. This entrypoint is used in +// `wasmtime_continuation_init` to bootstrap the execution of a new fiber. +// +// We also use this function as a persistent frame on the stack to emit dwarf +// information to unwind into the caller. This allows us to unwind from the +// fiber's stack back to the initial stack that the fiber was called from. We use +// special dwarf directives here to do so since this is a pretty nonstandard +// function. +// +// If you're curious a decent introduction to CFI things and unwinding is at +// https://www.imperialviolet.org/2017/01/18/cfi.html +// +// Note that this function is never called directly. It is only ever entered +// when a `stack_switch` instruction loads its address when switching to a stack +// prepared by `FiberStack::initialize`. +// +// Executing `stack_switch` on a stack prepared by `FiberStack::initialize` as +// described in the comment on `FiberStack::initialize` leads to the following +// values in various registers when execution of wasmtime_continuation_start begins: +// +// RSP: TOS - 0x40 - (16 * `args_capacity`) +// RBP: TOS - 0x10 +asm_func!( + "wasmtime_continuation_start", + " + // TODO(frank-emrich): Restore DWARF information for this function. In + // the meantime, debugging is possible using frame pointer walking. + + + // + // Note that the next 4 instructions amount to calling fiber_start + // with the following arguments: + // 1. func_ref + // 2. caller_vmctx + // 3. args (of type *mut ArrayRef) + // 4. return_value_count + // + + pop rcx // return_value_count + pop rdx // args + pop rsi // caller_vmctx + pop rdi // func_ref + // Note that RBP already contains the right frame pointer to build a + // frame pointer chain including the parent continuation: + // The current value of RBP is where we store the parent RBP in the + // control context! + call {fiber_start} + + // Return to the parent continuation. + // RBP is callee-saved (no matter if it's used as a frame pointe or + // not), so its value is still TOS - 0x10. + // Use that fact to obtain saved parent RBP, RSP, and RIP from control + // context near TOS. + mov rsi, 0x08[rbp] // putting new RIP in temp register + mov rsp, -0x08[rbp] + mov rbp, [rbp] + + // The stack_switch instruction uses register RDI for the payload. + // Here, the payload indicates that we are returning (value 0). + // See the test case below to keep this in sync with + // ControlEffect::return_() + mov rdi, 0 + + jmp rsi + ", + fiber_start = sym super::fiber_start, +); + +#[test] +fn test_return_payload() { + // The following assumption is baked into `wasmtime_continuation_start`. + assert_eq!(wasmtime_environ::CONTROL_EFFECT_RETURN_DISCRIMINANT, 0); +} diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs index 09ef06bfff50..9cfa186701d7 100644 --- a/crates/wasmtime/src/runtime/vm/table.rs +++ b/crates/wasmtime/src/runtime/vm/table.rs @@ -5,6 +5,7 @@ #![cfg_attr(feature = "gc", allow(irrefutable_let_patterns))] use crate::prelude::*; +use crate::runtime::vm::stack_switching::VMContObj; use crate::runtime::vm::vmcontext::{VMFuncRef, VMTableDefinition}; use crate::runtime::vm::{GcStore, SendSyncPtr, VMGcRef, VMStore}; use core::alloc::Layout; @@ -34,12 +35,16 @@ pub enum TableElement { /// (which has access to the info needed for lazy initialization) /// will replace it when fetched. UninitFunc, + + /// A `contref` + ContRef(Option), } #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum TableElementType { Func, GcRef, + Cont, } impl TableElementType { @@ -47,9 +52,19 @@ impl TableElementType { match (val, self) { (TableElement::FuncRef(_), TableElementType::Func) => true, (TableElement::GcRef(_), TableElementType::GcRef) => true, + (TableElement::ContRef(_), TableElementType::Cont) => true, _ => false, } } + + /// Returns the size required to actually store an element of this particular type + pub fn element_size(&self) -> usize { + match self { + TableElementType::Func => core::mem::size_of::(), + TableElementType::GcRef => core::mem::size_of::>(), + TableElementType::Cont => core::mem::size_of::(), + } + } } // The usage of `*mut VMFuncRef` is safe w.r.t. thread safety, this just relies @@ -74,6 +89,7 @@ impl TableElement { Self::FuncRef(e) => e, Self::UninitFunc => panic!("Uninitialized table element value outside of table slot"), Self::GcRef(_) => panic!("GC reference is not a function reference"), + Self::ContRef(_) => panic!("Continuation reference is not a function reference"), } } @@ -105,6 +121,18 @@ impl From for TableElement { } } +impl From> for TableElement { + fn from(c: Option) -> TableElement { + TableElement::ContRef(c) + } +} + +impl From for TableElement { + fn from(c: VMContObj) -> TableElement { + TableElement::ContRef(Some(c)) + } +} + #[derive(Copy, Clone)] #[repr(transparent)] struct TaggedFuncRef(*mut VMFuncRef); @@ -140,10 +168,36 @@ impl TaggedFuncRef { } pub type FuncTableElem = Option>; +pub type ContTableElem = Option; + +/// The maximum of the sizes of any of the table element types +#[allow(dead_code, reason = "Only used if pooling allocator is enabled")] +pub const NOMINAL_MAX_TABLE_ELEM_SIZE: usize = { + // ContTableElem intentionally excluded for "nominal" calculation. + let sizes = [ + core::mem::size_of::(), + core::mem::size_of::>(), + ]; + + // This is equivalent to `|data| {data.iter().reduce(std::cmp::max).unwrap()}`, + // but as a `const` function, so we can use it to define a constant. + const fn slice_max(data: &[usize]) -> usize { + match data { + [] => 0, + [head, tail @ ..] => { + let tail_max = slice_max(tail); + if *head >= tail_max { *head } else { tail_max } + } + } + } + + slice_max(&sizes) +}; pub enum StaticTable { Func(StaticFuncTable), GcRef(StaticGcRefTable), + Cont(StaticContTable), } impl From for StaticTable { @@ -158,6 +212,12 @@ impl From for StaticTable { } } +impl From for StaticTable { + fn from(value: StaticContTable) -> Self { + Self::Cont(value) + } +} + pub struct StaticFuncTable { /// Where data for this table is stored. The length of this list is the /// maximum size of the table. @@ -176,9 +236,18 @@ pub struct StaticGcRefTable { size: usize, } +pub struct StaticContTable { + /// Where data for this table is stored. The length of this list is the + /// maximum size of the table. + data: SendSyncPtr<[ContTableElem]>, + /// The current size of the table. + size: usize, +} + pub enum DynamicTable { Func(DynamicFuncTable), GcRef(DynamicGcRefTable), + Cont(DynamicContTable), } impl From for DynamicTable { @@ -193,6 +262,12 @@ impl From for DynamicTable { } } +impl From for DynamicTable { + fn from(value: DynamicContTable) -> Self { + Self::Cont(value) + } +} + pub struct DynamicFuncTable { /// Dynamically managed storage space for this table. The length of this /// vector is the current size of the table. @@ -211,6 +286,14 @@ pub struct DynamicGcRefTable { maximum: Option, } +pub struct DynamicContTable { + /// Dynamically managed storage space for this table. The length of this + /// vector is the current size of the table. + elements: Vec, + /// Maximum size that `elements` can grow to. + maximum: Option, +} + /// Represents an instance's table. pub enum Table { /// A "static" table where storage space is managed externally, currently @@ -241,6 +324,13 @@ impl From for Table { } } +impl From for Table { + fn from(value: StaticContTable) -> Self { + let t: StaticTable = value.into(); + t.into() + } +} + impl From for Table { fn from(value: DynamicTable) -> Self { Self::Dynamic(value) @@ -261,11 +351,18 @@ impl From for Table { } } -fn wasm_to_table_type(ty: WasmRefType) -> TableElementType { +impl From for Table { + fn from(value: DynamicContTable) -> Self { + let t: DynamicTable = value.into(); + t.into() + } +} + +pub(crate) fn wasm_to_table_type(ty: WasmRefType) -> TableElementType { match ty.heap_type.top() { WasmHeapTopType::Func => TableElementType::Func, WasmHeapTopType::Any | WasmHeapTopType::Extern => TableElementType::GcRef, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => TableElementType::Cont, } } @@ -326,6 +423,10 @@ impl Table { elements: unsafe { alloc_dynamic_table_elements(minimum)? }, maximum, })), + TableElementType::Cont => Ok(Self::from(DynamicContTable { + elements: vec![None; minimum], + maximum, + })), } } @@ -385,6 +486,26 @@ impl Table { )); Ok(Self::from(StaticGcRefTable { data, size })) } + TableElementType::Cont => { + let len = { + let data = data.as_non_null().as_ref(); + let (before, data, after) = data.align_to::(); + assert!(before.is_empty()); + assert!(after.is_empty()); + data.len() + }; + ensure!( + usize::try_from(ty.limits.min).unwrap() <= len, + "initial table size of {} exceeds the pooling allocator's \ + configured maximum table size of {len} elements", + ty.limits.min, + ); + let data = SendSyncPtr::new(NonNull::slice_from_raw_parts( + data.as_non_null().cast::(), + cmp::min(len, max), + )); + Ok(Self::from(StaticContTable { data, size })) + } } } @@ -441,6 +562,9 @@ impl Table { Table::Static(StaticTable::GcRef(_)) | Table::Dynamic(DynamicTable::GcRef(_)) => { TableElementType::GcRef } + Table::Static(StaticTable::Cont(_)) | Table::Dynamic(DynamicTable::Cont(_)) => { + TableElementType::Cont + } } } @@ -455,10 +579,12 @@ impl Table { match self { Table::Static(StaticTable::Func(StaticFuncTable { size, .. })) => *size, Table::Static(StaticTable::GcRef(StaticGcRefTable { size, .. })) => *size, + Table::Static(StaticTable::Cont(StaticContTable { size, .. })) => *size, Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => elements.len(), Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { elements.len() } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => elements.len(), } } @@ -470,10 +596,12 @@ impl Table { /// when it is being constrained by an instance allocator. pub fn maximum(&self) -> Option { match self { + Table::Static(StaticTable::Cont(StaticContTable { data, .. })) => Some(data.len()), Table::Static(StaticTable::Func(StaticFuncTable { data, .. })) => Some(data.len()), Table::Static(StaticTable::GcRef(StaticGcRefTable { data, .. })) => Some(data.len()), Table::Dynamic(DynamicTable::Func(DynamicFuncTable { maximum, .. })) => *maximum, Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { maximum, .. })) => *maximum, + Table::Dynamic(DynamicTable::Cont(DynamicContTable { maximum, .. })) => *maximum, } } @@ -579,6 +707,10 @@ impl Table { let (funcrefs, _lazy_init) = self.funcrefs_mut(); funcrefs[start..end].fill(TaggedFuncRef::UNINIT); } + TableElement::ContRef(c) => { + let contrefs = self.contrefs_mut(); + contrefs[start..end].fill(c); + } } Ok(()) @@ -659,6 +791,12 @@ impl Table { } *size = new_size; } + Table::Static(StaticTable::Cont(StaticContTable { data, size })) => { + unsafe { + debug_assert!(data.as_ref()[*size..new_size].iter().all(|x| x.is_none())); + } + *size = new_size; + } // These calls to `resize` could move the base address of // `elements`. If this table's limits declare it to be fixed-size, @@ -673,6 +811,9 @@ impl Table { Table::Dynamic(DynamicTable::GcRef(DynamicGcRefTable { elements, .. })) => { elements.resize_with(new_size, || None); } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => { + elements.resize(new_size, None); + } } self.fill( @@ -709,6 +850,11 @@ impl Table { TableElement::GcRef(r) }), + TableElementType::Cont => self + .contrefs() + .get(index) + .copied() + .map(|e| TableElement::ContRef(e)), } } @@ -736,6 +882,9 @@ impl Table { TableElement::GcRef(e) => { *self.gc_refs_mut().get_mut(index).ok_or(())? = e; } + TableElement::ContRef(c) => { + *self.contrefs_mut().get_mut(index).ok_or(())? = c; + } } Ok(()) } @@ -803,6 +952,10 @@ impl Table { current_elements: *size, } } + Table::Static(StaticTable::Cont(StaticContTable { data, size })) => VMTableDefinition { + base: data.cast().into(), + current_elements: *size, + }, Table::Dynamic(DynamicTable::Func(DynamicFuncTable { elements, .. })) => { VMTableDefinition { base: NonNull::<[FuncTableElem]>::from(&mut elements[..]) @@ -819,6 +972,14 @@ impl Table { current_elements: elements.len(), } } + Table::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => { + VMTableDefinition { + base: NonNull::<[Option]>::from(&mut elements[..]) + .cast() + .into(), + current_elements: elements.len(), + } + } } } @@ -883,6 +1044,32 @@ impl Table { } } + fn contrefs(&self) -> &[Option] { + assert_eq!(self.element_type(), TableElementType::Cont); + match self { + Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe { + slice::from_raw_parts(elements.as_ptr().cast(), elements.len()) + }, + Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe { + slice::from_raw_parts(data.as_ptr().cast(), *size) + }, + _ => unreachable!(), + } + } + + fn contrefs_mut(&mut self) -> &mut [Option] { + assert_eq!(self.element_type(), TableElementType::Cont); + match self { + Self::Dynamic(DynamicTable::Cont(DynamicContTable { elements, .. })) => unsafe { + slice::from_raw_parts_mut(elements.as_mut_ptr().cast(), elements.len()) + }, + Self::Static(StaticTable::Cont(StaticContTable { data, size })) => unsafe { + slice::from_raw_parts_mut(data.as_ptr().cast(), *size) + }, + _ => unreachable!(), + } + } + /// Get this table's GC references as a slice. /// /// Panics if this is not a table of GC references. @@ -931,6 +1118,11 @@ impl Table { ); } } + TableElementType::Cont => { + // `contref` are `Copy`, so just do a mempcy + dst_table.contrefs_mut()[dst_range] + .copy_from_slice(&src_table.contrefs()[src_range]); + } } } @@ -979,6 +1171,10 @@ impl Table { } } } + TableElementType::Cont => { + // `contref` are `Copy`, so just do a memmove + self.contrefs_mut().copy_within(src_range, dst_range.start); + } } } } diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 76148ff7f303..eadb1565a762 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -27,6 +27,9 @@ use core::ops::Range; use core::ptr::{self, NonNull}; pub use self::backtrace::Backtrace; +#[cfg(feature = "gc")] +pub use self::backtrace::Frame; + pub use self::coredump::CoreDumpStack; pub use self::tls::tls_eager_initialize; #[cfg(feature = "async")] @@ -430,7 +433,7 @@ where mod call_thread_state { use super::*; use crate::EntryStoreContext; - use crate::runtime::vm::Unwind; + use crate::runtime::vm::{Unwind, VMStackChain}; /// Temporary state stored on the stack which is registered in the `tls` /// module below for calls into wasm. @@ -532,6 +535,11 @@ mod call_thread_state { (&*self.old_state).last_wasm_entry_fp } + /// Get the saved `VMStackChain` for the previous `CallThreadState`. + pub unsafe fn old_stack_chain(&self) -> VMStackChain { + (&*self.old_state).stack_chain.clone() + } + /// Get the previous `CallThreadState`. pub fn prev(&self) -> tls::Ptr { self.prev.get() diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index b71002689458..cde668013540 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -23,10 +23,12 @@ use crate::prelude::*; use crate::runtime::store::StoreOpaque; +use crate::runtime::vm::stack_switching::VMStackChain; use crate::runtime::vm::{ Unwind, VMStoreContext, traphandlers::{CallThreadState, tls}, }; +use crate::vm::stack_switching::{VMContRef, VMStackState}; use core::ops::ControlFlow; /// A WebAssembly stack trace. @@ -107,6 +109,48 @@ impl Backtrace { }); } + // Walk the stack of the given continuation, which must be suspended, and + // all of its parent continuations (if any). + #[allow(dead_code, reason = "Only used by GC code at the moment")] + pub fn trace_suspended_continuation( + store: &StoreOpaque, + continuation: &VMContRef, + f: impl FnMut(Frame) -> ControlFlow<()>, + ) { + log::trace!("====== Capturing Backtrace (suspended continuation) ======"); + + assert_eq!( + continuation.common_stack_information.state, + VMStackState::Suspended + ); + + let unwind = store.unwinder(); + + let pc = continuation.stack.control_context_instruction_pointer(); + let fp = continuation.stack.control_context_frame_pointer(); + let trampoline_fp = continuation + .common_stack_information + .limits + .last_wasm_entry_fp; + + unsafe { + // FIXME(frank-emrich) Casting from *const to *mut pointer is + // terrible, but we won't actually modify any of the continuations + // here. + let stack_chain = + VMStackChain::Continuation(continuation as *const VMContRef as *mut VMContRef); + + if let ControlFlow::Break(()) = + Self::trace_through_continuations(unwind, stack_chain, pc, fp, trampoline_fp, f) + { + log::trace!("====== Done Capturing Backtrace (closure break) ======"); + return; + } + } + + log::trace!("====== Done Capturing Backtrace (reached end of stack chain) ======"); + } + /// Walk the current Wasm stack, calling `f` for each frame we walk. /// /// If Wasm hit a trap, and we calling this from the trap handler, then the @@ -141,7 +185,17 @@ impl Backtrace { } }; + let stack_chain = (*(*vm_store_context).stack_chain.get()).clone(); + + // The first value in `activations` is for the most recently running + // wasm. We thus provide the stack chain of `first_wasm_state` to + // traverse the potential continuation stacks. For the subsequent + // activations, we unconditionally use `None` as the corresponding stack + // chain. This is justified because only the most recent execution of + // wasm may execute off the initial stack (see comments in + // `wasmtime::invoke_wasm_and_catch_traps` for details). let activations = core::iter::once(( + stack_chain, last_wasm_exit_pc, last_wasm_exit_fp, *(*vm_store_context).last_wasm_entry_fp.get(), @@ -149,25 +203,31 @@ impl Backtrace { .chain( state .iter() + .flat_map(|state| state.iter()) .filter(|state| core::ptr::eq(vm_store_context, state.vm_store_context.as_ptr())) .map(|state| { ( + state.old_stack_chain(), state.old_last_wasm_exit_pc(), state.old_last_wasm_exit_fp(), state.old_last_wasm_entry_fp(), ) }), ) - .take_while(|&(pc, fp, sp)| { - if pc == 0 { - debug_assert_eq!(fp, 0); - debug_assert_eq!(sp, 0); + .take_while(|(chain, pc, fp, sp)| { + if *pc == 0 { + debug_assert_eq!(*fp, 0); + debug_assert_eq!(*sp, 0); + } else { + debug_assert_ne!(chain.clone(), VMStackChain::Absent) } - pc != 0 + *pc != 0 }); - for (pc, fp, sp) in activations { - if let ControlFlow::Break(()) = Self::trace_through_wasm(unwind, pc, fp, sp, &mut f) { + for (chain, pc, fp, sp) in activations { + if let ControlFlow::Break(()) = + Self::trace_through_continuations(unwind, chain, pc, fp, sp, &mut f) + { log::trace!("====== Done Capturing Backtrace (closure break) ======"); return; } @@ -176,6 +236,100 @@ impl Backtrace { log::trace!("====== Done Capturing Backtrace (reached end of activations) ======"); } + /// Traces through a sequence of stacks, creating a backtrace for each one, + /// beginning at the given `pc` and `fp`. + /// + /// If `chain` is `InitialStack`, we are tracing through the initial stack, + /// and this function behaves like `trace_through_wasm`. + /// Otherwise, we can interpret `chain` as a linked list of stacks, which + /// ends with the initial stack. We then trace through each of these stacks + /// individually, up to (and including) the initial stack. + unsafe fn trace_through_continuations( + unwind: &dyn Unwind, + chain: VMStackChain, + pc: usize, + fp: usize, + trampoline_fp: usize, + mut f: impl FnMut(Frame) -> ControlFlow<()>, + ) -> ControlFlow<()> { + use crate::runtime::vm::stack_switching::{VMContRef, VMStackLimits}; + + // Handle the stack that is currently running (which may be a + // continuation or the initial stack). + Self::trace_through_wasm(unwind, pc, fp, trampoline_fp, &mut f)?; + + // Note that the rest of this function has no effect if `chain` is + // `Some(VMStackChain::InitialStack(_))` (i.e., there is only one stack to + // trace through: the initial stack) + + assert_ne!(chain, VMStackChain::Absent); + let stack_limits_vec: Vec<*mut VMStackLimits> = + chain.clone().into_stack_limits_iter().collect(); + let continuations_vec: Vec<*mut VMContRef> = + chain.clone().into_continuation_iter().collect(); + + // The VMStackLimits of the currently running stack (whether that's a + // continuation or the initial stack) contains undefined data, the + // information about that stack is saved in the Store's + // `VMRuntimeLimits` and handled at the top of this function + // already. That's why we ignore `stack_limits_vec[0]`. + // + // Note that a continuation stack's control context stores + // information about how to resume execution *in its parent*. Thus, + // we combine the information from continuations_vec[i] with + // stack_limits_vec[i + 1] below to get information about a + // particular stack. + // + // There must be exactly one more `VMStackLimits` object than there + // are continuations, due to the initial stack having one, too. + assert_eq!(stack_limits_vec.len(), continuations_vec.len() + 1); + + for i in 0..continuations_vec.len() { + let (continuation, parent_continuation, parent_limits) = unsafe { + // The continuation whose control context we want to + // access, to get information about how to continue + // execution in its parent. + let continuation = &*continuations_vec[i]; + + // The stack limits describing the parent of `continuation`. + let parent_limits = &*stack_limits_vec[i + 1]; + + // The parent of `continuation`, if the parent is itself a + // continuation. Otherwise, if `continuation` is the last + // continuation (i.e., its parent is the initial stack), this is + // None. + let parent_continuation = if i + 1 < continuations_vec.len() { + Some(&*continuations_vec[i + 1]) + } else { + None + }; + (continuation, parent_continuation, parent_limits) + }; + let fiber_stack = continuation.fiber_stack(); + let resume_pc = fiber_stack.control_context_instruction_pointer(); + let resume_fp = fiber_stack.control_context_frame_pointer(); + + // If the parent is indeed a continuation, we know the + // boundaries of its stack and can perform some extra debugging + // checks. + let parent_stack_range = parent_continuation.and_then(|p| p.fiber_stack().range()); + parent_stack_range.inspect(|parent_stack_range| { + debug_assert!(parent_stack_range.contains(&resume_fp)); + debug_assert!(parent_stack_range.contains(&parent_limits.last_wasm_entry_fp)); + debug_assert!(parent_stack_range.contains(&parent_limits.stack_limit)); + }); + + Self::trace_through_wasm( + unwind, + resume_pc, + resume_fp, + parent_limits.last_wasm_entry_fp, + &mut f, + )? + } + ControlFlow::Continue(()) + } + /// Walk through a contiguous sequence of Wasm frames starting with the /// frame at the given PC and FP and ending at `trampoline_sp`. unsafe fn trace_through_wasm( diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 49563c4e4a22..1a868737fbaa 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -7,6 +7,7 @@ pub use self::vm_host_func_context::VMArrayCallHostFuncContext; use crate::prelude::*; use crate::runtime::vm::{GcStore, InterpreterRef, VMGcRef, VmPtr, VmSafe, f32x4, f64x2, i8x16}; use crate::store::StoreOpaque; +use crate::vm::stack_switching::VMStackChain; use core::cell::UnsafeCell; use core::ffi::c_void; use core::fmt; @@ -515,7 +516,7 @@ impl VMGlobalDefinition { global.init_gc_ref(store.gc_store_mut()?, r.as_ref()) } WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(), - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => *global.as_func_ref_mut() = raw.get_funcref().cast(), // TODO(#10248): temporary hack. }, } Ok(global) @@ -1078,6 +1079,10 @@ pub struct VMStoreContext { /// Used to find the end of a contiguous sequence of Wasm frames when /// walking the stack. pub last_wasm_entry_fp: UnsafeCell, + + /// Stack information used by stack switching instructions. See documentation + /// on `VMStackChain` for details. + pub stack_chain: UnsafeCell, } // The `VMStoreContext` type is a pod-type with no destructor, and we don't @@ -1103,6 +1108,7 @@ impl Default for VMStoreContext { last_wasm_exit_fp: UnsafeCell::new(0), last_wasm_exit_pc: UnsafeCell::new(0), last_wasm_entry_fp: UnsafeCell::new(0), + stack_chain: UnsafeCell::new(VMStackChain::Absent), } } } diff --git a/crates/winch/Cargo.toml b/crates/winch/Cargo.toml index 6be5ed09bbf8..b966107b829b 100644 --- a/crates/winch/Cargo.toml +++ b/crates/winch/Cargo.toml @@ -31,5 +31,6 @@ all-arch = ["winch-codegen/all-arch"] gc = ['winch-codegen/gc'] gc-drc = ['winch-codegen/gc-drc'] gc-null = ['winch-codegen/gc-null'] +stack-switching = ['winch-codegen/stack-switching'] threads = ['winch-codegen/threads'] wmemcheck = ['winch-codegen/wmemcheck'] diff --git a/tests/all/main.rs b/tests/all/main.rs index b8f1c98d0909..5676647b9b08 100644 --- a/tests/all/main.rs +++ b/tests/all/main.rs @@ -43,6 +43,7 @@ mod stack_overflow; mod store; mod structs; mod table; +#[cfg(all(feature = "stack-switching", unix, target_arch = "x86_64"))] mod tags; mod threads; mod traps; diff --git a/winch/codegen/Cargo.toml b/winch/codegen/Cargo.toml index e60429d66af0..d8b87aca2d9f 100644 --- a/winch/codegen/Cargo.toml +++ b/winch/codegen/Cargo.toml @@ -38,5 +38,6 @@ all-arch = [ gc = ['wasmtime-environ/gc'] gc-drc = ['wasmtime-environ/gc-drc'] gc-null = ['wasmtime-environ/gc-null'] +stack-switching = ['wasmtime-environ/stack-switching'] threads = ['wasmtime-environ/threads'] wmemcheck = ['wasmtime-environ/wmemcheck']