Lower fused adapter trap intrinsics to CLIF trap instructions

crates/cranelift/src/compiler/component.rs

@@ -689,13 +689,10 @@ impl<'a> TrampolineCompiler<'a> {
                     |_, _| {},
                 );
             }
-            Trampoline::Trap => {
-                self.translate_libcall(
-                    host::trap,
-                    TrapSentinel::Falsy,
-                    WasmArgs::InRegisters,
-                    |_, _| {},
-                );
+            Trampoline::Trap(code) => {
+                let code = crate::env_trap_to_clif_trap(*code);
+                let (mut traps, builder) = self.traps();
+                traps.trap(builder, code);
             }
             Trampoline::EnterSyncCall => {
                 self.translate_libcall(
@@ -1491,7 +1488,7 @@ impl<'a> TrampolineCompiler<'a> {
             | Trampoline::FutureTransfer
             | Trampoline::StreamTransfer
             | Trampoline::ErrorContextTransfer
-            | Trampoline::Trap
+            | Trampoline::Trap(_)
             | Trampoline::EnterSyncCall
             | Trampoline::ExitSyncCall
             | Trampoline::Transcoder { .. } => return,

crates/cranelift/src/func_environ.rs

@@ -9,7 +9,7 @@ use crate::translate::{
 };
 use crate::trap::TranslateTrap;
 use crate::{
-    BuiltinFunctionSignatures, TRAP_ARRAY_OUT_OF_BOUNDS, TRAP_GC_HEAP_CORRUPT,
+    BuiltinFunctionSignatures, Reachability, TRAP_ARRAY_OUT_OF_BOUNDS, TRAP_GC_HEAP_CORRUPT,
     TRAP_TABLE_OUT_OF_BOUNDS,
 };
 use cranelift_codegen::cursor::FuncCursor;
@@ -1807,7 +1807,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
         callee_index: FuncIndex,
         sig_ref: ir::SigRef,
         wasm_call_args: &[ir::Value],
-    ) -> WasmResult<CallRets> {
+    ) -> WasmResult<Reachability<CallRets>> {
         let mut real_call_args = Vec::with_capacity(wasm_call_args.len() + 2);
         let caller_vmctx = self
             .builder
@@ -1831,7 +1831,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
             let callee = self
                 .env
                 .get_or_create_defined_func_ref(self.builder.func, def_func_index);
-            return Ok(self.direct_call_inst(callee, &real_call_args));
+            return Ok(Reachability::Reachable(
+                self.direct_call_inst(callee, &real_call_args),
+            ));
         }
 
         // Handle direct calls to imported functions. We use an indirect call
@@ -1873,9 +1875,11 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
                     let result = intrinsic_compiler
                         .translate(*intrinsic, &real_call_args)
                         .unwrap();
-                    Ok(result.into_iter().collect())
+                    Ok(Reachability::Reachable(result.into_iter().collect()))
                 } else {
-                    Ok(self.direct_call_inst(callee, &real_call_args))
+                    Ok(Reachability::Reachable(
+                        self.direct_call_inst(callee, &real_call_args),
+                    ))
                 }
             }
 
@@ -1887,37 +1891,54 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
                 let callee = self
                     .env
                     .get_or_create_imported_func_ref(self.builder.func, callee_index);
-                Ok(self.direct_call_inst(callee, &real_call_args))
+                Ok(Reachability::Reachable(
+                    self.direct_call_inst(callee, &real_call_args),
+                ))
             }
 
-            // The guest-to-guest sync fast path: these adapter intrinsics are
-            // lowered inline rather than called, but only when concurrency
-            // support is enabled (the deferred thread state only exists then)
-            // and this isn't a tail call (the deferred frame must outlive the
-            // call). Otherwise fall back to the indirect call, which is also
-            // the out-of-line slow path the inline `exit` branches to.
+            // Fused adapter intrinsics that are lowered inline rather than
+            // called.
             Some(KnownFunc::FactIntrinsic(intrinsic)) => {
-                if self.env.tunables.concurrency_support {
-                    debug_assert!(!self.tail);
-                    match intrinsic {
-                        FactInlineIntrinsic::EnterSyncCall => {
-                            return Ok(self.lower_fact_enter_sync_call(&real_call_args));
-                        }
-                        FactInlineIntrinsic::ExitSyncCall => {
-                            return Ok(self.lower_fact_exit_sync_call(
-                                callee_index,
-                                sig_ref,
-                                &real_call_args,
-                            ));
-                        }
+                match intrinsic {
+                    FactInlineIntrinsic::Trap(trap) => {
+                        self.env
+                            .trap(self.builder, crate::env_trap_to_clif_trap(*trap));
+                        return Ok(Reachability::Unreachable);
                     }
+
+                    // The guest-to-guest sync fast path: these adapter
+                    // intrinsics are lowered inline rather than called, but
+                    // only when concurrency support is enabled (the deferred
+                    // thread state only exists then) and this isn't a tail call
+                    // (the deferred frame must outlive the call). Otherwise
+                    // fall back to the indirect call, which is also the
+                    // out-of-line slow path the inline `exit` branches to.
+                    FactInlineIntrinsic::EnterSyncCall if self.env.tunables.concurrency_support => {
+                        debug_assert!(!self.tail);
+                        return Ok(Reachability::Reachable(
+                            self.lower_fact_enter_sync_call(&real_call_args),
+                        ));
+                    }
+                    FactInlineIntrinsic::ExitSyncCall if self.env.tunables.concurrency_support => {
+                        debug_assert!(!self.tail);
+                        return Ok(Reachability::Reachable(self.lower_fact_exit_sync_call(
+                            callee_index,
+                            sig_ref,
+                            &real_call_args,
+                        )));
+                    }
+                    FactInlineIntrinsic::EnterSyncCall | FactInlineIntrinsic::ExitSyncCall => {}
                 }
                 let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(
                     &mut self.builder.cursor(),
                     vmctx,
                     callee_index,
                 );
-                Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))
+                Ok(Reachability::Reachable(self.indirect_call_inst(
+                    sig_ref,
+                    func_addr,
+                    &real_call_args,
+                )))
             }
 
             Some(key) => panic!("unexpected kind of known-import function: {key:?}"),
@@ -1931,7 +1952,11 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
                     vmctx,
                     callee_index,
                 );
-                Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))
+                Ok(Reachability::Reachable(self.indirect_call_inst(
+                    sig_ref,
+                    func_addr,
+                    &real_call_args,
+                )))
             }
         }
     }
@@ -3406,14 +3431,16 @@ impl FuncEnvironment<'_> {
         )
     }
 
+    /// Returns `None` when the call was lowered to an unconditional trap and so
+    /// everything after it is unreachable. See `Call::direct_call`.
     pub fn translate_call<'a>(
         &mut self,
         builder: &'a mut FunctionBuilder,
         srcloc: ir::SourceLoc,
         callee_index: FuncIndex,
         sig_ref: ir::SigRef,
         call_args: &[ir::Value],
-    ) -> WasmResult<CallRets> {
+    ) -> WasmResult<Reachability<CallRets>> {
         Call::new(builder, self, srcloc).direct_call(callee_index, sig_ref, call_args)
     }
 
@@ -3436,7 +3463,8 @@ impl FuncEnvironment<'_> {
         sig_ref: ir::SigRef,
         call_args: &[ir::Value],
     ) -> WasmResult<()> {
-        Call::new_tail(builder, self, srcloc).direct_call(callee_index, sig_ref, call_args)?;
+        let _ =
+            Call::new_tail(builder, self, srcloc).direct_call(callee_index, sig_ref, call_args)?;
         Ok(())
     }
 
@@ -6088,7 +6116,10 @@ impl FuncEnvironment<'_> {
             .signature
             .unwrap_module_type_index();
         let sig_ref = self.get_or_create_interned_sig_ref(builder.func, ty);
-        self.translate_call(builder, Default::default(), func, sig_ref, &[])?;
+        match self.translate_call(builder, Default::default(), func, sig_ref, &[])? {
+            Reachability::Reachable(_) => {}
+            Reachability::Unreachable => return Ok(()),
+        }
         if self.tunables.consume_fuel {
             self.fuel_load_into_var(builder);
         }

crates/cranelift/src/lib.rs

@@ -78,6 +78,16 @@ pub const TRAP_CAST_FAILURE: TrapCode =
 pub const TRAP_UNCAUGHT_EXCEPTION: TrapCode =
     TrapCode::unwrap_user(Trap::UncaughtException as u8 + TRAP_OFFSET);
 
+/// The CLIF trap code for a Wasmtime trap code.
+///
+/// This is the inverse of `clif_trap_to_env_trap`'s fallback arm, and is what
+/// all of the `TRAP_*` constants above compute for their particular trap. Use
+/// it for traps that don't have a constant above, e.g. the trap named by a
+/// fused adapter's `trap` intrinsic.
+const fn env_trap_to_clif_trap(trap: Trap) -> TrapCode {
+    TrapCode::unwrap_user(trap as u8 + TRAP_OFFSET)
+}
+
 /// Creates a new cranelift `Signature` with no wasm params/results for the
 /// given calling convention.
 ///

crates/cranelift/src/translate/code_translator.rs

@@ -715,13 +715,16 @@ pub fn translate_operator(
             let mut args = environ.stacks.peekn(num_args).to_vec();
             bitcast_wasm_params(environ, sig_ref, &mut args, builder);
 
-            let inst_results = environ.translate_call(
-                builder,
-                environ.next_srcloc,
-                function_index,
-                sig_ref,
-                &args,
-            )?;
+            let inst_results = unwrap_or_return_unreachable_state!(
+                environ,
+                environ.translate_call(
+                    builder,
+                    environ.next_srcloc,
+                    function_index,
+                    sig_ref,
+                    &args,
+                )?
+            );
 
             debug_assert_eq!(
                 inst_results.len(),

crates/environ/src/compile/module_environ.rs

@@ -8,7 +8,7 @@ use crate::{
     EngineOrModuleTypeIndex, EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType,
     MemoryIndex, MemoryInitializer, ModuleInternedTypeIndex, ModuleStartup, ModuleTypesBuilder,
     PanicOnOom as _, PassiveElemIndex, PrimaryMap, RuntimeDataIndex, StaticModuleIndex, TableIndex,
-    TableInitialValue, TableInitialization, Tag, TagIndex, Tunables, TypeConvert, TypeIndex,
+    TableInitialValue, TableInitialization, Tag, TagIndex, Trap, Tunables, TypeConvert, TypeIndex,
     WasmHeapTopType, WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter,
 };
 use alloc::borrow::Cow;
@@ -47,6 +47,8 @@ pub enum FactInlineIntrinsic {
     /// fall back to the out-of-line `exit-sync-call` libcall when the thread
     /// was promoted.
     ExitSyncCall,
+    /// `trap`: raise the given trap.
+    Trap(Trap),
 }
 
 /// A statically-known function import.

crates/environ/src/component.rs

@@ -202,8 +202,6 @@ macro_rules! foreach_builtin_component_function {
             #[cfg(feature = "component-model-async")]
             thread_yield_then_promote(vmctx: vmctx, caller_instance: u32, cancellable: u8, thread_idx: u32) -> u32;
 
-            trap(vmctx: vmctx, code: u32) -> bool;
-
             utf8_to_utf8(vmctx: vmctx, src: ptr_u8, len: size, dst: ptr_u8) -> bool;
             utf16_to_utf16(vmctx: vmctx, src: ptr_u16, len: size, dst: ptr_u16) -> bool;
             latin1_to_latin1(vmctx: vmctx, src: ptr_u8, len: size, dst: ptr_u8) -> bool;

crates/environ/src/component/dfg.rs

@@ -30,7 +30,7 @@
 use crate::component::*;
 use crate::error::Result;
 use crate::prelude::*;
-use crate::{EntityIndex, EntityRef, ModuleInternedTypeIndex, PrimaryMap, WasmValType};
+use crate::{EntityIndex, EntityRef, ModuleInternedTypeIndex, PrimaryMap, Trap, WasmValType};
 use cranelift_entity::packed_option::PackedOption;
 use indexmap::IndexMap;
 use info::LinearMemoryOptions;
@@ -469,7 +469,7 @@ pub enum Trampoline {
     FutureTransfer,
     StreamTransfer,
     ErrorContextTransfer,
-    Trap,
+    Trap(Trap),
     EnterSyncCall,
     ExitSyncCall,
     ThreadIndex {
@@ -1155,7 +1155,7 @@ impl LinearizeDfg<'_> {
             Trampoline::FutureTransfer => info::Trampoline::FutureTransfer,
             Trampoline::StreamTransfer => info::Trampoline::StreamTransfer,
             Trampoline::ErrorContextTransfer => info::Trampoline::ErrorContextTransfer,
-            Trampoline::Trap => info::Trampoline::Trap,
+            Trampoline::Trap(trap) => info::Trampoline::Trap(*trap),
             Trampoline::EnterSyncCall => info::Trampoline::EnterSyncCall,
             Trampoline::ExitSyncCall => info::Trampoline::ExitSyncCall,
             Trampoline::ThreadIndex { instance } => info::Trampoline::ThreadIndex {

crates/environ/src/component/info.rs

@@ -48,7 +48,7 @@
 
 use crate::component::*;
 use crate::prelude::*;
-use crate::{EntityIndex, ModuleInternedTypeIndex, PrimaryMap, WasmValType};
+use crate::{EntityIndex, ModuleInternedTypeIndex, PrimaryMap, Trap, WasmValType};
 use cranelift_entity::packed_option::PackedOption;
 use serde_derive::{Deserialize, Serialize};
 
@@ -1097,9 +1097,9 @@ pub enum Trampoline {
     /// component does not invalidate the handle in the original component.
     ErrorContextTransfer,
 
-    /// An intrinsic used by FACT-generated modules to trap with a specified
+    /// An intrinsic used by FACT-generated modules to trap with the specified
     /// code.
-    Trap,
+    Trap(Trap),
 
     /// An intrinsic used by FACT-generated modules to push a task onto the
     /// stack for a sync-to-sync, guest-to-guest call.
@@ -1247,7 +1247,7 @@ impl Trampoline {
             FutureTransfer => format!("future-transfer"),
             StreamTransfer => format!("stream-transfer"),
             ErrorContextTransfer => format!("error-context-transfer"),
-            Trap => format!("trap"),
+            Trap(trap) => format!("trap-{}", *trap as u8),
             EnterSyncCall => format!("enter-sync-call"),
             ExitSyncCall => format!("exit-sync-call"),
             ThreadIndex { .. } => format!("thread-index"),

crates/environ/src/component/translate.rs

@@ -635,6 +635,7 @@ impl<'a, 'data> Translator<'a, 'data> {
                     CoreDef::Trampoline(index) => match translation.trampolines[*index] {
                         Trampoline::EnterSyncCall => FactInlineIntrinsic::EnterSyncCall.into(),
                         Trampoline::ExitSyncCall => FactInlineIntrinsic::ExitSyncCall.into(),
+                        Trampoline::Trap(trap) => FactInlineIntrinsic::Trap(trap).into(),
                         _ => continue,
                     },
 

crates/environ/src/component/translate/adapt.rs

@@ -348,7 +348,7 @@ fn fact_import_to_core_def(
         fact::Import::ErrorContextTransfer => {
             simple_intrinsic(dfg::Trampoline::ErrorContextTransfer)
         }
-        fact::Import::Trap => simple_intrinsic(dfg::Trampoline::Trap),
+        fact::Import::Trap(trap) => simple_intrinsic(dfg::Trampoline::Trap(*trap)),
         fact::Import::EnterSyncCall => simple_intrinsic(dfg::Trampoline::EnterSyncCall),
         fact::Import::ExitSyncCall => simple_intrinsic(dfg::Trampoline::ExitSyncCall),
     }