Inline {enter,exit}_sync_call trampolines into sync-to-sync fused adapters (#13695)

* Add tests for guest-to-guest sync-call context preservation

* Add `FuncKey::FactInlineIntrinsic` for inline FACT adapter intrinsics

* Add `VMLazyThread`/`VMDeferredThread` to `VMStoreContext`

* Route host current-thread reads through `StoreOpaque::current_thread`

* Promote deferred guest threads on demand in `current_thread`

* Inline `{enter,exit}_sync_call` trampoline calls in sync adapters

This commit inlines fast paths for the `{enter,exit}_sync_call` component
trampolines in sync-to-sync fused adapters. For cross-component calls of trivial
functions, after inlining, we should avoid all calls in the fused adapter (other
than the `trap` libcall when `may_leave` is set).

The inlining happens during translation, rather than via the regular inliner,
since the lazy thread is stack-allocated within the fused adapter; there would
otherwise be no easy way to reference it.

Fixes #12311

* cargo fmt

* first set of review feedback

* Move the reset-current-thread-on-trap into the array-to-wasm trampoline

* fix compilation after rebase

* fix alias regions after rebase, use existing `vm_store_ctx`

* fix more disas tests after rebase

* fix ci

* cargo fmt

Author: fitzgen

crates/cranelift/src/alias_region.rs

@@ -21,6 +21,7 @@ enum VmType {
     VMMemoryDefinition,
     VMTableDefinition,
     VMComponentContext,
+    VMDeferredThread,
 }
 
 /// A key that uniquely identifies an alias region across an entire compilation.
@@ -111,6 +112,7 @@ impl AliasRegionKey {
     const VM_MEMORY_DEFINITION_KIND: u32 = Self::new_kind(0b1001);
     const VM_TABLE_DEFINITION_KIND: u32 = Self::new_kind(0b1010);
     const VM_COMPONENT_CONTEXT_KIND: u32 = Self::new_kind(0b1011);
+    const VM_DEFERRED_THREAD_KIND: u32 = Self::new_kind(0b1100);
 
     /// Encode this key into a raw `u32` suitable for use as an
     /// `AliasRegionData::user_id`.
@@ -124,6 +126,7 @@ impl AliasRegionKey {
                     VmType::VMMemoryDefinition => Self::VM_MEMORY_DEFINITION_KIND,
                     VmType::VMTableDefinition => Self::VM_TABLE_DEFINITION_KIND,
                     VmType::VMComponentContext => Self::VM_COMPONENT_CONTEXT_KIND,
+                    VmType::VMDeferredThread => Self::VM_DEFERRED_THREAD_KIND,
                 };
                 kind | (offset & Self::OFFSET_MASK)
             }
@@ -1099,6 +1102,44 @@ where
         )
     }
 
+    /// Load the `VMStoreContext::current_thread` field (the JIT-visible
+    /// deferred-thread pointer; see `VMLazyThread`).
+    pub fn vmstore_context_current_thread(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmstore_ctx: ir::Value,
+    ) -> ir::Value {
+        self.vmstore_context_load(
+            cursor,
+            self.pointer_type,
+            ir::MemFlagsData::trusted(),
+            vmstore_ctx,
+            self.offsets
+                .get_ptr_size()
+                .vmstore_context_current_thread()
+                .into(),
+        )
+    }
+
+    /// Store the `VMStoreContext::current_thread` field.
+    pub fn store_vmstore_context_current_thread(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmstore_ctx: ir::Value,
+        new_thread: ir::Value,
+    ) {
+        self.vmstore_context_store(
+            cursor,
+            ir::MemFlagsData::trusted(),
+            vmstore_ctx,
+            self.offsets
+                .get_ptr_size()
+                .vmstore_context_current_thread()
+                .into(),
+            new_thread,
+        )
+    }
+
     /// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).
     ///
     /// The caller supplies the base flags because whether the base pointer is
@@ -1333,6 +1374,194 @@ where
     }
 }
 
+/// `VMDeferredThread`-related methods.
+impl<Offsets> AliasRegions<Offsets>
+where
+    Offsets: GetPtrSize,
+{
+    fn vmdeferred_thread_region(
+        &mut self,
+        func: &mut ir::Function,
+        offset: u32,
+    ) -> ir::AliasRegion {
+        self.region(
+            func,
+            AliasRegionKey::Vm {
+                ty: VmType::VMDeferredThread,
+                offset,
+            },
+        )
+    }
+
+    fn vmdeferred_thread_load(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        ty: ir::Type,
+        base_flags: ir::MemFlagsData,
+        vmdeferred_thread_ptr: ir::Value,
+        offset: u32,
+    ) -> ir::Value {
+        let region = self.vmdeferred_thread_region(cursor.func, offset);
+        cursor.ins().load(
+            ty,
+            base_flags.with_alias_region(Some(region)),
+            vmdeferred_thread_ptr,
+            i32::try_from(offset).unwrap(),
+        )
+    }
+
+    fn vmdeferred_thread_store(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        base_flags: ir::MemFlagsData,
+        vmdeferred_thread_ptr: ir::Value,
+        offset: u32,
+        val: ir::Value,
+    ) {
+        let region = self.vmdeferred_thread_region(cursor.func, offset);
+        cursor.ins().store(
+            base_flags.with_alias_region(Some(region)),
+            val,
+            vmdeferred_thread_ptr,
+            i32::try_from(offset).unwrap(),
+        );
+    }
+
+    /// Load `VMDeferredThread::parent` (the current thread this frame replaced).
+    pub fn vmdeferred_thread_parent(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmdeferred_thread_ptr: ir::Value,
+    ) -> ir::Value {
+        self.vmdeferred_thread_load(
+            cursor,
+            self.pointer_type,
+            ir::MemFlagsData::trusted(),
+            vmdeferred_thread_ptr,
+            self.offsets
+                .get_ptr_size()
+                .vmdeferred_thread_parent()
+                .into(),
+        )
+    }
+
+    /// Store `VMDeferredThread::parent`.
+    pub fn store_vmdeferred_thread_parent(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmdeferred_thread_ptr: ir::Value,
+        parent: ir::Value,
+    ) {
+        self.vmdeferred_thread_store(
+            cursor,
+            ir::MemFlagsData::trusted(),
+            vmdeferred_thread_ptr,
+            self.offsets
+                .get_ptr_size()
+                .vmdeferred_thread_parent()
+                .into(),
+            parent,
+        )
+    }
+
+    /// Store `VMDeferredThread::caller_instance`.
+    pub fn store_vmdeferred_thread_caller_instance(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmdeferred_thread_ptr: ir::Value,
+        caller_instance: ir::Value,
+    ) {
+        self.vmdeferred_thread_store(
+            cursor,
+            ir::MemFlagsData::trusted(),
+            vmdeferred_thread_ptr,
+            self.offsets
+                .get_ptr_size()
+                .vmdeferred_thread_caller_instance()
+                .into(),
+            caller_instance,
+        )
+    }
+
+    /// Store `VMDeferredThread::callee_async`.
+    pub fn store_vmdeferred_thread_callee_async(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmdeferred_thread_ptr: ir::Value,
+        callee_async: ir::Value,
+    ) {
+        self.vmdeferred_thread_store(
+            cursor,
+            ir::MemFlagsData::trusted(),
+            vmdeferred_thread_ptr,
+            self.offsets
+                .get_ptr_size()
+                .vmdeferred_thread_callee_async()
+                .into(),
+            callee_async,
+        )
+    }
+
+    /// Store `VMDeferredThread::callee_instance`.
+    pub fn store_vmdeferred_thread_callee_instance(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmdeferred_thread_ptr: ir::Value,
+        callee_instance: ir::Value,
+    ) {
+        self.vmdeferred_thread_store(
+            cursor,
+            ir::MemFlagsData::trusted(),
+            vmdeferred_thread_ptr,
+            self.offsets
+                .get_ptr_size()
+                .vmdeferred_thread_callee_instance()
+                .into(),
+            callee_instance,
+        )
+    }
+
+    /// Load `VMDeferredThread::saved_context[i]` (a saved `context.{get,set}`
+    /// slot).
+    pub fn vmdeferred_thread_saved_context(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmdeferred_thread_ptr: ir::Value,
+        i: u8,
+    ) -> ir::Value {
+        self.vmdeferred_thread_load(
+            cursor,
+            ir::types::I32,
+            ir::MemFlagsData::trusted(),
+            vmdeferred_thread_ptr,
+            self.offsets
+                .get_ptr_size()
+                .vmdeferred_thread_saved_context(i)
+                .into(),
+        )
+    }
+
+    /// Store `VMDeferredThread::saved_context[i]`.
+    pub fn store_vmdeferred_thread_saved_context(
+        &mut self,
+        cursor: &mut FuncCursor<'_>,
+        vmdeferred_thread_ptr: ir::Value,
+        i: u8,
+        val: ir::Value,
+    ) {
+        self.vmdeferred_thread_store(
+            cursor,
+            ir::MemFlagsData::trusted(),
+            vmdeferred_thread_ptr,
+            self.offsets
+                .get_ptr_size()
+                .vmdeferred_thread_saved_context(i)
+                .into(),
+            val,
+        )
+    }
+}
+
 /// `VMMemoryDefinition`-related methods.
 ///
 /// The `base` and `current_length` fields are reached either directly through

crates/cranelift/src/compiler.rs

@@ -1523,6 +1523,20 @@ impl Compiler {
         // succeed". Note that register restoration is part of the `try_call`
         // and handler implementation.
         builder.switch_to_block(exceptional_return);
+        if cfg!(feature = "component-model") && self.tunables().concurrency_support {
+            // NB: A trap unwinds over any fused adapter frames without running
+            // their `exit-sync-call`s. That can leave
+            // `VMStoreContext::current_thread` pointing at an old, invalid
+            // `VMDeferredThread` inside an old, since-unwound stack
+            // frame. Therefore, we must reset `current_thread` to avoid
+            // potential use-after-free bugs.
+            let forced = builder.ins().iconst(pointer_type, 1);
+            alias_regions.store_vmstore_context_current_thread(
+                &mut builder.cursor(),
+                vm_store_ctx,
+                forced,
+            );
+        }
         let false_return = builder.ins().iconst(ir::types::I8, 0);
         builder.ins().return_(&[false_return]);