cranelift: per-table mutability tracking + call_indirect elisions on immutable funcref tables

crates/cranelift/src/func_environ.rs

@@ -33,8 +33,9 @@ use wasmtime_environ::{
     EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey,
     GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module,
     ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex,
-    TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,
-    WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,
+    TableInitialValue, TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets,
+    WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult,
+    WasmStorageType, WasmValType,
 };
 use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};
 
@@ -1656,20 +1657,26 @@ impl FuncEnvironment<'_> {
             self.reference_type(table.ref_type.heap_type).0.bytes()
         };
 
+        // A table is fixed-size if min == max or if translation proved it
+        // is never mutated. Either way, the base address and bound are
+        // constant for the instance's lifetime, so `base_gv` can be
+        // `readonly + can_move` and the bound check becomes a static
+        // compare against `table.limits.min`.
+        let immutable = !self.translation.module.tables_mutated[index];
+        let fixed_size = immutable || Some(table.limits.min) == table.limits.max;
+
         let base_gv = func.create_global_value(ir::GlobalValueData::Load {
             base: ptr,
             offset: Offset32::new(base_offset),
             global_type: pointer_type,
-            flags: if Some(table.limits.min) == table.limits.max {
-                // A fixed-size table can't be resized so its base address won't
-                // change.
+            flags: if fixed_size {
                 MemFlagsData::trusted().with_readonly().with_can_move()
             } else {
                 MemFlagsData::trusted()
             },
         });
 
-        let bound = if Some(table.limits.min) == table.limits.max {
+        let bound = if fixed_size {
             TableSize::Static {
                 bound: table.limits.min,
             }
@@ -1941,6 +1948,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
         callee: ir::Value,
         call_args: &[ir::Value],
     ) -> WasmResult<Option<CallRets>> {
+        // Fast path: if we can statically resolve this indirect call to a
+        // single defined function (immutable funcref table + constant
+        // callee index + matching signature), emit a direct call instead.
+        // See `try_static_resolve_indirect_call`.
+        if let Some(target) = self.try_static_resolve_indirect_call(table_index, ty_index, callee) {
+            return self.direct_call(target, sig_ref, call_args).map(Some);
+        }
+
         let (code_ptr, callee_vmctx) = match self.check_and_load_code_and_callee_vmctx(
             features,
             table_index,
@@ -1956,6 +1971,163 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
             .map(Some)
     }
 
+    /// Try to statically resolve a `call_indirect` to a direct call.
+    /// Succeeds when (1) the table is immutable, (2) the callee index is an
+    /// `iconst`, (3) the slot is precomputable from a static `elem`
+    /// segment, and (4) the resolved function's signature matches the
+    /// site's declared `ty_index`.
+    fn try_static_resolve_indirect_call(
+        &self,
+        table_index: TableIndex,
+        ty_index: TypeIndex,
+        callee: ir::Value,
+    ) -> Option<FuncIndex> {
+        let translation = self.env.translation;
+        let module = &translation.module;
+
+        // (1) Table must be provably immutable. Imports are pre-marked
+        //     mutated by `ModuleEnvironment::translate`.
+        if translation.module.tables_mutated[table_index] {
+            return None;
+        }
+        let defined_table = module.defined_table_index(table_index)?;
+
+        // (2) Callee must be a constant `iconst`.
+        let dfg = &self.builder.func.dfg;
+        let inst = dfg.value_def(callee).inst()?;
+        let imm = match dfg.insts[inst] {
+            ir::InstructionData::UnaryImm {
+                opcode: ir::Opcode::Iconst,
+                imm,
+            } => imm,
+            _ => return None,
+        };
+        let callee_ty = dfg.value_type(callee);
+        let callee_idx_u64 = imm
+            .zero_extend_from_width(callee_ty.bits())
+            .bits()
+            .cast_unsigned();
+
+        // (3) Slot must be precomputable.
+        let init = module
+            .table_initialization
+            .initial_values
+            .get(defined_table)?;
+        let precomputed = match init {
+            TableInitialValue::Null { precomputed } => precomputed,
+            // A fully-expression-driven initializer can't be resolved at
+            // compile time. Bail.
+            TableInitialValue::Expr(_) => return None,
+        };
+        let slot = usize::try_from(callee_idx_u64).ok()?;
+        if slot >= precomputed.len() {
+            return None;
+        }
+        let target = precomputed[slot];
+        // `FuncIndex::reserved_value()` is the "no entry" sentinel —
+        // this slot wasn't covered by any static `elem` segment.
+        if target.is_reserved_value() {
+            return None;
+        }
+
+        // (4) Signature match. The site's declared `ty_index` and the
+        //     target function's declared signature must intern to the same
+        //     module type index.
+        let expected_ty = module.types[ty_index].unwrap_module_type_index();
+        let target_ty = module.functions[target]
+            .signature
+            .unwrap_module_type_index();
+        if expected_ty != target_ty {
+            return None;
+        }
+
+        Some(target)
+    }
+
+    /// True iff `table_index`'s precomputed `elem` contents cover every
+    /// in-bounds slot with a concrete `FuncIndex`. Caller must have proven
+    /// the table is immutable; when this is true, the funcref-NULL check
+    /// on the call_indirect hot path is provably redundant (bounds checks
+    /// still emit and trap on OOB).
+    fn precomputed_table_has_no_null_slots(&self, table_index: TableIndex) -> bool {
+        let module = &self.env.translation.module;
+        let Some(defined_table) = module.defined_table_index(table_index) else {
+            return false;
+        };
+        let Some(init) = module
+            .table_initialization
+            .initial_values
+            .get(defined_table)
+        else {
+            return false;
+        };
+        let precomputed = match init {
+            TableInitialValue::Null { precomputed } => precomputed,
+            TableInitialValue::Expr(_) => return false,
+        };
+        if precomputed.is_empty() {
+            return false;
+        }
+        // Slots beyond `precomputed.len()` are null at runtime. Coverage
+        // up to `limits.min` is required (caller proved immutable, so the
+        // table can't be grown beyond min).
+        let table_min = module.tables[table_index].limits.min;
+        if (precomputed.len() as u64) < table_min {
+            return false;
+        }
+        precomputed.iter().all(|f| !f.is_reserved_value())
+    }
+
+    fn try_elide_sig_check_for_immutable_table(
+        &self,
+        table_index: TableIndex,
+        ty_index: TypeIndex,
+    ) -> bool {
+        let translation = self.env.translation;
+        let module = &translation.module;
+
+        if translation.module.tables_mutated[table_index] {
+            return false;
+        }
+        let defined_table = match module.defined_table_index(table_index) {
+            Some(d) => d,
+            None => return false,
+        };
+
+        let init = match module
+            .table_initialization
+            .initial_values
+            .get(defined_table)
+        {
+            Some(i) => i,
+            None => return false,
+        };
+        let precomputed = match init {
+            TableInitialValue::Null { precomputed } => precomputed,
+            TableInitialValue::Expr(_) => return false,
+        };
+
+        if precomputed.is_empty() {
+            return false;
+        }
+
+        let expected_ty = module.types[ty_index].unwrap_module_type_index();
+        for &func_idx in precomputed.iter() {
+            // Null slots will trap on the funcref-NULL load anyway.
+            if func_idx.is_reserved_value() {
+                continue;
+            }
+            let actual_ty = module.functions[func_idx]
+                .signature
+                .unwrap_module_type_index();
+            if actual_ty != expected_ty {
+                return false;
+            }
+        }
+
+        true
+    }
+
     fn check_and_load_code_and_callee_vmctx(
         &mut self,
         features: &WasmFeatures,
@@ -2018,6 +2190,20 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
         // table of typed functions and that type matches `ty_index`, then
         // there's no need to perform a typecheck.
         match table.ref_type.heap_type {
+            // Untyped `funcref` tables normally need a runtime sig check.
+            // Elide it when the table is immutable and every precomputed
+            // slot has the same signature as the call site's `ty_index`.
+            WasmHeapType::Func
+                if self.try_elide_sig_check_for_immutable_table(table_index, ty_index) =>
+            {
+                // If every slot is also non-null, drop `may_be_null` so
+                // the funcref-NULL check is elided too. Sound only under
+                // the immutability check above.
+                let may_be_null = table.ref_type.nullable
+                    && !self.precomputed_table_has_no_null_slots(table_index);
+                return CheckIndirectCallTypeSignature::StaticMatch { may_be_null };
+            }
+
             // Functions do not have a statically known type in the table, a
             // typecheck is required. Fall through to below to perform the
             // actual typecheck.

crates/environ/src/compile/module_environ.rs

@@ -247,6 +247,17 @@ impl<'a, 'data> ModuleEnvironment<'a, 'data> {
             self.translate_payload(payload?)?;
         }
 
+        // Precompute static funcref-table contents from `elem` segments
+        // before Cranelift lowering, so the `call_indirect` optimizations
+        // in `func_environ.rs` see the populated `precomputed` lists.
+        // Must run before `analyze_table_mutability` so that pass can
+        // mark tables with leftover (unfoldable) segments as mutated.
+        if self.tunables.table_lazy_init {
+            self.result.try_func_table_init();
+        }
+
+        analyze_table_mutability(&mut self.result)?;
+
         Ok(self.result)
     }
 
@@ -1358,3 +1369,67 @@ impl ModuleTranslation<'_> {
         self.module.table_initialization.segments = segments.try_collect().panic_on_oom();
     }
 }
+
+/// Populate `translation.module.tables_mutated`: a table is marked mutated
+/// if it's imported, exported, the destination of any `table.set` /
+/// `table.fill` / `table.copy` / `table.grow` / `table.init` opcode in any
+/// defined function body, or the target of an active `elem` segment that
+/// wasn't folded into `initial_values[t].precomputed` by
+/// `try_func_table_init` (`try_func_table_init` must have already run).
+fn analyze_table_mutability<'data>(translation: &mut ModuleTranslation<'data>) -> Result<()> {
+    let num_tables = translation.module.tables.len();
+    if num_tables == 0 {
+        return Ok(());
+    }
+
+    // Imports: caller can mutate via wasmtime API. Exports: host can
+    // mutate the same way. Both are conservatively non-stable.
+    let num_imported = translation.module.num_imported_tables;
+    for i in 0..num_imported {
+        translation.module.tables_mutated[TableIndex::from_u32(i as u32)] = true;
+    }
+    for (_, entity_index) in &translation.module.exports {
+        if let EntityIndex::Table(table_index) = entity_index {
+            translation.module.tables_mutated[*table_index] = true;
+        }
+    }
+
+    // Walk function bodies for mutation opcodes. O(total opcodes).
+    for (_, body_data) in &translation.function_body_inputs {
+        let mut reader = body_data.body.get_operators_reader()?;
+        while !reader.eof() {
+            use wasmparser::Operator;
+            match reader.read()? {
+                Operator::TableSet { table }
+                | Operator::TableFill { table }
+                | Operator::TableGrow { table } => {
+                    translation.module.tables_mutated[TableIndex::from_u32(table)] = true;
+                }
+                Operator::TableCopy {
+                    dst_table,
+                    src_table: _,
+                } => {
+                    // `src_table` is read-only in `table.copy`; only the
+                    // destination is mutated.
+                    translation.module.tables_mutated[TableIndex::from_u32(dst_table)] = true;
+                }
+                Operator::TableInit {
+                    table,
+                    elem_index: _,
+                } => {
+                    translation.module.tables_mutated[TableIndex::from_u32(table)] = true;
+                }
+                _ => {}
+            }
+        }
+    }
+
+    // Active `elem` segments left in the segments list (not folded into
+    // `precomputed` by `try_func_table_init`) run at instantiation and
+    // can overwrite slots arbitrarily.
+    for segment in translation.module.table_initialization.segments.iter() {
+        translation.module.tables_mutated[segment.table_index] = true;
+    }
+
+    Ok(())
+}