feat: add experimental tailcall vm backend

fix-vm/src/lib.rs

@@ -1,4 +1,9 @@
 #![warn(clippy::unwrap_used)]
+#![cfg_attr(feature = "tailcall", expect(incomplete_features))]
+#![cfg_attr(
+    feature = "tailcall",
+    feature(explicit_tail_calls, rust_preserve_none_cc)
+)]
 
 use std::path::PathBuf;
 
@@ -12,16 +17,17 @@ use hashbrown::HashMap;
 use num_enum::TryFromPrimitive;
 use smallvec::SmallVec;
 
-mod bytecode_reader;
 mod boxing;
+mod bytecode_reader;
+#[cfg(feature = "tailcall")]
+mod dispatch_tailcall;
 mod value;
-use value::*;
 pub use value::StaticValue;
-pub(crate) mod instructions;
+use value::*;
 mod helpers;
-use helpers::*;
-
+pub(crate) mod instructions;
 pub(crate) use bytecode_reader::BytecodeReader;
+use helpers::*;
 
 type VmResult<T> = std::result::Result<T, VmError>;
 
@@ -339,11 +345,7 @@ impl<'gc> Vm<'gc> {
         if let Some(thunk) = val.as_gc::<Thunk>() {
             let mut state = thunk.borrow_mut(mc);
             match *state {
-                ThunkState::Pending {
-                    ip,
-                    env,
-                    with_env,
-                } => {
+                ThunkState::Pending { ip, env, with_env } => {
                     *state = ThunkState::Blackhole;
                     drop(state);
                     StepResult::ForceThunk(ForceInfo {
@@ -381,6 +383,20 @@ impl<'gc> Vm<'gc> {
             other => Some(other),
         }
     }
+
+    #[inline(always)]
+    pub(crate) fn apply_force_thunk(&mut self, info: ForceInfo<'gc>) -> usize {
+        self.call_stack.push(CallFrame {
+            thunk: Some(info.thunk),
+            stack_depth: info.stack_depth,
+            pc: info.inst_start_pc,
+            env: self.env,
+            with_env: self.with_env,
+        });
+        self.env = info.env;
+        self.with_env = info.with_env;
+        info.ip
+    }
 }
 
 #[allow(dead_code)]
@@ -423,7 +439,7 @@ impl Vm<'_> {
         let mut pc = ip.0;
         let bytecode: Vec<u8> = ctx.bytecode().to_vec();
         loop {
-            match arena.mutate_root(|mc, root| root.execute_batch(&bytecode, &mut ctx, pc, mc)) {
+            match arena.mutate_root(|mc, root| root.dispatch_batch(&bytecode, &mut ctx, pc, mc)) {
                 Action::Continue { pc: new_pc } => {
                     pc = new_pc;
                     if arena.metrics().allocation_debt() > COLLECTOR_GRANULARITY {
@@ -442,6 +458,37 @@ impl Vm<'_> {
 
 impl<'gc> Vm<'gc> {
     #[inline(always)]
+    fn dispatch_batch<C: VmContext>(
+        &mut self,
+        bytecode: &[u8],
+        ctx: &mut C,
+        pc: usize,
+        mc: &Mutation<'gc>,
+    ) -> Action {
+        #[cfg(not(feature = "tailcall"))]
+        {
+            self.execute_batch(bytecode, ctx, pc, mc)
+        }
+        #[cfg(feature = "tailcall")]
+        {
+            use crate::dispatch_tailcall::{TailResult, run_tailcall};
+            match run_tailcall(self, mc, ctx, bytecode, pc as u32) {
+                TailResult::YieldFuel(new_pc) => Action::Continue {
+                    pc: new_pc as usize,
+                },
+                TailResult::ForceThunk(info) => {
+                    let new_pc = self.apply_force_thunk(info);
+                    Action::Continue { pc: new_pc }
+                }
+                TailResult::Done => {
+                    Action::Done(self.result.take().expect("TailResult::Done without result"))
+                }
+            }
+        }
+    }
+
+    #[inline(always)]
+    #[cfg(not(feature = "tailcall"))]
     fn execute_batch(
         &mut self,
         bytecode: &[u8],
@@ -537,16 +584,8 @@ impl<'gc> Vm<'gc> {
             match result {
                 StepResult::Continue => {}
                 StepResult::ForceThunk(info) => {
-                    self.call_stack.push(CallFrame {
-                        thunk: Some(info.thunk),
-                        stack_depth: info.stack_depth,
-                        pc: info.inst_start_pc,
-                        env: self.env,
-                        with_env: self.with_env,
-                    });
-                    reader.set_pc(info.ip);
-                    self.env = info.env;
-                    self.with_env = info.with_env;
+                    let new_pc = self.apply_force_thunk(info);
+                    reader.set_pc(new_pc);
                 }
                 StepResult::Done => {
                     return Action::Done(