perf: outpace CPython across the benchmark suite

.github/workflows/ci.yml

@@ -231,7 +231,13 @@ jobs:
         # rust-cache still primes dependency compilation, and the
         # merge-base is usually a recent main commit CI has built
         # before, so this step is mostly cache hits.
+        #
+        # The reference binary is only timed, so a lint that a newer
+        # toolchain added (one the merge-base predates) mustn't fail
+        # the build: lints are capped at warnings here.
         shell: bash
+        env:
+          RUSTFLAGS: "--cap-lints=warn"
         run: |
           BASE_SHA=$(git merge-base "${{ github.event.pull_request.base.sha }}" HEAD)
           echo "merge-base: $BASE_SHA"
@@ -263,6 +269,7 @@ jobs:
           python tools/bench_cold_jit.py \
             --base "$BASE_WEAVEPY" --new target/release/weavepy.exe \
             --samples 7 \
+            --fixtures sumvm:2000000,nested_loops:120,jitloop:1000,call_overhead:150000 \
             --out "target/performance/cold-jit-ci/${{ github.run_id }}-${{ github.run_attempt }}/report.json"
       - uses: actions/upload-artifact@v4
         if: always() && steps.cold_jit_diagnostic.outcome != 'skipped' && steps.cold_jit_diagnostic.outcome != ''

Cargo.toml

@@ -288,6 +288,8 @@ manual_map              = "allow"
 comparison_chain        = "allow"
 unnecessary_cast        = "allow"
 len_without_is_empty    = "allow"
+# New in Clippy 1.99; tests across the workspace assert emptiness this way.
+assert_is_empty         = "allow"
 # Interpreter compares IEEE-754 values exactly by design (Python `==`,
 # rounding, struct packing); an epsilon would be incorrect.
 float_cmp               = "allow"

crates/weavepy-capi/src/gc_bridge.rs

@@ -48,8 +48,8 @@ fn instance_type_ptr(obj: &Object) -> Option<*mut PyTypeObject> {
         // An instance that never crossed into C (no box, no inline body)
         // has no C-held edges for `tp_traverse`/`tp_clear` to report or
         // drop: the VM traverses its own fields. This keeps the bridge —
-        // a mint, a C call and a free per node — off the prompt-reap
-        // cascade of every native exception instance.
+        // a mint, a C call and a free per node — off every collection
+        // that visits a native exception instance.
         Object::Instance(i) if i.c_body.get() != 0 => crate::types::type_ptr_for_class(&i.cls()),
         _ => None,
     }
@@ -212,7 +212,7 @@ pub unsafe extern "C" fn PyObject_GC_Track(op: *mut c_void) {
     }
     let obj = unsafe { crate::object::clone_object(op as *mut PyObject) };
     if matches!(obj, Object::Instance(_)) {
-        weavepy_vm::gc_trace::track(obj);
+        weavepy_vm::gc_trace::track(&obj);
     }
 }
 

crates/weavepy-capi/src/instance.rs

@@ -329,30 +329,10 @@ pub unsafe fn release_c_ownership(p: *mut PyObject) {
         .lock()
         .ok()
         .and_then(|mut g| g.as_mut().and_then(|m| m.remove(&(p as usize))));
-    if let Some(inst) = &pinned {
+    if pinned.is_some() {
         // C just dropped what may be this instance's last program-visible
-        // reference (CPython would run `tp_dealloc` right here). If the VM
-        // side still pins it through GC-handle / weakref-registry clones, a
-        // plain `Rc` drop below cannot reap it — a *tracked* extension
-        // temporary that dies inside a C call (a Cython generator abandoned
-        // by `BlockManager.iget`, still holding `self` in its closure) would
-        // otherwise stay pinned by its own handle until the next full
-        // collection, keeping everything it references alive with it
-        // (pandas' `_is_view_after_cow_rules` then reads a stale live
-        // `Block` weakref). Park it for the eval loop's between-bytecodes
-        // reap; anything still genuinely alive fails the drain's
-        // refcount-dead test untouched.
-        //
-        // Only a *GC-tracked* instance needs the park: an untracked one has
-        // no handle pinning it, so the plain `Rc` drop below reclaims it
-        // immediately (running `tp_dealloc` through the free hook) — and a
-        // queued clone would instead keep it alive until the next eval-loop
-        // safe point, which never comes for a drop performed outside
-        // bytecode execution (an embedding host dropping its last handle).
-        let obj = weavepy_vm::object::Object::Instance(inst.clone());
-        if weavepy_vm::gc_trace::is_tracked(weavepy_vm::weakref_registry::id_of(&obj)) {
-            weavepy_vm::vm_singletons::queue_cext_dropped(&obj);
-        }
+        // reference; the plain `Rc` drop below reclaims it then (running
+        // `tp_dealloc` through the free hook), as CPython's would.
     } else if unsafe { crate::mirror::is_orphaned_instance_body(p) }
         && !body_free_in_flight(p as usize)
     {
@@ -618,8 +598,8 @@ unsafe fn dealloc_and_free_body(p: *mut PyObject) {
                     (*ty).tp_basicsize = orig_basicsize.wrapping_add(8);
                     // A `tp_dealloc` is extension code: its decref chains can
                     // re-enter the VM, and any bytecode that runs beneath it
-                    // must see a live C frame (RFC 0047 — the prompt reaper's
-                    // borrowed-pointer window).
+                    // must see a live C frame (RFC 0047 — the borrowed-pointer
+                    // window that defers parked C drops).
                     let _cext_guard = weavepy_vm::vm_singletons::enter_cext_call();
                     // Consent protocol: the dealloc must reach a `tp_free`
                     // (absorbed by `PyObject_Free`/`PyObject_GC_Del`, which

crates/weavepy-capi/src/interp.rs

@@ -145,11 +145,10 @@ pub fn ensure_active<R>(body: impl FnOnce() -> R) -> R {
     // RFC 0047 (wave 5): mark a C-extension call live on this thread for
     // the duration of `body`. Every bridged C invocation (dunder shims,
     // `PyCFunction`s, descriptors, tp_call, foreign hooks) funnels through
-    // here, so the VM's prompt reaper can tell "plain bytecode, no
-    // extension frame below" (`depth == 0` — reclaiming a dead escaped
-    // subgraph is as safe as CPython's own `tp_dealloc`) from "inside a C
-    // call that may hold borrowed body pointers" (`depth > 0` — defer
-    // memory reclamation to the tracing collector).
+    // here, so the VM can tell "plain bytecode, no extension frame below"
+    // (`depth == 0` — releasing objects C dropped is as safe as CPython's
+    // own `tp_dealloc`) from "inside a C call that may hold borrowed body
+    // pointers" (`depth > 0` — keep them parked).
     let _cext_guard = weavepy_vm::vm_singletons::enter_cext_call();
     if current_interpreter_mut().is_some() {
         if ea_trace_enabled() {

crates/weavepy-capi/src/object.rs

@@ -1659,14 +1659,13 @@ pub(crate) unsafe fn free_box(p: *mut PyObject) {
         evict_property_box(weavepy_vm::sync::Rc::as_ptr(rc) as usize, p);
     }
     // RFC 0047 (wave 5): C is dropping what may be the payload's last
-    // program-visible reference, from inside an extension call the VM's
-    // prompt reaper cannot see. Park instance/container payloads for a
-    // refcount-guarded reap at the next eval-loop safe point so anything
-    // that died here gets its weakrefs cleared with CPython's timing
-    // (pandas' `BlockValuesRefs.has_reference` prunes dead block
-    // weakrefs the very next Python-level call after a Cython-internal
-    // `self.blocks = ...` rebind). A payload that is still alive
-    // elsewhere fails the reap's deadness test untouched.
+    // program-visible reference, from inside an extension call that may
+    // still hold borrowed body pointers. Park instance/container payloads
+    // and release them at the next eval-loop safe point outside any C
+    // call, so anything that dies there gets its weakrefs cleared before
+    // the next Python-level read (pandas' `BlockValuesRefs.has_reference`
+    // prunes dead block weakrefs right after a Cython-internal
+    // `self.blocks = ...` rebind).
     weavepy_vm::vm_singletons::queue_cext_dropped(&bx.payload.obj);
     if let Some(d) = bx.payload.destructor {
         let raw = Box::into_raw(bx);

crates/weavepy-capi/src/types.rs

@@ -2585,7 +2585,7 @@ fn inherit_inline_base_layout(
     };
 
     let bases = t.bases.borrow();
-    let first_ptr = bases.first().and_then(&resolve);
+    let first_ptr = bases.first().and_then(resolve);
 
     // CPython's `best_base()`: the "solid base" is the base with the widest
     // instance layout (`tp_basicsize`), scanned across *all* bases — not

crates/weavepy-compiler/src/cpython_code.rs

@@ -1480,7 +1480,7 @@ fn encode_exception_table(code: &CodeObject, starts: &[usize]) -> Vec<u8> {
 /// depths for inlined comprehensions; entries the walk never reaches
 /// stay -1. Handlers whose depth is still the sentinel do not seed
 /// the walk (their depth is exactly what's being computed).
-pub(crate) fn compute_startdepths(code: &CodeObject) -> Vec<i64> {
+pub fn compute_startdepths(code: &CodeObject) -> Vec<i64> {
     use OpCode as O;
     let n = code.instructions.len();
     let mut startdepth: Vec<i64> = vec![-1; n];

crates/weavepy-compiler/src/flowgraph.rs

@@ -3092,7 +3092,7 @@ impl Cfg {
         }
         co.instructions = instructions;
         co.linetable = linetable;
-        co.coltable = coltable;
+        co.coltable = coltable.into();
         co.exception_table = table;
         co.constants = self.consts;
         no_interrupt.sort_unstable();

crates/weavepy-compiler/src/lib.rs

@@ -280,6 +280,14 @@ impl JitHint {
         true
     }
 
+    /// Whether the tier-2 state has no further interest in this code's
+    /// back edges (see [`Self::set_backedge_quiet`]).
+    #[inline]
+    pub fn is_backedge_quiet(&self) -> bool {
+        self.backedge_quiet
+            .load(std::sync::atomic::Ordering::Relaxed)
+    }
+
     pub fn set_backedge_quiet(&self) {
         self.backedge_quiet
             .store(true, std::sync::atomic::Ordering::Relaxed);
@@ -504,8 +512,9 @@ pub struct CodeObject {
     /// PEP-657 fine-grained column spans, one per instruction (same length
     /// as `instructions` once emission finishes). Drives the column fields
     /// of `co_positions()`. Empty when never populated (e.g. code objects
-    /// reconstructed from marshal, which doesn't carry columns).
-    pub coltable: Vec<ColSpan>,
+    /// reconstructed from marshal, which doesn't carry columns). Decoded
+    /// from the code cache on first use (see [`ColTable`]).
+    pub coltable: ColTable,
     /// Number of positional + keyword arguments (excluding `*args`/`**kwargs`).
     pub arg_count: u32,
     /// Number of positional-only arguments.
@@ -625,6 +634,74 @@ pub struct WireOverrides {
 
 /// A per-instruction source-column span (PEP-657). `col`/`end_col` are
 /// 0-based UTF-8 byte offsets within their respective source lines, and
+/// A code object's column spans (see [`CodeObject::coltable`]): a
+/// vector, or the code cache's encoding of one, decoded on first use.
+/// Most code never reports a column (only tracebacks and
+/// `co_positions()` read them), so a module loaded from the cache skips
+/// the work until something does.
+#[derive(Clone, Default)]
+pub struct ColTable {
+    spans: std::sync::OnceLock<Vec<ColSpan>>,
+    encoded: Option<std::sync::Arc<[u8]>>,
+}
+
+impl ColTable {
+    /// The spans `encoded` holds in the code cache's form (see
+    /// `native_code`), decoded when first read.
+    pub(crate) fn encoded(encoded: std::sync::Arc<[u8]>) -> Self {
+        ColTable {
+            spans: std::sync::OnceLock::new(),
+            encoded: Some(encoded),
+        }
+    }
+
+    fn spans(&self) -> &Vec<ColSpan> {
+        self.spans.get_or_init(|| {
+            self.encoded
+                .as_deref()
+                .and_then(native_code::decode_coltable)
+                .unwrap_or_default()
+        })
+    }
+}
+
+impl From<Vec<ColSpan>> for ColTable {
+    fn from(spans: Vec<ColSpan>) -> Self {
+        ColTable {
+            spans: std::sync::OnceLock::from(spans),
+            encoded: None,
+        }
+    }
+}
+
+impl std::ops::Deref for ColTable {
+    type Target = Vec<ColSpan>;
+
+    fn deref(&self) -> &Vec<ColSpan> {
+        self.spans()
+    }
+}
+
+impl std::ops::DerefMut for ColTable {
+    fn deref_mut(&mut self) -> &mut Vec<ColSpan> {
+        self.spans();
+        self.encoded = None;
+        self.spans.get_mut().expect("decoded above")
+    }
+}
+
+impl PartialEq for ColTable {
+    fn eq(&self, other: &Self) -> bool {
+        **self == **other
+    }
+}
+
+impl std::fmt::Debug for ColTable {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        (**self).fmt(f)
+    }
+}
+
 /// are `-1` when the column was not tracked. `end_lineno` is `0` when
 /// unknown (callers fall back to the instruction's start line).
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -2623,7 +2700,8 @@ impl Compiler {
             }
         };
         self.co.linetable.push(line);
-        self.co.coltable.push(self.resolve_colspan());
+        let span = self.resolve_colspan();
+        self.co.coltable.push(span);
         offset
     }
 

crates/weavepy-compiler/src/native_code.rs

@@ -16,7 +16,7 @@ use crate::bytecode::{CacheTable, Instruction, OpCode};
 use crate::{CodeObject, ColSpan, Constant, ExcHandler};
 
 /// The layout revision; bump it whenever the encoding changes.
-pub const VERSION: u8 = 2;
+pub const VERSION: u8 = 3;
 
 /// Encode `code` (and its nested code objects). `None` for a code object
 /// the format doesn't carry: one with raw CPython wire overrides, or with
@@ -28,6 +28,21 @@ pub fn encode(code: &CodeObject) -> Option<Vec<u8>> {
     Some(w.0)
 }
 
+/// Decode a column-span block [`encode`] wrote (see `ColTable`).
+pub(crate) fn decode_coltable(bytes: &[u8]) -> Option<Vec<ColSpan>> {
+    let mut r = Reader { bytes, pos: 0 };
+    let end_lines = r.deltas()?;
+    let mut spans = Vec::with_capacity(end_lines.len());
+    for end_lineno in end_lines {
+        spans.push(ColSpan {
+            end_lineno,
+            col: r.i32()?,
+            end_col: r.i32()?,
+        });
+    }
+    (r.pos == bytes.len()).then_some(spans)
+}
+
 /// Decode what [`encode`] wrote, stamping `filename` on every code
 /// object. `None` for input it didn't write (or a different version's).
 pub fn decode(bytes: &[u8], filename: &str) -> Option<CodeObject> {
@@ -116,11 +131,15 @@ impl Writer {
         // Line numbers as deltas from the previous entry: mostly zero, so
         // one byte each whatever the line.
         self.deltas(c.linetable.iter().copied());
-        self.deltas(c.coltable.iter().map(|s| s.end_lineno));
-        for s in &c.coltable {
-            self.int(i64::from(s.col));
-            self.int(i64::from(s.end_col));
+        // The column spans as one length-prefixed block, which a reader
+        // keeps encoded until something reads a column (see `ColTable`).
+        let mut cols = Writer(Vec::new());
+        cols.deltas(c.coltable.iter().map(|s| s.end_lineno));
+        for s in c.coltable.iter() {
+            cols.int(i64::from(s.col));
+            cols.int(i64::from(s.end_col));
         }
+        self.bytes(&cols.0);
         self.uint(u64::from(c.arg_count));
         self.uint(u64::from(c.posonly_count));
         self.uint(u64::from(c.kwonly_count));
@@ -360,15 +379,7 @@ impl Reader<'_> {
             });
         }
         let linetable = self.deltas()?;
-        let end_lines = self.deltas()?;
-        let mut coltable = Vec::with_capacity(end_lines.len());
-        for end_lineno in end_lines {
-            coltable.push(ColSpan {
-                end_lineno,
-                col: self.i32()?,
-                end_col: self.i32()?,
-            });
-        }
+        let coltable = crate::ColTable::encoded(Arc::from(self.raw()?));
         let arg_count = self.u32()?;
         let posonly_count = self.u32()?;
         let kwonly_count = self.u32()?;
@@ -472,7 +483,7 @@ mod tests {
             constants: vec![Constant::Int(-5)],
             varnames: vec!["x".to_owned()],
             linetable: vec![1, 2, 2],
-            coltable: vec![ColSpan::default(); 3],
+            coltable: vec![ColSpan::default(); 3].into(),
             arg_count: 1,
             is_generator: true,
             is_nested: true,